Skip to content

Commit d7f1d16

Browse files
crusaderkycrusaderky
committed
Repeat A/B tests
1 parent 00d2f08 commit d7f1d16

10 files changed

Lines changed: 131 additions & 63 deletions

File tree

‎.github/workflows/ab_tests.yml‎

Lines changed: 51 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -20,16 +20,25 @@ jobs:
2020
name: Discover A/B environments
2121
runs-on: ubuntu-latest
2222
steps:
23-
- uses: actions/checkout@v2
23+
- name: Checkout
24+
uses: actions/checkout@v2
2425
with:
2526
fetch-depth: 0
26-
- uses: actions/setup-python@v4
27+
28+
- name: Install Python
29+
uses: actions/setup-python@v4
2730
with:
2831
python-version: '3.10'
29-
- id: set-matrix
32+
33+
- name: Install dependencies
34+
run: pip install PyYaml
35+
36+
- name: Generate dynamic matrix
37+
id: set-matrix
3038
run: echo "::set-output name=matrix::$(python ci/scripts/discover_ab_environments.py)"
39+
3140
outputs:
32-
matrix: ${{ steps.set-matrix.outputs.matrix }}
41+
matrix: ${{ steps.set-matrix.outputs.matrix }}
3342

3443
# Everything below this point runs iff there are files matching
3544
# AB_environments/AB_*.conda.yaml
@@ -39,15 +48,16 @@ jobs:
3948
name: Setup
4049
runs-on: ubuntu-latest
4150
needs: discover_ab_envs
42-
if: ${{ fromJson(needs.discover_ab_envs.outputs.matrix) }}
51+
if: ${{ fromJson(needs.discover_ab_envs.outputs.matrix).runtime }}
4352
strategy:
4453
fail-fast: false
4554
matrix:
4655
python-version: ["3.9"]
47-
runtime-version: ${{ fromJson(needs.discover_ab_envs.outputs.matrix) }}
56+
runtime-version: ${{ fromJson(needs.discover_ab_envs.outputs.matrix).runtime }}
4857

4958
steps:
50-
- uses: actions/checkout@v2
59+
- name: Checkout
60+
uses: actions/checkout@v2
5161
with:
5262
fetch-depth: 0
5363

@@ -111,10 +121,12 @@ jobs:
111121
matrix:
112122
os: [ubuntu-latest]
113123
python-version: ["3.9"]
114-
runtime-version: ${{ fromJson(needs.discover_ab_envs.outputs.matrix) }}
124+
runtime-version: ${{ fromJson(needs.discover_ab_envs.outputs.matrix).runtime }}
125+
repeat: ${{ fromJson(needs.discover_ab_envs.outputs.matrix).repeat }}
115126

116127
steps:
117-
- uses: actions/checkout@v2
128+
- name: Checkout
129+
uses: actions/checkout@v2
118130
with:
119131
fetch-depth: 0
120132

@@ -153,8 +165,8 @@ jobs:
153165
uses: actions/upload-artifact@v3
154166
if: always()
155167
with:
156-
name: runtime-${{ matrix.os }}-${{ matrix.runtime-version }}-py${{ matrix.python-version }}
157-
path: runtime-${{ matrix.os }}-${{ matrix.runtime-version }}-py${{ matrix.python-version }}.db
168+
name: runtime-${{ matrix.os }}-${{ matrix.runtime-version }}-${{ matrix.repeat }}-py${{ matrix.python-version }}
169+
path: runtime-${{ matrix.os }}-${{ matrix.runtime-version }}-${{ matrix.repeat }}-py${{ matrix.python-version }}.db
158170

159171
benchmarks:
160172
name: Benchmarks - ${{ matrix.os }}, Python ${{ matrix.python-version }}, Runtime ${{ matrix.runtime-version }}
@@ -166,10 +178,12 @@ jobs:
166178
matrix:
167179
os: [ubuntu-latest]
168180
python-version: ["3.9"]
169-
runtime-version: ${{ fromJson(needs.discover_ab_envs.outputs.matrix) }}
181+
runtime-version: ${{ fromJson(needs.discover_ab_envs.outputs.matrix).runtime }}
182+
repeat: ${{ fromJson(needs.discover_ab_envs.outputs.matrix).repeat }}
170183

171184
steps:
172-
- uses: actions/checkout@v2
185+
- name: Checkout
186+
uses: actions/checkout@v2
173187
with:
174188
fetch-depth: 0
175189

@@ -207,8 +221,8 @@ jobs:
207221
uses: actions/upload-artifact@v3
208222
if: always()
209223
with:
210-
name: benchmark-${{ matrix.os }}-${{ matrix.runtime-version }}-py${{ matrix.python-version }}
211-
path: benchmark-${{ matrix.os }}-${{ matrix.runtime-version }}-py${{ matrix.python-version }}.db
224+
name: benchmark-${{ matrix.os }}-${{ matrix.runtime-version }}-${{ matrix.repeat }}-py${{ matrix.python-version }}
225+
path: benchmark-${{ matrix.os }}-${{ matrix.runtime-version }}-${{ matrix.repeat }}-py${{ matrix.python-version }}.db
212226

213227
stability:
214228
name: Stability - ${{ matrix.os }}, Python ${{ matrix.python-version }}, Runtime ${{ matrix.runtime-version }}
@@ -220,10 +234,12 @@ jobs:
220234
matrix:
221235
os: [ubuntu-latest]
222236
python-version: ["3.9"]
223-
runtime-version: ${{ fromJson(needs.discover_ab_envs.outputs.matrix) }}
237+
runtime-version: ${{ fromJson(needs.discover_ab_envs.outputs.matrix).runtime }}
238+
repeat: ${{ fromJson(needs.discover_ab_envs.outputs.matrix).repeat }}
224239

225240
steps:
226-
- uses: actions/checkout@v2
241+
- name: Checkout
242+
uses: actions/checkout@v2
227243
with:
228244
fetch-depth: 0
229245

@@ -263,22 +279,23 @@ jobs:
263279
uses: actions/upload-artifact@v3
264280
if: always()
265281
with:
266-
name: stability-${{ matrix.os }}-${{ matrix.runtime-version }}-py${{ matrix.python-version }}
267-
path: stability-${{ matrix.os }}-${{ matrix.runtime-version }}-py${{ matrix.python-version }}.db
282+
name: stability-${{ matrix.os }}-${{ matrix.runtime-version }}-${{ matrix.repeat }}-py${{ matrix.python-version }}
283+
path: stability-${{ matrix.os }}-${{ matrix.runtime-version }}-${{ matrix.repeat }}-py${{ matrix.python-version }}.db
268284

269285
cleanup:
270286
needs: [discover_ab_envs, software, runtime, benchmarks, stability]
271-
if: always() && ${{ fromJson(needs.discover_ab_envs.outputs.matrix) }}
287+
if: always() && ${{ fromJson(needs.discover_ab_envs.outputs.matrix).runtime }}
272288
name: Cleanup
273289
runs-on: ubuntu-latest
274290
strategy:
275291
fail-fast: false
276292
matrix:
277293
python-version: ["3.9"]
278-
runtime-version: ${{ fromJson(needs.discover_ab_envs.outputs.matrix) }}
294+
runtime-version: ${{ fromJson(needs.discover_ab_envs.outputs.matrix).runtime }}
279295

280296
steps:
281-
- uses: actions/checkout@v2
297+
- name: Checkout
298+
uses: actions/checkout@v2
282299

283300
- name: Set up Python
284301
uses: actions/setup-python@v4
@@ -304,22 +321,25 @@ jobs:
304321
process-results:
305322
needs: [discover_ab_envs, runtime, benchmarks, stability]
306323
name: Combine separate benchmark results
307-
if: always() && ${{ fromJson(needs.discover_ab_envs.outputs.matrix) }}
324+
if: always() && ${{ fromJson(needs.discover_ab_envs.outputs.matrix).runtime }}
308325
runs-on: ubuntu-latest
309326
concurrency:
310327
# Fairly strict concurrency rule to avoid stepping on benchmark db.
311328
# Could eventually replace with a real db in coiled, RDS, or litestream
312329
group: process-benchmarks
313330
cancel-in-progress: false
314331
steps:
315-
- uses: actions/checkout@v2
332+
- name: Checkout
333+
uses: actions/checkout@v2
316334

317-
- uses: actions/setup-python@v4
335+
- name: Install Python
336+
uses: actions/setup-python@v4
318337

319338
- name: Install dependencies
320339
run: pip install alembic
321340

322-
- uses: actions/download-artifact@v3
341+
- name: Download artifacts
342+
uses: actions/download-artifact@v3
323343
with:
324344
path: benchmarks
325345

@@ -337,15 +357,17 @@ jobs:
337357
static-site:
338358
needs: [discover_ab_envs, process-results]
339359
# Always generate the site, as this can be skipped even if an indirect dependency fails (like a test run)
340-
if: always() && ${{ fromJson(needs.discover_ab_envs.outputs.matrix) }}
360+
if: always() && ${{ fromJson(needs.discover_ab_envs.outputs.matrix).runtime }}
341361
name: Build static dashboards
342362
runs-on: ubuntu-latest
343363
steps:
344-
- uses: actions/checkout@v2
364+
- name: Checkout
365+
uses: actions/checkout@v2
345366
with:
346367
fetch-depth: 0
347368

348-
- uses: actions/download-artifact@v3
369+
- name: Download artifacts
370+
uses: actions/download-artifact@v3
349371
with:
350372
name: benchmark.db
351373

‎.github/workflows/tests.yml‎

Lines changed: 24 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,8 @@ on:
55
- main
66
tags:
77
- '*'
8-
pull_request:
8+
# TODO REVERT
9+
# pull_request:
910
schedule:
1011
# Runs "At 00:01" (see https://crontab.guru)
1112
- cron: "1 0 * * *"
@@ -31,7 +32,8 @@ jobs:
3132
python-version: ["3.8", "3.9", "3.10"]
3233

3334
steps:
34-
- uses: actions/checkout@v2
35+
- name: Checkout
36+
uses: actions/checkout@v2
3537
with:
3638
fetch-depth: 0
3739

@@ -122,7 +124,8 @@ jobs:
122124
runtime-version: ["latest", "0.0.4", "0.1.0"]
123125

124126
steps:
125-
- uses: actions/checkout@v2
127+
- name: Checkout
128+
uses: actions/checkout@v2
126129
with:
127130
fetch-depth: 0
128131

@@ -177,7 +180,8 @@ jobs:
177180
runtime-version: ["latest", "0.0.4", "0.1.0"]
178181

179182
steps:
180-
- uses: actions/checkout@v2
183+
- name: Checkout
184+
uses: actions/checkout@v2
181185
with:
182186
fetch-depth: 0
183187

@@ -239,7 +243,8 @@ jobs:
239243
os: "macos-latest"
240244

241245
steps:
242-
- uses: actions/checkout@v2
246+
- name: Checkout
247+
uses: actions/checkout@v2
243248
with:
244249
fetch-depth: 0
245250

@@ -292,7 +297,8 @@ jobs:
292297
matrix:
293298
python-version: ["3.8", "3.9", "3.10"]
294299
steps:
295-
- uses: actions/checkout@v2
300+
- name: Checkout
301+
uses: actions/checkout@v2
296302

297303
- name: Set up Python
298304
uses: actions/setup-python@v4
@@ -326,14 +332,17 @@ jobs:
326332
group: process-benchmarks
327333
cancel-in-progress: false
328334
steps:
329-
- uses: actions/checkout@v2
335+
- name: Checkout
336+
uses: actions/checkout@v2
330337

331-
- uses: actions/setup-python@v4
338+
- name: Install Python
339+
uses: actions/setup-python@v4
332340

333341
- name: Install dependencies
334342
run: pip install alembic
335343

336-
- uses: actions/download-artifact@v3
344+
- name: Download artifacts
345+
uses: actions/download-artifact@v3
337346
with:
338347
path: benchmarks
339348

@@ -374,7 +383,8 @@ jobs:
374383
name: Detect regressions
375384
runs-on: ubuntu-latest
376385
steps:
377-
- uses: actions/checkout@v2
386+
- name: Checkout
387+
uses: actions/checkout@v2
378388
with:
379389
fetch-depth: 0
380390

@@ -445,11 +455,13 @@ jobs:
445455
name: Build static dashboards
446456
runs-on: ubuntu-latest
447457
steps:
448-
- uses: actions/checkout@v2
458+
- name: Checkout
459+
uses: actions/checkout@v2
449460
with:
450461
fetch-depth: 0
451462

452-
- uses: actions/download-artifact@v3
463+
- name: Download tests database
464+
uses: actions/download-artifact@v3
453465
with:
454466
name: benchmark.db
455467

AB_environments/AB_baseline.conda.yaml.rename_me renamed to AB_environments/AB_baseline.conda.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# Special environment file for A/B testing, used as the baseline environment.
2-
# Change contents as needed and remove the .rename_me suffix.
2+
# Change contents, but do not rename.
33
channels:
44
- conda-forge
55
dependencies:
@@ -14,6 +14,6 @@ dependencies:
1414
# - You can point to your own git fork instead
1515
# For example, if you want to test a PR before it's merged into main, you should
1616
# change this to the dask/dask and/or dask/distributed git tip
17-
- dask==2022.8.1
18-
- distributed=2022.8.1
17+
- dask==2022.9.0
18+
- distributed==2022.9.0
1919
# - git+https://github.com/dask/distributed@dd81b424971e81616e1a52fa09ce4698a5002d41

AB_environments/AB_baseline.dask.yaml.rename_me renamed to AB_environments/AB_baseline.dask.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
# Special environment file for A/B testing, used as the baseline environment.
2-
# Change contents as needed and remove the .rename_me suffix.
2+
# Change contents, but do not rename.
33
# Leave empty if you don't want to override anything.

AB_environments/AB_sample.conda.yaml.rename_me renamed to AB_environments/AB_sample.conda.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,5 @@ dependencies:
1010
- python=3.9
1111
- coiled-runtime=0.1.0
1212
- pip:
13-
- dask==2022.8.1
14-
- git+https://github.com/dask/distributed@dd81b424971e81616e1a52fa09ce4698a5002d41
13+
- dask==2022.9.0
14+
- git+https://github.com/dask/distributed@1fd07f03cacee6fde81d13282568a727bce789b9

AB_environments/AB_sample.dask.yaml.rename_me renamed to AB_environments/AB_sample.dask.yaml

File renamed without changes.

0 commit comments

Comments
 (0)