Skip to content

Commit 45f0661

Browse files
crusaderkycrusaderky
committed
Repeat A/B tests
1 parent 57f01ac commit 45f0661

10 files changed

Lines changed: 499 additions & 442 deletions

File tree

‎.github/workflows/ab_tests.yml‎

Lines changed: 56 additions & 140 deletions
Original file line numberDiff line numberDiff line change
@@ -20,34 +20,44 @@ jobs:
2020
name: Discover A/B environments
2121
runs-on: ubuntu-latest
2222
steps:
23-
- uses: actions/checkout@v2
23+
- name: Checkout
24+
uses: actions/checkout@v2
2425
with:
2526
fetch-depth: 0
26-
- uses: actions/setup-python@v4
27+
28+
- name: Install Python
29+
uses: actions/setup-python@v4
2730
with:
2831
python-version: '3.10'
29-
- id: set-matrix
32+
33+
- name: Install dependencies
34+
run: pip install PyYaml
35+
36+
- name: Generate dynamic matrix
37+
id: set-matrix
3038
run: echo "::set-output name=matrix::$(python ci/scripts/discover_ab_environments.py)"
39+
3140
outputs:
32-
matrix: ${{ steps.set-matrix.outputs.matrix }}
41+
matrix: ${{ steps.set-matrix.outputs.matrix }}
3342

3443
# Everything below this point runs iff there are files matching
35-
# AB_environments/AB_*.conda.yaml
36-
# AB_environments/AB_*.dask.yaml
44+
# AB_environments/AB_*.{conda,dask}.yaml
45+
# and AB_environments/config.yaml set repeat > 0
3746

3847
software:
39-
name: Setup
48+
name: Setup - ${{ matrix.runtime-version }} py${{ matrix.python-version }}
4049
runs-on: ubuntu-latest
4150
needs: discover_ab_envs
42-
if: ${{ fromJson(needs.discover_ab_envs.outputs.matrix) }}
51+
if: ${{ fromJson(needs.discover_ab_envs.outputs.matrix).runtime }}
4352
strategy:
4453
fail-fast: false
4554
matrix:
4655
python-version: ["3.9"]
47-
runtime-version: ${{ fromJson(needs.discover_ab_envs.outputs.matrix) }}
56+
runtime-version: ${{ fromJson(needs.discover_ab_envs.outputs.matrix).runtime }}
4857

4958
steps:
50-
- uses: actions/checkout@v2
59+
- name: Checkout
60+
uses: actions/checkout@v2
5161
with:
5262
fetch-depth: 0
5363

@@ -60,6 +70,13 @@ jobs:
6070
python-version: ${{ matrix.python-version }}
6171
environment-file: ci/environment.yml
6272

73+
- name: Create null hypothesis as a copy of baseline
74+
if: matrix.runtime-version == 'AB_null_hypothesis'
75+
run: |
76+
cd AB_environments
77+
cp AB_baseline.conda.yaml AB_null_hypothesis.conda.yaml
78+
cp AB_baseline.dask.yaml AB_null_hypothesis.dask.yaml
79+
6380
- name: Build Coiled Software Environment
6481
env:
6582
DASK_COILED__TOKEN: ${{ secrets.COILED_BENCHMARK_BOT_TOKEN }}
@@ -101,8 +118,8 @@ jobs:
101118
software_name.txt
102119
test_upstream.txt
103120
104-
runtime:
105-
name: Runtime - ${{ matrix.os }}, Python ${{ matrix.python-version }}, Runtime ${{ matrix.runtime-version }}
121+
tests:
122+
name: A/B Tests - ${{ matrix.category }} ${{ matrix.runtime-version }} ${{ matrix.os }} py${{ matrix.python-version }}
106123
needs: [discover_ab_envs, software]
107124
runs-on: ${{ matrix.os }}
108125
timeout-minutes: 120
@@ -111,10 +128,13 @@ jobs:
111128
matrix:
112129
os: [ubuntu-latest]
113130
python-version: ["3.9"]
114-
runtime-version: ${{ fromJson(needs.discover_ab_envs.outputs.matrix) }}
131+
category: ${{ fromJson(needs.discover_ab_envs.outputs.matrix).category }}
132+
runtime-version: ${{ fromJson(needs.discover_ab_envs.outputs.matrix).runtime }}
133+
repeat: ${{ fromJson(needs.discover_ab_envs.outputs.matrix).repeat }}
115134

116135
steps:
117-
- uses: actions/checkout@v2
136+
- name: Checkout
137+
uses: actions/checkout@v2
118138
with:
119139
fetch-depth: 0
120140

@@ -128,7 +148,6 @@ jobs:
128148
environment-file: ci/environment.yml
129149

130150
- name: Download software environment assets
131-
if: matrix.runtime-version == 'latest' || startsWith(matrix.runtime-version, 'AB_')
132151
uses: actions/download-artifact@v3
133152
with:
134153
name: software-environment-${{ matrix.runtime-version }}-py${{ matrix.python-version }}
@@ -145,140 +164,32 @@ jobs:
145164
AWS_ACCESS_KEY_ID: ${{ secrets.RUNTIME_CI_BOT_AWS_ACCESS_KEY_ID }}
146165
AWS_SECRET_ACCESS_KEY: ${{ secrets.RUNTIME_CI_BOT_AWS_SECRET_ACCESS_KEY }}
147166
COILED_RUNTIME_VERSION: ${{ matrix.runtime-version }}
148-
DB_NAME: runtime-${{ matrix.os }}-${{ matrix.runtime-version }}-py${{ matrix.python-version }}.db
149-
BENCHMARK: true
150-
run: bash ci/scripts/run_tests.sh tests/runtime
151-
152-
- name: Upload benchmark results
153-
uses: actions/upload-artifact@v3
154-
if: always()
155-
with:
156-
name: runtime-${{ matrix.os }}-${{ matrix.runtime-version }}-py${{ matrix.python-version }}
157-
path: runtime-${{ matrix.os }}-${{ matrix.runtime-version }}-py${{ matrix.python-version }}.db
158-
159-
benchmarks:
160-
name: Benchmarks - ${{ matrix.os }}, Python ${{ matrix.python-version }}, Runtime ${{ matrix.runtime-version }}
161-
needs: [discover_ab_envs, software]
162-
runs-on: ${{ matrix.os }}
163-
timeout-minutes: 120
164-
strategy:
165-
fail-fast: false
166-
matrix:
167-
os: [ubuntu-latest]
168-
python-version: ["3.9"]
169-
runtime-version: ${{ fromJson(needs.discover_ab_envs.outputs.matrix) }}
170-
171-
steps:
172-
- uses: actions/checkout@v2
173-
with:
174-
fetch-depth: 0
175-
176-
- name: Set up environment
177-
uses: conda-incubator/setup-miniconda@v2
178-
with:
179-
miniforge-variant: Mambaforge
180-
use-mamba: true
181-
condarc-file: ci/condarc
182-
python-version: ${{ matrix.python-version }}
183-
environment-file: ci/environment.yml
184-
185-
- name: Download software environment assets
186-
uses: actions/download-artifact@v3
187-
with:
188-
name: software-environment-${{ matrix.runtime-version }}-py${{ matrix.python-version }}
189-
190-
- name: Install coiled-runtime
191-
env:
192-
COILED_RUNTIME_VERSION: ${{ matrix.runtime-version }}
193-
run: source ci/scripts/install_coiled_runtime.sh
194-
195-
- name: Run benchmarking tests
196-
id: benchmarking_tests
197-
env:
198-
DASK_COILED__TOKEN: ${{ secrets.COILED_BENCHMARK_BOT_TOKEN }}
199-
AWS_ACCESS_KEY_ID: ${{ secrets.RUNTIME_CI_BOT_AWS_ACCESS_KEY_ID }}
200-
AWS_SECRET_ACCESS_KEY: ${{ secrets.RUNTIME_CI_BOT_AWS_SECRET_ACCESS_KEY }}
201-
COILED_RUNTIME_VERSION: ${{ matrix.runtime-version }}
202-
DB_NAME: benchmark-${{ matrix.os }}-${{ matrix.runtime-version }}-py${{ matrix.python-version }}.db
203-
BENCHMARK: true
204-
run: bash ci/scripts/run_tests.sh tests/benchmarks
205-
206-
- name: Upload benchmark results
207-
uses: actions/upload-artifact@v3
208-
if: always()
209-
with:
210-
name: benchmark-${{ matrix.os }}-${{ matrix.runtime-version }}-py${{ matrix.python-version }}
211-
path: benchmark-${{ matrix.os }}-${{ matrix.runtime-version }}-py${{ matrix.python-version }}.db
212-
213-
stability:
214-
name: Stability - ${{ matrix.os }}, Python ${{ matrix.python-version }}, Runtime ${{ matrix.runtime-version }}
215-
needs: [discover_ab_envs, software]
216-
runs-on: ${{ matrix.os }}
217-
timeout-minutes: 120
218-
strategy:
219-
fail-fast: false
220-
matrix:
221-
os: [ubuntu-latest]
222-
python-version: ["3.9"]
223-
runtime-version: ${{ fromJson(needs.discover_ab_envs.outputs.matrix) }}
224-
225-
steps:
226-
- uses: actions/checkout@v2
227-
with:
228-
fetch-depth: 0
229-
230-
- name: Set up environment
231-
uses: conda-incubator/setup-miniconda@v2
232-
with:
233-
miniforge-variant: Mambaforge
234-
use-mamba: true
235-
condarc-file: ci/condarc
236-
python-version: ${{ matrix.python-version }}
237-
environment-file: ci/environment.yml
238-
239-
- name: Download software environment assets
240-
if: matrix.runtime-version == 'latest' || startsWith(matrix.runtime-version, 'AB_')
241-
uses: actions/download-artifact@v3
242-
with:
243-
name: software-environment-${{ matrix.runtime-version }}-py${{ matrix.python-version }}
244-
245-
- name: Install coiled-runtime
246-
env:
247-
COILED_RUNTIME_VERSION: ${{ matrix.runtime-version }}
248-
run: source ci/scripts/install_coiled_runtime.sh
249-
250-
- name: Run stability tests
251-
id: stability_tests
252-
env:
253-
DASK_COILED__TOKEN: ${{ secrets.COILED_BENCHMARK_BOT_TOKEN }}
254-
AWS_ACCESS_KEY_ID: ${{ secrets.RUNTIME_CI_BOT_AWS_ACCESS_KEY_ID }}
255-
AWS_SECRET_ACCESS_KEY: ${{ secrets.RUNTIME_CI_BOT_AWS_SECRET_ACCESS_KEY }}
256-
COILED_RUNTIME_VERSION: ${{ matrix.runtime-version }}
257-
DB_NAME: stability-${{ matrix.os }}-${{ matrix.runtime-version }}-py${{ matrix.python-version }}.db
167+
DB_NAME: ${{ matrix.category }}-${{ matrix.os }}-${{ matrix.runtime-version }}-${{ matrix.repeat }}-py${{ matrix.python-version }}.db
258168
BENCHMARK: true
259169
CLUSTER_DUMP: true
260-
run: bash ci/scripts/run_tests.sh tests/stability
170+
run: bash ci/scripts/run_tests.sh tests/${{ matrix.category }}
261171

262172
- name: Upload benchmark results
263173
uses: actions/upload-artifact@v3
264174
if: always()
265175
with:
266-
name: stability-${{ matrix.os }}-${{ matrix.runtime-version }}-py${{ matrix.python-version }}
267-
path: stability-${{ matrix.os }}-${{ matrix.runtime-version }}-py${{ matrix.python-version }}.db
176+
name: ${{ matrix.category }}-${{ matrix.os }}-${{ matrix.runtime-version }}-${{ matrix.repeat }}-py${{ matrix.python-version }}
177+
path: ${{ matrix.category }}-${{ matrix.os }}-${{ matrix.runtime-version }}-${{ matrix.repeat }}-py${{ matrix.python-version }}.db
268178

269179
cleanup:
270-
needs: [discover_ab_envs, software, runtime, benchmarks, stability]
271-
if: always() && ${{ fromJson(needs.discover_ab_envs.outputs.matrix) }}
180+
needs: [discover_ab_envs, software, tests]
181+
if: always() && ${{ fromJson(needs.discover_ab_envs.outputs.matrix).runtime }}
272182
name: Cleanup
273183
runs-on: ubuntu-latest
274184
strategy:
275185
fail-fast: false
276186
matrix:
277187
python-version: ["3.9"]
278-
runtime-version: ${{ fromJson(needs.discover_ab_envs.outputs.matrix) }}
188+
runtime-version: ${{ fromJson(needs.discover_ab_envs.outputs.matrix).runtime }}
279189

280190
steps:
281-
- uses: actions/checkout@v2
191+
- name: Checkout
192+
uses: actions/checkout@v2
282193

283194
- name: Set up Python
284195
uses: actions/setup-python@v4
@@ -302,24 +213,27 @@ jobs:
302213
coiled env delete $SOFTWARE_NAME
303214
304215
process-results:
305-
needs: [discover_ab_envs, runtime, benchmarks, stability]
216+
needs: [discover_ab_envs, tests]
306217
name: Combine separate benchmark results
307-
if: always() && ${{ fromJson(needs.discover_ab_envs.outputs.matrix) }}
218+
if: always() && ${{ fromJson(needs.discover_ab_envs.outputs.matrix).runtime }}
308219
runs-on: ubuntu-latest
309220
concurrency:
310221
# Fairly strict concurrency rule to avoid stepping on benchmark db.
311222
# Could eventually replace with a real db in coiled, RDS, or litestream
312223
group: process-benchmarks
313224
cancel-in-progress: false
314225
steps:
315-
- uses: actions/checkout@v2
226+
- name: Checkout
227+
uses: actions/checkout@v2
316228

317-
- uses: actions/setup-python@v4
229+
- name: Install Python
230+
uses: actions/setup-python@v4
318231

319232
- name: Install dependencies
320233
run: pip install alembic
321234

322-
- uses: actions/download-artifact@v3
235+
- name: Download artifacts
236+
uses: actions/download-artifact@v3
323237
with:
324238
path: benchmarks
325239

@@ -337,15 +251,17 @@ jobs:
337251
static-site:
338252
needs: [discover_ab_envs, process-results]
339253
# Always generate the site, as this can be skipped even if an indirect dependency fails (like a test run)
340-
if: always() && ${{ fromJson(needs.discover_ab_envs.outputs.matrix) }}
254+
if: always() && ${{ fromJson(needs.discover_ab_envs.outputs.matrix).runtime }}
341255
name: Build static dashboards
342256
runs-on: ubuntu-latest
343257
steps:
344-
- uses: actions/checkout@v2
258+
- name: Checkout
259+
uses: actions/checkout@v2
345260
with:
346261
fetch-depth: 0
347262

348-
- uses: actions/download-artifact@v3
263+
- name: Download artifacts
264+
uses: actions/download-artifact@v3
349265
with:
350266
name: benchmark.db
351267

0 commit comments

Comments
 (0)