Skip to content

Commit 2ad2a80

Browse files
crusaderkycrusaderky
committed
Repeat A/B tests
1 parent 00d2f08 commit 2ad2a80

10 files changed

Lines changed: 150 additions & 305 deletions

File tree

‎.github/workflows/ab_tests.yml‎

Lines changed: 49 additions & 140 deletions
Original file line numberDiff line numberDiff line change
@@ -20,34 +20,44 @@ jobs:
2020
name: Discover A/B environments
2121
runs-on: ubuntu-latest
2222
steps:
23-
- uses: actions/checkout@v2
23+
- name: Checkout
24+
uses: actions/checkout@v2
2425
with:
2526
fetch-depth: 0
26-
- uses: actions/setup-python@v4
27+
28+
- name: Install Python
29+
uses: actions/setup-python@v4
2730
with:
2831
python-version: '3.10'
29-
- id: set-matrix
32+
33+
- name: Install dependencies
34+
run: pip install PyYaml
35+
36+
- name: Generate dynamic matrix
37+
id: set-matrix
3038
run: echo "::set-output name=matrix::$(python ci/scripts/discover_ab_environments.py)"
39+
3140
outputs:
32-
matrix: ${{ steps.set-matrix.outputs.matrix }}
41+
matrix: ${{ steps.set-matrix.outputs.matrix }}
3342

3443
# Everything below this point runs iff there are files matching
35-
# AB_environments/AB_*.conda.yaml
36-
# AB_environments/AB_*.dask.yaml
44+
# AB_environments/AB_*.{conda,dask}.yaml
45+
# and AB_environments/config.yaml set repeat > 0
3746

3847
software:
39-
name: Setup
48+
name: Setup - ${{ matrix.runtime-version }} py${{ matrix.python-version }}
4049
runs-on: ubuntu-latest
4150
needs: discover_ab_envs
42-
if: ${{ fromJson(needs.discover_ab_envs.outputs.matrix) }}
51+
if: ${{ fromJson(needs.discover_ab_envs.outputs.matrix).runtime }}
4352
strategy:
4453
fail-fast: false
4554
matrix:
4655
python-version: ["3.9"]
47-
runtime-version: ${{ fromJson(needs.discover_ab_envs.outputs.matrix) }}
56+
runtime-version: ${{ fromJson(needs.discover_ab_envs.outputs.matrix).runtime }}
4857

4958
steps:
50-
- uses: actions/checkout@v2
59+
- name: Checkout
60+
uses: actions/checkout@v2
5161
with:
5262
fetch-depth: 0
5363

@@ -101,8 +111,8 @@ jobs:
101111
software_name.txt
102112
test_upstream.txt
103113
104-
runtime:
105-
name: Runtime - ${{ matrix.os }}, Python ${{ matrix.python-version }}, Runtime ${{ matrix.runtime-version }}
114+
tests:
115+
name: A/B Tests - ${{ matrix.category }} ${{ matrix.runtime-version }} ${{ matrix.os }} py${{ matrix.python-version }}
106116
needs: [discover_ab_envs, software]
107117
runs-on: ${{ matrix.os }}
108118
timeout-minutes: 120
@@ -111,10 +121,13 @@ jobs:
111121
matrix:
112122
os: [ubuntu-latest]
113123
python-version: ["3.9"]
114-
runtime-version: ${{ fromJson(needs.discover_ab_envs.outputs.matrix) }}
124+
category: [runtime, benchmarks, stability]
125+
runtime-version: ${{ fromJson(needs.discover_ab_envs.outputs.matrix).runtime }}
126+
repeat: ${{ fromJson(needs.discover_ab_envs.outputs.matrix).repeat }}
115127

116128
steps:
117-
- uses: actions/checkout@v2
129+
- name: Checkout
130+
uses: actions/checkout@v2
118131
with:
119132
fetch-depth: 0
120133

@@ -128,7 +141,6 @@ jobs:
128141
environment-file: ci/environment.yml
129142

130143
- name: Download software environment assets
131-
if: matrix.runtime-version == 'latest' || startsWith(matrix.runtime-version, 'AB_')
132144
uses: actions/download-artifact@v3
133145
with:
134146
name: software-environment-${{ matrix.runtime-version }}-py${{ matrix.python-version }}
@@ -145,140 +157,32 @@ jobs:
145157
AWS_ACCESS_KEY_ID: ${{ secrets.RUNTIME_CI_BOT_AWS_ACCESS_KEY_ID }}
146158
AWS_SECRET_ACCESS_KEY: ${{ secrets.RUNTIME_CI_BOT_AWS_SECRET_ACCESS_KEY }}
147159
COILED_RUNTIME_VERSION: ${{ matrix.runtime-version }}
148-
DB_NAME: runtime-${{ matrix.os }}-${{ matrix.runtime-version }}-py${{ matrix.python-version }}.db
149-
BENCHMARK: true
150-
run: bash ci/scripts/run_tests.sh tests/runtime
151-
152-
- name: Upload benchmark results
153-
uses: actions/upload-artifact@v3
154-
if: always()
155-
with:
156-
name: runtime-${{ matrix.os }}-${{ matrix.runtime-version }}-py${{ matrix.python-version }}
157-
path: runtime-${{ matrix.os }}-${{ matrix.runtime-version }}-py${{ matrix.python-version }}.db
158-
159-
benchmarks:
160-
name: Benchmarks - ${{ matrix.os }}, Python ${{ matrix.python-version }}, Runtime ${{ matrix.runtime-version }}
161-
needs: [discover_ab_envs, software]
162-
runs-on: ${{ matrix.os }}
163-
timeout-minutes: 120
164-
strategy:
165-
fail-fast: false
166-
matrix:
167-
os: [ubuntu-latest]
168-
python-version: ["3.9"]
169-
runtime-version: ${{ fromJson(needs.discover_ab_envs.outputs.matrix) }}
170-
171-
steps:
172-
- uses: actions/checkout@v2
173-
with:
174-
fetch-depth: 0
175-
176-
- name: Set up environment
177-
uses: conda-incubator/setup-miniconda@v2
178-
with:
179-
miniforge-variant: Mambaforge
180-
use-mamba: true
181-
condarc-file: ci/condarc
182-
python-version: ${{ matrix.python-version }}
183-
environment-file: ci/environment.yml
184-
185-
- name: Download software environment assets
186-
uses: actions/download-artifact@v3
187-
with:
188-
name: software-environment-${{ matrix.runtime-version }}-py${{ matrix.python-version }}
189-
190-
- name: Install coiled-runtime
191-
env:
192-
COILED_RUNTIME_VERSION: ${{ matrix.runtime-version }}
193-
run: source ci/scripts/install_coiled_runtime.sh
194-
195-
- name: Run benchmarking tests
196-
id: benchmarking_tests
197-
env:
198-
DASK_COILED__TOKEN: ${{ secrets.COILED_BENCHMARK_BOT_TOKEN }}
199-
AWS_ACCESS_KEY_ID: ${{ secrets.RUNTIME_CI_BOT_AWS_ACCESS_KEY_ID }}
200-
AWS_SECRET_ACCESS_KEY: ${{ secrets.RUNTIME_CI_BOT_AWS_SECRET_ACCESS_KEY }}
201-
COILED_RUNTIME_VERSION: ${{ matrix.runtime-version }}
202-
DB_NAME: benchmark-${{ matrix.os }}-${{ matrix.runtime-version }}-py${{ matrix.python-version }}.db
203-
BENCHMARK: true
204-
run: bash ci/scripts/run_tests.sh tests/benchmarks
205-
206-
- name: Upload benchmark results
207-
uses: actions/upload-artifact@v3
208-
if: always()
209-
with:
210-
name: benchmark-${{ matrix.os }}-${{ matrix.runtime-version }}-py${{ matrix.python-version }}
211-
path: benchmark-${{ matrix.os }}-${{ matrix.runtime-version }}-py${{ matrix.python-version }}.db
212-
213-
stability:
214-
name: Stability - ${{ matrix.os }}, Python ${{ matrix.python-version }}, Runtime ${{ matrix.runtime-version }}
215-
needs: [discover_ab_envs, software]
216-
runs-on: ${{ matrix.os }}
217-
timeout-minutes: 120
218-
strategy:
219-
fail-fast: false
220-
matrix:
221-
os: [ubuntu-latest]
222-
python-version: ["3.9"]
223-
runtime-version: ${{ fromJson(needs.discover_ab_envs.outputs.matrix) }}
224-
225-
steps:
226-
- uses: actions/checkout@v2
227-
with:
228-
fetch-depth: 0
229-
230-
- name: Set up environment
231-
uses: conda-incubator/setup-miniconda@v2
232-
with:
233-
miniforge-variant: Mambaforge
234-
use-mamba: true
235-
condarc-file: ci/condarc
236-
python-version: ${{ matrix.python-version }}
237-
environment-file: ci/environment.yml
238-
239-
- name: Download software environment assets
240-
if: matrix.runtime-version == 'latest' || startsWith(matrix.runtime-version, 'AB_')
241-
uses: actions/download-artifact@v3
242-
with:
243-
name: software-environment-${{ matrix.runtime-version }}-py${{ matrix.python-version }}
244-
245-
- name: Install coiled-runtime
246-
env:
247-
COILED_RUNTIME_VERSION: ${{ matrix.runtime-version }}
248-
run: source ci/scripts/install_coiled_runtime.sh
249-
250-
- name: Run stability tests
251-
id: stability_tests
252-
env:
253-
DASK_COILED__TOKEN: ${{ secrets.COILED_BENCHMARK_BOT_TOKEN }}
254-
AWS_ACCESS_KEY_ID: ${{ secrets.RUNTIME_CI_BOT_AWS_ACCESS_KEY_ID }}
255-
AWS_SECRET_ACCESS_KEY: ${{ secrets.RUNTIME_CI_BOT_AWS_SECRET_ACCESS_KEY }}
256-
COILED_RUNTIME_VERSION: ${{ matrix.runtime-version }}
257-
DB_NAME: stability-${{ matrix.os }}-${{ matrix.runtime-version }}-py${{ matrix.python-version }}.db
160+
DB_NAME: ${{ matrix.category }}-${{ matrix.os }}-${{ matrix.runtime-version }}-${{ matrix.repeat }}-py${{ matrix.python-version }}.db
258161
BENCHMARK: true
259162
CLUSTER_DUMP: true
260-
run: bash ci/scripts/run_tests.sh tests/stability
163+
run: bash ci/scripts/run_tests.sh tests/${{ matrix.category }}
261164

262165
- name: Upload benchmark results
263166
uses: actions/upload-artifact@v3
264167
if: always()
265168
with:
266-
name: stability-${{ matrix.os }}-${{ matrix.runtime-version }}-py${{ matrix.python-version }}
267-
path: stability-${{ matrix.os }}-${{ matrix.runtime-version }}-py${{ matrix.python-version }}.db
169+
name: ${{ matrix.category }}-${{ matrix.os }}-${{ matrix.runtime-version }}-${{ matrix.repeat }}-py${{ matrix.python-version }}
170+
path: ${{ matrix.category }}-${{ matrix.os }}-${{ matrix.runtime-version }}-${{ matrix.repeat }}-py${{ matrix.python-version }}.db
268171

269172
cleanup:
270-
needs: [discover_ab_envs, software, runtime, benchmarks, stability]
271-
if: always() && ${{ fromJson(needs.discover_ab_envs.outputs.matrix) }}
173+
needs: [discover_ab_envs, software, tests]
174+
if: always() && ${{ fromJson(needs.discover_ab_envs.outputs.matrix).runtime }}
272175
name: Cleanup
273176
runs-on: ubuntu-latest
274177
strategy:
275178
fail-fast: false
276179
matrix:
277180
python-version: ["3.9"]
278-
runtime-version: ${{ fromJson(needs.discover_ab_envs.outputs.matrix) }}
181+
runtime-version: ${{ fromJson(needs.discover_ab_envs.outputs.matrix).runtime }}
279182

280183
steps:
281-
- uses: actions/checkout@v2
184+
- name: Checkout
185+
uses: actions/checkout@v2
282186

283187
- name: Set up Python
284188
uses: actions/setup-python@v4
@@ -302,24 +206,27 @@ jobs:
302206
coiled env delete $SOFTWARE_NAME
303207
304208
process-results:
305-
needs: [discover_ab_envs, runtime, benchmarks, stability]
209+
needs: [discover_ab_envs, tests]
306210
name: Combine separate benchmark results
307-
if: always() && ${{ fromJson(needs.discover_ab_envs.outputs.matrix) }}
211+
if: always() && ${{ fromJson(needs.discover_ab_envs.outputs.matrix).runtime }}
308212
runs-on: ubuntu-latest
309213
concurrency:
310214
# Fairly strict concurrency rule to avoid stepping on benchmark db.
311215
# Could eventually replace with a real db in coiled, RDS, or litestream
312216
group: process-benchmarks
313217
cancel-in-progress: false
314218
steps:
315-
- uses: actions/checkout@v2
219+
- name: Checkout
220+
uses: actions/checkout@v2
316221

317-
- uses: actions/setup-python@v4
222+
- name: Install Python
223+
uses: actions/setup-python@v4
318224

319225
- name: Install dependencies
320226
run: pip install alembic
321227

322-
- uses: actions/download-artifact@v3
228+
- name: Download artifacts
229+
uses: actions/download-artifact@v3
323230
with:
324231
path: benchmarks
325232

@@ -337,15 +244,17 @@ jobs:
337244
static-site:
338245
needs: [discover_ab_envs, process-results]
339246
# Always generate the site, as this can be skipped even if an indirect dependency fails (like a test run)
340-
if: always() && ${{ fromJson(needs.discover_ab_envs.outputs.matrix) }}
247+
if: always() && ${{ fromJson(needs.discover_ab_envs.outputs.matrix).runtime }}
341248
name: Build static dashboards
342249
runs-on: ubuntu-latest
343250
steps:
344-
- uses: actions/checkout@v2
251+
- name: Checkout
252+
uses: actions/checkout@v2
345253
with:
346254
fetch-depth: 0
347255

348-
- uses: actions/download-artifact@v3
256+
- name: Download artifacts
257+
uses: actions/download-artifact@v3
349258
with:
350259
name: benchmark.db
351260

0 commit comments

Comments
 (0)