Skip to content

Commit 346feba

Browse files
QiliangCuiQiliangCui
committed
Move "skip" out of test command.
Signed-off-by: Qiliang Cui <derrhein@gmail.com>
1 parent 5644ffb commit 346feba

File tree

2 files changed

+45
-99
lines changed

2 files changed

+45
-99
lines changed

‎.buildkite/pipeline_jax.yml‎

Lines changed: 41 additions & 99 deletions
Original file line numberDiff line numberDiff line change
@@ -19,34 +19,24 @@ steps:
1919
- label: "E2E MLPerf tests for JAX models with quantization"
2020
key: test_1
2121
soft_fail: true
22+
if: build.env("NIGHTLY") == "1"
2223
env:
2324
QUANTIZATION: "True"
2425
agents:
2526
queue: tpu_v6e_queue
2627
commands:
27-
- |
28-
if [[ "$$NIGHTLY" == "1" ]]; then
29-
.buildkite/scripts/run_in_docker.sh bash /workspace/tpu_inference/tests/e2e/benchmarking/mlperf.sh
30-
else
31-
echo "Skipping: NIGHTLY environment variable not set"
32-
exit 0
33-
fi
28+
- .buildkite/scripts/run_in_docker.sh bash /workspace/tpu_inference/tests/e2e/benchmarking/mlperf.sh
3429

3530
- label: "E2E MLPerf tests for JAX new models"
3631
key: test_2
3732
soft_fail: true
33+
if: build.env("NIGHTLY") == "1"
3834
env:
3935
NEW_MODEL_DESIGN: "1"
4036
agents:
4137
queue: tpu_v6e_queue
4238
commands:
43-
- |
44-
if [[ "$$NIGHTLY" == "1" ]]; then
45-
.buildkite/scripts/run_in_docker.sh bash /workspace/tpu_inference/tests/e2e/benchmarking/mlperf.sh
46-
else
47-
echo "Skipping: NIGHTLY environment variable not set"
48-
exit 0
49-
fi
39+
- .buildkite/scripts/run_in_docker.sh bash /workspace/tpu_inference/tests/e2e/benchmarking/mlperf.sh
5040

5141
- label: "E2E MLPerf tests for JAX + vLLM models on single chip"
5242
key: test_3
@@ -61,52 +51,35 @@ steps:
6151
- label: "E2E MLperf tests for Llama4 models"
6252
key: test_4
6353
soft_fail: true
54+
if: build.env("NIGHTLY") == "1"
6455
env:
6556
NEW_MODEL_DESIGN: "1"
6657
USE_V6E8_QUEUE: "True"
6758
agents:
6859
queue: tpu_v6e_8_queue
6960
commands:
70-
- |
71-
if [[ "$$NIGHTLY" == "1" ]]; then
72-
.buildkite/scripts/run_in_docker.sh bash /workspace/tpu_inference/tests/e2e/benchmarking/mlperf.sh
73-
else
74-
echo "Skipping: NIGHTLY environment variable not set"
75-
exit 0
76-
fi
77-
61+
- .buildkite/scripts/run_in_docker.sh bash /workspace/tpu_inference/tests/e2e/benchmarking/mlperf.sh
7862

7963
- label: "E2E multi modality test"
8064
key: test_5
8165
soft_fail: true
66+
if: build.env("NIGHTLY") == "1"
8267
agents:
8368
queue: tpu_v6e_queue
8469
commands:
85-
- |
86-
if [[ "$$NIGHTLY" == "1" ]]; then
87-
.buildkite/scripts/run_in_docker.sh \
88-
bash -c 'python3 -m pytest -s -v -x /workspace/tpu_inference/tests/e2e/test_multi_modal_inference.py && \
89-
bash /workspace/tpu_inference/tests/e2e/benchmarking/mm_bench.sh'
90-
else
91-
echo "Skipping: NIGHTLY environment variable not set"
92-
exit 0
93-
fi
70+
- .buildkite/scripts/run_in_docker.sh \
71+
bash -c 'python3 -m pytest -s -v -x /workspace/tpu_inference/tests/e2e/test_multi_modal_inference.py && \
72+
bash /workspace/tpu_inference/tests/e2e/benchmarking/mm_bench.sh'
9473

9574
- label: "E2E speculative decoding test"
9675
key: test_6
9776
soft_fail: true
77+
if: build.env("NIGHTLY") == "1"
9878
agents:
9979
queue: tpu_v6e_queue
10080
commands:
101-
- |
102-
if [[ "$$NIGHTLY" == "1" ]]; then
103-
.buildkite/scripts/run_in_docker.sh \
104-
bash -c 'python3 -m pytest -s -v -x /workspace/tpu_inference/tests/e2e/test_speculative_decoding.py'
105-
else
106-
echo "Skipping: NIGHTLY environment variable not set"
107-
exit 0
108-
fi
109-
81+
- .buildkite/scripts/run_in_docker.sh \
82+
bash -c 'python3 -m pytest -s -v -x /workspace/tpu_inference/tests/e2e/test_speculative_decoding.py'
11083

11184
- label: "JAX unit tests"
11285
key: test_7
@@ -143,71 +116,56 @@ steps:
143116
- label: "JAX unit tests - kernels"
144117
key: test_8
145118
soft_fail: true
119+
if: |
120+
build.env("NIGHTLY") == "1" ||
121+
buildkite.git.hasChangedFiles("tpu_inference/kernels/**") ||
122+
buildkite.git.hasChangedFiles("tests/kernels/**") ||
123+
buildkite.git.hasChangedFiles("requirements.txt")
146124
agents:
147125
queue: tpu_v6e_queue
148126
commands:
149-
- |
150-
if [[ "$$NIGHTLY" == "1" ]] || git diff --name-only HEAD~1 | grep -qE '^(tpu_inference/kernels|tests/kernels|requirements\.txt)'; then
151-
.buildkite/scripts/run_in_docker.sh \
152-
python3 -m pytest -s -v -x /workspace/tpu_inference/tests/kernels \
153-
--ignore=/workspace/tpu_inference/tests/kernels/ragged_paged_attention_kernel_v2_test.py \
154-
--ignore=/workspace/tpu_inference/tests/kernels/ragged_kv_cache_update_v2_test.py \
155-
--ignore=/workspace/tpu_inference/tests/kernels/collectives \
156-
--ignore=/workspace/tpu_inference/tests/kernels/fused_moe_v1_test.py
157-
else
158-
echo "Skipping: no changes detected in kernels, tests/kernels, or requirements.txt"
159-
exit 0
160-
fi
127+
- .buildkite/scripts/run_in_docker.sh \
128+
python3 -m pytest -s -v -x /workspace/tpu_inference/tests/kernels \
129+
--ignore=/workspace/tpu_inference/tests/kernels/ragged_paged_attention_kernel_v2_test.py \
130+
--ignore=/workspace/tpu_inference/tests/kernels/ragged_kv_cache_update_v2_test.py \
131+
--ignore=/workspace/tpu_inference/tests/kernels/collectives \
132+
--ignore=/workspace/tpu_inference/tests/kernels/fused_moe_v1_test.py
161133

162134
- label: "JAX unit tests - collective kernels"
163135
key: test_9
164136
soft_fail: true
137+
if: build.env("NIGHTLY") == "1"
165138
agents:
166139
queue: tpu_v6e_8_queue
167140
commands:
168-
- |
169-
if [[ "$$NIGHTLY" == "1" ]] || git diff --name-only HEAD~1 | grep -qE '^(tpu_inference/kernels/collectives|tests/kernels/collectives|requirements\.txt)'; then
170-
.buildkite/scripts/run_in_docker.sh \
171-
python3 -m pytest -s -v -x /workspace/tpu_inference/tests/kernels/collectives
172-
else
173-
echo "Skipping: no changes detected in kernels/collectives, tests/kernels/collectives, or requirements.txt"
174-
exit 0
175-
fi
141+
- .buildkite/scripts/run_in_docker.sh \
142+
python3 -m pytest -s -v -x /workspace/tpu_inference/tests/kernels/collectives
176143

177144
- label: "lora e2e tests for JAX + vLLM models single chip"
178145
key: test_10
179146
soft_fail: true
147+
if: build.env("NIGHTLY") == "1"
180148
agents:
181149
queue: tpu_v6e_queue
182150
commands:
183-
- |
184-
if [[ "$$NIGHTLY" == "1" ]]; then
185-
.buildkite/scripts/run_in_docker.sh \
186-
bash -c 'MODEL_IMPL_TYPE=vllm TPU_BACKEND_TYPE=jax python3 -m pytest -s -v -x /workspace/tpu_inference/tests/lora/test_lora.py'
187-
else
188-
echo "Skipping: NIGHTLY environment variable not set"
189-
exit 0
190-
fi
151+
- .buildkite/scripts/run_in_docker.sh \
152+
bash -c 'MODEL_IMPL_TYPE=vllm TPU_BACKEND_TYPE=jax python3 -m pytest -s -v -x /workspace/tpu_inference/tests/lora/test_lora.py'
191153

192154
- label: "E2E MLPerf tests for JAX + vLLM models on multiple chips"
193155
key: test_11
194156
soft_fail: true
157+
if: build.env("NIGHTLY") == "1"
195158
env:
196159
MODEL_IMPL_TYPE: "vllm"
197160
agents:
198161
queue: tpu_v6e_8_queue
199162
commands:
200-
- |
201-
if [[ "$$NIGHTLY" == "1" ]]; then
202-
.buildkite/scripts/run_in_docker.sh bash /workspace/tpu_inference/tests/e2e/benchmarking/mlperf.sh
203-
else
204-
echo "Skipping: NIGHTLY environment variable not set"
205-
exit 0
206-
fi
163+
- .buildkite/scripts/run_in_docker.sh bash /workspace/tpu_inference/tests/e2e/benchmarking/mlperf.sh
207164

208165
- label: "E2E MLperf tests for DeepSeek-R1 (no accuracy, 12-decoder layers only)"
209166
key: test_12
210167
soft_fail: true
168+
if: build.env("NIGHTLY") == "1"
211169
env:
212170
NEW_MODEL_DESIGN: "1"
213171
USE_V6E8_QUEUE: "True"
@@ -216,32 +174,20 @@ steps:
216174
agents:
217175
queue: tpu_v6e_8_queue
218176
commands:
219-
- |
220-
if [[ "$$NIGHTLY" == "1" ]]; then
221-
.buildkite/scripts/run_in_docker.sh bash /workspace/tpu_inference/tests/e2e/benchmarking/mlperf.sh -m deepseek-ai/DeepSeek-R1-0528 --use-dummy-weights
222-
else
223-
echo "Skipping: NIGHTLY environment variable not set"
224-
exit 0
225-
fi
177+
- .buildkite/scripts/run_in_docker.sh bash /workspace/tpu_inference/tests/e2e/benchmarking/mlperf.sh -m deepseek-ai/DeepSeek-R1-0528 --use-dummy-weights
226178

227179
- label: "lora e2e tests for JAX + vLLM models multi chips"
228180
key: test_13
229181
soft_fail: true
182+
if: build.env("NIGHTLY") == "1"
230183
env:
231184
USE_V6E8_QUEUE: "True"
232185
VLLM_LOG_LEVEL: "INFO"
233186
agents:
234187
queue: tpu_v6e_8_queue
235188
commands:
236-
- |
237-
if [[ "$$NIGHTLY" == "1" ]]; then
238-
.buildkite/scripts/run_in_docker.sh \
239-
bash -c 'MODEL_IMPL_TYPE=vllm TPU_BACKEND_TYPE=jax python3 -m pytest -s -v -x /workspace/tpu_inference/tests/lora/test_lora.py'
240-
else
241-
echo "Skipping: NIGHTLY environment variable not set"
242-
exit 0
243-
fi
244-
189+
- .buildkite/scripts/run_in_docker.sh \
190+
bash -c 'MODEL_IMPL_TYPE=vllm TPU_BACKEND_TYPE=jax python3 -m pytest -s -v -x /workspace/tpu_inference/tests/lora/test_lora.py'
245191

246192
- label: "lora unit tests on single chip"
247193
key: test_15
@@ -257,20 +203,16 @@ steps:
257203
- label: "lora unit tests on multi chips"
258204
key: test_16
259205
soft_fail: true
206+
if: build.env("NIGHTLY") == "1"
260207
env:
261208
USE_V6E8_QUEUE: "True"
262209
VLLM_LOG_LEVEL: "INFO"
263210
agents:
264211
queue: tpu_v6e_8_queue
265212
commands:
266-
- |
267-
if [[ "$$NIGHTLY" == "1" ]]; then
268-
.buildkite/scripts/run_in_docker.sh \
269-
bash -c 'python3 -m pytest -s -v -x /workspace/tpu_inference/tests/lora/test_layers.py'
270-
else
271-
echo "Skipping: NIGHTLY environment variable not set"
272-
exit 0
273-
fi
213+
- .buildkite/scripts/run_in_docker.sh \
214+
bash -c 'python3 -m pytest -s -v -x /workspace/tpu_inference/tests/lora/test_layers.py'
215+
274216
# -----------------------------------------------------------------
275217
# NOTIFICATION STEP
276218
# -----------------------------------------------------------------

‎.buildkite/scripts/check_results.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,10 @@ echo "--- Checking Test Outcomes"
1414

1515
for KEY in "$@"; do
1616
OUTCOME=$(buildkite-agent step get "outcome" --step "${KEY}" || echo "skipped")
17+
if [ -z "$OUTCOME" ]; then
18+
OUTCOME="skipped"
19+
fi
20+
1721
echo "Step ${KEY} outcome: ${OUTCOME}"
1822

1923
if [ "${OUTCOME}" != "passed" ] && [ "${OUTCOME}" != "skipped" ] ; then

0 commit comments

Comments
 (0)