diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b200_vllm.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b200_vllm.sh index 2cafd4b32..011f9dedb 100755 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b200_vllm.sh +++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b200_vllm.sh @@ -44,9 +44,11 @@ fi GMU_ARGS=() MOE_ARGS=() EPLB_ARGS=() +PREFILL_SCHEDULE_ARGS=() if [ "${DP_ATTENTION}" = "true" ]; then MOE_ARGS=(--moe-backend deep_gemm_mega_moe) EPLB_ARGS=(--enable-eplb --eplb-config '{"communicator":"torch_nccl", "use_async": false}') + PREFILL_SCHEDULE_ARGS=(--prefill-schedule-interval 4) fi if [ "${ISL}" -eq 8192 ] && [ "${CONC}" -le 128 ]; then @@ -81,6 +83,7 @@ vllm serve "$MODEL" --host 0.0.0.0 --port "$PORT" \ "${GMU_ARGS[@]}" \ "${MOE_ARGS[@]}" \ "${EPLB_ARGS[@]}" \ + "${PREFILL_SCHEDULE_ARGS[@]}" \ --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' \ --attention_config.use_fp4_indexer_cache=True \ --tokenizer-mode deepseek_v4 \ diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b200_vllm_mtp.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b200_vllm_mtp.sh index 6846223e8..2a1fa4145 100755 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b200_vllm_mtp.sh +++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b200_vllm_mtp.sh @@ -47,9 +47,11 @@ fi # FULL_AND_PIECEWISE compilation config. GMU_ARGS=() MOE_ARGS=() +PREFILL_SCHEDULE_ARGS=() if [ "${DP_ATTENTION}" = "true" ]; then - GMU_ARGS=(--gpu-memory-utilization 0.95) + GMU_ARGS=(--gpu-memory-utilization 0.9) MOE_ARGS=(--moe-backend deep_gemm_mega_moe) + PREFILL_SCHEDULE_ARGS=(--prefill-schedule-interval 4) fi MAX_NUM_BATCHED_TOKENS=$(( ISL * 2 )) @@ -79,6 +81,7 @@ vllm serve "$MODEL" --host 0.0.0.0 --port "$PORT" \ "${EP_ARGS[@]}" \ "${GMU_ARGS[@]}" \ "${MOE_ARGS[@]}" \ + "${PREFILL_SCHEDULE_ARGS[@]}" \ --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' \ --attention_config.use_fp4_indexer_cache=True \ --tokenizer-mode deepseek_v4 \ diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 6be8e006f..82334dd34 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1739,10 +1739,10 @@ dsv4-fp4-b200-sglang: - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 1024 } dsv4-fp4-b200-vllm: - image: vllm/vllm-openai:nightly-3f0a91bb96f8d72e0498b95c166e817deae14d62 + image: vllm/vllm-openai:v0.25.0 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 - runner: b200-dsv4 + runner: cluster:b200-dgxc precision: fp4 framework: vllm multinode: false @@ -1752,13 +1752,14 @@ dsv4-fp4-b200-vllm: osl: 1024 search-space: - { tp: 8, conc-start: 1, conc-end: 64 } - - { tp: 8, ep: 8, conc-start: 128, conc-end: 128 } + - { tp: 8, ep: 8, conc-start: 128, conc-end: 256 } - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 1024 } - { tp: 8, ep: 8, dp-attn: true, conc-start: 4096, conc-end: 4096 } - isl: 8192 osl: 1024 search-space: - - { tp: 8, conc-start: 1, conc-end: 32 } + - { tp: 8, conc-start: 1, conc-end: 64 } + - { tp: 8, ep: 8, conc-start: 128, conc-end: 256 } - { tp: 8, ep: 8, dp-attn: true, conc-start: 64, conc-end: 1024 } dsv4-fp4-b200-vllm-agentic: @@ -1827,10 +1828,10 @@ dsv4-fp4-b200-trt-mtp: # MTP variant of dsv4-fp4-b200-vllm. Mirrors the base search space and adds # --speculative-config '{"method":"mtp","num_speculative_tokens":2}'. dsv4-fp4-b200-vllm-mtp: - image: vllm/vllm-openai:v0.21.0 + image: vllm/vllm-openai:v0.25.0 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 - runner: b200-dsv4 + runner: cluster:b200-dgxc precision: fp4 framework: vllm multinode: false @@ -1840,14 +1841,13 @@ dsv4-fp4-b200-vllm-mtp: osl: 1024 search-space: - { tp: 8, conc-start: 1, conc-end: 64, spec-decoding: mtp } - - { tp: 8, ep: 8, conc-start: 128, conc-end: 128, spec-decoding: mtp } + - { tp: 8, ep: 8, conc-start: 128, conc-end: 256, spec-decoding: mtp } - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 4096, spec-decoding: mtp } - isl: 8192 osl: 1024 search-space: - # 8k/1k TP=8 caps at conc=16 (not 32) to avoid OOM observed at conc=32: - # https://github.com/SemiAnalysisAI/InferenceX/actions/runs/25134892257/job/73670854021 - - { tp: 8, conc-start: 1, conc-end: 16, spec-decoding: mtp } + - { tp: 8, conc-start: 1, conc-end: 64, spec-decoding: mtp } + - { tp: 8, ep: 8, conc-start: 128, conc-end: 256, spec-decoding: mtp } - { tp: 8, ep: 8, dp-attn: true, conc-start: 64, conc-end: 256, spec-decoding: mtp } # NOTE: At the time of submission, https://cookbook.sglang.io/autoregressive/DeepSeek/DeepSeek-R1 @@ -3181,10 +3181,10 @@ dsv4-fp8-h200-sglang-mtp: # field, so dp-attn=true is used as the existing vLLM script switch for DP4 # layouts on 4 allocated GPUs. dsv4-fp4-b300-vllm: - image: vllm/vllm-openai:v0.22.0 + image: vllm/vllm-openai:v0.25.0 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 - runner: b300 + runner: cluster:b300-nv precision: fp4 framework: vllm multinode: false @@ -3276,10 +3276,10 @@ dsv4-fp4-b300-trt-mtp: - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 1024, spec-decoding: mtp } dsv4-fp4-b300-vllm-mtp: - image: vllm/vllm-openai:v0.21.0 + image: vllm/vllm-openai:v0.25.0 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 - runner: b300 + runner: cluster:b300-nv precision: fp4 framework: vllm multinode: false diff --git a/perf-changelog.yaml b/perf-changelog.yaml index be28208ab..f6accfcca 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4716,3 +4716,12 @@ - "Clean the export envs" - "Enable two batch overlap" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2093 + +- config-keys: + - dsv4-fp4-b200-vllm + - dsv4-fp4-b200-vllm-mtp + - dsv4-fp4-b300-vllm + - dsv4-fp4-b300-vllm-mtp + description: + - "Bump vLLM image to v0.25.0 for DeepSeek-V4-Pro FP4 on B200 and B300." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2169