From 0e741f29de139940feba543403d09b929d405fd1 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 10 Jul 2026 10:12:13 -0500 Subject: [PATCH] tune DeepSeek-V4 vLLM AgentX configs --- .../single_node/agentic/dsv4_fp4_b200_vllm.sh | 20 ++++-- .../single_node/agentic/dsv4_fp4_b300_vllm.sh | 19 ++++-- .../single_node/agentic/patch_vllm_pr45406.py | 63 +++++++++++++++++++ configs/nvidia-master.yaml | 27 ++++---- perf-changelog.yaml | 9 +++ 5 files changed, 113 insertions(+), 25 deletions(-) create mode 100755 benchmarks/single_node/agentic/patch_vllm_pr45406.py diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh index d723754ac8..317eab4bd3 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh @@ -13,10 +13,9 @@ set -x # experts EP-sharded across DP ranks (per the vLLM blog recipe). # Highest aggregate throughput at large CONC. # -# Image is configured in nvidia-master.yaml. block_size=256, -# kv-cache-dtype=fp8, FP4 indexer cache enabled, FULL_AND_PIECEWISE cudagraph -# capture with custom_ops=all (per the vLLM blog recipe at -# https://vllm.ai/blog/deepseek-v4). +# Image is configured in nvidia-master.yaml. The serving flags follow the +# DeepSeek-V4 low-latency vLLM recipe with sparse MLA attention, Mega-MoE, +# FP8 KV cache, and full decode-only CUDA graphs. # # Required env vars: # MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR @@ -118,6 +117,7 @@ if require_agentic_kv_offload_backend mooncake; then agentic_pip_install --quiet --no-cache-dir --no-deps \ --force-reinstall "mooncake-transfer-engine-cuda13==$MOONCAKE_VERSION" python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null + python3 "$(dirname "$0")/patch_vllm_pr45406.py" MOONCAKE_MASTER_PORT=$((PORT + 12000)) MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json" @@ -178,7 +178,10 @@ fi EP_ARGS=() if [ "$EP_SIZE" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) + EP_ARGS=( + --enable-expert-parallel + --moe-backend deep_gemm_mega_moe + ) fi # AgentX concurrency counts live session trees, not individual requests. @@ -202,15 +205,20 @@ VLLM_CMD=( "${PARALLEL_ARGS[@]}" "${VLLM_CP_ARGS[@]}" "${EP_ARGS[@]}" - --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' + --prefill-schedule-interval 8 + --numa-bind + --compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + --attention-config '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true}' --attention_config.use_fp4_indexer_cache=True --tokenizer-mode deepseek_v4 --tool-call-parser deepseek_v4 --enable-auto-tool-choice --reasoning-parser deepseek_v4 + --no-enable-flashinfer-autotune --enable-prefix-caching --no-disable-hybrid-kv-cache-manager --max-num-seqs "$MAX_NUM_SEQS" + --disable-uvicorn-access-log "${OFFLOAD_ARGS[@]}" ) printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh index 74dc2129be..6f4367b5c3 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh @@ -13,9 +13,9 @@ set -x # experts EP-sharded across DP ranks (per the vLLM blog recipe). # Highest aggregate throughput at large CONC. # -# Image is vllm/vllm-openai:v0.20.0-cu130. block_size=256, kv-cache-dtype=fp8, -# FP4 indexer cache enabled, FULL_AND_PIECEWISE cudagraph capture with -# custom_ops=all (per the vLLM blog recipe at https://vllm.ai/blog/deepseek-v4). +# Image is configured in nvidia-master.yaml. The serving flags follow the +# DeepSeek-V4 low-latency vLLM recipe with sparse MLA attention, Mega-MoE, +# FP8 KV cache, and full decode-only CUDA graphs. # # Required env vars: # MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR @@ -122,6 +122,7 @@ if require_agentic_kv_offload_backend mooncake; then agentic_pip_install --quiet --no-cache-dir --no-deps \ --force-reinstall "mooncake-transfer-engine-cuda13==$MOONCAKE_VERSION" python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null + python3 "$(dirname "$0")/patch_vllm_pr45406.py" MOONCAKE_MASTER_PORT=$((PORT + 12000)) MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json" @@ -180,7 +181,10 @@ fi EP_ARGS=() if [ "$EP_SIZE" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) + EP_ARGS=( + --enable-expert-parallel + --moe-backend deep_gemm_mega_moe + ) fi # AgentX concurrency counts live session trees, not individual requests. @@ -205,15 +209,20 @@ vllm serve "$MODEL_PATH" --served-model-name "$MODEL" \ "${PARALLEL_ARGS[@]}" \ "${VLLM_CP_ARGS[@]}" \ "${EP_ARGS[@]}" \ ---compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' \ +--prefill-schedule-interval 8 \ +--numa-bind \ +--compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' \ +--attention-config '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true}' \ --attention_config.use_fp4_indexer_cache=True \ --tokenizer-mode deepseek_v4 \ --tool-call-parser deepseek_v4 \ --enable-auto-tool-choice \ --reasoning-parser deepseek_v4 \ +--no-enable-flashinfer-autotune \ --enable-prefix-caching \ --no-disable-hybrid-kv-cache-manager \ --max-num-seqs "$MAX_NUM_SEQS" \ +--disable-uvicorn-access-log \ "${OFFLOAD_ARGS[@]}" > "$SERVER_LOG" 2>&1 & SERVER_PID=$! echo "Server PID: $SERVER_PID" diff --git a/benchmarks/single_node/agentic/patch_vllm_pr45406.py b/benchmarks/single_node/agentic/patch_vllm_pr45406.py new file mode 100755 index 0000000000..4b41b60c30 --- /dev/null +++ b/benchmarks/single_node/agentic/patch_vllm_pr45406.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Backport the scheduler fix from vllm-project/vllm PR #45406.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + + +PATCH_MARKER = "See https://github.com/vllm-project/vllm/issues/45388" + + +def main() -> None: + spec = importlib.util.find_spec("vllm") + if spec is None or not spec.submodule_search_locations: + raise RuntimeError("Could not locate the installed vllm package") + + package_root = Path(next(iter(spec.submodule_search_locations))) + scheduler = package_root / "v1" / "core" / "sched" / "scheduler.py" + text = scheduler.read_text() + if PATCH_MARKER in text: + print(f"vLLM PR #45406 backport already present in {scheduler}") + return + + old = """ if request.has_encoder_inputs: + self.encoder_cache_manager.free(request) + break + + # KVTransfer: the connector uses this info to determine +""" + new = """ if request.has_encoder_inputs: + self.encoder_cache_manager.free(request) + if self.running: + # Running requests will free blocks when they + # complete; stop here to preserve queue-order + # admission. + break + # Nothing is running, so no future event frees blocks and + # stopping at this request would freeze this state + # permanently. Requests behind this one may hold blocks + # while parked (async KV loads in WAITING_FOR_REMOTE_KVS) + # and are only promoted when this traversal reaches them. + # Keep scanning so they can be promoted, scheduled, and + # eventually free the blocks this request needs. + # See https://github.com/vllm-project/vllm/issues/45388 + request_queue.pop_request() + step_skipped_waiting.prepend_request(request) + continue + + # KVTransfer: the connector uses this info to determine +""" + if text.count(old) != 1: + raise RuntimeError( + f"Expected exactly one vLLM PR #45406 patch target in {scheduler}" + ) + + scheduler.write_text(text.replace(old, new, 1)) + compile(scheduler.read_text(), str(scheduler), "exec") + print(f"Applied vLLM PR #45406 backport to {scheduler}") + + +if __name__ == "__main__": + main() diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 6be8e006f9..6736653554 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1762,7 +1762,7 @@ dsv4-fp4-b200-vllm: - { tp: 8, ep: 8, dp-attn: true, conc-start: 64, conc-end: 1024 } dsv4-fp4-b200-vllm-agentic: - image: vllm/vllm-openai:v0.23.0 + image: vllm/vllm-openai:nightly-95ed0feaa5cd7fb16d72c53ce04950aaf07c4698 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:b200-dgxc @@ -1775,12 +1775,11 @@ dsv4-fp4-b200-vllm-agentic: search-space: # Pure TP is only competitive at very low concurrency. - { tp: 8, kv-offloading: none, conc-list: [1, 2, 3, 4, 5] } - # Sample the useful MooncakeStore range without repeating its collapsed - # high-concurrency tail. - - { tp: 8, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [8, 10, 16] } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [16, 32, 38, 44, 50] } + # Cover the external-cache transition region. + - { tp: 8, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [8, 10, 16, 32, 40, 44] } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [24, 32, 38, 44, 48, 50, 52] } # Retain the external-cache transition and peak-throughput region. - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [16, 38, 44, 56, 64, 66, 68] } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [32, 38, 44, 50, 56, 64, 66, 68] } dsv4-fp4-b200-trt: image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066 @@ -3207,7 +3206,7 @@ dsv4-fp4-b300-vllm: - { tp: 8, ep: 8, dp-attn: true, conc-start: 2048, conc-end: 2048 } dsv4-fp4-b300-vllm-agentic: - image: vllm/vllm-openai:v0.23.0 + image: vllm/vllm-openai:nightly-95ed0feaa5cd7fb16d72c53ce04950aaf07c4698 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:b300-nv @@ -3219,15 +3218,15 @@ dsv4-fp4-b300-vllm-agentic: - dram-utilization: 0.80 search-space: # Compare native GPU-cache and MooncakeStore CPU-offload cliffs. - - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 16] } + - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 16, 20, 24, 32] } - { tp: 4, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [16, 18, 20, 24] } - # TP8 remains cache-resident through conc 52. - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 16, 32, 40, 48, 52] } + # Cover the TP8 cache-transition region. + - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 16, 32, 40, 48, 52, 56, 60, 64, 72] } - { tp: 8, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [52] } - - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, conc-list: [8, 16] } - - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [32] } - # TP8 DEP retains representative low, peak, and transition points. - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [52, 72, 100, 128, 144] } + - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, conc-list: [8, 16, 24, 32, 40, 64] } + - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [32, 40, 48, 56, 64, 72, 80, 88, 96, 128] } + # Cover the TP8 DEP throughput region. + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [52, 72, 100, 128, 144, 196, 512] } dsv4-fp4-b300-trt: image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f4a77dd6a1..fe1f988cc9 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4707,3 +4707,12 @@ - "Clean the export envs" - "Enable two batch overlap" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2093 + +- config-keys: + - dsv4-fp4-b200-vllm-agentic + - dsv4-fp4-b300-vllm-agentic + description: + - "Tune DeepSeek-V4 AgentX on B200 and B300 with the vLLM nightly at 95ed0fe." + - "Enable sparse MLA, EP-gated Mega-MoE, full decode-only CUDA graphs, NUMA binding, FP8 KV cache, and the vLLM PR #45406 scheduler fix." + - "Expand native-cache and MooncakeStore concurrency coverage." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2138