From ec0a5f3885dad38bc508f1b3a931d61f3620b99e Mon Sep 17 00:00:00 2001 From: Ankur-singh Date: Thu, 9 Jul 2026 13:48:47 -0700 Subject: [PATCH 1/3] perf(dsv4): refresh GB200 Dynamo vLLM stack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update the vLLM image and Dynamo wheel, and enable multiple frontends across all five GB200 DeepSeek-V4-Pro topologies. 中文：更新 vLLM 镜像和 Dynamo wheel，并为 DeepSeek-V4-Pro 的全部五种 GB200 拓扑启用多个前端。 --- .../deepseek-v4/8k1k/disagg-gb200-high-tpt-megamoe.yaml | 6 +++--- .../vllm/deepseek-v4/8k1k/disagg-gb200-low-latency.yaml | 6 +++--- .../deepseek-v4/8k1k/disagg-gb200-low-middle-curve.yaml | 6 +++--- .../deepseek-v4/8k1k/disagg-gb200-max-tpt-megamoe.yaml | 6 +++--- .../deepseek-v4/8k1k/disagg-gb200-mid-curve-megamoe.yaml | 6 +++--- configs/nvidia-master.yaml | 2 +- perf-changelog.yaml | 8 ++++++++ 7 files changed, 24 insertions(+), 16 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-high-tpt-megamoe.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-high-tpt-megamoe.yaml index a78b83ce2e..751b552f61 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-high-tpt-megamoe.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-high-tpt-megamoe.yaml @@ -18,12 +18,12 @@ name: "svf-vllm-disagg-gb200-high-tpt-megamoe" # absorb cold-cache /mnt/numa1 model loads. model: path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.20.0-ubuntu2404" + container: "vllm/vllm-openai:dsv4-megamoe-mxfp4-arm64-cu130-4ba0a72" precision: "fp4" dynamo: install: true - wheel: "1.2.0.dev20260426" + wheel: "1.3.0.dev1" setup_script: vllm-container-deps.sh @@ -48,7 +48,7 @@ infra: frontend: type: dynamo - enable_multiple_frontends: false + enable_multiple_frontends: true backend: type: vllm connector: null diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-latency.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-latency.yaml index 6c5dec41d5..a3c6d4d369 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-latency.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-latency.yaml @@ -17,12 +17,12 @@ name: "svf-vllm-disagg-gb200-low-latency" # absorb cold-cache /mnt/numa1 model loads. model: path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.20.0-ubuntu2404" + container: "vllm/vllm-openai:dsv4-megamoe-mxfp4-arm64-cu130-4ba0a72" precision: "fp4" dynamo: install: true - wheel: "1.2.0.dev20260426" + wheel: "1.3.0.dev1" setup_script: vllm-container-deps.sh @@ -47,7 +47,7 @@ infra: frontend: type: dynamo - enable_multiple_frontends: false + enable_multiple_frontends: true backend: type: vllm connector: null diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-middle-curve.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-middle-curve.yaml index 98ec9acdad..d31bd0b5df 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-middle-curve.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-middle-curve.yaml @@ -18,12 +18,12 @@ name: "svf-vllm-disagg-gb200-low-middle-curve" # absorb cold-cache /mnt/numa1 model loads. model: path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.20.0-ubuntu2404" + container: "vllm/vllm-openai:dsv4-megamoe-mxfp4-arm64-cu130-4ba0a72" precision: "fp4" dynamo: install: true - wheel: "1.2.0.dev20260426" + wheel: "1.3.0.dev1" setup_script: vllm-container-deps.sh @@ -48,7 +48,7 @@ infra: frontend: type: dynamo - enable_multiple_frontends: false + enable_multiple_frontends: true backend: type: vllm connector: null diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-max-tpt-megamoe.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-max-tpt-megamoe.yaml index db3caef440..2186657c58 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-max-tpt-megamoe.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-max-tpt-megamoe.yaml @@ -18,12 +18,12 @@ name: "svf-vllm-disagg-gb200-max-tpt-megamoe" # absorb cold-cache /mnt/numa1 model loads. model: path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.20.0-ubuntu2404" + container: "vllm/vllm-openai:dsv4-megamoe-mxfp4-arm64-cu130-4ba0a72" precision: "fp4" dynamo: install: true - wheel: "1.2.0.dev20260426" + wheel: "1.3.0.dev1" setup_script: vllm-container-deps.sh @@ -48,7 +48,7 @@ infra: frontend: type: dynamo - enable_multiple_frontends: false + enable_multiple_frontends: true backend: type: vllm connector: null diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-mid-curve-megamoe.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-mid-curve-megamoe.yaml index a1f7cfd5a6..e1f120c158 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-mid-curve-megamoe.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-mid-curve-megamoe.yaml @@ -18,12 +18,12 @@ name: "svf-vllm-disagg-gb200-mid-curve-megamoe" # absorb cold-cache /mnt/numa1 model loads. model: path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.20.0-ubuntu2404" + container: "vllm/vllm-openai:dsv4-megamoe-mxfp4-arm64-cu130-4ba0a72" precision: "fp4" dynamo: install: true - wheel: "1.2.0.dev20260426" + wheel: "1.3.0.dev1" setup_script: vllm-container-deps.sh @@ -48,7 +48,7 @@ infra: frontend: type: dynamo - enable_multiple_frontends: false + enable_multiple_frontends: true backend: type: vllm connector: null diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 6be8e006f9..a3fbe93a2b 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8587,7 +8587,7 @@ dsv4-fp4-b200-dynamo-vllm: dp-attn: true dsv4-fp4-gb200-dynamo-vllm: - image: vllm/vllm-openai:v0.20.0-ubuntu2404 + image: vllm/vllm-openai:dsv4-megamoe-mxfp4-arm64-cu130-4ba0a72 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: gb200 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c1a3c3aee8..7229de96e2 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4699,3 +4699,11 @@ - "Clean the export envs" - "Enable two batch overlap" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2093 + +- config-keys: + - dsv4-fp4-gb200-dynamo-vllm + description: + - "Update the Dynamo wheel from 1.2.0.dev20260426 to 1.3.0.dev1" + - "Enable multiple Dynamo frontends across all DeepSeek-V4-Pro FP4 GB200 topologies" + - "Update the vLLM image from v0.20.0-ubuntu2404 to dsv4-megamoe-mxfp4-arm64-cu130-4ba0a72" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX From eccc729a15c8d7430b65dd9cba7e8b3da4fa2393 Mon Sep 17 00:00:00 2001 From: Ankur-singh Date: Thu, 9 Jul 2026 13:49:25 -0700 Subject: [PATCH 2/3] chore: link performance changelog to PR #2139 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backfill the canonical pull request URL after creating the draft PR. 中文：创建草稿 PR 后，将正式 PR 链接回填至性能变更日志。 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 7229de96e2..602139beba 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4706,4 +4706,4 @@ - "Update the Dynamo wheel from 1.2.0.dev20260426 to 1.3.0.dev1" - "Enable multiple Dynamo frontends across all DeepSeek-V4-Pro FP4 GB200 topologies" - "Update the vLLM image from v0.20.0-ubuntu2404 to dsv4-megamoe-mxfp4-arm64-cu130-4ba0a72" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2139 From 94d15da95c1c500eda00ca048670a42612ef212c Mon Sep 17 00:00:00 2001 From: Ankur-singh Date: Thu, 9 Jul 2026 14:03:52 -0700 Subject: [PATCH 3/3] perf(dsv4): use MessagePack for Dynamo request plane MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Set DYN_REQUEST_PLANE_CODEC=msgpack on every GB200 Dynamo frontend, prefill worker, and decode worker. 中文：为所有 GB200 Dynamo 前端、预填充 worker 和解码 worker 设置 DYN_REQUEST_PLANE_CODEC=msgpack。 --- .../vllm/deepseek-v4/8k1k/disagg-gb200-high-tpt-megamoe.yaml | 4 ++++ .../vllm/deepseek-v4/8k1k/disagg-gb200-low-latency.yaml | 4 ++++ .../vllm/deepseek-v4/8k1k/disagg-gb200-low-middle-curve.yaml | 4 ++++ .../vllm/deepseek-v4/8k1k/disagg-gb200-max-tpt-megamoe.yaml | 4 ++++ .../vllm/deepseek-v4/8k1k/disagg-gb200-mid-curve-megamoe.yaml | 4 ++++ perf-changelog.yaml | 1 + 6 files changed, 21 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-high-tpt-megamoe.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-high-tpt-megamoe.yaml index 751b552f61..99f85702b3 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-high-tpt-megamoe.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-high-tpt-megamoe.yaml @@ -49,10 +49,13 @@ infra: frontend: type: dynamo enable_multiple_frontends: true + env: + DYN_REQUEST_PLANE_CODEC: "msgpack" backend: type: vllm connector: null prefill_environment: + DYN_REQUEST_PLANE_CODEC: "msgpack" TILELANG_CLEANUP_TEMP_FILES: "1" VLLM_USE_NCCL_SYMM_MEM: "1" TORCH_SYMMMEM: "NVSHMEM" @@ -70,6 +73,7 @@ backend: UCX_CUDA_IPC_ENABLE_MNNVL: "y" NCCL_P2P_LEVEL: NVL decode_environment: + DYN_REQUEST_PLANE_CODEC: "msgpack" TILELANG_CLEANUP_TEMP_FILES: "1" VLLM_USE_NCCL_SYMM_MEM: "1" TORCH_SYMMMEM: "NVSHMEM" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-latency.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-latency.yaml index a3c6d4d369..1e202b2281 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-latency.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-latency.yaml @@ -48,10 +48,13 @@ infra: frontend: type: dynamo enable_multiple_frontends: true + env: + DYN_REQUEST_PLANE_CODEC: "msgpack" backend: type: vllm connector: null prefill_environment: + DYN_REQUEST_PLANE_CODEC: "msgpack" TILELANG_CLEANUP_TEMP_FILES: "1" VLLM_USE_NCCL_SYMM_MEM: "1" NCCL_CUMEM_ENABLE: "1" @@ -68,6 +71,7 @@ backend: UCX_CUDA_IPC_ENABLE_MNNVL: "y" NCCL_P2P_LEVEL: NVL decode_environment: + DYN_REQUEST_PLANE_CODEC: "msgpack" TILELANG_CLEANUP_TEMP_FILES: "1" VLLM_USE_NCCL_SYMM_MEM: "1" NCCL_CUMEM_ENABLE: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-middle-curve.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-middle-curve.yaml index d31bd0b5df..cc4f142296 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-middle-curve.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-middle-curve.yaml @@ -49,10 +49,13 @@ infra: frontend: type: dynamo enable_multiple_frontends: true + env: + DYN_REQUEST_PLANE_CODEC: "msgpack" backend: type: vllm connector: null prefill_environment: + DYN_REQUEST_PLANE_CODEC: "msgpack" TILELANG_CLEANUP_TEMP_FILES: "1" VLLM_USE_NCCL_SYMM_MEM: "1" NCCL_CUMEM_ENABLE: "1" @@ -69,6 +72,7 @@ backend: UCX_CUDA_IPC_ENABLE_MNNVL: "y" NCCL_P2P_LEVEL: NVL decode_environment: + DYN_REQUEST_PLANE_CODEC: "msgpack" TILELANG_CLEANUP_TEMP_FILES: "1" VLLM_USE_NCCL_SYMM_MEM: "1" NCCL_CUMEM_ENABLE: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-max-tpt-megamoe.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-max-tpt-megamoe.yaml index 2186657c58..9a46c6997d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-max-tpt-megamoe.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-max-tpt-megamoe.yaml @@ -49,10 +49,13 @@ infra: frontend: type: dynamo enable_multiple_frontends: true + env: + DYN_REQUEST_PLANE_CODEC: "msgpack" backend: type: vllm connector: null prefill_environment: + DYN_REQUEST_PLANE_CODEC: "msgpack" TILELANG_CLEANUP_TEMP_FILES: "1" VLLM_USE_NCCL_SYMM_MEM: "1" TORCH_SYMMMEM: "NVSHMEM" @@ -70,6 +73,7 @@ backend: UCX_CUDA_IPC_ENABLE_MNNVL: "y" NCCL_P2P_LEVEL: NVL decode_environment: + DYN_REQUEST_PLANE_CODEC: "msgpack" TILELANG_CLEANUP_TEMP_FILES: "1" VLLM_USE_NCCL_SYMM_MEM: "1" TORCH_SYMMMEM: "NVSHMEM" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-mid-curve-megamoe.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-mid-curve-megamoe.yaml index e1f120c158..8a37a551cc 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-mid-curve-megamoe.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-mid-curve-megamoe.yaml @@ -49,10 +49,13 @@ infra: frontend: type: dynamo enable_multiple_frontends: true + env: + DYN_REQUEST_PLANE_CODEC: "msgpack" backend: type: vllm connector: null prefill_environment: + DYN_REQUEST_PLANE_CODEC: "msgpack" TILELANG_CLEANUP_TEMP_FILES: "1" VLLM_USE_NCCL_SYMM_MEM: "1" TORCH_SYMMMEM: "NVSHMEM" @@ -70,6 +73,7 @@ backend: UCX_CUDA_IPC_ENABLE_MNNVL: "y" NCCL_P2P_LEVEL: NVL decode_environment: + DYN_REQUEST_PLANE_CODEC: "msgpack" TILELANG_CLEANUP_TEMP_FILES: "1" VLLM_USE_NCCL_SYMM_MEM: "1" TORCH_SYMMMEM: "NVSHMEM" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 602139beba..924d0cc1b2 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4705,5 +4705,6 @@ description: - "Update the Dynamo wheel from 1.2.0.dev20260426 to 1.3.0.dev1" - "Enable multiple Dynamo frontends across all DeepSeek-V4-Pro FP4 GB200 topologies" + - "Use MessagePack encoding for the Dynamo request plane on frontends and workers" - "Update the vLLM image from v0.20.0-ubuntu2404 to dsv4-megamoe-mxfp4-arm64-cu130-4ba0a72" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2139