[AgentX][vLLM][NVIDIA] chore: tune vLLM DeepSeek v4 agentic B200+B300

benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh

@@ -13,10 +13,9 @@ set -x
 #       experts EP-sharded across DP ranks (per the vLLM blog recipe).
 #       Highest aggregate throughput at large CONC.
 #
-# Image is configured in nvidia-master.yaml. block_size=256,
-# kv-cache-dtype=fp8, FP4 indexer cache enabled, FULL_AND_PIECEWISE cudagraph
-# capture with custom_ops=all (per the vLLM blog recipe at
-# https://vllm.ai/blog/deepseek-v4).
+# Image is configured in nvidia-master.yaml. The serving flags follow the
+# DeepSeek-V4 low-latency vLLM recipe with sparse MLA attention, Mega-MoE,
+# FP8 KV cache, and full decode-only CUDA graphs.
 #
 # Required env vars:
 #   MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR
@@ -118,6 +117,7 @@ if require_agentic_kv_offload_backend mooncake; then
         agentic_pip_install --quiet --no-cache-dir --no-deps \
             --force-reinstall "mooncake-transfer-engine-cuda13==$MOONCAKE_VERSION"
         python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null
+        python3 "$(dirname "$0")/patch_vllm_pr45406.py"
 
         MOONCAKE_MASTER_PORT=$((PORT + 12000))
         MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json"
@@ -178,7 +178,10 @@ fi
 
 EP_ARGS=()
 if [ "$EP_SIZE" -gt 1 ]; then
-    EP_ARGS=(--enable-expert-parallel)
+    EP_ARGS=(
+        --enable-expert-parallel
+        --moe-backend deep_gemm_mega_moe
+    )
 fi
 
 # AgentX concurrency counts live session trees, not individual requests.
@@ -202,15 +205,20 @@ VLLM_CMD=(
     "${PARALLEL_ARGS[@]}"
     "${VLLM_CP_ARGS[@]}"
     "${EP_ARGS[@]}"
-    --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'
+    --prefill-schedule-interval 8
+    --numa-bind
+    --compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}'
+    --attention-config '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true}'
     --attention_config.use_fp4_indexer_cache=True
     --tokenizer-mode deepseek_v4
     --tool-call-parser deepseek_v4
     --enable-auto-tool-choice
     --reasoning-parser deepseek_v4
+    --no-enable-flashinfer-autotune
     --enable-prefix-caching
     --no-disable-hybrid-kv-cache-manager
     --max-num-seqs "$MAX_NUM_SEQS"
+    --disable-uvicorn-access-log
     "${OFFLOAD_ARGS[@]}"
 )
 printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt"

benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh

@@ -13,9 +13,9 @@ set -x
 #       experts EP-sharded across DP ranks (per the vLLM blog recipe).
 #       Highest aggregate throughput at large CONC.
 #
-# Image is vllm/vllm-openai:v0.20.0-cu130. block_size=256, kv-cache-dtype=fp8,
-# FP4 indexer cache enabled, FULL_AND_PIECEWISE cudagraph capture with
-# custom_ops=all (per the vLLM blog recipe at https://vllm.ai/blog/deepseek-v4).
+# Image is configured in nvidia-master.yaml. The serving flags follow the
+# DeepSeek-V4 low-latency vLLM recipe with sparse MLA attention, Mega-MoE,
+# FP8 KV cache, and full decode-only CUDA graphs.
 #
 # Required env vars:
 #   MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR
@@ -122,6 +122,7 @@ if require_agentic_kv_offload_backend mooncake; then
         agentic_pip_install --quiet --no-cache-dir --no-deps \
             --force-reinstall "mooncake-transfer-engine-cuda13==$MOONCAKE_VERSION"
         python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null
+        python3 "$(dirname "$0")/patch_vllm_pr45406.py"
 
         MOONCAKE_MASTER_PORT=$((PORT + 12000))
         MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json"
@@ -180,7 +181,10 @@ fi
 
 EP_ARGS=()
 if [ "$EP_SIZE" -gt 1 ]; then
-    EP_ARGS=(--enable-expert-parallel)
+    EP_ARGS=(
+        --enable-expert-parallel
+        --moe-backend deep_gemm_mega_moe
+    )
 fi
 
 # AgentX concurrency counts live session trees, not individual requests.
@@ -205,15 +209,20 @@ vllm serve "$MODEL_PATH" --served-model-name "$MODEL" \
 "${PARALLEL_ARGS[@]}" \
 "${VLLM_CP_ARGS[@]}" \
 "${EP_ARGS[@]}" \
---compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' \
+--prefill-schedule-interval 8 \
+--numa-bind \
+--compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' \
+--attention-config '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true}' \
 --attention_config.use_fp4_indexer_cache=True \
 --tokenizer-mode deepseek_v4 \
 --tool-call-parser deepseek_v4 \
 --enable-auto-tool-choice \
 --reasoning-parser deepseek_v4 \
+--no-enable-flashinfer-autotune \
 --enable-prefix-caching \
 --no-disable-hybrid-kv-cache-manager \
 --max-num-seqs "$MAX_NUM_SEQS" \
+--disable-uvicorn-access-log \
 "${OFFLOAD_ARGS[@]}" > "$SERVER_LOG" 2>&1 &
 SERVER_PID=$!
 echo "Server PID: $SERVER_PID"

benchmarks/single_node/agentic/patch_vllm_pr45406.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+"""Backport the scheduler fix from vllm-project/vllm PR #45406."""
+
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+
+
+PATCH_MARKER = "See https://github.com/vllm-project/vllm/issues/45388"
+
+
+def main() -> None:
+    spec = importlib.util.find_spec("vllm")
+    if spec is None or not spec.submodule_search_locations:
+        raise RuntimeError("Could not locate the installed vllm package")
+
+    package_root = Path(next(iter(spec.submodule_search_locations)))
+    scheduler = package_root / "v1" / "core" / "sched" / "scheduler.py"
+    text = scheduler.read_text()
+    if PATCH_MARKER in text:
+        print(f"vLLM PR #45406 backport already present in {scheduler}")
+        return
+
+    old = """                    if request.has_encoder_inputs:
+                        self.encoder_cache_manager.free(request)
+                    break
+
+                # KVTransfer: the connector uses this info to determine
+"""
+    new = """                    if request.has_encoder_inputs:
+                        self.encoder_cache_manager.free(request)
+                    if self.running:
+                        # Running requests will free blocks when they
+                        # complete; stop here to preserve queue-order
+                        # admission.
+                        break
+                    # Nothing is running, so no future event frees blocks and
+                    # stopping at this request would freeze this state
+                    # permanently. Requests behind this one may hold blocks
+                    # while parked (async KV loads in WAITING_FOR_REMOTE_KVS)
+                    # and are only promoted when this traversal reaches them.
+                    # Keep scanning so they can be promoted, scheduled, and
+                    # eventually free the blocks this request needs.
+                    # See https://github.com/vllm-project/vllm/issues/45388
+                    request_queue.pop_request()
+                    step_skipped_waiting.prepend_request(request)
+                    continue
+
+                # KVTransfer: the connector uses this info to determine
+"""
+    if text.count(old) != 1:
+        raise RuntimeError(
+            f"Expected exactly one vLLM PR #45406 patch target in {scheduler}"
+        )
+
+    scheduler.write_text(text.replace(old, new, 1))
+    compile(scheduler.read_text(), str(scheduler), "exec")
+    print(f"Applied vLLM PR #45406 backport to {scheduler}")
+
+
+if __name__ == "__main__":
+    main()

configs/nvidia-master.yaml

@@ -1762,7 +1762,7 @@ dsv4-fp4-b200-vllm:
       - { tp: 8, ep: 8, dp-attn: true, conc-start: 64, conc-end: 1024 }
 
 dsv4-fp4-b200-vllm-agentic:
-  image: vllm/vllm-openai:v0.23.0
+  image: vllm/vllm-openai:nightly-95ed0feaa5cd7fb16d72c53ce04950aaf07c4698
   model: deepseek-ai/DeepSeek-V4-Pro
   model-prefix: dsv4
   runner: cluster:b200-dgxc
@@ -1775,12 +1775,11 @@ dsv4-fp4-b200-vllm-agentic:
       search-space:
       # Pure TP is only competitive at very low concurrency.
       - { tp: 8, kv-offloading: none,  conc-list: [1, 2, 3, 4, 5] }
-      # Sample the useful MooncakeStore range without repeating its collapsed
-      # high-concurrency tail.
-      - { tp: 8, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [8, 10, 16] }
-      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none,  conc-list: [16, 32, 38, 44, 50] }
+      # Cover the external-cache transition region.
+      - { tp: 8, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [8, 10, 16, 32, 40, 44] }
+      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none,  conc-list: [24, 32, 38, 44, 48, 50, 52] }
       # Retain the external-cache transition and peak-throughput region.
-      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [16, 38, 44, 56, 64, 66, 68] }
+      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [32, 38, 44, 50, 56, 64, 66, 68] }
 
 dsv4-fp4-b200-trt:
   image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066
@@ -3207,7 +3206,7 @@ dsv4-fp4-b300-vllm:
       - { tp: 8, ep: 8, dp-attn: true, conc-start: 2048, conc-end: 2048 }
 
 dsv4-fp4-b300-vllm-agentic:
-  image: vllm/vllm-openai:v0.23.0
+  image: vllm/vllm-openai:nightly-95ed0feaa5cd7fb16d72c53ce04950aaf07c4698
   model: deepseek-ai/DeepSeek-V4-Pro
   model-prefix: dsv4
   runner: cluster:b300-nv
@@ -3219,15 +3218,15 @@ dsv4-fp4-b300-vllm-agentic:
     - dram-utilization: 0.80
       search-space:
       # Compare native GPU-cache and MooncakeStore CPU-offload cliffs.
-      - { tp: 4, kv-offloading: none,  conc-list: [1, 2, 4, 6, 8, 16] }
+      - { tp: 4, kv-offloading: none,  conc-list: [1, 2, 4, 6, 8, 16, 20, 24, 32] }
       - { tp: 4, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [16, 18, 20, 24] }
-      # TP8 remains cache-resident through conc 52.
-      - { tp: 8, kv-offloading: none,  conc-list: [1, 2, 4, 6, 8, 16, 32, 40, 48, 52] }
+      # Cover the TP8 cache-transition region.
+      - { tp: 8, kv-offloading: none,  conc-list: [1, 2, 4, 6, 8, 16, 32, 40, 48, 52, 56, 60, 64, 72] }
       - { tp: 8, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [52] }
-      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none,  conc-list: [8, 16] }
-      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [32] }
-      # TP8 DEP retains representative low, peak, and transition points.
-      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none,  conc-list: [52, 72, 100, 128, 144] }
+      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none,  conc-list: [8, 16, 24, 32, 40, 64] }
+      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [32, 40, 48, 56, 64, 72, 80, 88, 96, 128] }
+      # Cover the TP8 DEP throughput region.
+      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none,  conc-list: [52, 72, 100, 128, 144, 196, 512] }
 
 dsv4-fp4-b300-trt:
   image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066

perf-changelog.yaml

@@ -4707,3 +4707,12 @@
     - "Clean the export envs"
     - "Enable two batch overlap"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2093
+
+- config-keys:
+    - dsv4-fp4-b200-vllm-agentic
+    - dsv4-fp4-b300-vllm-agentic
+  description:
+    - "Tune DeepSeek-V4 AgentX on B200 and B300 with the vLLM nightly at 95ed0fe."
+    - "Enable sparse MLA, EP-gated Mega-MoE, full decode-only CUDA graphs, NUMA binding, FP8 KV cache, and the vLLM PR #45406 scheduler fix."
+    - "Expand native-cache and MooncakeStore concurrency coverage."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2138