From 10e66391bc515724d6b28ca7441d3f9567b7156d Mon Sep 17 00:00:00 2001 From: ApostaC Date: Fri, 10 Jul 2026 09:44:42 -0700 Subject: [PATCH 1/5] Add LMCache configs for dsv4 vllm b200/b300 Signed-off-by: ApostaC --- .../single_node/agentic/dsv4_fp4_b200_vllm.sh | 86 ++++++++++++++++++- .../single_node/agentic/dsv4_fp4_b300_vllm.sh | 86 ++++++++++++++++++- configs/nvidia-master.yaml | 39 +++++++++ perf-changelog.yaml | 9 ++ 4 files changed, 216 insertions(+), 4 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh index d723754ac8..718a7e3f4a 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh @@ -21,7 +21,7 @@ set -x # Required env vars: # MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR # -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=mooncake. +# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=mooncake or lmcache. source "$(dirname "$0")/../../benchmark_lib.sh" @@ -101,15 +101,20 @@ export VLLM_PREFIX_CACHE_RETENTION_INTERVAL=32768 SERVER_LOG="$RESULT_DIR/server.log" ROUTER_LOG="$RESULT_DIR/router.log" MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log" +LMCACHE_SERVER_LOG="$RESULT_DIR/lmcache_server.log" mkdir -p "$RESULT_DIR" SERVER_PID="" ROUTER_PID="" MOONCAKE_MASTER_PID="" +LMCACHE_SERVER_PID="" OFFLOAD_ARGS=() -if require_agentic_kv_offload_backend mooncake; then +if agentic_kv_offload_enabled; then + case "$KV_OFFLOAD_BACKEND" in + mooncake) + require_agentic_kv_offload_backend mooncake # Embedded mode contributes one segment per GPU rank to a shared # distributed store, so pre-divide the aggregate host-memory budget. PER_RANK_GB=$((TOTAL_CPU_DRAM_GB / GPU_COUNT)) @@ -169,6 +174,83 @@ EOF --kv-transfer-config '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}' ) + ;; + lmcache) + require_agentic_kv_offload_backend lmcache + # The LMCache MP server owns the host-DRAM KV pool as one shared + # tier; vLLM ranks attach via LMCacheMPConnector, so the aggregate + # host budget is passed through undivided (unlike Mooncake's + # per-rank segments). Follows the LMCache DeepSeek-V4 recipe + # (docs.lmcache.ai/recipes/deepseek_v4_flash.html); LMCache handles + # DSV4's Sparse-MLA hybrid KV geometries automatically. + LMCACHE_VERSION=0.5.1 + agentic_pip_install --quiet --no-cache-dir "lmcache==$LMCACHE_VERSION" + python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null + + LMCACHE_HOST=127.0.0.1 + LMCACHE_PORT=$((PORT + 12000)) + LMCACHE_HTTP_PORT=$((PORT + 13000)) + # LMCacheMPConnector concatenates lmcache.mp.host and port into the + # ZMQ endpoint. Bind the server to a raw host, but pass the connector + # a ZMQ-style host string. + LMCACHE_CONNECT_HOST="tcp://$LMCACHE_HOST" + # The pool grows lazily from the initial allocation, so the full + # --l1-size-gb budget is not pinned at startup. + LMCACHE_L1_INIT_SIZE_GB=20 + LMCACHE_MQ_TIMEOUT=300 + # Identical prefixes must hash to identical cache keys across DP ranks. + export PYTHONHASHSEED=0 + + echo "Starting LMCache MP server on port $LMCACHE_PORT..." + # One GPU-side transfer worker avoids concurrent-GPU-transfer stalls + # under heavy async-load pressure; CPU-side workers stay at 8. + lmcache server \ + --host "$LMCACHE_HOST" \ + --port "$LMCACHE_PORT" \ + --http-host "$LMCACHE_HOST" \ + --http-port "$LMCACHE_HTTP_PORT" \ + --l1-size-gb "$TOTAL_CPU_DRAM_GB" \ + --l1-init-size-gb "$LMCACHE_L1_INIT_SIZE_GB" \ + --max-gpu-workers 1 \ + --max-cpu-workers 8 \ + --chunk-size 1024 \ + --l1-align-bytes 16384 \ + --eviction-trigger-watermark 0.85 \ + --eviction-ratio 0.10 \ + --eviction-policy LRU \ + > "$LMCACHE_SERVER_LOG" 2>&1 & + LMCACHE_SERVER_PID=$! + LMCACHE_READY=0 + for _ in $(seq 1 60); do + if ! kill -0 "$LMCACHE_SERVER_PID" 2>/dev/null; then + echo "LMCache server died during startup." >&2 + cat "$LMCACHE_SERVER_LOG" >&2 + exit 1 + fi + if curl --output /dev/null --silent --fail \ + "http://127.0.0.1:$LMCACHE_HTTP_PORT/healthcheck"; then + LMCACHE_READY=1 + break + fi + sleep 2 + done + if [ "$LMCACHE_READY" -ne 1 ]; then + echo "LMCache server did not become healthy in time." >&2 + cat "$LMCACHE_SERVER_LOG" >&2 + exit 1 + fi + + unset VLLM_USE_SIMPLE_KV_OFFLOAD + OFFLOAD_ARGS=( + --kv-transfer-config + "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.host\":\"$LMCACHE_CONNECT_HOST\",\"lmcache.mp.port\":$LMCACHE_PORT,\"lmcache.mp.mq_timeout\":$LMCACHE_MQ_TIMEOUT}}" + ) + ;; + *) + echo "Error: unsupported KV_OFFLOAD_BACKEND '$KV_OFFLOAD_BACKEND' (expected one of: mooncake, lmcache)" >&2 + exit 1 + ;; + esac fi PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh index 74dc2129be..c2846060d5 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh @@ -20,7 +20,7 @@ set -x # Required env vars: # MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR # -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=mooncake. +# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=mooncake or lmcache. source "$(dirname "$0")/../../benchmark_lib.sh" @@ -105,14 +105,19 @@ export VLLM_PREFIX_CACHE_RETENTION_INTERVAL=32768 SERVER_LOG="$RESULT_DIR/server.log" ROUTER_LOG="$RESULT_DIR/router.log" MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log" +LMCACHE_SERVER_LOG="$RESULT_DIR/lmcache_server.log" mkdir -p "$RESULT_DIR" SERVER_PID="" ROUTER_PID="" MOONCAKE_MASTER_PID="" +LMCACHE_SERVER_PID="" OFFLOAD_ARGS=() -if require_agentic_kv_offload_backend mooncake; then +if agentic_kv_offload_enabled; then + case "$KV_OFFLOAD_BACKEND" in + mooncake) + require_agentic_kv_offload_backend mooncake # Mooncake embedded mode contributes one global segment per GPU rank to # a shared distributed store. Pre-divide the aggregate host budget # across those rank-contributed segments. @@ -171,6 +176,83 @@ EOF --kv-transfer-config '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}' ) + ;; + lmcache) + require_agentic_kv_offload_backend lmcache + # The LMCache MP server owns the host-DRAM KV pool as one shared + # tier; vLLM ranks attach via LMCacheMPConnector, so the aggregate + # host budget is passed through undivided (unlike Mooncake's + # per-rank segments). Follows the LMCache DeepSeek-V4 recipe + # (docs.lmcache.ai/recipes/deepseek_v4_flash.html); LMCache handles + # DSV4's Sparse-MLA hybrid KV geometries automatically. + LMCACHE_VERSION=0.5.1 + agentic_pip_install --quiet --no-cache-dir "lmcache==$LMCACHE_VERSION" + python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null + + LMCACHE_HOST=127.0.0.1 + LMCACHE_PORT=$((PORT + 12000)) + LMCACHE_HTTP_PORT=$((PORT + 13000)) + # LMCacheMPConnector concatenates lmcache.mp.host and port into the + # ZMQ endpoint. Bind the server to a raw host, but pass the connector + # a ZMQ-style host string. + LMCACHE_CONNECT_HOST="tcp://$LMCACHE_HOST" + # The pool grows lazily from the initial allocation, so the full + # --l1-size-gb budget is not pinned at startup. + LMCACHE_L1_INIT_SIZE_GB=20 + LMCACHE_MQ_TIMEOUT=300 + # Identical prefixes must hash to identical cache keys across DP ranks. + export PYTHONHASHSEED=0 + + echo "Starting LMCache MP server on port $LMCACHE_PORT..." + # One GPU-side transfer worker avoids concurrent-GPU-transfer stalls + # under heavy async-load pressure; CPU-side workers stay at 8. + lmcache server \ + --host "$LMCACHE_HOST" \ + --port "$LMCACHE_PORT" \ + --http-host "$LMCACHE_HOST" \ + --http-port "$LMCACHE_HTTP_PORT" \ + --l1-size-gb "$TOTAL_CPU_DRAM_GB" \ + --l1-init-size-gb "$LMCACHE_L1_INIT_SIZE_GB" \ + --max-gpu-workers 1 \ + --max-cpu-workers 8 \ + --chunk-size 1024 \ + --l1-align-bytes 16384 \ + --eviction-trigger-watermark 0.85 \ + --eviction-ratio 0.10 \ + --eviction-policy LRU \ + > "$LMCACHE_SERVER_LOG" 2>&1 & + LMCACHE_SERVER_PID=$! + LMCACHE_READY=0 + for _ in $(seq 1 60); do + if ! kill -0 "$LMCACHE_SERVER_PID" 2>/dev/null; then + echo "LMCache server died during startup." >&2 + cat "$LMCACHE_SERVER_LOG" >&2 + exit 1 + fi + if curl --output /dev/null --silent --fail \ + "http://127.0.0.1:$LMCACHE_HTTP_PORT/healthcheck"; then + LMCACHE_READY=1 + break + fi + sleep 2 + done + if [ "$LMCACHE_READY" -ne 1 ]; then + echo "LMCache server did not become healthy in time." >&2 + cat "$LMCACHE_SERVER_LOG" >&2 + exit 1 + fi + + unset VLLM_USE_SIMPLE_KV_OFFLOAD + OFFLOAD_ARGS=( + --kv-transfer-config + "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.host\":\"$LMCACHE_CONNECT_HOST\",\"lmcache.mp.port\":$LMCACHE_PORT,\"lmcache.mp.mq_timeout\":$LMCACHE_MQ_TIMEOUT}}" + ) + ;; + *) + echo "Error: unsupported KV_OFFLOAD_BACKEND '$KV_OFFLOAD_BACKEND' (expected one of: mooncake, lmcache)" >&2 + exit 1 + ;; + esac fi PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 6be8e006f9..85f9be5e57 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1782,6 +1782,25 @@ dsv4-fp4-b200-vllm-agentic: # Retain the external-cache transition and peak-throughput region. - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [16, 38, 44, 56, 64, 66, 68] } +# LMCache CPU-offload arm of dsv4-fp4-b200-vllm-agentic, split into its own +# config so it can be triggered/tested independently of the Mooncake curve. +# Points mirror the Mooncake offload points; the GPU-cache (kv-offloading: +# none) baselines live in the parent config only. +dsv4-fp4-b200-vllm-agentic-lmcache: + image: vllm/vllm-openai:v0.23.0 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:b200-dgxc + precision: fp4 + framework: vllm + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - { tp: 8, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [8, 10, 16] } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [16, 38, 44, 56, 64, 66, 68] } + dsv4-fp4-b200-trt: image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066 model: deepseek-ai/DeepSeek-V4-Pro @@ -3229,6 +3248,26 @@ dsv4-fp4-b300-vllm-agentic: # TP8 DEP retains representative low, peak, and transition points. - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [52, 72, 100, 128, 144] } +# LMCache CPU-offload arm of dsv4-fp4-b300-vllm-agentic, split into its own +# config so it can be triggered/tested independently of the Mooncake curve. +# Points mirror the Mooncake offload points; the GPU-cache (kv-offloading: +# none) baselines live in the parent config only. +dsv4-fp4-b300-vllm-agentic-lmcache: + image: vllm/vllm-openai:v0.23.0 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:b300-nv + precision: fp4 + framework: vllm + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - { tp: 4, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [16, 18, 20, 24] } + - { tp: 8, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [52] } + - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [32] } + dsv4-fp4-b300-trt: image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066 model: deepseek-ai/DeepSeek-V4-Pro diff --git a/perf-changelog.yaml b/perf-changelog.yaml index be28208ab4..76f70a7dce 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4716,3 +4716,12 @@ - "Clean the export envs" - "Enable two batch overlap" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2093 + +- config-keys: + - dsv4-fp4-b200-vllm-agentic-lmcache + - dsv4-fp4-b300-vllm-agentic-lmcache + description: + - "Add LMCache 0.5.1 DRAM KV-offload configs (kv-offload-backend: lmcache) as standalone sections alongside the Mooncake parents" + - "LMCache MP server + LMCacheMPConnector with one shared host-DRAM pool sized to the aggregate TOTAL_CPU_DRAM_GB budget" + - "LMCache points mirror the Mooncake offload points: B300 TP4 conc 16-24, TP8 conc 52, TP4-DEP4 conc 32; B200 TP8 conc 8-16, TP8-DEP8 conc 16-68" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX From 73137f194dd8457a590f2782a5d354053f04447c Mon Sep 17 00:00:00 2001 From: ApostaC Date: Fri, 10 Jul 2026 09:50:27 -0700 Subject: [PATCH 2/5] update perf changelog Signed-off-by: ApostaC --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 76f70a7dce..3ffc8a4203 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4724,4 +4724,4 @@ - "Add LMCache 0.5.1 DRAM KV-offload configs (kv-offload-backend: lmcache) as standalone sections alongside the Mooncake parents" - "LMCache MP server + LMCacheMPConnector with one shared host-DRAM pool sized to the aggregate TOTAL_CPU_DRAM_GB budget" - "LMCache points mirror the Mooncake offload points: B300 TP4 conc 16-24, TP8 conc 52, TP4-DEP4 conc 32; B200 TP8 conc 8-16, TP8-DEP8 conc 16-68" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2153 From 4244f91fcdabfbf26513c9249f36dc2096393439 Mon Sep 17 00:00:00 2001 From: ApostaC Date: Fri, 10 Jul 2026 12:04:20 -0700 Subject: [PATCH 3/5] lmcache: derate L1 pool to 75%, move to vLLM v0.24.0, defer B300 section Signed-off-by: ApostaC --- .../single_node/agentic/dsv4_fp4_b200_vllm.sh | 19 +++++++++++--- .../single_node/agentic/dsv4_fp4_b300_vllm.sh | 19 +++++++++++--- configs/nvidia-master.yaml | 25 +++---------------- perf-changelog.yaml | 10 +++++--- 4 files changed, 39 insertions(+), 34 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh index 718a7e3f4a..e644b9fa79 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh @@ -70,8 +70,12 @@ export AIPERF_AGENTIC_CACHE_WARMUP_DURATION=600 # vLLM v0.22.1 can ship CUTLASS DSL 4.5.2 with stale native MLIR bindings, # which fails DSV4 indexer compilation with mlir_global_dtors(..., data). # Reinstall the matching native wheel until NVIDIA/cutlass#3259 is resolved. -agentic_pip_install --quiet --force-reinstall --no-deps \ - 'nvidia-cutlass-dsl-libs-cu13==4.5.2' +# The v0.24 images ship fixed bindings; reinstalling there would downgrade. +VLLM_INSTALLED_VERSION=$(python3 -c "from importlib.metadata import version; print(version('vllm'))") +if [ "$(printf '%s\n' "$VLLM_INSTALLED_VERSION" 0.24.0 | sort -V | head -n1)" != "0.24.0" ]; then + agentic_pip_install --quiet --force-reinstall --no-deps \ + 'nvidia-cutlass-dsl-libs-cu13==4.5.2' +fi # vllm-project/router expands the one HTTP backend into one logical worker per # DP rank and sends X-data-parallel-rank on forwarded requests. aiperf's @@ -194,8 +198,15 @@ EOF # ZMQ endpoint. Bind the server to a raw host, but pass the connector # a ZMQ-style host string. LMCACHE_CONNECT_HOST="tcp://$LMCACHE_HOST" + # Pool target derated to 75% of the aggregate budget: pinned host + # memory is unswappable and also consumes GPU-side mapping + # resources, so leave headroom for vLLM host buffers and the OS. + # Full-budget targets OOM-killed the node (host OOM-killer or + # cudaErrorMemoryAllocation) as the cache filled past ~2 TB during + # PR #2153 bring-up. + LMCACHE_L1_SIZE_GB=$((TOTAL_CPU_DRAM_GB * 3 / 4)) # The pool grows lazily from the initial allocation, so the full - # --l1-size-gb budget is not pinned at startup. + # --l1-size-gb target is not pinned at startup. LMCACHE_L1_INIT_SIZE_GB=20 LMCACHE_MQ_TIMEOUT=300 # Identical prefixes must hash to identical cache keys across DP ranks. @@ -209,7 +220,7 @@ EOF --port "$LMCACHE_PORT" \ --http-host "$LMCACHE_HOST" \ --http-port "$LMCACHE_HTTP_PORT" \ - --l1-size-gb "$TOTAL_CPU_DRAM_GB" \ + --l1-size-gb "$LMCACHE_L1_SIZE_GB" \ --l1-init-size-gb "$LMCACHE_L1_INIT_SIZE_GB" \ --max-gpu-workers 1 \ --max-cpu-workers 8 \ diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh index c2846060d5..9aced4c74d 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh @@ -73,8 +73,12 @@ export AIPERF_AGENTIC_CACHE_WARMUP_DURATION=600 # vLLM v0.22.1 can ship CUTLASS DSL 4.5.2 with stale native MLIR bindings, # which fails DSV4 indexer compilation with mlir_global_dtors(..., data). # Reinstall the matching native wheel until NVIDIA/cutlass#3259 is resolved. -agentic_pip_install --quiet --force-reinstall --no-deps \ - 'nvidia-cutlass-dsl-libs-cu13==4.5.2' +# The v0.24 images ship fixed bindings; reinstalling there would downgrade. +VLLM_INSTALLED_VERSION=$(python3 -c "from importlib.metadata import version; print(version('vllm'))") +if [ "$(printf '%s\n' "$VLLM_INSTALLED_VERSION" 0.24.0 | sort -V | head -n1)" != "0.24.0" ]; then + agentic_pip_install --quiet --force-reinstall --no-deps \ + 'nvidia-cutlass-dsl-libs-cu13==4.5.2' +fi # vllm-project/router expands the one HTTP backend into one logical worker per # DP rank and sends X-data-parallel-rank on forwarded requests. aiperf's @@ -196,8 +200,15 @@ EOF # ZMQ endpoint. Bind the server to a raw host, but pass the connector # a ZMQ-style host string. LMCACHE_CONNECT_HOST="tcp://$LMCACHE_HOST" + # Pool target derated to 75% of the aggregate budget: pinned host + # memory is unswappable and also consumes GPU-side mapping + # resources, so leave headroom for vLLM host buffers and the OS. + # Full-budget targets OOM-killed the node (host OOM-killer or + # cudaErrorMemoryAllocation) as the cache filled past ~2 TB during + # PR #2153 bring-up. + LMCACHE_L1_SIZE_GB=$((TOTAL_CPU_DRAM_GB * 3 / 4)) # The pool grows lazily from the initial allocation, so the full - # --l1-size-gb budget is not pinned at startup. + # --l1-size-gb target is not pinned at startup. LMCACHE_L1_INIT_SIZE_GB=20 LMCACHE_MQ_TIMEOUT=300 # Identical prefixes must hash to identical cache keys across DP ranks. @@ -211,7 +222,7 @@ EOF --port "$LMCACHE_PORT" \ --http-host "$LMCACHE_HOST" \ --http-port "$LMCACHE_HTTP_PORT" \ - --l1-size-gb "$TOTAL_CPU_DRAM_GB" \ + --l1-size-gb "$LMCACHE_L1_SIZE_GB" \ --l1-init-size-gb "$LMCACHE_L1_INIT_SIZE_GB" \ --max-gpu-workers 1 \ --max-cpu-workers 8 \ diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 85f9be5e57..5a53b17709 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1785,9 +1785,10 @@ dsv4-fp4-b200-vllm-agentic: # LMCache CPU-offload arm of dsv4-fp4-b200-vllm-agentic, split into its own # config so it can be triggered/tested independently of the Mooncake curve. # Points mirror the Mooncake offload points; the GPU-cache (kv-offloading: -# none) baselines live in the parent config only. +# none) baselines live in the parent config only. Runs the v0.24.0 image +# (the LMCache-0.5.x-validated pairing) while the parent stays on v0.23.0. dsv4-fp4-b200-vllm-agentic-lmcache: - image: vllm/vllm-openai:v0.23.0 + image: vllm/vllm-openai:v0.24.0 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:b200-dgxc @@ -3248,26 +3249,6 @@ dsv4-fp4-b300-vllm-agentic: # TP8 DEP retains representative low, peak, and transition points. - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [52, 72, 100, 128, 144] } -# LMCache CPU-offload arm of dsv4-fp4-b300-vllm-agentic, split into its own -# config so it can be triggered/tested independently of the Mooncake curve. -# Points mirror the Mooncake offload points; the GPU-cache (kv-offloading: -# none) baselines live in the parent config only. -dsv4-fp4-b300-vllm-agentic-lmcache: - image: vllm/vllm-openai:v0.23.0 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: cluster:b300-nv - precision: fp4 - framework: vllm - multinode: false - scenarios: - agentic-coding: - - dram-utilization: 0.80 - search-space: - - { tp: 4, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [16, 18, 20, 24] } - - { tp: 8, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [52] } - - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [32] } - dsv4-fp4-b300-trt: image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066 model: deepseek-ai/DeepSeek-V4-Pro diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 79c8bd07b2..a7e631118f 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4726,9 +4726,11 @@ - config-keys: - dsv4-fp4-b200-vllm-agentic-lmcache - - dsv4-fp4-b300-vllm-agentic-lmcache + scenario-type: + - agentic-coding description: - - "Add LMCache 0.5.1 DRAM KV-offload configs (kv-offload-backend: lmcache) as standalone sections alongside the Mooncake parents" - - "LMCache MP server + LMCacheMPConnector with one shared host-DRAM pool sized to the aggregate TOTAL_CPU_DRAM_GB budget" - - "LMCache points mirror the Mooncake offload points: B300 TP4 conc 16-24, TP8 conc 52, TP4-DEP4 conc 32; B200 TP8 conc 8-16, TP8-DEP8 conc 16-68" + - "Add LMCache 0.5.1 DRAM KV-offload config (kv-offload-backend: lmcache) as a standalone section alongside the Mooncake parent (B300 section deferred)" + - "Image vllm/vllm-openai:v0.24.0 for the LMCache section (parent stays on v0.23.0); CUTLASS DSL cu13 reinstall gated to vLLM < 0.24" + - "LMCache MP server + LMCacheMPConnector; L1 pool derated to 75% of TOTAL_CPU_DRAM_GB after full-budget pinned pools OOM-killed nodes in bring-up" + - "LMCache points mirror the Mooncake offload points: B200 TP8 conc 8-16, TP8-DEP8 conc 16-68" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2153 From 380ee5a38303fdba588eb9dfb225b46181fa734e Mon Sep 17 00:00:00 2001 From: ApostaC Date: Fri, 10 Jul 2026 15:37:33 -0700 Subject: [PATCH 4/5] fixing the comments and picking optimizations from #2138 Signed-off-by: ApostaC --- .../single_node/agentic/dsv4_fp4_b200_vllm.sh | 27 +++++--- .../single_node/agentic/dsv4_fp4_b300_vllm.sh | 27 +++++--- .../single_node/agentic/patch_vllm_pr45406.py | 63 +++++++++++++++++++ 3 files changed, 101 insertions(+), 16 deletions(-) create mode 100755 benchmarks/single_node/agentic/patch_vllm_pr45406.py diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh index e644b9fa79..068f10100b 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh @@ -70,12 +70,8 @@ export AIPERF_AGENTIC_CACHE_WARMUP_DURATION=600 # vLLM v0.22.1 can ship CUTLASS DSL 4.5.2 with stale native MLIR bindings, # which fails DSV4 indexer compilation with mlir_global_dtors(..., data). # Reinstall the matching native wheel until NVIDIA/cutlass#3259 is resolved. -# The v0.24 images ship fixed bindings; reinstalling there would downgrade. -VLLM_INSTALLED_VERSION=$(python3 -c "from importlib.metadata import version; print(version('vllm'))") -if [ "$(printf '%s\n' "$VLLM_INSTALLED_VERSION" 0.24.0 | sort -V | head -n1)" != "0.24.0" ]; then - agentic_pip_install --quiet --force-reinstall --no-deps \ - 'nvidia-cutlass-dsl-libs-cu13==4.5.2' -fi +agentic_pip_install --quiet --force-reinstall --no-deps \ + 'nvidia-cutlass-dsl-libs-cu13==4.5.2' # vllm-project/router expands the one HTTP backend into one logical worker per # DP rank and sends X-data-parallel-rank on forwarded requests. aiperf's @@ -127,6 +123,7 @@ if agentic_kv_offload_enabled; then agentic_pip_install --quiet --no-cache-dir --no-deps \ --force-reinstall "mooncake-transfer-engine-cuda13==$MOONCAKE_VERSION" python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null + python3 "$(dirname "$0")/patch_vllm_pr45406.py" MOONCAKE_MASTER_PORT=$((PORT + 12000)) MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json" @@ -190,6 +187,11 @@ EOF LMCACHE_VERSION=0.5.1 agentic_pip_install --quiet --no-cache-dir "lmcache==$LMCACHE_VERSION" python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null + # Async KV loads park requests in WAITING_FOR_REMOTE_KVS holding + # blocks; without the PR #45406 scheduler fix the waiting-queue scan + # can freeze permanently once nothing is running (hung warmup + # stragglers in PR #2153 bring-up). + python3 "$(dirname "$0")/patch_vllm_pr45406.py" LMCACHE_HOST=127.0.0.1 LMCACHE_PORT=$((PORT + 12000)) @@ -229,6 +231,7 @@ EOF --eviction-trigger-watermark 0.85 \ --eviction-ratio 0.10 \ --eviction-policy LRU \ + --supported-transfer-mode lmcache_driven \ > "$LMCACHE_SERVER_LOG" 2>&1 & LMCACHE_SERVER_PID=$! LMCACHE_READY=0 @@ -271,7 +274,10 @@ fi EP_ARGS=() if [ "$EP_SIZE" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) + EP_ARGS=( + --enable-expert-parallel + --moe-backend deep_gemm_mega_moe + ) fi # AgentX concurrency counts live session trees, not individual requests. @@ -295,15 +301,20 @@ VLLM_CMD=( "${PARALLEL_ARGS[@]}" "${VLLM_CP_ARGS[@]}" "${EP_ARGS[@]}" - --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' + --prefill-schedule-interval 8 + --numa-bind + --compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + --attention-config '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true}' --attention_config.use_fp4_indexer_cache=True --tokenizer-mode deepseek_v4 --tool-call-parser deepseek_v4 --enable-auto-tool-choice --reasoning-parser deepseek_v4 + --no-enable-flashinfer-autotune --enable-prefix-caching --no-disable-hybrid-kv-cache-manager --max-num-seqs "$MAX_NUM_SEQS" + --disable-uvicorn-access-log "${OFFLOAD_ARGS[@]}" ) printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh index 9aced4c74d..79c6a0591a 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh @@ -73,12 +73,8 @@ export AIPERF_AGENTIC_CACHE_WARMUP_DURATION=600 # vLLM v0.22.1 can ship CUTLASS DSL 4.5.2 with stale native MLIR bindings, # which fails DSV4 indexer compilation with mlir_global_dtors(..., data). # Reinstall the matching native wheel until NVIDIA/cutlass#3259 is resolved. -# The v0.24 images ship fixed bindings; reinstalling there would downgrade. -VLLM_INSTALLED_VERSION=$(python3 -c "from importlib.metadata import version; print(version('vllm'))") -if [ "$(printf '%s\n' "$VLLM_INSTALLED_VERSION" 0.24.0 | sort -V | head -n1)" != "0.24.0" ]; then - agentic_pip_install --quiet --force-reinstall --no-deps \ - 'nvidia-cutlass-dsl-libs-cu13==4.5.2' -fi +agentic_pip_install --quiet --force-reinstall --no-deps \ + 'nvidia-cutlass-dsl-libs-cu13==4.5.2' # vllm-project/router expands the one HTTP backend into one logical worker per # DP rank and sends X-data-parallel-rank on forwarded requests. aiperf's @@ -131,6 +127,7 @@ if agentic_kv_offload_enabled; then agentic_pip_install --quiet --no-cache-dir --no-deps \ --force-reinstall "mooncake-transfer-engine-cuda13==$MOONCAKE_VERSION" python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null + python3 "$(dirname "$0")/patch_vllm_pr45406.py" MOONCAKE_MASTER_PORT=$((PORT + 12000)) MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json" @@ -192,6 +189,11 @@ EOF LMCACHE_VERSION=0.5.1 agentic_pip_install --quiet --no-cache-dir "lmcache==$LMCACHE_VERSION" python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null + # Async KV loads park requests in WAITING_FOR_REMOTE_KVS holding + # blocks; without the PR #45406 scheduler fix the waiting-queue scan + # can freeze permanently once nothing is running (hung warmup + # stragglers in PR #2153 bring-up). + python3 "$(dirname "$0")/patch_vllm_pr45406.py" LMCACHE_HOST=127.0.0.1 LMCACHE_PORT=$((PORT + 12000)) @@ -231,6 +233,7 @@ EOF --eviction-trigger-watermark 0.85 \ --eviction-ratio 0.10 \ --eviction-policy LRU \ + --supported-transfer-mode lmcache_driven \ > "$LMCACHE_SERVER_LOG" 2>&1 & LMCACHE_SERVER_PID=$! LMCACHE_READY=0 @@ -273,7 +276,10 @@ fi EP_ARGS=() if [ "$EP_SIZE" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) + EP_ARGS=( + --enable-expert-parallel + --moe-backend deep_gemm_mega_moe + ) fi # AgentX concurrency counts live session trees, not individual requests. @@ -298,15 +304,20 @@ vllm serve "$MODEL_PATH" --served-model-name "$MODEL" \ "${PARALLEL_ARGS[@]}" \ "${VLLM_CP_ARGS[@]}" \ "${EP_ARGS[@]}" \ ---compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' \ +--prefill-schedule-interval 8 \ +--numa-bind \ +--compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' \ +--attention-config '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true}' \ --attention_config.use_fp4_indexer_cache=True \ --tokenizer-mode deepseek_v4 \ --tool-call-parser deepseek_v4 \ --enable-auto-tool-choice \ --reasoning-parser deepseek_v4 \ +--no-enable-flashinfer-autotune \ --enable-prefix-caching \ --no-disable-hybrid-kv-cache-manager \ --max-num-seqs "$MAX_NUM_SEQS" \ +--disable-uvicorn-access-log \ "${OFFLOAD_ARGS[@]}" > "$SERVER_LOG" 2>&1 & SERVER_PID=$! echo "Server PID: $SERVER_PID" diff --git a/benchmarks/single_node/agentic/patch_vllm_pr45406.py b/benchmarks/single_node/agentic/patch_vllm_pr45406.py new file mode 100755 index 0000000000..4b41b60c30 --- /dev/null +++ b/benchmarks/single_node/agentic/patch_vllm_pr45406.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Backport the scheduler fix from vllm-project/vllm PR #45406.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + + +PATCH_MARKER = "See https://github.com/vllm-project/vllm/issues/45388" + + +def main() -> None: + spec = importlib.util.find_spec("vllm") + if spec is None or not spec.submodule_search_locations: + raise RuntimeError("Could not locate the installed vllm package") + + package_root = Path(next(iter(spec.submodule_search_locations))) + scheduler = package_root / "v1" / "core" / "sched" / "scheduler.py" + text = scheduler.read_text() + if PATCH_MARKER in text: + print(f"vLLM PR #45406 backport already present in {scheduler}") + return + + old = """ if request.has_encoder_inputs: + self.encoder_cache_manager.free(request) + break + + # KVTransfer: the connector uses this info to determine +""" + new = """ if request.has_encoder_inputs: + self.encoder_cache_manager.free(request) + if self.running: + # Running requests will free blocks when they + # complete; stop here to preserve queue-order + # admission. + break + # Nothing is running, so no future event frees blocks and + # stopping at this request would freeze this state + # permanently. Requests behind this one may hold blocks + # while parked (async KV loads in WAITING_FOR_REMOTE_KVS) + # and are only promoted when this traversal reaches them. + # Keep scanning so they can be promoted, scheduled, and + # eventually free the blocks this request needs. + # See https://github.com/vllm-project/vllm/issues/45388 + request_queue.pop_request() + step_skipped_waiting.prepend_request(request) + continue + + # KVTransfer: the connector uses this info to determine +""" + if text.count(old) != 1: + raise RuntimeError( + f"Expected exactly one vLLM PR #45406 patch target in {scheduler}" + ) + + scheduler.write_text(text.replace(old, new, 1)) + compile(scheduler.read_text(), str(scheduler), "exec") + print(f"Applied vLLM PR #45406 backport to {scheduler}") + + +if __name__ == "__main__": + main() From d9209ddce981bfcc079ccafc335f655ba09d4f57 Mon Sep 17 00:00:00 2001 From: ApostaC Date: Fri, 10 Jul 2026 17:58:31 -0700 Subject: [PATCH 5/5] revert to 0.24.0 Signed-off-by: ApostaC --- benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh | 12 ++---------- benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh | 12 ++---------- perf-changelog.yaml | 2 +- 3 files changed, 5 insertions(+), 21 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh index 068f10100b..8598de93f3 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh @@ -274,10 +274,7 @@ fi EP_ARGS=() if [ "$EP_SIZE" -gt 1 ]; then - EP_ARGS=( - --enable-expert-parallel - --moe-backend deep_gemm_mega_moe - ) + EP_ARGS=(--enable-expert-parallel) fi # AgentX concurrency counts live session trees, not individual requests. @@ -301,20 +298,15 @@ VLLM_CMD=( "${PARALLEL_ARGS[@]}" "${VLLM_CP_ARGS[@]}" "${EP_ARGS[@]}" - --prefill-schedule-interval 8 - --numa-bind - --compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - --attention-config '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true}' + --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' --attention_config.use_fp4_indexer_cache=True --tokenizer-mode deepseek_v4 --tool-call-parser deepseek_v4 --enable-auto-tool-choice --reasoning-parser deepseek_v4 - --no-enable-flashinfer-autotune --enable-prefix-caching --no-disable-hybrid-kv-cache-manager --max-num-seqs "$MAX_NUM_SEQS" - --disable-uvicorn-access-log "${OFFLOAD_ARGS[@]}" ) printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh index 79c6a0591a..701d29adc5 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh @@ -276,10 +276,7 @@ fi EP_ARGS=() if [ "$EP_SIZE" -gt 1 ]; then - EP_ARGS=( - --enable-expert-parallel - --moe-backend deep_gemm_mega_moe - ) + EP_ARGS=(--enable-expert-parallel) fi # AgentX concurrency counts live session trees, not individual requests. @@ -304,20 +301,15 @@ vllm serve "$MODEL_PATH" --served-model-name "$MODEL" \ "${PARALLEL_ARGS[@]}" \ "${VLLM_CP_ARGS[@]}" \ "${EP_ARGS[@]}" \ ---prefill-schedule-interval 8 \ ---numa-bind \ ---compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' \ ---attention-config '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true}' \ +--compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' \ --attention_config.use_fp4_indexer_cache=True \ --tokenizer-mode deepseek_v4 \ --tool-call-parser deepseek_v4 \ --enable-auto-tool-choice \ --reasoning-parser deepseek_v4 \ ---no-enable-flashinfer-autotune \ --enable-prefix-caching \ --no-disable-hybrid-kv-cache-manager \ --max-num-seqs "$MAX_NUM_SEQS" \ ---disable-uvicorn-access-log \ "${OFFLOAD_ARGS[@]}" > "$SERVER_LOG" 2>&1 & SERVER_PID=$! echo "Server PID: $SERVER_PID" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index a5144018f3..4bef8a1709 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4725,6 +4725,6 @@ - "Add LMCache 0.5.1 DRAM KV-offload config (kv-offload-backend: lmcache) as a standalone section alongside the Mooncake parent (B300 section deferred)" - "Image vllm/vllm-openai:v0.24.0 for the LMCache section (parent stays on v0.23.0)" - "LMCache MP server (lmcache_driven transfer mode) + LMCacheMPConnector; L1 pool derated to 75% of TOTAL_CPU_DRAM_GB after full-budget pinned pools OOM-killed nodes in bring-up" - - "Adopt the vLLM perf flags from #2138 (Mega-MoE, FULL_DECODE_ONLY cudagraphs, sparse MLA attention, NUMA binding) and the vLLM PR #45406 scheduler-freeze backport for async KV loads (waiver: docs/waiver/2153.md)" + - "Apply the vLLM PR #45406 scheduler-freeze backport for async KV loads (waiver: docs/waiver/2153.md); serving flags stay on the stock v0.23-era recipe" - "LMCache points mirror the Mooncake offload points: B200 TP8 conc 8-16, TP8-DEP8 conc 16-68" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2153