diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh index d723754ac8..8598de93f3 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh @@ -21,7 +21,7 @@ set -x # Required env vars: # MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR # -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=mooncake. +# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=mooncake or lmcache. source "$(dirname "$0")/../../benchmark_lib.sh" @@ -101,15 +101,20 @@ export VLLM_PREFIX_CACHE_RETENTION_INTERVAL=32768 SERVER_LOG="$RESULT_DIR/server.log" ROUTER_LOG="$RESULT_DIR/router.log" MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log" +LMCACHE_SERVER_LOG="$RESULT_DIR/lmcache_server.log" mkdir -p "$RESULT_DIR" SERVER_PID="" ROUTER_PID="" MOONCAKE_MASTER_PID="" +LMCACHE_SERVER_PID="" OFFLOAD_ARGS=() -if require_agentic_kv_offload_backend mooncake; then +if agentic_kv_offload_enabled; then + case "$KV_OFFLOAD_BACKEND" in + mooncake) + require_agentic_kv_offload_backend mooncake # Embedded mode contributes one segment per GPU rank to a shared # distributed store, so pre-divide the aggregate host-memory budget. PER_RANK_GB=$((TOTAL_CPU_DRAM_GB / GPU_COUNT)) @@ -118,6 +123,7 @@ if require_agentic_kv_offload_backend mooncake; then agentic_pip_install --quiet --no-cache-dir --no-deps \ --force-reinstall "mooncake-transfer-engine-cuda13==$MOONCAKE_VERSION" python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null + python3 "$(dirname "$0")/patch_vllm_pr45406.py" MOONCAKE_MASTER_PORT=$((PORT + 12000)) MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json" @@ -169,6 +175,96 @@ EOF --kv-transfer-config '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}' ) + ;; + lmcache) + require_agentic_kv_offload_backend lmcache + # The LMCache MP server owns the host-DRAM KV pool as one shared + # tier; vLLM ranks attach via LMCacheMPConnector, so the aggregate + # host budget is passed through undivided (unlike Mooncake's + # per-rank segments). Follows the LMCache DeepSeek-V4 recipe + # (docs.lmcache.ai/recipes/deepseek_v4_flash.html); LMCache handles + # DSV4's Sparse-MLA hybrid KV geometries automatically. + LMCACHE_VERSION=0.5.1 + agentic_pip_install --quiet --no-cache-dir "lmcache==$LMCACHE_VERSION" + python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null + # Async KV loads park requests in WAITING_FOR_REMOTE_KVS holding + # blocks; without the PR #45406 scheduler fix the waiting-queue scan + # can freeze permanently once nothing is running (hung warmup + # stragglers in PR #2153 bring-up). + python3 "$(dirname "$0")/patch_vllm_pr45406.py" + + LMCACHE_HOST=127.0.0.1 + LMCACHE_PORT=$((PORT + 12000)) + LMCACHE_HTTP_PORT=$((PORT + 13000)) + # LMCacheMPConnector concatenates lmcache.mp.host and port into the + # ZMQ endpoint. Bind the server to a raw host, but pass the connector + # a ZMQ-style host string. + LMCACHE_CONNECT_HOST="tcp://$LMCACHE_HOST" + # Pool target derated to 75% of the aggregate budget: pinned host + # memory is unswappable and also consumes GPU-side mapping + # resources, so leave headroom for vLLM host buffers and the OS. + # Full-budget targets OOM-killed the node (host OOM-killer or + # cudaErrorMemoryAllocation) as the cache filled past ~2 TB during + # PR #2153 bring-up. + LMCACHE_L1_SIZE_GB=$((TOTAL_CPU_DRAM_GB * 3 / 4)) + # The pool grows lazily from the initial allocation, so the full + # --l1-size-gb target is not pinned at startup. + LMCACHE_L1_INIT_SIZE_GB=20 + LMCACHE_MQ_TIMEOUT=300 + # Identical prefixes must hash to identical cache keys across DP ranks. + export PYTHONHASHSEED=0 + + echo "Starting LMCache MP server on port $LMCACHE_PORT..." + # One GPU-side transfer worker avoids concurrent-GPU-transfer stalls + # under heavy async-load pressure; CPU-side workers stay at 8. + lmcache server \ + --host "$LMCACHE_HOST" \ + --port "$LMCACHE_PORT" \ + --http-host "$LMCACHE_HOST" \ + --http-port "$LMCACHE_HTTP_PORT" \ + --l1-size-gb "$LMCACHE_L1_SIZE_GB" \ + --l1-init-size-gb "$LMCACHE_L1_INIT_SIZE_GB" \ + --max-gpu-workers 1 \ + --max-cpu-workers 8 \ + --chunk-size 1024 \ + --l1-align-bytes 16384 \ + --eviction-trigger-watermark 0.85 \ + --eviction-ratio 0.10 \ + --eviction-policy LRU \ + --supported-transfer-mode lmcache_driven \ + > "$LMCACHE_SERVER_LOG" 2>&1 & + LMCACHE_SERVER_PID=$! + LMCACHE_READY=0 + for _ in $(seq 1 60); do + if ! kill -0 "$LMCACHE_SERVER_PID" 2>/dev/null; then + echo "LMCache server died during startup." >&2 + cat "$LMCACHE_SERVER_LOG" >&2 + exit 1 + fi + if curl --output /dev/null --silent --fail \ + "http://127.0.0.1:$LMCACHE_HTTP_PORT/healthcheck"; then + LMCACHE_READY=1 + break + fi + sleep 2 + done + if [ "$LMCACHE_READY" -ne 1 ]; then + echo "LMCache server did not become healthy in time." >&2 + cat "$LMCACHE_SERVER_LOG" >&2 + exit 1 + fi + + unset VLLM_USE_SIMPLE_KV_OFFLOAD + OFFLOAD_ARGS=( + --kv-transfer-config + "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.host\":\"$LMCACHE_CONNECT_HOST\",\"lmcache.mp.port\":$LMCACHE_PORT,\"lmcache.mp.mq_timeout\":$LMCACHE_MQ_TIMEOUT}}" + ) + ;; + *) + echo "Error: unsupported KV_OFFLOAD_BACKEND '$KV_OFFLOAD_BACKEND' (expected one of: mooncake, lmcache)" >&2 + exit 1 + ;; + esac fi PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh index 74dc2129be..701d29adc5 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh @@ -20,7 +20,7 @@ set -x # Required env vars: # MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR # -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=mooncake. +# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=mooncake or lmcache. source "$(dirname "$0")/../../benchmark_lib.sh" @@ -105,14 +105,19 @@ export VLLM_PREFIX_CACHE_RETENTION_INTERVAL=32768 SERVER_LOG="$RESULT_DIR/server.log" ROUTER_LOG="$RESULT_DIR/router.log" MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log" +LMCACHE_SERVER_LOG="$RESULT_DIR/lmcache_server.log" mkdir -p "$RESULT_DIR" SERVER_PID="" ROUTER_PID="" MOONCAKE_MASTER_PID="" +LMCACHE_SERVER_PID="" OFFLOAD_ARGS=() -if require_agentic_kv_offload_backend mooncake; then +if agentic_kv_offload_enabled; then + case "$KV_OFFLOAD_BACKEND" in + mooncake) + require_agentic_kv_offload_backend mooncake # Mooncake embedded mode contributes one global segment per GPU rank to # a shared distributed store. Pre-divide the aggregate host budget # across those rank-contributed segments. @@ -122,6 +127,7 @@ if require_agentic_kv_offload_backend mooncake; then agentic_pip_install --quiet --no-cache-dir --no-deps \ --force-reinstall "mooncake-transfer-engine-cuda13==$MOONCAKE_VERSION" python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null + python3 "$(dirname "$0")/patch_vllm_pr45406.py" MOONCAKE_MASTER_PORT=$((PORT + 12000)) MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json" @@ -171,6 +177,96 @@ EOF --kv-transfer-config '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}' ) + ;; + lmcache) + require_agentic_kv_offload_backend lmcache + # The LMCache MP server owns the host-DRAM KV pool as one shared + # tier; vLLM ranks attach via LMCacheMPConnector, so the aggregate + # host budget is passed through undivided (unlike Mooncake's + # per-rank segments). Follows the LMCache DeepSeek-V4 recipe + # (docs.lmcache.ai/recipes/deepseek_v4_flash.html); LMCache handles + # DSV4's Sparse-MLA hybrid KV geometries automatically. + LMCACHE_VERSION=0.5.1 + agentic_pip_install --quiet --no-cache-dir "lmcache==$LMCACHE_VERSION" + python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null + # Async KV loads park requests in WAITING_FOR_REMOTE_KVS holding + # blocks; without the PR #45406 scheduler fix the waiting-queue scan + # can freeze permanently once nothing is running (hung warmup + # stragglers in PR #2153 bring-up). + python3 "$(dirname "$0")/patch_vllm_pr45406.py" + + LMCACHE_HOST=127.0.0.1 + LMCACHE_PORT=$((PORT + 12000)) + LMCACHE_HTTP_PORT=$((PORT + 13000)) + # LMCacheMPConnector concatenates lmcache.mp.host and port into the + # ZMQ endpoint. Bind the server to a raw host, but pass the connector + # a ZMQ-style host string. + LMCACHE_CONNECT_HOST="tcp://$LMCACHE_HOST" + # Pool target derated to 75% of the aggregate budget: pinned host + # memory is unswappable and also consumes GPU-side mapping + # resources, so leave headroom for vLLM host buffers and the OS. + # Full-budget targets OOM-killed the node (host OOM-killer or + # cudaErrorMemoryAllocation) as the cache filled past ~2 TB during + # PR #2153 bring-up. + LMCACHE_L1_SIZE_GB=$((TOTAL_CPU_DRAM_GB * 3 / 4)) + # The pool grows lazily from the initial allocation, so the full + # --l1-size-gb target is not pinned at startup. + LMCACHE_L1_INIT_SIZE_GB=20 + LMCACHE_MQ_TIMEOUT=300 + # Identical prefixes must hash to identical cache keys across DP ranks. + export PYTHONHASHSEED=0 + + echo "Starting LMCache MP server on port $LMCACHE_PORT..." + # One GPU-side transfer worker avoids concurrent-GPU-transfer stalls + # under heavy async-load pressure; CPU-side workers stay at 8. + lmcache server \ + --host "$LMCACHE_HOST" \ + --port "$LMCACHE_PORT" \ + --http-host "$LMCACHE_HOST" \ + --http-port "$LMCACHE_HTTP_PORT" \ + --l1-size-gb "$LMCACHE_L1_SIZE_GB" \ + --l1-init-size-gb "$LMCACHE_L1_INIT_SIZE_GB" \ + --max-gpu-workers 1 \ + --max-cpu-workers 8 \ + --chunk-size 1024 \ + --l1-align-bytes 16384 \ + --eviction-trigger-watermark 0.85 \ + --eviction-ratio 0.10 \ + --eviction-policy LRU \ + --supported-transfer-mode lmcache_driven \ + > "$LMCACHE_SERVER_LOG" 2>&1 & + LMCACHE_SERVER_PID=$! + LMCACHE_READY=0 + for _ in $(seq 1 60); do + if ! kill -0 "$LMCACHE_SERVER_PID" 2>/dev/null; then + echo "LMCache server died during startup." >&2 + cat "$LMCACHE_SERVER_LOG" >&2 + exit 1 + fi + if curl --output /dev/null --silent --fail \ + "http://127.0.0.1:$LMCACHE_HTTP_PORT/healthcheck"; then + LMCACHE_READY=1 + break + fi + sleep 2 + done + if [ "$LMCACHE_READY" -ne 1 ]; then + echo "LMCache server did not become healthy in time." >&2 + cat "$LMCACHE_SERVER_LOG" >&2 + exit 1 + fi + + unset VLLM_USE_SIMPLE_KV_OFFLOAD + OFFLOAD_ARGS=( + --kv-transfer-config + "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.host\":\"$LMCACHE_CONNECT_HOST\",\"lmcache.mp.port\":$LMCACHE_PORT,\"lmcache.mp.mq_timeout\":$LMCACHE_MQ_TIMEOUT}}" + ) + ;; + *) + echo "Error: unsupported KV_OFFLOAD_BACKEND '$KV_OFFLOAD_BACKEND' (expected one of: mooncake, lmcache)" >&2 + exit 1 + ;; + esac fi PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) diff --git a/benchmarks/single_node/agentic/patch_vllm_pr45406.py b/benchmarks/single_node/agentic/patch_vllm_pr45406.py new file mode 100755 index 0000000000..4b41b60c30 --- /dev/null +++ b/benchmarks/single_node/agentic/patch_vllm_pr45406.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Backport the scheduler fix from vllm-project/vllm PR #45406.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + + +PATCH_MARKER = "See https://github.com/vllm-project/vllm/issues/45388" + + +def main() -> None: + spec = importlib.util.find_spec("vllm") + if spec is None or not spec.submodule_search_locations: + raise RuntimeError("Could not locate the installed vllm package") + + package_root = Path(next(iter(spec.submodule_search_locations))) + scheduler = package_root / "v1" / "core" / "sched" / "scheduler.py" + text = scheduler.read_text() + if PATCH_MARKER in text: + print(f"vLLM PR #45406 backport already present in {scheduler}") + return + + old = """ if request.has_encoder_inputs: + self.encoder_cache_manager.free(request) + break + + # KVTransfer: the connector uses this info to determine +""" + new = """ if request.has_encoder_inputs: + self.encoder_cache_manager.free(request) + if self.running: + # Running requests will free blocks when they + # complete; stop here to preserve queue-order + # admission. + break + # Nothing is running, so no future event frees blocks and + # stopping at this request would freeze this state + # permanently. Requests behind this one may hold blocks + # while parked (async KV loads in WAITING_FOR_REMOTE_KVS) + # and are only promoted when this traversal reaches them. + # Keep scanning so they can be promoted, scheduled, and + # eventually free the blocks this request needs. + # See https://github.com/vllm-project/vllm/issues/45388 + request_queue.pop_request() + step_skipped_waiting.prepend_request(request) + continue + + # KVTransfer: the connector uses this info to determine +""" + if text.count(old) != 1: + raise RuntimeError( + f"Expected exactly one vLLM PR #45406 patch target in {scheduler}" + ) + + scheduler.write_text(text.replace(old, new, 1)) + compile(scheduler.read_text(), str(scheduler), "exec") + print(f"Applied vLLM PR #45406 backport to {scheduler}") + + +if __name__ == "__main__": + main() diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 6be8e006f9..5a53b17709 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1782,6 +1782,26 @@ dsv4-fp4-b200-vllm-agentic: # Retain the external-cache transition and peak-throughput region. - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [16, 38, 44, 56, 64, 66, 68] } +# LMCache CPU-offload arm of dsv4-fp4-b200-vllm-agentic, split into its own +# config so it can be triggered/tested independently of the Mooncake curve. +# Points mirror the Mooncake offload points; the GPU-cache (kv-offloading: +# none) baselines live in the parent config only. Runs the v0.24.0 image +# (the LMCache-0.5.x-validated pairing) while the parent stays on v0.23.0. +dsv4-fp4-b200-vllm-agentic-lmcache: + image: vllm/vllm-openai:v0.24.0 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:b200-dgxc + precision: fp4 + framework: vllm + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - { tp: 8, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [8, 10, 16] } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [16, 38, 44, 56, 64, 66, 68] } + dsv4-fp4-b200-trt: image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066 model: deepseek-ai/DeepSeek-V4-Pro diff --git a/docs/waiver/2153.md b/docs/waiver/2153.md new file mode 100644 index 0000000000..5adcdabde8 --- /dev/null +++ b/docs/waiver/2153.md @@ -0,0 +1,54 @@ +# Inference-engine patch waiver — PR #2153 + +## What is patched + +`benchmarks/single_node/agentic/patch_vllm_pr45406.py` rewrites +`vllm/v1/core/sched/scheduler.py` inside the pinned `vllm/vllm-openai:v0.24.0` +image before the server starts. It is invoked only from the `lmcache` arm of +`benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh` and +`dsv4_fp4_b300_vllm.sh`, so Mooncake and GPU-cache baseline runs execute the +image as shipped. The patch is a verbatim backport of upstream +vllm-project/vllm PR #45406 and is idempotent (it no-ops when the fix is +already present and fails loudly if the target code has drifted). + +## Why the unmodified image cannot run this benchmark + +The `dsv4-fp4-b200-vllm-agentic-lmcache` config uses `LMCacheMPConnector` +with asynchronous KV loads. Requests park in `WAITING_FOR_REMOTE_KVS` while +holding KV blocks. In v0.24.0 the scheduler's waiting-queue traversal stops +at the first request that cannot allocate blocks even when nothing is +running; parked requests behind it are never promoted, so the queue freezes +permanently. In PR #2153 bring-up this reproduced as hung agentic warmup +stragglers (e.g. 35 in-flight requests never returning at DEP conc 68), +aborting the benchmark with `warmup_failure`. The deadlock is upstream issue +vllm-project/vllm#45388; no released image up to and including v0.24.0 +contains the fix. + +## Upstream reference + +- Issue: https://github.com/vllm-project/vllm/issues/45388 +- Fix: https://github.com/vllm-project/vllm/pull/45406 (merged upstream) + +## Removal plan + +Remove `patch_vllm_pr45406.py` and its invocation when the LMCache section's +pinned image first includes upstream PR #45406 (the first vLLM release or +nightly after v0.24.0 that ships it). Because the patch script detects an +already-fixed scheduler and no-ops, an image bump cannot silently double-apply +it; the removal is a plain deletion in the same PR as that image bump. + +## 中文说明 + +**补丁内容**：`patch_vllm_pr45406.py` 在服务启动前修改固定镜像 +`vllm/vllm-openai:v0.24.0` 中的 `vllm/v1/core/sched/scheduler.py`，为上游 +vllm-project/vllm PR #45406 的逐字回移（backport），仅在 LMCache 分支中调用； +Mooncake 与 GPU 缓存基线均按原始镜像运行。 + +**为何原始镜像无法运行本基准测试**：LMCacheMPConnector 的异步 KV 加载会让请求 +在 `WAITING_FOR_REMOTE_KVS` 状态下持有 KV block。v0.24.0 的调度器在等待队列 +遍历中遇到无法分配 block 的请求即停止，即使当前没有运行中的请求，其后被挂起 +的请求永远不会被调度，队列永久冻结（上游 issue #45388）。在 PR #2153 调试中 +表现为 agentic 预热请求挂起、基准测试以 warmup_failure 中止。 + +**移除计划**：当 LMCache 配置的固定镜像包含上游 PR #45406 后，在镜像升级的 +同一 PR 中删除该补丁脚本及其调用。补丁脚本具备幂等检测，不会重复应用。 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index be28208ab4..4bef8a1709 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4716,3 +4716,15 @@ - "Clean the export envs" - "Enable two batch overlap" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2093 + +- config-keys: + - dsv4-fp4-b200-vllm-agentic-lmcache + scenario-type: + - agentic-coding + description: + - "Add LMCache 0.5.1 DRAM KV-offload config (kv-offload-backend: lmcache) as a standalone section alongside the Mooncake parent (B300 section deferred)" + - "Image vllm/vllm-openai:v0.24.0 for the LMCache section (parent stays on v0.23.0)" + - "LMCache MP server (lmcache_driven transfer mode) + LMCacheMPConnector; L1 pool derated to 75% of TOTAL_CPU_DRAM_GB after full-budget pinned pools OOM-killed nodes in bring-up" + - "Apply the vLLM PR #45406 scheduler-freeze backport for async KV loads (waiver: docs/waiver/2153.md); serving flags stay on the stock v0.23-era recipe" + - "LMCache points mirror the Mooncake offload points: B200 TP8 conc 8-16, TP8-DEP8 conc 16-68" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2153