Skip to content

[LMCache] Add LMCache configs for dsv4 vllm b200/b300 agentix setups #2153

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ApostaC wants to merge 7 commits into
base: main
Choose a base branch
from
Open

[LMCache] Add LMCache configs for dsv4 vllm b200/b300 agentix setups #2153

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
10e6639
Add LMCache configs for dsv4 vllm b200/b300
ApostaC Jul 10, 2026
73137f1
update perf changelog
ApostaC Jul 10, 2026
951f2c8
Merge origin/main into lmcache/agentix, re-append lmcache changelog e…
ApostaC Jul 10, 2026
4244f91
lmcache: derate L1 pool to 75%, move to vLLM v0.24.0, defer B300 section
ApostaC Jul 10, 2026
380ee5a
fixing the comments and picking optimizations from #2138
ApostaC Jul 10, 2026
6623385
merging upstream updates
ApostaC Jul 10, 2026
d9209dd
revert to 0.24.0
ApostaC Jul 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 98 additions & 2 deletions benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ set -x
# Required env vars:
# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR
#
# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=mooncake.
# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=mooncake or lmcache.

source "$(dirname "$0")/../../benchmark_lib.sh"

Expand Down Expand Up @@ -101,15 +101,20 @@ export VLLM_PREFIX_CACHE_RETENTION_INTERVAL=32768
SERVER_LOG="$RESULT_DIR/server.log"
ROUTER_LOG="$RESULT_DIR/router.log"
MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log"
LMCACHE_SERVER_LOG="$RESULT_DIR/lmcache_server.log"
mkdir -p "$RESULT_DIR"

SERVER_PID=""
ROUTER_PID=""
MOONCAKE_MASTER_PID=""
LMCACHE_SERVER_PID=""

OFFLOAD_ARGS=()

if require_agentic_kv_offload_backend mooncake; then
if agentic_kv_offload_enabled; then
case "$KV_OFFLOAD_BACKEND" in
mooncake)
require_agentic_kv_offload_backend mooncake
# Embedded mode contributes one segment per GPU rank to a shared
# distributed store, so pre-divide the aggregate host-memory budget.
PER_RANK_GB=$((TOTAL_CPU_DRAM_GB / GPU_COUNT))
Expand All @@ -118,6 +123,7 @@ if require_agentic_kv_offload_backend mooncake; then
agentic_pip_install --quiet --no-cache-dir --no-deps \
--force-reinstall "mooncake-transfer-engine-cuda13==$MOONCAKE_VERSION"
python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null
python3 "$(dirname "$0")/patch_vllm_pr45406.py"

@functionstackx functionstackx Jul 10, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

generally we wanna avoid patches too

@ApostaC ApostaC Jul 10, 2026

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, this is what I picked from #2138. Once that is in, I will remove this.


MOONCAKE_MASTER_PORT=$((PORT + 12000))
MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json"
Expand Down Expand Up @@ -169,6 +175,96 @@ EOF
--kv-transfer-config
'{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}'
)
;;
lmcache)
require_agentic_kv_offload_backend lmcache
# The LMCache MP server owns the host-DRAM KV pool as one shared
# tier; vLLM ranks attach via LMCacheMPConnector, so the aggregate
# host budget is passed through undivided (unlike Mooncake's
# per-rank segments). Follows the LMCache DeepSeek-V4 recipe
# (docs.lmcache.ai/recipes/deepseek_v4_flash.html); LMCache handles
# DSV4's Sparse-MLA hybrid KV geometries automatically.
LMCACHE_VERSION=0.5.1
agentic_pip_install --quiet --no-cache-dir "lmcache==$LMCACHE_VERSION"
python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null
# Async KV loads park requests in WAITING_FOR_REMOTE_KVS holding
# blocks; without the PR #45406 scheduler fix the waiting-queue scan
# can freeze permanently once nothing is running (hung warmup
# stragglers in PR #2153 bring-up).
python3 "$(dirname "$0")/patch_vllm_pr45406.py"

LMCACHE_HOST=127.0.0.1
LMCACHE_PORT=$((PORT + 12000))
LMCACHE_HTTP_PORT=$((PORT + 13000))
# LMCacheMPConnector concatenates lmcache.mp.host and port into the
# ZMQ endpoint. Bind the server to a raw host, but pass the connector
# a ZMQ-style host string.
LMCACHE_CONNECT_HOST="tcp://$LMCACHE_HOST"
# Pool target derated to 75% of the aggregate budget: pinned host
# memory is unswappable and also consumes GPU-side mapping
# resources, so leave headroom for vLLM host buffers and the OS.
# Full-budget targets OOM-killed the node (host OOM-killer or
# cudaErrorMemoryAllocation) as the cache filled past ~2 TB during
# PR #2153 bring-up.
LMCACHE_L1_SIZE_GB=$((TOTAL_CPU_DRAM_GB * 3 / 4))
# The pool grows lazily from the initial allocation, so the full
# --l1-size-gb target is not pinned at startup.
LMCACHE_L1_INIT_SIZE_GB=20
LMCACHE_MQ_TIMEOUT=300
# Identical prefixes must hash to identical cache keys across DP ranks.
export PYTHONHASHSEED=0

echo "Starting LMCache MP server on port $LMCACHE_PORT..."
# One GPU-side transfer worker avoids concurrent-GPU-transfer stalls
# under heavy async-load pressure; CPU-side workers stay at 8.
lmcache server \
--host "$LMCACHE_HOST" \
--port "$LMCACHE_PORT" \
--http-host "$LMCACHE_HOST" \
--http-port "$LMCACHE_HTTP_PORT" \
--l1-size-gb "$LMCACHE_L1_SIZE_GB" \
--l1-init-size-gb "$LMCACHE_L1_INIT_SIZE_GB" \
--max-gpu-workers 1 \
--max-cpu-workers 8 \
--chunk-size 1024 \
--l1-align-bytes 16384 \
--eviction-trigger-watermark 0.85 \
--eviction-ratio 0.10 \
--eviction-policy LRU \
--supported-transfer-mode lmcache_driven \
> "$LMCACHE_SERVER_LOG" 2>&1 &
LMCACHE_SERVER_PID=$!
LMCACHE_READY=0
for _ in $(seq 1 60); do
if ! kill -0 "$LMCACHE_SERVER_PID" 2>/dev/null; then
echo "LMCache server died during startup." >&2
cat "$LMCACHE_SERVER_LOG" >&2
exit 1
fi
if curl --output /dev/null --silent --fail \
"http://127.0.0.1:$LMCACHE_HTTP_PORT/healthcheck"; then
LMCACHE_READY=1
break
fi
sleep 2
done
if [ "$LMCACHE_READY" -ne 1 ]; then
echo "LMCache server did not become healthy in time." >&2
cat "$LMCACHE_SERVER_LOG" >&2
exit 1
fi

unset VLLM_USE_SIMPLE_KV_OFFLOAD
OFFLOAD_ARGS=(
--kv-transfer-config
"{\"kv_connector\":\"LMCacheMPConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.host\":\"$LMCACHE_CONNECT_HOST\",\"lmcache.mp.port\":$LMCACHE_PORT,\"lmcache.mp.mq_timeout\":$LMCACHE_MQ_TIMEOUT}}"
)
;;
*)
echo "Error: unsupported KV_OFFLOAD_BACKEND '$KV_OFFLOAD_BACKEND' (expected one of: mooncake, lmcache)" >&2
exit 1
;;
esac
Comment on lines +263 to +267

@claude claude Bot Jul 10, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🔴 The new --kv-transfer-config JSON for the lmcache arm omits the top-level "kv_connector_module_path":"lmcache.integration.vllm.lmcache_mp_connector" key. Both other in-tree LMCacheMPConnector call-sites include it (benchmarks/single_node/agentic/kimik2.5_fp4_b200.sh:150, benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm.sh:350); vLLMs KVConnectorFactory pre-registers built-in connectors like MooncakeStoreConnector by bare name but not the third-party LMCacheMPConnector, so vllm serve will fail to resolve the class at engine startup on every point of both the new B200 and B300 -lmcache configs. The same fix is needed at the equivalent OFFLOAD_ARGS block in dsv4_fp4_b300_vllm.sh.

Extended reasoning...

What the bug is

In the new lmcache branch of the KV_OFFLOAD_BACKEND case, the --kv-transfer-config JSON is built as:

{
  "kv_connector": "LMCacheMPConnector",
  "kv_role": "kv_both",
  "kv_connector_extra_config": {
    "lmcache.mp.host": "...",
    "lmcache.mp.port": ...,
    "lmcache.mp.mq_timeout": ...
  }
}

It is missing the top-level "kv_connector_module_path": "lmcache.integration.vllm.lmcache_mp_connector" key.

Why the existing in-tree pattern says this is required

Every other in-tree call-site of LMCacheMPConnector includes that module path explicitly:

  • benchmarks/single_node/agentic/kimik2.5_fp4_b200.sh:150 (vLLM v0.22.0)
  • benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm.sh:350 (AMD lmcache path)

Those two sites were authored by independent PRs and both pass the module path. The consistency of that pattern is strong evidence the key is load-bearing, not decorative.

By contrast, MooncakeStoreConnector is passed with a bare name across every in-tree site (including the sibling mooncake) branch in these same two scripts) with no module path — because Mooncake is pre-registered in vLLMs built-in KVConnectorFactory registry. LMCacheMPConnector is a third-party connector shipped by the lmcache PyPI package at lmcache.integration.vllm.lmcache_mp_connector; vLLM has no built-in entry for it, so KVConnectorFactory needs the module path to import the class.

Why the pre-existing import check does not save us

The script does run python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" on the way up, but that runs in a separate subprocess and does not seed the vllm serve engine subprocess registry. Even if LMCache 0.5.1 ships entry-point auto-registration for vLLM 0.23.0, both other in-tree sites — one of them on the same LMCache generation — chose not to rely on it, so it is at best not guaranteed on this stack.

Impact and step-by-step failure

  1. Sweep drives a point with kv-offload-backend: lmcache in either dsv4-fp4-b200-vllm-agentic-lmcache or dsv4-fp4-b300-vllm-agentic-lmcache.
  2. The lmcache) branch runs lmcache server in the background and the healthcheck passes (this is the LMCache process, not vLLM).
  3. vllm serve starts with --kv-transfer-config '{"kv_connector":"LMCacheMPConnector",...}' (no module path).
  4. Inside the engine, KVConnectorFactory.create_connector(...) looks up the bare name "LMCacheMPConnector", does not find it in the built-in registry, has no kv_connector_module_path to import from, and raises — the engine dies at startup, SERVER_PID exits, wait_for_server_ready fails, and every point in the new -lmcache configs never produces results.

Because both dsv4-fp4-b200-vllm-agentic-lmcache and dsv4-fp4-b300-vllm-agentic-lmcache are 100% kv-offload-backend: lmcache, this blocks the PRs entire core purpose.

Fix

Add the module path in the JSON built at benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh around line 251 and the equivalent block in benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh around line 253:

OFFLOAD_ARGS=(
    --kv-transfer-config
    "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_connector_module_path\":\"lmcache.integration.vllm.lmcache_mp_connector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.host\":\"$LMCACHE_CONNECT_HOST\",\"lmcache.mp.port\":$LMCACHE_PORT,\"lmcache.mp.mq_timeout\":$LMCACHE_MQ_TIMEOUT}}"
)

This mirrors the pattern already established at kimik2.5_fp4_b200.sh:150 and dsv4_fp4_mi355x_vllm.sh:350.

@ApostaC ApostaC Jul 10, 2026

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No we don't need this in newer versions of vLLM

fi

PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1)
Expand Down
100 changes: 98 additions & 2 deletions benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ set -x
# Required env vars:
# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR
#
# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=mooncake.
# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=mooncake or lmcache.

source "$(dirname "$0")/../../benchmark_lib.sh"

Expand Down Expand Up @@ -105,14 +105,19 @@ export VLLM_PREFIX_CACHE_RETENTION_INTERVAL=32768
SERVER_LOG="$RESULT_DIR/server.log"
ROUTER_LOG="$RESULT_DIR/router.log"
MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log"
LMCACHE_SERVER_LOG="$RESULT_DIR/lmcache_server.log"
mkdir -p "$RESULT_DIR"

SERVER_PID=""
ROUTER_PID=""
MOONCAKE_MASTER_PID=""
LMCACHE_SERVER_PID=""

OFFLOAD_ARGS=()
if require_agentic_kv_offload_backend mooncake; then
if agentic_kv_offload_enabled; then
case "$KV_OFFLOAD_BACKEND" in
mooncake)
require_agentic_kv_offload_backend mooncake
# Mooncake embedded mode contributes one global segment per GPU rank to
# a shared distributed store. Pre-divide the aggregate host budget
# across those rank-contributed segments.
Expand All @@ -122,6 +127,7 @@ if require_agentic_kv_offload_backend mooncake; then
agentic_pip_install --quiet --no-cache-dir --no-deps \
--force-reinstall "mooncake-transfer-engine-cuda13==$MOONCAKE_VERSION"
python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null
python3 "$(dirname "$0")/patch_vllm_pr45406.py"

MOONCAKE_MASTER_PORT=$((PORT + 12000))
MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json"
Expand Down Expand Up @@ -171,6 +177,96 @@ EOF
--kv-transfer-config
'{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}'
)
;;
lmcache)
require_agentic_kv_offload_backend lmcache
# The LMCache MP server owns the host-DRAM KV pool as one shared
# tier; vLLM ranks attach via LMCacheMPConnector, so the aggregate
# host budget is passed through undivided (unlike Mooncake's
# per-rank segments). Follows the LMCache DeepSeek-V4 recipe
# (docs.lmcache.ai/recipes/deepseek_v4_flash.html); LMCache handles
# DSV4's Sparse-MLA hybrid KV geometries automatically.
LMCACHE_VERSION=0.5.1
agentic_pip_install --quiet --no-cache-dir "lmcache==$LMCACHE_VERSION"
python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null
# Async KV loads park requests in WAITING_FOR_REMOTE_KVS holding
# blocks; without the PR #45406 scheduler fix the waiting-queue scan
# can freeze permanently once nothing is running (hung warmup
# stragglers in PR #2153 bring-up).
python3 "$(dirname "$0")/patch_vllm_pr45406.py"

LMCACHE_HOST=127.0.0.1
LMCACHE_PORT=$((PORT + 12000))
LMCACHE_HTTP_PORT=$((PORT + 13000))
# LMCacheMPConnector concatenates lmcache.mp.host and port into the
# ZMQ endpoint. Bind the server to a raw host, but pass the connector
# a ZMQ-style host string.
LMCACHE_CONNECT_HOST="tcp://$LMCACHE_HOST"
# Pool target derated to 75% of the aggregate budget: pinned host
# memory is unswappable and also consumes GPU-side mapping
# resources, so leave headroom for vLLM host buffers and the OS.
# Full-budget targets OOM-killed the node (host OOM-killer or
# cudaErrorMemoryAllocation) as the cache filled past ~2 TB during
# PR #2153 bring-up.
LMCACHE_L1_SIZE_GB=$((TOTAL_CPU_DRAM_GB * 3 / 4))
# The pool grows lazily from the initial allocation, so the full
# --l1-size-gb target is not pinned at startup.
LMCACHE_L1_INIT_SIZE_GB=20
LMCACHE_MQ_TIMEOUT=300
# Identical prefixes must hash to identical cache keys across DP ranks.
export PYTHONHASHSEED=0

echo "Starting LMCache MP server on port $LMCACHE_PORT..."
# One GPU-side transfer worker avoids concurrent-GPU-transfer stalls
# under heavy async-load pressure; CPU-side workers stay at 8.
lmcache server \
--host "$LMCACHE_HOST" \
--port "$LMCACHE_PORT" \
--http-host "$LMCACHE_HOST" \
--http-port "$LMCACHE_HTTP_PORT" \
--l1-size-gb "$LMCACHE_L1_SIZE_GB" \
--l1-init-size-gb "$LMCACHE_L1_INIT_SIZE_GB" \
--max-gpu-workers 1 \
--max-cpu-workers 8 \
--chunk-size 1024 \
--l1-align-bytes 16384 \
--eviction-trigger-watermark 0.85 \
--eviction-ratio 0.10 \
--eviction-policy LRU \
--supported-transfer-mode lmcache_driven \
> "$LMCACHE_SERVER_LOG" 2>&1 &
LMCACHE_SERVER_PID=$!
LMCACHE_READY=0
for _ in $(seq 1 60); do
if ! kill -0 "$LMCACHE_SERVER_PID" 2>/dev/null; then
echo "LMCache server died during startup." >&2
cat "$LMCACHE_SERVER_LOG" >&2
exit 1
fi
if curl --output /dev/null --silent --fail \
"http://127.0.0.1:$LMCACHE_HTTP_PORT/healthcheck"; then
LMCACHE_READY=1
break
fi
sleep 2
done
if [ "$LMCACHE_READY" -ne 1 ]; then
echo "LMCache server did not become healthy in time." >&2
cat "$LMCACHE_SERVER_LOG" >&2
exit 1
fi

unset VLLM_USE_SIMPLE_KV_OFFLOAD
OFFLOAD_ARGS=(
--kv-transfer-config
"{\"kv_connector\":\"LMCacheMPConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.host\":\"$LMCACHE_CONNECT_HOST\",\"lmcache.mp.port\":$LMCACHE_PORT,\"lmcache.mp.mq_timeout\":$LMCACHE_MQ_TIMEOUT}}"
)
;;
*)
echo "Error: unsupported KV_OFFLOAD_BACKEND '$KV_OFFLOAD_BACKEND' (expected one of: mooncake, lmcache)" >&2
exit 1
;;
esac
fi

PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1)
Expand Down
63 changes: 63 additions & 0 deletions benchmarks/single_node/agentic/patch_vllm_pr45406.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
#!/usr/bin/env python3
"""Backport the scheduler fix from vllm-project/vllm PR #45406."""

from __future__ import annotations

import importlib.util
from pathlib import Path


PATCH_MARKER = "See https://github.com/vllm-project/vllm/issues/45388"


def main() -> None:
spec = importlib.util.find_spec("vllm")
if spec is None or not spec.submodule_search_locations:
raise RuntimeError("Could not locate the installed vllm package")

package_root = Path(next(iter(spec.submodule_search_locations)))
scheduler = package_root / "v1" / "core" / "sched" / "scheduler.py"
text = scheduler.read_text()
if PATCH_MARKER in text:
print(f"vLLM PR #45406 backport already present in {scheduler}")
return

old = """ if request.has_encoder_inputs:
self.encoder_cache_manager.free(request)
break

# KVTransfer: the connector uses this info to determine
"""
new = """ if request.has_encoder_inputs:
self.encoder_cache_manager.free(request)
if self.running:
# Running requests will free blocks when they
# complete; stop here to preserve queue-order
# admission.
break
# Nothing is running, so no future event frees blocks and
# stopping at this request would freeze this state
# permanently. Requests behind this one may hold blocks
# while parked (async KV loads in WAITING_FOR_REMOTE_KVS)
# and are only promoted when this traversal reaches them.
# Keep scanning so they can be promoted, scheduled, and
# eventually free the blocks this request needs.
# See https://github.com/vllm-project/vllm/issues/45388
request_queue.pop_request()
step_skipped_waiting.prepend_request(request)
continue

# KVTransfer: the connector uses this info to determine
"""
if text.count(old) != 1:
raise RuntimeError(
f"Expected exactly one vLLM PR #45406 patch target in {scheduler}"
)

scheduler.write_text(text.replace(old, new, 1))
compile(scheduler.read_text(), str(scheduler), "exec")
print(f"Applied vLLM PR #45406 backport to {scheduler}")


if __name__ == "__main__":
main()
20 changes: 20 additions & 0 deletions configs/nvidia-master.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1782,6 +1782,26 @@ dsv4-fp4-b200-vllm-agentic:
# Retain the external-cache transition and peak-throughput region.
- { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [16, 38, 44, 56, 64, 66, 68] }

# LMCache CPU-offload arm of dsv4-fp4-b200-vllm-agentic, split into its own
# config so it can be triggered/tested independently of the Mooncake curve.
# Points mirror the Mooncake offload points; the GPU-cache (kv-offloading:
# none) baselines live in the parent config only. Runs the v0.24.0 image
# (the LMCache-0.5.x-validated pairing) while the parent stays on v0.23.0.
dsv4-fp4-b200-vllm-agentic-lmcache:
image: vllm/vllm-openai:v0.24.0
model: deepseek-ai/DeepSeek-V4-Pro
model-prefix: dsv4
runner: cluster:b200-dgxc
precision: fp4
framework: vllm
multinode: false
scenarios:
agentic-coding:
- dram-utilization: 0.80
search-space:
- { tp: 8, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [8, 10, 16] }
- { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [16, 38, 44, 56, 64, 66, 68] }

dsv4-fp4-b200-trt:
image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066
model: deepseek-ai/DeepSeek-V4-Pro
Expand Down
Loading