From a007c03f164c463c4bdb3e31a05cad2c4f38798f Mon Sep 17 00:00:00 2001 From: ajith-sirra-amd Date: Wed, 8 Jul 2026 17:00:13 +0530 Subject: [PATCH 01/17] [AMD] MINIMAX-M3 FP4 vLLM Agentic Support Signed-off-by: ajith-sirra-amd --- .../agentic/minimaxm3_fp4_mi355x.sh | 271 ++++++++++++++++++ configs/amd-master.yaml | 17 ++ perf-changelog.yaml | 8 + 3 files changed, 296 insertions(+) create mode 100644 benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh new file mode 100644 index 0000000000..877fe11d41 --- /dev/null +++ b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh @@ -0,0 +1,271 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +# Agentic trace replay benchmark for Kimi-K2.5 FP4 on MI355X using vLLM. +# +# Required env vars: +# MODEL, TP, CONC, OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR +# +# OFFLOADING values: +# none - vLLM GPU KV only. +# cpu - vLLM native CPU offload. +# lmcache - LMCache MP server + vLLM LMCacheMPConnector. + +source "$(dirname "$0")/../../benchmark_lib.sh" + +check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION + +echo "MODEL=$MODEL TP=$TP CONC=$CONC KV_OFFLOADING=$KV_OFFLOADING TOTAL_CPU_DRAM_GB=$TOTAL_CPU_DRAM_GB RESULT_DIR=$RESULT_DIR DURATION=$DURATION EP_SIZE=$EP_SIZE DP_ATTENTION=$DP_ATTENTION" + +PORT=${PORT:-8888} +DURATION=${DURATION:-1800} +EP_SIZE=${EP_SIZE:-1} + +if [[ -n "${SLURM_JOB_ID:-}" ]]; then + echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" +fi + +# ROCR/HIP visibility for vLLM 0.14+ +if [ -n "${ROCR_VISIBLE_DEVICES:-}" ]; then + export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" +fi + +if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi +rocm-smi || true +amd-smi || true + +# `hf download` creates the target dir if missing and is itself idempotent. +# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE +# Either way, MODEL_PATH is what the server is launched with. +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi + +# ---- Resolve traces and install deps ---------------------------------------- +# MiniMax-M2.5 servers run at max_model_len ~256k; the unfiltered 052726 +# corpus has requests up to ~1M proxy tokens that would be rejected. +# Switch to the 256k-capped variant (470 traces, max in+out <= 256k). +#export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_with_subagents_256k +#060226 +# export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_with_subagents_060226_256k + +resolve_trace_source +install_agentic_deps + +# export AIPERF_AGENTIC_CACHE_WARMUP_DURATION=1200 + +# ---- Server config ---------------------------------------------------------- +SERVER_LOG="$RESULT_DIR/server.log" +LMCACHE_LOG="$RESULT_DIR/lmcache_server.log" +mkdir -p "$RESULT_DIR" + +OFFLOAD_ARGS=(--no-enable-prefix-caching) + +# ---- Lmcache config ---------------------------------------------------------- +LMCACHE_PID="" + +cleanup_lmcache_server() { + if [[ -n "$LMCACHE_PID" ]] && kill -0 "$LMCACHE_PID" 2>/dev/null; then + kill "$LMCACHE_PID" 2>/dev/null || true + wait "$LMCACHE_PID" 2>/dev/null || true + fi +} + +trap cleanup_lmcache_server EXIT + +wait_for_lmcache_ready() { + { set +x; } 2>/dev/null + local attempts="${LMCACHE_READY_ATTEMPTS:-120}" + local tail_pid="" + + while [ ! -f "$LMCACHE_LOG" ]; do + if [[ -n "$LMCACHE_PID" ]] && ! kill -0 "$LMCACHE_PID" 2>/dev/null; then + echo "LMCache server died before creating log file. Exiting." >&2 + exit 1 + fi + sleep 1 + done + + tail -f -n +1 "$LMCACHE_LOG" & + tail_pid=$! + + for ((i = 1; i <= attempts; i++)); do + if curl --output /dev/null --silent --fail "http://127.0.0.1:${LMCACHE_HTTP_PORT}/healthcheck"; then + kill "$tail_pid" 2>/dev/null || true + wait "$tail_pid" 2>/dev/null || true + return 0 + fi + if [[ -n "$LMCACHE_PID" ]] && ! kill -0 "$LMCACHE_PID" 2>/dev/null; then + echo "LMCache server died before becoming healthy. Log follows:" >&2 + kill "$tail_pid" 2>/dev/null || true + wait "$tail_pid" 2>/dev/null || true + cat "$LMCACHE_LOG" >&2 || true + exit 1 + fi + sleep 1 + done + + echo "Timed out waiting for LMCache server healthcheck. Log follows:" >&2 + kill "$tail_pid" 2>/dev/null || true + wait "$tail_pid" 2>/dev/null || true + cat "$LMCACHE_LOG" >&2 || true + exit 1 +} + +case "$KV_OFFLOAD_BACKEND" in + native) + unset VLLM_USE_SIMPLE_KV_OFFLOAD + # MI355X nodes have ~2.7 TiB of host DRAM available for offload; + # reserve 2.5 TB for the offload pool (leaves ~200 GB headroom for + # worker RSS / page cache / slurm cgroup). + TOTAL_CPU_DRAM_GB="${TOTAL_CPU_DRAM_GB:-3000}" + TOTAL_CPU_DRAM_PARTITION_GB="${TOTAL_CPU_DRAM_PARTITION_GB:-${TOTAL_CPU_DRAM_GB}}" + # Use vLLM's regular native KV-offload path (OffloadingConnector), + # NOT the SimpleCPUOffloadConnector. The "native" backend resolves to + # OffloadingConnector by default; setting VLLM_USE_SIMPLE_KV_OFFLOAD=1 + # would switch it to SimpleCPUOffloadConnector. We intentionally leave + # that env var UNSET here so the regular OffloadingConnector path is + # used. The shortcut --kv_offloading_backend native + --kv_offloading_size + # form constructs the KVTransferConfig at engine startup + # (vllm/config/vllm.py:662). + + # Remove --disable-hybrid-kv-cache-manager and enable hybrid kv cache manager (default) + # This gives extra cache hit than disabling hybrid kv cache manager + OFFLOAD_ARGS=( + --kv_offloading_backend native + --kv_offloading_size "$TOTAL_CPU_DRAM_PARTITION_GB" + ) + ;; + lmcache) + unset VLLM_USE_SIMPLE_KV_OFFLOAD + + git clone https://github.com/LMCache/LMCache.git + cd LMCache + pip install -r requirements/build.txt + CXX=hipcc BUILD_WITH_HIP=1 pip install -e . --no-build-isolation + cd .. + + python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null + + # Let the external MP server own the full CPU KV pool so vLLM does not + # split --kv-offloading-size across TP ranks through the integrated + # LMCache backend. + TOTAL_CPU_DRAM_GB="${TOTAL_CPU_DRAM_GB:-3000}" + TOTAL_CPU_DRAM_PARTITION_GB="${TOTAL_CPU_DRAM_PARTITION_GB:-${TOTAL_CPU_DRAM_GB}}" + LMCACHE_HOST="${LMCACHE_HOST:-127.0.0.1}" + LMCACHE_PORT="${LMCACHE_PORT:-5555}" + LMCACHE_HTTP_PORT="${LMCACHE_HTTP_PORT:-8080}" + # LMCacheMPConnector concatenates lmcache.mp.host and port into the + # ZMQ endpoint. Bind the server to a raw host, but pass the connector a + # ZMQ-style host string. + LMCACHE_CONNECT_HOST="${LMCACHE_CONNECT_HOST:-tcp://$LMCACHE_HOST}" + LMCACHE_L1_SIZE_GB="${LMCACHE_L1_SIZE_GB:-$((TOTAL_CPU_DRAM_PARTITION_GB))}" + LMCACHE_L1_INIT_SIZE_GB="${LMCACHE_L1_INIT_SIZE_GB:-20}" + # LMCache read locks are leases on chunks that lookup has promised + # vLLM can retrieve. The default 300s TTL is too short for this + # long-context agentic queue: TP8/conc32 can spend >300s between + # lookup and retrieve while GPU KV is saturated, which leaves the + # object present in L1 but no longer readable. Keep the 2.5 TB pool + # size unchanged and only extend the lookup-to-retrieve lease. + LMCACHE_L1_READ_TTL_SECONDS="${LMCACHE_L1_READ_TTL_SECONDS:-7200}" + # (srok) check 256 vs 32 + #LMCACHE_CHUNK_SIZE="${LMCACHE_CHUNK_SIZE:-32}" + LMCACHE_CHUNK_SIZE="${LMCACHE_CHUNK_SIZE:-256}" + LMCACHE_MAX_WORKERS="${LMCACHE_MAX_WORKERS:-$((TP * 2))}" + export PYTHONHASHSEED="${PYTHONHASHSEED:-0}" + export LMCACHE_BLOCKING_TIMEOUT_SECS=60 + + echo "Starting LMCache MP server..." + LMCACHE_CMD=( + lmcache server + --host "$LMCACHE_HOST" + --port "$LMCACHE_PORT" + --http-host "$LMCACHE_HOST" + --http-port "$LMCACHE_HTTP_PORT" + --l1-size-gb "$LMCACHE_L1_SIZE_GB" + --l1-init-size-gb "$LMCACHE_L1_INIT_SIZE_GB" + --l1-read-ttl-seconds "$LMCACHE_L1_READ_TTL_SECONDS" + --chunk-size "$LMCACHE_CHUNK_SIZE" + --max-workers "$LMCACHE_MAX_WORKERS" + --eviction-policy LRU + ) + printf '%q ' "${LMCACHE_CMD[@]}" > "$RESULT_DIR/lmcache_command.txt" + printf '\n' >> "$RESULT_DIR/lmcache_command.txt" + "${LMCACHE_CMD[@]}" > "$LMCACHE_LOG" 2>&1 & + LMCACHE_PID=$! + echo "LMCache server PID: $LMCACHE_PID" + wait_for_lmcache_ready + + # Remove --disable-hybrid-kv-cache-manager and enable hybrid kv cache manager (default) + # This gives extra cache hit than disabling hybrid kv cache manager + OFFLOAD_ARGS=( + --kv-transfer-config + "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_connector_module_path\":\"lmcache.integration.vllm.lmcache_mp_connector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.host\":\"$LMCACHE_CONNECT_HOST\",\"lmcache.mp.port\":$LMCACHE_PORT}}" + ) + ;; +esac + +# ---- LLM server config ---------------------------------------------------------- +PARALLEL_ARGS=(--tensor-parallel-size "$TP") +if [ "${DP_ATTENTION}" = "true" ]; then + PARALLEL_ARGS=( + --tensor-parallel-size 1 + --data-parallel-size "$TP" + --enable-expert-parallel + ) +elif [ "$EP_SIZE" -gt 1 ]; then + PARALLEL_ARGS+=(--enable-expert-parallel) +fi + +echo "Starting vllm server..." +export PYTHONNOUSERSITE=1 + +export VLLM_ENGINE_READY_TIMEOUT_S=3600 +export VLLM_USE_BREAKABLE_CUDAGRAPH=0 +export VLLM_ROCM_USE_AITER=1 +export VLLM_ROCM_USE_AITER_MOE=1 +export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 +# INT4 quantized all-reduce for the (~1.5 MB) decode all-reduces, which are the +# single biggest decode kernel at high concurrency. The MIN_SIZE_KB override is +# required: vLLM's default INT4 quick-reduce size gate for (bf16, TP4) is 16 MB, +# so it never fires for decode-sized tensors without it. +export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 +export VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16=0 +export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB=256 + +VLLM_CMD=( + vllm serve "$MODEL" + --host 0.0.0.0 + --port "$PORT" + "${PARALLEL_ARGS[@]}" + --trust-remote-code + --block-size 128 + --gpu-memory-utilization 0.85 + --language-model-only + --attention-backend TRITON_ATTN + --moe-backend aiter + --kv-cache-dtype fp8 + --tool-call-parser minimax_m3 + --enable-auto-tool-choice + --reasoning-parser minimax_m3 + --max-num-seqs "$CONC" + "${OFFLOAD_ARGS[@]}" +) +printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" +printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" +"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & +SERVER_PID=$! +echo "Server PID: $SERVER_PID" + +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +# ---- Run benchmark ---------------------------------------------------------- +build_replay_cmd "$RESULT_DIR" + +run_agentic_replay_and_write_outputs "$RESULT_DIR" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 72514a7756..e0067fb0e3 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -3041,3 +3041,20 @@ minimaxm3-fp8-mi325x-vllm-agentic: - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: native, conc-list: [3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] } - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: native, conc-list: [10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 32] } - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: native, conc-list: [24, 32, 36, 40, 44, 48, 52, 56, 60, 64, 72, 80, 96] } + +minimaxm3-fp4-mi355x-vllm-agentic: + image: vllm/vllm-openai-rocm:nightly-69715823df89b11ee684b84066390cbb9092d5c1 + model: amd/MiniMax-M3-MXFP4 + model-prefix: minimaxm3 + runner: cluster:mi355x-amds + precision: fp4 + framework: vllm + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - { tp: 4, kv-offloading: none, conc-list: [1, 8, 16, 24, 32, 40, 48, 56, 64, 72, 96] } + - { tp: 4, kv-offloading: dram, kv-offload-backend: native, conc-list: [1, 8, 16, 24, 32, 40, 48, 56, 64, 72, 96] } + - { tp: 4, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [1, 8, 16, 24, 32, 40, 48, 56, 64, 72, 96] } + \ No newline at end of file diff --git a/perf-changelog.yaml b/perf-changelog.yaml index cd5a77d19f..4b76f36542 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4601,3 +4601,11 @@ - "Image: lmsysorg/sglang:v0.5.12" - "14 topologies across 1k/1k and 8k/1k: prefill TP8 STP + decode wide-EP (DEP16/DEP32 high-throughput) and per-node TP8 low-latency, recipes under benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1895 + +- config-keys: + - minimaxm3-fp4-mi355x-vllm-agentic + description: + - "Add Minimax-M3 FP4 vLLM Single Node Agentic Support" + - "Image: vllm/vllm-openai-rocm:nightly-69715823df89b11ee684b84066390cbb9092d5c1" + pr-link: To-Be-Added + \ No newline at end of file From d6be9de68bfe2f607da4243f2e976f952a3243ed Mon Sep 17 00:00:00 2001 From: ajith-sirra-amd Date: Wed, 8 Jul 2026 17:07:47 +0530 Subject: [PATCH 02/17] [AMD] MINIMAX-M3 FP4 vLLM Agentic Support. Added PR Link Signed-off-by: ajith-sirra-amd --- perf-changelog.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 4b76f36542..e4e851e1ea 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4607,5 +4607,4 @@ description: - "Add Minimax-M3 FP4 vLLM Single Node Agentic Support" - "Image: vllm/vllm-openai-rocm:nightly-69715823df89b11ee684b84066390cbb9092d5c1" - pr-link: To-Be-Added - \ No newline at end of file + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2118 From cdac231acd3f286a273746f4e2a60d0e502cbfde Mon Sep 17 00:00:00 2001 From: ajith-sirra-amd Date: Wed, 8 Jul 2026 17:11:59 +0530 Subject: [PATCH 03/17] [AMD] MINIMAX-M3 FP4 vLLM Agentic Support. Search Space Modification. Signed-off-by: ajith-sirra-amd --- configs/amd-master.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index e0067fb0e3..5cf63bf83b 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -3054,7 +3054,6 @@ minimaxm3-fp4-mi355x-vllm-agentic: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, kv-offloading: none, conc-list: [1, 8, 16, 24, 32, 40, 48, 56, 64, 72, 96] } - - { tp: 4, kv-offloading: dram, kv-offload-backend: native, conc-list: [1, 8, 16, 24, 32, 40, 48, 56, 64, 72, 96] } - - { tp: 4, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [1, 8, 16, 24, 32, 40, 48, 56, 64, 72, 96] } - \ No newline at end of file + - { tp: 4, kv-offloading: none, conc-list: [24, 32, 40, 48, 56] } + - { tp: 4, kv-offloading: dram, kv-offload-backend: native, conc-list: [24, 32, 40, 48, 56] } + - { tp: 4, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [24, 32, 40, 48, 56] } From 89daabed9f968bc1aa370a3a1d7168d6ebc625f8 Mon Sep 17 00:00:00 2001 From: ajith-sirra-amd Date: Wed, 8 Jul 2026 17:18:48 +0530 Subject: [PATCH 04/17] [AMD] MINIMAX-M3 FP4 vLLM Agentic Support. Search Space Modification. Signed-off-by: ajith-sirra-amd --- configs/amd-master.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 5cf63bf83b..69b29fe2d7 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -3054,6 +3054,5 @@ minimaxm3-fp4-mi355x-vllm-agentic: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, kv-offloading: none, conc-list: [24, 32, 40, 48, 56] } - { tp: 4, kv-offloading: dram, kv-offload-backend: native, conc-list: [24, 32, 40, 48, 56] } - { tp: 4, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [24, 32, 40, 48, 56] } From c54a19095134ecbf095042b74880adb00034cac9 Mon Sep 17 00:00:00 2001 From: ajith-sirra-amd Date: Wed, 8 Jul 2026 17:21:54 +0530 Subject: [PATCH 05/17] [AMD] MINIMAX-M3 FP4 vLLM Agentic Support. Search Space Modification. Signed-off-by: ajith-sirra-amd --- configs/amd-master.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 69b29fe2d7..4ec3123a7a 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -3054,5 +3054,5 @@ minimaxm3-fp4-mi355x-vllm-agentic: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, kv-offloading: dram, kv-offload-backend: native, conc-list: [24, 32, 40, 48, 56] } - - { tp: 4, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [24, 32, 40, 48, 56] } + - { tp: 4, kv-offloading: dram, kv-offload-backend: native, conc-list: [32, 40, 48] } + - { tp: 4, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [32, 40, 48] } From bf6d1698911054fc64dd58800ffa25eca957ae35 Mon Sep 17 00:00:00 2001 From: ajith-sirra-amd Date: Wed, 8 Jul 2026 17:28:52 +0530 Subject: [PATCH 06/17] [AMD] MINIMAX-M3 FP4 vLLM Agentic Support. Search Space Modification & Fixing MODEL_PATH Arg in Serve Command. Signed-off-by: ajith-sirra-amd --- benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh index 877fe11d41..00ae044fe4 100644 --- a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh +++ b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh @@ -240,7 +240,8 @@ export VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16=0 export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB=256 VLLM_CMD=( - vllm serve "$MODEL" + vllm serve "$MODEL_PATH" + --served-model-name "$MODEL" --host 0.0.0.0 --port "$PORT" "${PARALLEL_ARGS[@]}" From 8bf107f8698188e9babb27c08209cb407f0b0e14 Mon Sep 17 00:00:00 2001 From: ajith-sirra-amd Date: Wed, 8 Jul 2026 17:33:57 +0530 Subject: [PATCH 07/17] [AMD] MINIMAX-M3 FP4 vLLM Agentic Support. Search Space Modification & Fixing MODEL_PATH Arg in Serve Command. Signed-off-by: ajith-sirra-amd --- benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh index 00ae044fe4..2040603e2e 100644 --- a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh +++ b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh @@ -2,7 +2,7 @@ set -euo pipefail set -x -# Agentic trace replay benchmark for Kimi-K2.5 FP4 on MI355X using vLLM. +# Agentic trace replay benchmark for Minimax-M3 FP4 on MI355X using vLLM. # # Required env vars: # MODEL, TP, CONC, OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR From 02cf9fdf6ed219218c002245afb09e7e579e2654 Mon Sep 17 00:00:00 2001 From: ajith-sirra-amd Date: Wed, 8 Jul 2026 17:47:14 +0530 Subject: [PATCH 08/17] [AMD] MINIMAX-M3 FP4 vLLM Agentic Support. Search Space Modification. Signed-off-by: ajith-sirra-amd --- configs/amd-master.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 4ec3123a7a..264ed54ef8 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -3054,5 +3054,5 @@ minimaxm3-fp4-mi355x-vllm-agentic: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, kv-offloading: dram, kv-offload-backend: native, conc-list: [32, 40, 48] } - - { tp: 4, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [32, 40, 48] } + - { tp: 4, kv-offloading: dram, kv-offload-backend: native, conc-list: [1, 4, 8, 16, 32] } + - { tp: 4, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [48, 56] } From 205732dbd3f74ed9688c9fca2bb60e1930978941 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Thu, 9 Jul 2026 10:26:00 +0900 Subject: [PATCH 09/17] chore: remove LMCache backend from MiniMax-M3 FP4 MI355X agentic benchmark Drop lmcache case branch, helper functions, and config entry; keep native-only offload path. Add OFFLOAD_ARGS fallback, enable prefix caching and hybrid KV cache manager, bump max-num-seqs to 2x CONC. Co-Authored-By: Claude Opus 4.6 --- .../agentic/minimaxm3_fp4_mi355x.sh | 178 +++--------------- configs/amd-master.yaml | 3 +- 2 files changed, 24 insertions(+), 157 deletions(-) diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh index 2040603e2e..181aef6b72 100644 --- a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh +++ b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh @@ -10,7 +10,6 @@ set -x # OFFLOADING values: # none - vLLM GPU KV only. # cpu - vLLM native CPU offload. -# lmcache - LMCache MP server + vLLM LMCacheMPConnector. source "$(dirname "$0")/../../benchmark_lib.sh" @@ -48,13 +47,6 @@ else fi # ---- Resolve traces and install deps ---------------------------------------- -# MiniMax-M2.5 servers run at max_model_len ~256k; the unfiltered 052726 -# corpus has requests up to ~1M proxy tokens that would be rejected. -# Switch to the 256k-capped variant (470 traces, max in+out <= 256k). -#export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_with_subagents_256k -#060226 -# export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_with_subagents_060226_256k - resolve_trace_source install_agentic_deps @@ -62,154 +54,22 @@ install_agentic_deps # ---- Server config ---------------------------------------------------------- SERVER_LOG="$RESULT_DIR/server.log" -LMCACHE_LOG="$RESULT_DIR/lmcache_server.log" mkdir -p "$RESULT_DIR" -OFFLOAD_ARGS=(--no-enable-prefix-caching) - -# ---- Lmcache config ---------------------------------------------------------- -LMCACHE_PID="" - -cleanup_lmcache_server() { - if [[ -n "$LMCACHE_PID" ]] && kill -0 "$LMCACHE_PID" 2>/dev/null; then - kill "$LMCACHE_PID" 2>/dev/null || true - wait "$LMCACHE_PID" 2>/dev/null || true - fi -} - -trap cleanup_lmcache_server EXIT - -wait_for_lmcache_ready() { - { set +x; } 2>/dev/null - local attempts="${LMCACHE_READY_ATTEMPTS:-120}" - local tail_pid="" - - while [ ! -f "$LMCACHE_LOG" ]; do - if [[ -n "$LMCACHE_PID" ]] && ! kill -0 "$LMCACHE_PID" 2>/dev/null; then - echo "LMCache server died before creating log file. Exiting." >&2 - exit 1 - fi - sleep 1 - done - - tail -f -n +1 "$LMCACHE_LOG" & - tail_pid=$! - - for ((i = 1; i <= attempts; i++)); do - if curl --output /dev/null --silent --fail "http://127.0.0.1:${LMCACHE_HTTP_PORT}/healthcheck"; then - kill "$tail_pid" 2>/dev/null || true - wait "$tail_pid" 2>/dev/null || true - return 0 - fi - if [[ -n "$LMCACHE_PID" ]] && ! kill -0 "$LMCACHE_PID" 2>/dev/null; then - echo "LMCache server died before becoming healthy. Log follows:" >&2 - kill "$tail_pid" 2>/dev/null || true - wait "$tail_pid" 2>/dev/null || true - cat "$LMCACHE_LOG" >&2 || true - exit 1 - fi - sleep 1 - done - - echo "Timed out waiting for LMCache server healthcheck. Log follows:" >&2 - kill "$tail_pid" 2>/dev/null || true - wait "$tail_pid" 2>/dev/null || true - cat "$LMCACHE_LOG" >&2 || true - exit 1 -} - -case "$KV_OFFLOAD_BACKEND" in - native) - unset VLLM_USE_SIMPLE_KV_OFFLOAD - # MI355X nodes have ~2.7 TiB of host DRAM available for offload; - # reserve 2.5 TB for the offload pool (leaves ~200 GB headroom for - # worker RSS / page cache / slurm cgroup). - TOTAL_CPU_DRAM_GB="${TOTAL_CPU_DRAM_GB:-3000}" - TOTAL_CPU_DRAM_PARTITION_GB="${TOTAL_CPU_DRAM_PARTITION_GB:-${TOTAL_CPU_DRAM_GB}}" - # Use vLLM's regular native KV-offload path (OffloadingConnector), - # NOT the SimpleCPUOffloadConnector. The "native" backend resolves to - # OffloadingConnector by default; setting VLLM_USE_SIMPLE_KV_OFFLOAD=1 - # would switch it to SimpleCPUOffloadConnector. We intentionally leave - # that env var UNSET here so the regular OffloadingConnector path is - # used. The shortcut --kv_offloading_backend native + --kv_offloading_size - # form constructs the KVTransferConfig at engine startup - # (vllm/config/vllm.py:662). - - # Remove --disable-hybrid-kv-cache-manager and enable hybrid kv cache manager (default) - # This gives extra cache hit than disabling hybrid kv cache manager - OFFLOAD_ARGS=( - --kv_offloading_backend native - --kv_offloading_size "$TOTAL_CPU_DRAM_PARTITION_GB" - ) - ;; - lmcache) - unset VLLM_USE_SIMPLE_KV_OFFLOAD - - git clone https://github.com/LMCache/LMCache.git - cd LMCache - pip install -r requirements/build.txt - CXX=hipcc BUILD_WITH_HIP=1 pip install -e . --no-build-isolation - cd .. - - python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null - - # Let the external MP server own the full CPU KV pool so vLLM does not - # split --kv-offloading-size across TP ranks through the integrated - # LMCache backend. - TOTAL_CPU_DRAM_GB="${TOTAL_CPU_DRAM_GB:-3000}" - TOTAL_CPU_DRAM_PARTITION_GB="${TOTAL_CPU_DRAM_PARTITION_GB:-${TOTAL_CPU_DRAM_GB}}" - LMCACHE_HOST="${LMCACHE_HOST:-127.0.0.1}" - LMCACHE_PORT="${LMCACHE_PORT:-5555}" - LMCACHE_HTTP_PORT="${LMCACHE_HTTP_PORT:-8080}" - # LMCacheMPConnector concatenates lmcache.mp.host and port into the - # ZMQ endpoint. Bind the server to a raw host, but pass the connector a - # ZMQ-style host string. - LMCACHE_CONNECT_HOST="${LMCACHE_CONNECT_HOST:-tcp://$LMCACHE_HOST}" - LMCACHE_L1_SIZE_GB="${LMCACHE_L1_SIZE_GB:-$((TOTAL_CPU_DRAM_PARTITION_GB))}" - LMCACHE_L1_INIT_SIZE_GB="${LMCACHE_L1_INIT_SIZE_GB:-20}" - # LMCache read locks are leases on chunks that lookup has promised - # vLLM can retrieve. The default 300s TTL is too short for this - # long-context agentic queue: TP8/conc32 can spend >300s between - # lookup and retrieve while GPU KV is saturated, which leaves the - # object present in L1 but no longer readable. Keep the 2.5 TB pool - # size unchanged and only extend the lookup-to-retrieve lease. - LMCACHE_L1_READ_TTL_SECONDS="${LMCACHE_L1_READ_TTL_SECONDS:-7200}" - # (srok) check 256 vs 32 - #LMCACHE_CHUNK_SIZE="${LMCACHE_CHUNK_SIZE:-32}" - LMCACHE_CHUNK_SIZE="${LMCACHE_CHUNK_SIZE:-256}" - LMCACHE_MAX_WORKERS="${LMCACHE_MAX_WORKERS:-$((TP * 2))}" - export PYTHONHASHSEED="${PYTHONHASHSEED:-0}" - export LMCACHE_BLOCKING_TIMEOUT_SECS=60 - - echo "Starting LMCache MP server..." - LMCACHE_CMD=( - lmcache server - --host "$LMCACHE_HOST" - --port "$LMCACHE_PORT" - --http-host "$LMCACHE_HOST" - --http-port "$LMCACHE_HTTP_PORT" - --l1-size-gb "$LMCACHE_L1_SIZE_GB" - --l1-init-size-gb "$LMCACHE_L1_INIT_SIZE_GB" - --l1-read-ttl-seconds "$LMCACHE_L1_READ_TTL_SECONDS" - --chunk-size "$LMCACHE_CHUNK_SIZE" - --max-workers "$LMCACHE_MAX_WORKERS" - --eviction-policy LRU - ) - printf '%q ' "${LMCACHE_CMD[@]}" > "$RESULT_DIR/lmcache_command.txt" - printf '\n' >> "$RESULT_DIR/lmcache_command.txt" - "${LMCACHE_CMD[@]}" > "$LMCACHE_LOG" 2>&1 & - LMCACHE_PID=$! - echo "LMCache server PID: $LMCACHE_PID" - wait_for_lmcache_ready - - # Remove --disable-hybrid-kv-cache-manager and enable hybrid kv cache manager (default) - # This gives extra cache hit than disabling hybrid kv cache manager - OFFLOAD_ARGS=( - --kv-transfer-config - "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_connector_module_path\":\"lmcache.integration.vllm.lmcache_mp_connector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.host\":\"$LMCACHE_CONNECT_HOST\",\"lmcache.mp.port\":$LMCACHE_PORT}}" - ) - ;; -esac +OFFLOAD_ARGS=() +if require_agentic_kv_offload_backend native; then + unset VLLM_USE_SIMPLE_KV_OFFLOAD + # MI355X nodes have ~2.7 TiB of host DRAM available for offload; + # reserve 2.5 TB for the offload pool (leaves ~200 GB headroom for + # worker RSS / page cache / slurm cgroup). + TOTAL_CPU_DRAM_GB="${TOTAL_CPU_DRAM_GB:-3000}" + TOTAL_CPU_DRAM_PARTITION_GB="${TOTAL_CPU_DRAM_PARTITION_GB:-${TOTAL_CPU_DRAM_GB}}" + + OFFLOAD_ARGS=( + --kv_offloading_backend native + --kv_offloading_size "$TOTAL_CPU_DRAM_PARTITION_GB" + ) +fi # ---- LLM server config ---------------------------------------------------------- PARALLEL_ARGS=(--tensor-parallel-size "$TP") @@ -223,6 +83,12 @@ elif [ "$EP_SIZE" -gt 1 ]; then PARALLEL_ARGS+=(--enable-expert-parallel) fi +# AgentX concurrency counts live session trees, not individual requests. +# Subagent fan-out can push instantaneous request concurrency above CONC, so +# leave 2x headroom rather than clipping those bursts at the scheduler. +MAX_NUM_SEQS=$((2 * CONC)) + +set -x echo "Starting vllm server..." export PYTHONNOUSERSITE=1 @@ -255,7 +121,9 @@ VLLM_CMD=( --tool-call-parser minimax_m3 --enable-auto-tool-choice --reasoning-parser minimax_m3 - --max-num-seqs "$CONC" + --enable-prefix-caching + --no-disable-hybrid-kv-cache-manager + --max-num-seqs "$MAX_NUM_SEQS" "${OFFLOAD_ARGS[@]}" ) printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 264ed54ef8..4a8066abf4 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -3054,5 +3054,4 @@ minimaxm3-fp4-mi355x-vllm-agentic: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, kv-offloading: dram, kv-offload-backend: native, conc-list: [1, 4, 8, 16, 32] } - - { tp: 4, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [48, 56] } + - { tp: 4, kv-offloading: dram, kv-offload-backend: native, conc-list: [1, 4, 8, 16, 32, 40] } From 82466e299b15230639c0b09eb799b54bc231b4e1 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Thu, 9 Jul 2026 12:34:54 +0900 Subject: [PATCH 10/17] chore: revert max-num-seqs to CONC and drop hybrid KV cache flag Co-Authored-By: Claude Opus 4.6 --- benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh index 181aef6b72..b6736e01a9 100644 --- a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh +++ b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh @@ -83,11 +83,6 @@ elif [ "$EP_SIZE" -gt 1 ]; then PARALLEL_ARGS+=(--enable-expert-parallel) fi -# AgentX concurrency counts live session trees, not individual requests. -# Subagent fan-out can push instantaneous request concurrency above CONC, so -# leave 2x headroom rather than clipping those bursts at the scheduler. -MAX_NUM_SEQS=$((2 * CONC)) - set -x echo "Starting vllm server..." export PYTHONNOUSERSITE=1 @@ -122,8 +117,7 @@ VLLM_CMD=( --enable-auto-tool-choice --reasoning-parser minimax_m3 --enable-prefix-caching - --no-disable-hybrid-kv-cache-manager - --max-num-seqs "$MAX_NUM_SEQS" + --max-num-seqs "$CONC" "${OFFLOAD_ARGS[@]}" ) printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" From 4bff323c3a750a47767af789c365a266a918be28 Mon Sep 17 00:00:00 2001 From: ajith-sirra-amd Date: Thu, 9 Jul 2026 10:29:37 +0530 Subject: [PATCH 11/17] [AMD] MINIMAX-M3 FP4 vLLM Agentic Support. Search Space Modification. Signed-off-by: ajith-sirra-amd --- configs/amd-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 4a8066abf4..26427b392e 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -3054,4 +3054,4 @@ minimaxm3-fp4-mi355x-vllm-agentic: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, kv-offloading: dram, kv-offload-backend: native, conc-list: [1, 4, 8, 16, 32, 40] } + - { tp: 4, kv-offloading: dram, kv-offload-backend: native, conc-list: [1, 4, 8, 16, 32] } From f3415720439fa5e4caf52db110c1c450b1d65319 Mon Sep 17 00:00:00 2001 From: ajith-sirra-amd Date: Thu, 9 Jul 2026 10:30:49 +0530 Subject: [PATCH 12/17] [AMD] MINIMAX-M3 FP4 vLLM Agentic Support. Removing redundant prefix cache flag. Signed-off-by: ajith-sirra-amd --- benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh index b6736e01a9..1ae03886c9 100644 --- a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh +++ b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh @@ -116,7 +116,6 @@ VLLM_CMD=( --tool-call-parser minimax_m3 --enable-auto-tool-choice --reasoning-parser minimax_m3 - --enable-prefix-caching --max-num-seqs "$CONC" "${OFFLOAD_ARGS[@]}" ) From 9f628cc020d31e23efe9854d5ecd557686255af1 Mon Sep 17 00:00:00 2001 From: ajith-sirra-amd Date: Thu, 9 Jul 2026 17:27:15 +0530 Subject: [PATCH 13/17] [AMD] MINIMAX-M3 FP4 vLLM Agentic Support. Modifying Search Space. Signed-off-by: ajith-sirra-amd --- .../agentic/minimaxm3_fp4_mi355x.sh | 165 ++++++++++++++++-- configs/amd-master.yaml | 3 +- 2 files changed, 149 insertions(+), 19 deletions(-) diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh index 1ae03886c9..501f919688 100644 --- a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh +++ b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh @@ -10,6 +10,7 @@ set -x # OFFLOADING values: # none - vLLM GPU KV only. # cpu - vLLM native CPU offload. +# lmcache - LMCache MP server + vLLM LMCacheMPConnector. source "$(dirname "$0")/../../benchmark_lib.sh" @@ -46,30 +47,159 @@ else export MODEL_PATH="$MODEL" fi -# ---- Resolve traces and install deps ---------------------------------------- resolve_trace_source install_agentic_deps -# export AIPERF_AGENTIC_CACHE_WARMUP_DURATION=1200 - # ---- Server config ---------------------------------------------------------- SERVER_LOG="$RESULT_DIR/server.log" +LMCACHE_LOG="$RESULT_DIR/lmcache_server.log" mkdir -p "$RESULT_DIR" -OFFLOAD_ARGS=() -if require_agentic_kv_offload_backend native; then - unset VLLM_USE_SIMPLE_KV_OFFLOAD - # MI355X nodes have ~2.7 TiB of host DRAM available for offload; - # reserve 2.5 TB for the offload pool (leaves ~200 GB headroom for - # worker RSS / page cache / slurm cgroup). - TOTAL_CPU_DRAM_GB="${TOTAL_CPU_DRAM_GB:-3000}" - TOTAL_CPU_DRAM_PARTITION_GB="${TOTAL_CPU_DRAM_PARTITION_GB:-${TOTAL_CPU_DRAM_GB}}" - - OFFLOAD_ARGS=( - --kv_offloading_backend native - --kv_offloading_size "$TOTAL_CPU_DRAM_PARTITION_GB" - ) -fi +OFFLOAD_ARGS=(--no-enable-prefix-caching) + +# ---- Lmcache config ---------------------------------------------------------- +LMCACHE_PID="" + +cleanup_lmcache_server() { + if [[ -n "$LMCACHE_PID" ]] && kill -0 "$LMCACHE_PID" 2>/dev/null; then + kill "$LMCACHE_PID" 2>/dev/null || true + wait "$LMCACHE_PID" 2>/dev/null || true + fi +} + +trap cleanup_lmcache_server EXIT + +wait_for_lmcache_ready() { + { set +x; } 2>/dev/null + local attempts="${LMCACHE_READY_ATTEMPTS:-120}" + local tail_pid="" + + while [ ! -f "$LMCACHE_LOG" ]; do + if [[ -n "$LMCACHE_PID" ]] && ! kill -0 "$LMCACHE_PID" 2>/dev/null; then + echo "LMCache server died before creating log file. Exiting." >&2 + exit 1 + fi + sleep 1 + done + + tail -f -n +1 "$LMCACHE_LOG" & + tail_pid=$! + + for ((i = 1; i <= attempts; i++)); do + if curl --output /dev/null --silent --fail "http://127.0.0.1:${LMCACHE_HTTP_PORT}/healthcheck"; then + kill "$tail_pid" 2>/dev/null || true + wait "$tail_pid" 2>/dev/null || true + return 0 + fi + if [[ -n "$LMCACHE_PID" ]] && ! kill -0 "$LMCACHE_PID" 2>/dev/null; then + echo "LMCache server died before becoming healthy. Log follows:" >&2 + kill "$tail_pid" 2>/dev/null || true + wait "$tail_pid" 2>/dev/null || true + cat "$LMCACHE_LOG" >&2 || true + exit 1 + fi + sleep 1 + done + + echo "Timed out waiting for LMCache server healthcheck. Log follows:" >&2 + kill "$tail_pid" 2>/dev/null || true + wait "$tail_pid" 2>/dev/null || true + cat "$LMCACHE_LOG" >&2 || true + exit 1 +} + +case "$KV_OFFLOAD_BACKEND" in + native) + unset VLLM_USE_SIMPLE_KV_OFFLOAD + # MI355X nodes have ~2.7 TiB of host DRAM available for offload; + # reserve 2.5 TB for the offload pool (leaves ~200 GB headroom for + # worker RSS / page cache / slurm cgroup). + TOTAL_CPU_DRAM_GB="${TOTAL_CPU_DRAM_GB:-3000}" + TOTAL_CPU_DRAM_PARTITION_GB="${TOTAL_CPU_DRAM_PARTITION_GB:-${TOTAL_CPU_DRAM_GB}}" + # Use vLLM's regular native KV-offload path (OffloadingConnector), + # NOT the SimpleCPUOffloadConnector. The "native" backend resolves to + # OffloadingConnector by default; setting VLLM_USE_SIMPLE_KV_OFFLOAD=1 + # would switch it to SimpleCPUOffloadConnector. We intentionally leave + # that env var UNSET here so the regular OffloadingConnector path is + # used. The shortcut --kv_offloading_backend native + --kv_offloading_size + # form constructs the KVTransferConfig at engine startup + # (vllm/config/vllm.py:662). + + # Remove --disable-hybrid-kv-cache-manager and enable hybrid kv cache manager (default) + # This gives extra cache hit than disabling hybrid kv cache manager + OFFLOAD_ARGS=( + --kv_offloading_backend native + --kv_offloading_size "$TOTAL_CPU_DRAM_PARTITION_GB" + ) + ;; + lmcache) + unset VLLM_USE_SIMPLE_KV_OFFLOAD + + git clone https://github.com/LMCache/LMCache.git + cd LMCache + pip install -r requirements/build.txt + CXX=hipcc BUILD_WITH_HIP=1 pip install -e . --no-build-isolation + cd .. + + python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null + + # Let the external MP server own the full CPU KV pool so vLLM does not + # split --kv-offloading-size across TP ranks through the integrated + # LMCache backend. + TOTAL_CPU_DRAM_GB="${TOTAL_CPU_DRAM_GB:-3000}" + TOTAL_CPU_DRAM_PARTITION_GB="${TOTAL_CPU_DRAM_PARTITION_GB:-${TOTAL_CPU_DRAM_GB}}" + LMCACHE_HOST="${LMCACHE_HOST:-127.0.0.1}" + LMCACHE_PORT="${LMCACHE_PORT:-5555}" + LMCACHE_HTTP_PORT="${LMCACHE_HTTP_PORT:-8080}" + # LMCacheMPConnector concatenates lmcache.mp.host and port into the + # ZMQ endpoint. Bind the server to a raw host, but pass the connector a + # ZMQ-style host string. + LMCACHE_CONNECT_HOST="${LMCACHE_CONNECT_HOST:-tcp://$LMCACHE_HOST}" + LMCACHE_L1_SIZE_GB="${LMCACHE_L1_SIZE_GB:-$((TOTAL_CPU_DRAM_PARTITION_GB))}" + LMCACHE_L1_INIT_SIZE_GB="${LMCACHE_L1_INIT_SIZE_GB:-20}" + # LMCache read locks are leases on chunks that lookup has promised + # vLLM can retrieve. The default 300s TTL is too short for this + # long-context agentic queue: TP8/conc32 can spend >300s between + # lookup and retrieve while GPU KV is saturated, which leaves the + # object present in L1 but no longer readable. Keep the 2.5 TB pool + # size unchanged and only extend the lookup-to-retrieve lease. + LMCACHE_L1_READ_TTL_SECONDS="${LMCACHE_L1_READ_TTL_SECONDS:-7200}" + # (srok) check 256 vs 32 + #LMCACHE_CHUNK_SIZE="${LMCACHE_CHUNK_SIZE:-32}" + LMCACHE_CHUNK_SIZE="${LMCACHE_CHUNK_SIZE:-256}" + LMCACHE_MAX_WORKERS="${LMCACHE_MAX_WORKERS:-$((TP * 2))}" + export PYTHONHASHSEED="${PYTHONHASHSEED:-0}" + export LMCACHE_BLOCKING_TIMEOUT_SECS=60 + + echo "Starting LMCache MP server..." + LMCACHE_CMD=( + lmcache server + --host "$LMCACHE_HOST" + --port "$LMCACHE_PORT" + --http-host "$LMCACHE_HOST" + --http-port "$LMCACHE_HTTP_PORT" + --l1-size-gb "$LMCACHE_L1_SIZE_GB" + --l1-init-size-gb "$LMCACHE_L1_INIT_SIZE_GB" + --l1-read-ttl-seconds "$LMCACHE_L1_READ_TTL_SECONDS" + --chunk-size "$LMCACHE_CHUNK_SIZE" + --max-workers "$LMCACHE_MAX_WORKERS" + --eviction-policy LRU + ) + printf '%q ' "${LMCACHE_CMD[@]}" > "$RESULT_DIR/lmcache_command.txt" + printf '\n' >> "$RESULT_DIR/lmcache_command.txt" + "${LMCACHE_CMD[@]}" > "$LMCACHE_LOG" 2>&1 & + LMCACHE_PID=$! + echo "LMCache server PID: $LMCACHE_PID" + wait_for_lmcache_ready + + # Remove --disable-hybrid-kv-cache-manager and enable hybrid kv cache manager (default) + # This gives extra cache hit than disabling hybrid kv cache manager + OFFLOAD_ARGS=( + --kv-transfer-config + "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_connector_module_path\":\"lmcache.integration.vllm.lmcache_mp_connector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.host\":\"$LMCACHE_CONNECT_HOST\",\"lmcache.mp.port\":$LMCACHE_PORT}}" + ) + ;; +esac # ---- LLM server config ---------------------------------------------------------- PARALLEL_ARGS=(--tensor-parallel-size "$TP") @@ -83,7 +213,6 @@ elif [ "$EP_SIZE" -gt 1 ]; then PARALLEL_ARGS+=(--enable-expert-parallel) fi -set -x echo "Starting vllm server..." export PYTHONNOUSERSITE=1 diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 26427b392e..79f9dfb229 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -3054,4 +3054,5 @@ minimaxm3-fp4-mi355x-vllm-agentic: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, kv-offloading: dram, kv-offload-backend: native, conc-list: [1, 4, 8, 16, 32] } + - { tp: 4, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [1, 4] } + - { tp: 4, kv-offloading: dram, kv-offload-backend: native, conc-list: [8, 16, 32] } From ccb54cd269c97947d7f1fa2bf13d9dd63fe6eba7 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 10 Jul 2026 11:14:23 -0500 Subject: [PATCH 14/17] fix(agentic): preserve matrix CPU DRAM value --- benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh index 501f919688..ccb5d2143e 100644 --- a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh +++ b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh @@ -114,7 +114,6 @@ case "$KV_OFFLOAD_BACKEND" in # MI355X nodes have ~2.7 TiB of host DRAM available for offload; # reserve 2.5 TB for the offload pool (leaves ~200 GB headroom for # worker RSS / page cache / slurm cgroup). - TOTAL_CPU_DRAM_GB="${TOTAL_CPU_DRAM_GB:-3000}" TOTAL_CPU_DRAM_PARTITION_GB="${TOTAL_CPU_DRAM_PARTITION_GB:-${TOTAL_CPU_DRAM_GB}}" # Use vLLM's regular native KV-offload path (OffloadingConnector), # NOT the SimpleCPUOffloadConnector. The "native" backend resolves to @@ -146,7 +145,6 @@ case "$KV_OFFLOAD_BACKEND" in # Let the external MP server own the full CPU KV pool so vLLM does not # split --kv-offloading-size across TP ranks through the integrated # LMCache backend. - TOTAL_CPU_DRAM_GB="${TOTAL_CPU_DRAM_GB:-3000}" TOTAL_CPU_DRAM_PARTITION_GB="${TOTAL_CPU_DRAM_PARTITION_GB:-${TOTAL_CPU_DRAM_GB}}" LMCACHE_HOST="${LMCACHE_HOST:-127.0.0.1}" LMCACHE_PORT="${LMCACHE_PORT:-5555}" From cf4bba6418283b272bb589e448eca7279aaab2ba Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 10 Jul 2026 11:17:20 -0500 Subject: [PATCH 15/17] fix(agentic): use explicit MI355X server port --- benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh index ccb5d2143e..dc81680735 100644 --- a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh +++ b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh @@ -18,9 +18,7 @@ check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION echo "MODEL=$MODEL TP=$TP CONC=$CONC KV_OFFLOADING=$KV_OFFLOADING TOTAL_CPU_DRAM_GB=$TOTAL_CPU_DRAM_GB RESULT_DIR=$RESULT_DIR DURATION=$DURATION EP_SIZE=$EP_SIZE DP_ATTENTION=$DP_ATTENTION" -PORT=${PORT:-8888} -DURATION=${DURATION:-1800} -EP_SIZE=${EP_SIZE:-1} +PORT=8888 if [[ -n "${SLURM_JOB_ID:-}" ]]; then echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" From 060ec35721a69b34fc6d8da9888e21e1604561bc Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 10 Jul 2026 11:37:24 -0500 Subject: [PATCH 16/17] fix(agentic): remove recipe env fallbacks --- .../agentic/minimaxm3_fp4_mi355x.sh | 60 +++++++------------ 1 file changed, 20 insertions(+), 40 deletions(-) diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh index dc81680735..8db475a6da 100644 --- a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh +++ b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh @@ -5,44 +5,31 @@ set -x # Agentic trace replay benchmark for Minimax-M3 FP4 on MI355X using vLLM. # # Required env vars: -# MODEL, TP, CONC, OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR -# -# OFFLOADING values: -# none - vLLM GPU KV only. -# cpu - vLLM native CPU offload. -# lmcache - LMCache MP server + vLLM LMCacheMPConnector. +# MODEL, MODEL_PATH, TP, CONC, KV_OFFLOADING, KV_OFFLOAD_BACKEND, +# TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION, EP_SIZE, DP_ATTENTION source "$(dirname "$0")/../../benchmark_lib.sh" -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION +check_env_vars MODEL MODEL_PATH TP CONC KV_OFFLOADING KV_OFFLOAD_BACKEND TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION echo "MODEL=$MODEL TP=$TP CONC=$CONC KV_OFFLOADING=$KV_OFFLOADING TOTAL_CPU_DRAM_GB=$TOTAL_CPU_DRAM_GB RESULT_DIR=$RESULT_DIR DURATION=$DURATION EP_SIZE=$EP_SIZE DP_ATTENTION=$DP_ATTENTION" PORT=8888 -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" +if [[ -n "${SLURM_JOB_ID+x}" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" fi # ROCR/HIP visibility for vLLM 0.14+ -if [ -n "${ROCR_VISIBLE_DEVICES:-}" ]; then +if [[ -n "${ROCR_VISIBLE_DEVICES+x}" ]]; then export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" fi -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi rocm-smi || true amd-smi || true -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" +if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" fi resolve_trace_source @@ -69,7 +56,7 @@ trap cleanup_lmcache_server EXIT wait_for_lmcache_ready() { { set +x; } 2>/dev/null - local attempts="${LMCACHE_READY_ATTEMPTS:-120}" + local attempts=120 local tail_pid="" while [ ! -f "$LMCACHE_LOG" ]; do @@ -109,10 +96,6 @@ wait_for_lmcache_ready() { case "$KV_OFFLOAD_BACKEND" in native) unset VLLM_USE_SIMPLE_KV_OFFLOAD - # MI355X nodes have ~2.7 TiB of host DRAM available for offload; - # reserve 2.5 TB for the offload pool (leaves ~200 GB headroom for - # worker RSS / page cache / slurm cgroup). - TOTAL_CPU_DRAM_PARTITION_GB="${TOTAL_CPU_DRAM_PARTITION_GB:-${TOTAL_CPU_DRAM_GB}}" # Use vLLM's regular native KV-offload path (OffloadingConnector), # NOT the SimpleCPUOffloadConnector. The "native" backend resolves to # OffloadingConnector by default; setting VLLM_USE_SIMPLE_KV_OFFLOAD=1 @@ -126,7 +109,7 @@ case "$KV_OFFLOAD_BACKEND" in # This gives extra cache hit than disabling hybrid kv cache manager OFFLOAD_ARGS=( --kv_offloading_backend native - --kv_offloading_size "$TOTAL_CPU_DRAM_PARTITION_GB" + --kv_offloading_size "$TOTAL_CPU_DRAM_GB" ) ;; lmcache) @@ -143,28 +126,25 @@ case "$KV_OFFLOAD_BACKEND" in # Let the external MP server own the full CPU KV pool so vLLM does not # split --kv-offloading-size across TP ranks through the integrated # LMCache backend. - TOTAL_CPU_DRAM_PARTITION_GB="${TOTAL_CPU_DRAM_PARTITION_GB:-${TOTAL_CPU_DRAM_GB}}" - LMCACHE_HOST="${LMCACHE_HOST:-127.0.0.1}" - LMCACHE_PORT="${LMCACHE_PORT:-5555}" - LMCACHE_HTTP_PORT="${LMCACHE_HTTP_PORT:-8080}" + LMCACHE_HOST=127.0.0.1 + LMCACHE_PORT=5555 + LMCACHE_HTTP_PORT=8080 # LMCacheMPConnector concatenates lmcache.mp.host and port into the # ZMQ endpoint. Bind the server to a raw host, but pass the connector a # ZMQ-style host string. - LMCACHE_CONNECT_HOST="${LMCACHE_CONNECT_HOST:-tcp://$LMCACHE_HOST}" - LMCACHE_L1_SIZE_GB="${LMCACHE_L1_SIZE_GB:-$((TOTAL_CPU_DRAM_PARTITION_GB))}" - LMCACHE_L1_INIT_SIZE_GB="${LMCACHE_L1_INIT_SIZE_GB:-20}" + LMCACHE_CONNECT_HOST="tcp://$LMCACHE_HOST" + LMCACHE_L1_SIZE_GB="$TOTAL_CPU_DRAM_GB" + LMCACHE_L1_INIT_SIZE_GB=20 # LMCache read locks are leases on chunks that lookup has promised # vLLM can retrieve. The default 300s TTL is too short for this # long-context agentic queue: TP8/conc32 can spend >300s between # lookup and retrieve while GPU KV is saturated, which leaves the # object present in L1 but no longer readable. Keep the 2.5 TB pool # size unchanged and only extend the lookup-to-retrieve lease. - LMCACHE_L1_READ_TTL_SECONDS="${LMCACHE_L1_READ_TTL_SECONDS:-7200}" - # (srok) check 256 vs 32 - #LMCACHE_CHUNK_SIZE="${LMCACHE_CHUNK_SIZE:-32}" - LMCACHE_CHUNK_SIZE="${LMCACHE_CHUNK_SIZE:-256}" - LMCACHE_MAX_WORKERS="${LMCACHE_MAX_WORKERS:-$((TP * 2))}" - export PYTHONHASHSEED="${PYTHONHASHSEED:-0}" + LMCACHE_L1_READ_TTL_SECONDS=7200 + LMCACHE_CHUNK_SIZE=256 + LMCACHE_MAX_WORKERS=$((TP * 2)) + export PYTHONHASHSEED=0 export LMCACHE_BLOCKING_TIMEOUT_SECS=60 echo "Starting LMCache MP server..." From e8ff5ba8f26ebe59b189eacc590153dae6884996 Mon Sep 17 00:00:00 2001 From: "claude[bot]" <41898282+claude[bot]@users.noreply.github.com> Date: Fri, 10 Jul 2026 17:35:11 +0000 Subject: [PATCH 17/17] fix(configs): restore indentation of dsv4 decode additional-settings The extra leading space broke YAML parsing of amd-master.yaml. Co-authored-by: Cameron Quilici <60715037+cquil11@users.noreply.github.com> Co-Authored-By: Claude Fable 5 --- configs/amd-master.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 4bbbb7bea0..269c6aae74 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -3167,6 +3167,6 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache: tp: 8 ep: 1 dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MTP_SIZE=0"