From 4d3f1e1ef07df778a4c79c5a473436c95eb63e5c Mon Sep 17 00:00:00 2001 From: ajith-sirra-amd Date: Thu, 9 Jul 2026 19:36:22 +0530 Subject: [PATCH 1/5] [AMD] MINIMAX-M3 FP4 vLLM Agentic Support. Signed-off-by: ajith-sirra-amd --- .../agentic/minimaxm3_fp4_mi355x.sh | 206 ++++++++++++++++++ configs/amd-master.yaml | 14 ++ perf-changelog.yaml | 7 + 3 files changed, 227 insertions(+) create mode 100644 benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh new file mode 100644 index 0000000000..affe97d044 --- /dev/null +++ b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh @@ -0,0 +1,206 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x +# Agentic trace replay benchmark for Minimax-M3 FP4 on MI355X using vLLM. +# +# Required env vars: +# MODEL, TP, CONC, OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR +# +# OFFLOADING values: +# none - vLLM GPU KV only. +# cpu - vLLM native CPU offload. +# lmcache - LMCache MP server + vLLM LMCacheMPConnector. +source "$(dirname "$0")/../../benchmark_lib.sh" +check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION +echo "MODEL=$MODEL TP=$TP CONC=$CONC KV_OFFLOADING=$KV_OFFLOADING TOTAL_CPU_DRAM_GB=$TOTAL_CPU_DRAM_GB RESULT_DIR=$RESULT_DIR DURATION=$DURATION EP_SIZE=$EP_SIZE DP_ATTENTION=$DP_ATTENTION" +PORT=${PORT:-8888} +DURATION=${DURATION:-1800} +EP_SIZE=${EP_SIZE:-1} +if [[ -n "${SLURM_JOB_ID:-}" ]]; then + echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" +fi +# ROCR/HIP visibility for vLLM 0.14+ +if [ -n "${ROCR_VISIBLE_DEVICES:-}" ]; then + export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" +fi +if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi +rocm-smi || true +amd-smi || true +# `hf download` creates the target dir if missing and is itself idempotent. +# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE +# Either way, MODEL_PATH is what the server is launched with. +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi +resolve_trace_source +install_agentic_deps +# ---- Server config ---------------------------------------------------------- +SERVER_LOG="$RESULT_DIR/server.log" +LMCACHE_LOG="$RESULT_DIR/lmcache_server.log" +mkdir -p "$RESULT_DIR" +OFFLOAD_ARGS=(--no-enable-prefix-caching) +# ---- Lmcache config ---------------------------------------------------------- +LMCACHE_PID="" +cleanup_lmcache_server() { + if [[ -n "$LMCACHE_PID" ]] && kill -0 "$LMCACHE_PID" 2>/dev/null; then + kill "$LMCACHE_PID" 2>/dev/null || true + wait "$LMCACHE_PID" 2>/dev/null || true + fi +} +trap cleanup_lmcache_server EXIT +wait_for_lmcache_ready() { + { set +x; } 2>/dev/null + local attempts="${LMCACHE_READY_ATTEMPTS:-120}" + local tail_pid="" + while [ ! -f "$LMCACHE_LOG" ]; do + if [[ -n "$LMCACHE_PID" ]] && ! kill -0 "$LMCACHE_PID" 2>/dev/null; then + echo "LMCache server died before creating log file. Exiting." >&2 + exit 1 + fi + sleep 1 + done + tail -f -n +1 "$LMCACHE_LOG" & + tail_pid=$! + for ((i = 1; i <= attempts; i++)); do + if curl --output /dev/null --silent --fail "http://127.0.0.1:${LMCACHE_HTTP_PORT}/healthcheck"; then + kill "$tail_pid" 2>/dev/null || true + wait "$tail_pid" 2>/dev/null || true + return 0 + fi + if [[ -n "$LMCACHE_PID" ]] && ! kill -0 "$LMCACHE_PID" 2>/dev/null; then + echo "LMCache server died before becoming healthy. Log follows:" >&2 + kill "$tail_pid" 2>/dev/null || true + wait "$tail_pid" 2>/dev/null || true + cat "$LMCACHE_LOG" >&2 || true + exit 1 + fi + sleep 1 + done + echo "Timed out waiting for LMCache server healthcheck. Log follows:" >&2 + kill "$tail_pid" 2>/dev/null || true + wait "$tail_pid" 2>/dev/null || true + cat "$LMCACHE_LOG" >&2 || true + exit 1 +} +case "$KV_OFFLOAD_BACKEND" in + lmcache) + unset VLLM_USE_SIMPLE_KV_OFFLOAD + git clone https://github.com/LMCache/LMCache.git + cd LMCache + pip install -r requirements/build.txt + CXX=hipcc BUILD_WITH_HIP=1 pip install -e . --no-build-isolation + cd .. + python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null + # Let the external MP server own the full CPU KV pool so vLLM does not + # split --kv-offloading-size across TP ranks through the integrated + # LMCache backend. + TOTAL_CPU_DRAM_GB="${TOTAL_CPU_DRAM_GB:-3000}" + TOTAL_CPU_DRAM_PARTITION_GB="${TOTAL_CPU_DRAM_PARTITION_GB:-${TOTAL_CPU_DRAM_GB}}" + LMCACHE_HOST="${LMCACHE_HOST:-127.0.0.1}" + LMCACHE_PORT="${LMCACHE_PORT:-5555}" + LMCACHE_HTTP_PORT="${LMCACHE_HTTP_PORT:-8080}" + # LMCacheMPConnector concatenates lmcache.mp.host and port into the + # ZMQ endpoint. Bind the server to a raw host, but pass the connector a + # ZMQ-style host string. + LMCACHE_CONNECT_HOST="${LMCACHE_CONNECT_HOST:-tcp://$LMCACHE_HOST}" + LMCACHE_L1_SIZE_GB="${LMCACHE_L1_SIZE_GB:-$((TOTAL_CPU_DRAM_PARTITION_GB))}" + LMCACHE_L1_INIT_SIZE_GB="${LMCACHE_L1_INIT_SIZE_GB:-20}" + # LMCache read locks are leases on chunks that lookup has promised + # vLLM can retrieve. The default 300s TTL is too short for this + # long-context agentic queue: TP8/conc32 can spend >300s between + # lookup and retrieve while GPU KV is saturated, which leaves the + # object present in L1 but no longer readable. Keep the 2.5 TB pool + # size unchanged and only extend the lookup-to-retrieve lease. + LMCACHE_L1_READ_TTL_SECONDS="${LMCACHE_L1_READ_TTL_SECONDS:-7200}" + # (srok) check 256 vs 32 + #LMCACHE_CHUNK_SIZE="${LMCACHE_CHUNK_SIZE:-32}" + LMCACHE_CHUNK_SIZE="${LMCACHE_CHUNK_SIZE:-256}" + LMCACHE_MAX_WORKERS="${LMCACHE_MAX_WORKERS:-$((TP * 2))}" + export PYTHONHASHSEED="${PYTHONHASHSEED:-0}" + export LMCACHE_BLOCKING_TIMEOUT_SECS=60 + echo "Starting LMCache MP server..." + LMCACHE_CMD=( + lmcache server + --host "$LMCACHE_HOST" + --port "$LMCACHE_PORT" + --http-host "$LMCACHE_HOST" + --http-port "$LMCACHE_HTTP_PORT" + --l1-size-gb "$LMCACHE_L1_SIZE_GB" + --l1-init-size-gb "$LMCACHE_L1_INIT_SIZE_GB" + --l1-read-ttl-seconds "$LMCACHE_L1_READ_TTL_SECONDS" + --chunk-size "$LMCACHE_CHUNK_SIZE" + --max-workers "$LMCACHE_MAX_WORKERS" + --eviction-policy LRU + ) + printf '%q ' "${LMCACHE_CMD[@]}" > "$RESULT_DIR/lmcache_command.txt" + printf '\n' >> "$RESULT_DIR/lmcache_command.txt" + "${LMCACHE_CMD[@]}" > "$LMCACHE_LOG" 2>&1 & + LMCACHE_PID=$! + echo "LMCache server PID: $LMCACHE_PID" + wait_for_lmcache_ready + # Remove --disable-hybrid-kv-cache-manager and enable hybrid kv cache manager (default) + # This gives extra cache hit than disabling hybrid kv cache manager + OFFLOAD_ARGS=( + --kv-transfer-config + "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_connector_module_path\":\"lmcache.integration.vllm.lmcache_mp_connector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.host\":\"$LMCACHE_CONNECT_HOST\",\"lmcache.mp.port\":$LMCACHE_PORT}}" + ) + ;; +esac +# ---- LLM server config ---------------------------------------------------------- +PARALLEL_ARGS=(--tensor-parallel-size "$TP") +if [ "${DP_ATTENTION}" = "true" ]; then + PARALLEL_ARGS=( + --tensor-parallel-size 1 + --data-parallel-size "$TP" + --enable-expert-parallel + ) +elif [ "$EP_SIZE" -gt 1 ]; then + PARALLEL_ARGS+=(--enable-expert-parallel) +fi +echo "Starting vllm server..." +export PYTHONNOUSERSITE=1 +export VLLM_ENGINE_READY_TIMEOUT_S=3600 +export VLLM_USE_BREAKABLE_CUDAGRAPH=0 +export VLLM_ROCM_USE_AITER=1 +export VLLM_ROCM_USE_AITER_MOE=1 +export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 +# INT4 quantized all-reduce for the (~1.5 MB) decode all-reduces, which are the +# single biggest decode kernel at high concurrency. The MIN_SIZE_KB override is +# required: vLLM's default INT4 quick-reduce size gate for (bf16, TP4) is 16 MB, +# so it never fires for decode-sized tensors without it. +export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 +export VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16=0 +export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB=256 +VLLM_CMD=( + vllm serve "$MODEL_PATH" + --served-model-name "$MODEL" + --host 0.0.0.0 + --port "$PORT" + "${PARALLEL_ARGS[@]}" + --trust-remote-code + --block-size 128 + --gpu-memory-utilization 0.85 + --language-model-only + --attention-backend TRITON_ATTN + --moe-backend aiter + --kv-cache-dtype fp8 + --tool-call-parser minimax_m3 + --enable-auto-tool-choice + --reasoning-parser minimax_m3 + --max-num-seqs "$CONC" + "${OFFLOAD_ARGS[@]}" +) +printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" +printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" +"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & +SERVER_PID=$! +echo "Server PID: $SERVER_PID" +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" +# ---- Run benchmark ---------------------------------------------------------- +build_replay_cmd "$RESULT_DIR" +run_agentic_replay_and_write_outputs "$RESULT_DIR" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 72514a7756..fd323f010e 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -3041,3 +3041,17 @@ minimaxm3-fp8-mi325x-vllm-agentic: - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: native, conc-list: [3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] } - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: native, conc-list: [10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 32] } - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: native, conc-list: [24, 32, 36, 40, 44, 48, 52, 56, 60, 64, 72, 80, 96] } + +minimaxm3-fp4-mi355x-vllm-agentic: + image: vllm/vllm-openai-rocm:nightly-69715823df89b11ee684b84066390cbb9092d5c1 + model: amd/MiniMax-M3-MXFP4 + model-prefix: minimaxm3 + runner: cluster:mi355x-amds + precision: fp4 + framework: vllm + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - { tp: 4, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [1, 4, 8, 16, 32] } diff --git a/perf-changelog.yaml b/perf-changelog.yaml index cd5a77d19f..28ca7377d7 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4601,3 +4601,10 @@ - "Image: lmsysorg/sglang:v0.5.12" - "14 topologies across 1k/1k and 8k/1k: prefill TP8 STP + decode wide-EP (DEP16/DEP32 high-throughput) and per-node TP8 low-latency, recipes under benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1895 + +- config-keys: + - minimaxm3-fp4-mi355x-vllm-agentic + description: + - "Add Minimax-M3 FP4 vLLM Single Node Agentic Support" + - "Image: vllm/vllm-openai-rocm:nightly-69715823df89b11ee684b84066390cbb9092d5c1" + pr-link: To-Be-Added \ No newline at end of file From e46b5fdd3b3c6136fac05d15fb1ede96fb78de83 Mon Sep 17 00:00:00 2001 From: ajith-sirra-amd Date: Thu, 9 Jul 2026 19:47:12 +0530 Subject: [PATCH 2/5] [AMD] MINIMAX-M3 FP4 vLLM Agentic Support. Signed-off-by: ajith-sirra-amd --- perf-changelog.yaml | 43 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 28ca7377d7..2dac5fe596 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4602,9 +4602,50 @@ - "14 topologies across 1k/1k and 8k/1k: prefill TP8 STP + decode wide-EP (DEP16/DEP32 high-throughput) and per-node TP8 low-latency, recipes under benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1895 +- config-keys: + - glm5-fp4-gb200-dynamo-sglang + description: + - "Initial submission: GLM-5 FP4 disagg on GB200 with Dynamo SGLang (8k1k + 1k1k, STP / no-MTP)." + - "Model: nvidia/GLM-5.1-NVFP4 (GB200 Lustre path: /mnt/lustre01/models/GLM-5.1-NVFP4)." + - "Image: lmsysorg/sglang:v0.5.11-cu130" + - "Recipes ported from srt-slurm PR #211 (recipes/gb200-fp4/glm5.yaml), split per topology under recipes/sglang/glm5/gb200-fp4/." + - "8k1k: 4 wide-EP (TP=32) max-throughput + 6 per-node (TP=4) low-latency topologies." + - "1k1k: 1 wide-EP max-throughput + 6 low-latency (DEP16/DEP32 + TP=4) topologies." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1780 + +- config-keys: + - dsr1-fp4-b200-dynamo-sglang-mtp + description: + - "Prune the 8k/1k MTP disagg low-latency points: 1p5d-tp8 to conc 4-32, 1p3d-tp8 to conc 32-64, and 1p1d-tp8 to conc 32." + - "Add 8k/1k MTP disagg high-throughput points: 1p1d and 4p1d (DEP4 prefill / DEP8 decode) at conc 512." + - "Add an 8k/1k MTP disagg 4p1d (DEP4 prefill / DEP8 decode) high-throughput point at conc 2048 with a dedicated engine tune (max-running-requests 2048, decode scheduler-recv-interval 1, prefill/decode stream-interval 100/34, SGLANG_HACK_SEQ_BOOTSTRAP_ROOM)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2113 + +- config-keys: + - glm5-fp4-gb200-dynamo-sglang-mtp + description: + - "Add GLM-5.1 NVFP4 GB200 disaggregated dynamo-sglang MTP config covering 8k1k and 1k1k, high-throughput and low-latency topologies" + - "Image: lmsysorg/sglang:v0.5.13.post1-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2115 + +- config-keys: + - glm5-fp4-gb300-dynamo-sglang-mtp + description: + - "Add GLM-5.1 NVFP4 GB300 disaggregated dynamo-sglang MTP config covering 8k1k and 1k1k, high-throughput and low-latency topologies" + - "Image: lmsysorg/sglang:v0.5.13.post1-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2114 + +- config-keys: + - glm5-fp4-gb300-dynamo-trt-mtp + description: + - "Add GLM-5 NVFP4 GB300 disaggregated TRT-LLM MTP (spec-decoding) benchmarks via Dynamo (23 MTP configs: 13 for 1K/1K, 10 for 8K/1K)" + - "Container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13" + - "Recipes sourced from NVIDIA/srt-slurm branch sa-submission-q2-2026 (gb300_nvfp4 MTP recipes)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1799 + - config-keys: - minimaxm3-fp4-mi355x-vllm-agentic description: - "Add Minimax-M3 FP4 vLLM Single Node Agentic Support" - "Image: vllm/vllm-openai-rocm:nightly-69715823df89b11ee684b84066390cbb9092d5c1" - pr-link: To-Be-Added \ No newline at end of file + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2129 From a090492f2b36a02e41e7b7acbb3bd217e168289d Mon Sep 17 00:00:00 2001 From: ajith-sirra-amd Date: Thu, 9 Jul 2026 21:29:50 +0530 Subject: [PATCH 3/5] [AMD] MINIMAX-M3 FP4 vLLM Agentic Support. Update Perf Change Log Yaml File. Signed-off-by: ajith-sirra-amd --- perf-changelog.yaml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 2dac5fe596..622334d018 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4644,8 +4644,16 @@ pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1799 - config-keys: + - dsv4-fp4-b200-sglang-agentic-hicache + - dsv4-fp4-b300-sglang-agentic-hicache + description: + - "Update SGLang image from lmsysorg/sglang:v0.5.13-cu130 to lmsysorg/sglang:nightly-dev-cu13-20260707-b4155233" + - "Enable SGLANG_ENABLE_UNIFIED_RADIX_TREE=1 unconditionally (previously hicache-only) and SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS=1 to free out-of-window SWA KV slots during chunked prefill, relieving SWA pool pressure and restoring prefix-cache hit rate on multi-turn agentic workloads" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2112 + + - config-keys: - minimaxm3-fp4-mi355x-vllm-agentic description: - "Add Minimax-M3 FP4 vLLM Single Node Agentic Support" - "Image: vllm/vllm-openai-rocm:nightly-69715823df89b11ee684b84066390cbb9092d5c1" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2129 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2129 \ No newline at end of file From 07a66fa0dd5abfd332b5f3ead9feb17b197d7f7a Mon Sep 17 00:00:00 2001 From: ajith-sirra-amd Date: Thu, 9 Jul 2026 21:33:12 +0530 Subject: [PATCH 4/5] [AMD] MINIMAX-M3 FP4 vLLM Agentic Support. Update Perf Change Log Yaml File. Signed-off-by: ajith-sirra-amd --- perf-changelog.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 622334d018..9f5f91c8db 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4656,4 +4656,5 @@ description: - "Add Minimax-M3 FP4 vLLM Single Node Agentic Support" - "Image: vllm/vllm-openai-rocm:nightly-69715823df89b11ee684b84066390cbb9092d5c1" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2129 \ No newline at end of file + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2129 + \ No newline at end of file From 92fe81b5149ef5030032f71ab6f898e5d20a0aaf Mon Sep 17 00:00:00 2001 From: ajith-sirra-amd Date: Thu, 9 Jul 2026 21:37:26 +0530 Subject: [PATCH 5/5] [AMD] MINIMAX-M3 FP4 vLLM Agentic Support. Update Perf Change Log Yaml File. Signed-off-by: ajith-sirra-amd --- perf-changelog.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 9f5f91c8db..c39299bc8b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4651,10 +4651,9 @@ - "Enable SGLANG_ENABLE_UNIFIED_RADIX_TREE=1 unconditionally (previously hicache-only) and SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS=1 to free out-of-window SWA KV slots during chunked prefill, relieving SWA pool pressure and restoring prefix-cache hit rate on multi-turn agentic workloads" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2112 - - config-keys: +- config-keys: - minimaxm3-fp4-mi355x-vllm-agentic description: - "Add Minimax-M3 FP4 vLLM Single Node Agentic Support" - "Image: vllm/vllm-openai-rocm:nightly-69715823df89b11ee684b84066390cbb9092d5c1" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2129 - \ No newline at end of file