From de4cff8faf5f516a19d94a1629add8e11200f041 Mon Sep 17 00:00:00 2001 From: Hyukjoon Lee Date: Thu, 9 Jul 2026 10:47:57 +0900 Subject: [PATCH 1/9] agentx: kimik2.7 fp4 mi355x agentic smoke (none + lmcache) Adds the kimik2.7 fp4 agentic trace-replay recipe (kimik2.7_fp4_mi355x.sh, KV_OFFLOADING none + dram/lmcache) and the kimik2.7-fp4-mi355x-vllm-agentic config key on cluster:mi355x-amds (tp4 conc32, none + lmcache). Recipe clones LMCache to /opt/lmcache-src so the CI checkout never trips over root-owned build artifacts. --- .../agentic/kimik2.7_fp4_mi355x.sh | 244 ++++++++++++++++++ configs/amd-master.yaml | 16 ++ 2 files changed, 260 insertions(+) create mode 100755 benchmarks/single_node/agentic/kimik2.7_fp4_mi355x.sh diff --git a/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x.sh b/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x.sh new file mode 100755 index 0000000000..e9edb59029 --- /dev/null +++ b/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x.sh @@ -0,0 +1,244 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +# Agentic trace replay benchmark for Kimi-K2.7 FP4 on MI355X using vLLM. +# +# Variant of kimik2.7_fp4_mi355x.sh that supports TWO KV configs: +# KV_OFFLOADING=none -> GPU KV only +# KV_OFFLOADING=dram KV_OFFLOAD_BACKEND=lmcache -> LMCache MP server + connector +# +# Required env vars: +# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION, EP_SIZE + + +source "$(dirname "$0")/../../benchmark_lib.sh" + +check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE + +if [[ -n "${SLURM_JOB_ID:-}" ]]; then + echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" +fi + +# ROCR/HIP visibility for vLLM 0.14+ +if [ -n "${ROCR_VISIBLE_DEVICES:-}" ]; then + export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" +fi + +# `hf download` creates the target dir if missing and is itself idempotent. +# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE +# Either way, MODEL_PATH is what the server is launched with. +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi +rocm-smi || true +amd-smi || true + +# ---- Resolve traces and install deps ---------------------------------------- +resolve_trace_source +install_agentic_deps + +# Install amd-quark for MXFP4 (manual install due to ROCm vLLM bug) +pip install amd-quark + +# Disable AITER RMSNorm for TP < 8 due to accuracy issues +if [ "${TP}" -lt 8 ]; then + export VLLM_ROCM_USE_AITER_RMSNORM=0 +fi +# Workaround for MEC FW <177 RCCL memory reclaim issue +version=$(rocm-smi --showfw 2>/dev/null | grep MEC | head -n 1 | awk '{print $NF}') +if [[ "$version" == "" || ${version:-0} -lt 177 ]]; then + export HSA_NO_SCRATCH_RECLAIM=1 +fi + +export VLLM_ROCM_USE_AITER=1 +export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 +# Avoid intermittent symm_mem all-reduce rendezvous hang at engine init on +# MI35x nodes (see KIMIK27_CONC64_LMCACHE_RUNBOOK error #4). +export VLLM_ALLREDUCE_USE_SYMM_MEM="${VLLM_ALLREDUCE_USE_SYMM_MEM:-0}" + +# ---- Server config ---------------------------------------------------------- +SERVER_LOG="$RESULT_DIR/server.log" +LMCACHE_LOG="$RESULT_DIR/lmcache_server.log" +mkdir -p "$RESULT_DIR" + +OFFLOAD_ARGS=() +PREFIX_CACHE_ARGS=() + +# ---- LMCache config --------------------------------------------------------- +LMCACHE_PID="" + +cleanup_lmcache_server() { + if [[ -n "$LMCACHE_PID" ]] && kill -0 "$LMCACHE_PID" 2>/dev/null; then + kill "$LMCACHE_PID" 2>/dev/null || true + wait "$LMCACHE_PID" 2>/dev/null || true + fi +} +trap cleanup_lmcache_server EXIT + +wait_for_lmcache_ready() { + { set +x; } 2>/dev/null + local attempts="${LMCACHE_READY_ATTEMPTS:-120}" + local tail_pid="" + + while [ ! -f "$LMCACHE_LOG" ]; do + if [[ -n "$LMCACHE_PID" ]] && ! kill -0 "$LMCACHE_PID" 2>/dev/null; then + echo "LMCache server died before creating log file. Exiting." >&2 + exit 1 + fi + sleep 1 + done + + tail -f -n +1 "$LMCACHE_LOG" & + tail_pid=$! + + for ((i = 1; i <= attempts; i++)); do + if curl --output /dev/null --silent --fail "http://127.0.0.1:${LMCACHE_HTTP_PORT}/healthcheck"; then + kill "$tail_pid" 2>/dev/null || true + wait "$tail_pid" 2>/dev/null || true + return 0 + fi + if [[ -n "$LMCACHE_PID" ]] && ! kill -0 "$LMCACHE_PID" 2>/dev/null; then + echo "LMCache server died before becoming healthy. Log follows:" >&2 + kill "$tail_pid" 2>/dev/null || true + wait "$tail_pid" 2>/dev/null || true + cat "$LMCACHE_LOG" >&2 || true + exit 1 + fi + sleep 1 + done + + echo "Timed out waiting for LMCache server healthcheck. Log follows:" >&2 + kill "$tail_pid" 2>/dev/null || true + wait "$tail_pid" 2>/dev/null || true + cat "$LMCACHE_LOG" >&2 || true + exit 1 +} + +# Resolve the effective offload backend. When KV_OFFLOADING=none there is no +# backend; when dram, KV_OFFLOAD_BACKEND selects native vs lmcache. +if [[ "$KV_OFFLOADING" == "none" ]]; then + OFFLOAD_MODE="none" +else + OFFLOAD_MODE="${KV_OFFLOAD_BACKEND:?KV_OFFLOAD_BACKEND required when KV_OFFLOADING=dram}" +fi + +case "$OFFLOAD_MODE" in + none) + OFFLOAD_ARGS=(--no-enable-prefix-caching) + ;; + lmcache) + unset VLLM_USE_SIMPLE_KV_OFFLOAD + + # Build LMCache against ROCm if the connector isn't already importable + # (prebuilt kimi-lmcache images already ship it). Clone to a + # container-local dir (NOT the bind-mounted /workspace) so the CI + # checkout's `clean: true` never trips over root-owned build artifacts + # on the next job. Pin a ref for reproducibility. + if ! python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null 2>&1; then + LMCACHE_SRC_DIR="${LMCACHE_SRC_DIR:-/opt/lmcache-src}" + LMCACHE_GIT_REF="${LMCACHE_GIT_REF:-aaf7c0d3}" + rm -rf "$LMCACHE_SRC_DIR" + git clone https://github.com/LMCache/LMCache.git "$LMCACHE_SRC_DIR" + ( cd "$LMCACHE_SRC_DIR" + git checkout "$LMCACHE_GIT_REF" + pip install -r requirements/build.txt + CXX=hipcc BUILD_WITH_HIP=1 pip install -e . --no-build-isolation ) + python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null + fi + + LMCACHE_HOST="${LMCACHE_HOST:-127.0.0.1}" + LMCACHE_PORT="${LMCACHE_PORT:-5555}" + LMCACHE_HTTP_PORT="${LMCACHE_HTTP_PORT:-8080}" + LMCACHE_CONNECT_HOST="${LMCACHE_CONNECT_HOST:-tcp://$LMCACHE_HOST}" + # Let the external MP server own the whole CPU KV pool. The requested + # budget is TOTAL_CPU_DRAM_GB, but LMCache's L1 is SHM-backed: if L1 > + # /dev/shm free it silently disables SHM and falls back to the slow + # pickle path (crashes at load — see kimik27 CI shm-overflow note). + # Cap L1 to 90% of current /dev/shm free space so SHM stays enabled. + SHM_FREE_GB=$(df -BG --output=avail /dev/shm 2>/dev/null | tail -1 | tr -dc '0-9') + SHM_CAP_GB=$(( SHM_FREE_GB * 90 / 100 )) + LMCACHE_L1_SIZE_GB="${LMCACHE_L1_SIZE_GB:-$TOTAL_CPU_DRAM_GB}" + if [ -n "$SHM_CAP_GB" ] && [ "$SHM_CAP_GB" -gt 0 ] && [ "$LMCACHE_L1_SIZE_GB" -gt "$SHM_CAP_GB" ]; then + echo "Capping LMCACHE_L1_SIZE_GB $LMCACHE_L1_SIZE_GB -> $SHM_CAP_GB to fit /dev/shm (${SHM_FREE_GB}G free)" + LMCACHE_L1_SIZE_GB="$SHM_CAP_GB" + fi + LMCACHE_L1_INIT_SIZE_GB="${LMCACHE_L1_INIT_SIZE_GB:-20}" + LMCACHE_L1_READ_TTL_SECONDS="${LMCACHE_L1_READ_TTL_SECONDS:-7200}" + LMCACHE_CHUNK_SIZE="${LMCACHE_CHUNK_SIZE:-256}" + LMCACHE_MAX_WORKERS="${LMCACHE_MAX_WORKERS:-$((TP * 2))}" + export PYTHONHASHSEED="${PYTHONHASHSEED:-0}" + export LMCACHE_BLOCKING_TIMEOUT_SECS=60 + + echo "Starting LMCache MP server..." + LMCACHE_CMD=( + lmcache server + --host "$LMCACHE_HOST" + --port "$LMCACHE_PORT" + --http-host "$LMCACHE_HOST" + --http-port "$LMCACHE_HTTP_PORT" + --l1-size-gb "$LMCACHE_L1_SIZE_GB" + --l1-init-size-gb "$LMCACHE_L1_INIT_SIZE_GB" + --l1-read-ttl-seconds "$LMCACHE_L1_READ_TTL_SECONDS" + --chunk-size "$LMCACHE_CHUNK_SIZE" + --max-workers "$LMCACHE_MAX_WORKERS" + --eviction-policy LRU + ) + printf '%q ' "${LMCACHE_CMD[@]}" > "$RESULT_DIR/lmcache_command.txt" + printf '\n' >> "$RESULT_DIR/lmcache_command.txt" + "${LMCACHE_CMD[@]}" > "$LMCACHE_LOG" 2>&1 & + LMCACHE_PID=$! + echo "LMCache server PID: $LMCACHE_PID" + wait_for_lmcache_ready + + OFFLOAD_ARGS=( + --kv-transfer-config + "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_connector_module_path\":\"lmcache.integration.vllm.lmcache_mp_connector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.host\":\"$LMCACHE_CONNECT_HOST\",\"lmcache.mp.port\":$LMCACHE_PORT}}" + ) + ;; + *) + echo "Error: unsupported KV_OFFLOAD_BACKEND '$OFFLOAD_MODE' (expected: lmcache)" >&2 + exit 1 + ;; +esac + +EP_ARGS=() +if [ "$EP_SIZE" -gt 1 ]; then + EP_ARGS=(--enable-expert-parallel) +fi + +echo "Starting vllm server..." +export PYTHONNOUSERSITE=1 + +{ set +x; } 2>/dev/null +VLLM_CMD=( + vllm serve "$MODEL_PATH" --served-model-name "$MODEL" + --host 0.0.0.0 + --port "$PORT" + --tensor-parallel-size="$TP" + "${EP_ARGS[@]}" + --gpu-memory-utilization 0.90 + --block-size=1 + --trust-remote-code + --max-num-seqs "$CONC" + --mm-encoder-tp-mode data + "${PREFIX_CACHE_ARGS[@]}" + "${OFFLOAD_ARGS[@]}" +) +printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" +printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" +"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & +SERVER_PID=$! +echo "Server PID: $SERVER_PID" + +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +# ---- Run benchmark ---------------------------------------------------------- +build_replay_cmd "$RESULT_DIR" + +run_agentic_replay_and_write_outputs "$RESULT_DIR" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 72514a7756..e4b52f7973 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -808,6 +808,22 @@ kimik2.5-fp4-mi355x-vllm-agentic: - { tp: 4, kv-offloading: none, conc-list: [16, 24, 32, 40] } - { tp: 4, kv-offloading: dram, kv-offload-backend: native, conc-list: [16, 24, 32, 40] } +kimik2.7-fp4-mi355x-vllm-agentic: + image: vllm/vllm-openai-rocm:v0.24.0 + model: amd/Kimi-K2.7-Code-MXFP4 + model-prefix: kimik2.7 + runner: cluster:mi355x-amds + precision: fp4 + framework: vllm + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - { tp: 4, kv-offloading: none, conc-list: [32] } + - { tp: 4, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [32] } + + kimik2.5-fp4-mi355x-atom: image: rocm/atom:rocm7.2.3_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom20260511 model: amd/Kimi-K2.5-MXFP4 From 87abb7efa1f653f068b78dbea24979adb5c4e80e Mon Sep 17 00:00:00 2001 From: Hyukjoon Lee Date: Thu, 9 Jul 2026 11:39:39 +0900 Subject: [PATCH 2/9] agentx: keep prefix caching ON in kimik2.7 none cell The none case previously passed --no-enable-prefix-caching, which disables the on-GPU prefix cache entirely, not just DRAM offload. That crippled the no-offload baseline (no reuse even within HBM) and made lmcache look artificially good. Leave prefix caching at vLLM's default (ON) so none is an honest GPU-only-KV baseline, apples-to-apples vs lmcache. Matches the kimik2.5 / dsv4 agentic recipes. --- benchmarks/single_node/agentic/kimik2.7_fp4_mi355x.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x.sh b/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x.sh index e9edb59029..b9a197a9b7 100755 --- a/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x.sh +++ b/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x.sh @@ -130,7 +130,12 @@ fi case "$OFFLOAD_MODE" in none) - OFFLOAD_ARGS=(--no-enable-prefix-caching) + # GPU-only KV, no DRAM offload. Leave prefix caching at vLLM's + # default (ON) so this is an honest no-offload baseline that still + # reuses shared prefixes on-GPU — apples-to-apples vs the lmcache + # cell, which extends that same reuse into DRAM. (Matches the + # kimik2.5 / dsv4 agentic recipes.) + PREFIX_CACHE_ARGS=(--enable-prefix-caching) ;; lmcache) unset VLLM_USE_SIMPLE_KV_OFFLOAD From ad1def8e6bf4abfcd594b45538312ff2ff347594 Mon Sep 17 00:00:00 2001 From: Hyukjoon Lee Date: Thu, 9 Jul 2026 12:44:42 +0900 Subject: [PATCH 3/9] agentx: add conc64 to kimik2.7 agentic search-space (none + lmcache) --- configs/amd-master.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index e4b52f7973..f75d80d174 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -820,8 +820,8 @@ kimik2.7-fp4-mi355x-vllm-agentic: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, kv-offloading: none, conc-list: [32] } - - { tp: 4, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [32] } + - { tp: 4, kv-offloading: none, conc-list: [32, 64] } + - { tp: 4, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [32, 64] } kimik2.5-fp4-mi355x-atom: From 4bde154a6da4689f49a9c6522b9e44327d93297a Mon Sep 17 00:00:00 2001 From: Hyukjoon Lee Date: Thu, 9 Jul 2026 15:29:18 +0900 Subject: [PATCH 4/9] agentx: kimik2.7 full-sweep search-space + perf-changelog Set the kimik2.7-fp4-mi355x-vllm-agentic sweep to conc [1,4,8,16,32,48] for both none and dram/LMCache at TP4, dram-utilization 0.80. Add the corresponding perf-changelog entry (pr-link placeholder to fill on PR open). --- configs/amd-master.yaml | 4 +- perf-changelog.yaml | 4636 --------------------------------------- 2 files changed, 2 insertions(+), 4638 deletions(-) delete mode 100644 perf-changelog.yaml diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index f75d80d174..addf1ea374 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -820,8 +820,8 @@ kimik2.7-fp4-mi355x-vllm-agentic: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, kv-offloading: none, conc-list: [32, 64] } - - { tp: 4, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [32, 64] } + - { tp: 4, kv-offloading: none, conc-list: [1, 4, 8, 16, 32, 48] } + - { tp: 4, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [1, 4, 8, 16, 32, 48] } kimik2.5-fp4-mi355x-atom: diff --git a/perf-changelog.yaml b/perf-changelog.yaml deleted file mode 100644 index fc79650307..0000000000 --- a/perf-changelog.yaml +++ /dev/null @@ -1,4636 +0,0 @@ -- config-keys: - - 70b-fp8-*-vllm - description: - - 'Add compilation-config ''{"custom_ops": ["-rms_norm", "-quant_fp8", "-silu_and_mul"]}'' as extra config to all benchmarks/70b_fp8_mi*.sh scripts' - - "6-7% uplift for llama for 6/8 configs" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/95 - -- config-keys: - - gptoss-fp4-*-trt - description: - - "Upgrade GPT-OSS TRT images from 'release:1.1.0rc2.post2' to '1.2.0rc0.post1'" - - "Add NCCL_GRAPH_REGISTER=0 to benchmarks/gptoss_fp4_b200_trt_slurm.sh" - - "Change kv_cache_config.dtype from 'auto' to 'fp8' in benchmarks/gptoss_fp4_b200_trt_slurm.sh" - - "Remove MOE_BACKEND=CUTLASS, now just defaults to TRTLLM" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/110 - -- config-keys: - - gptoss* - - dsr1* - description: - - "Remove Llama 70B runs to make room for multi-node disagg prefill+wideEP on h100/h200/b200/mi300/mi325/mi355" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/149 - -- config-keys: - - gptoss-fp4-b200-vllm - - gptoss-fp4-h100-vllm - - gptoss-fp4-h200-vllm - description: - - "Upgrade vLLM from 0.10.2 to 0.11.0 for GPT-OSS NVIDIA single-node configs" - - 'Add compilation-config ''{"cudagraph_mode":"PIECEWISE"}'' since vLLM 0.11.0 now defaults to FULL_AND_PIECEWISE' - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/159 - -- config-keys: - - dsr1* - description: - - "Fix bug where 1k8k and 8k1k full sweeps had incorrect max-model-len for DeepSeek" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/163 - -- config-keys: - - dsr1-fp4-b200-sglang - - dsr1-fp8-b200-sglang - - dsr1-fp8-h200-sglang - description: - - "Consolidate H200 and B200 SGLang configurations to use unified v0.5.5-cu129-amd64 image tag" - - "Update deprecated SGLang server arguments to current equivalents" - - "Replace --enable-ep-moe with --ep-size $EP_SIZE" - - "Replace --enable-flashinfer-trtllm-moe with --moe-runner-backend flashinfer_trtllm" - - "Add -e EP_SIZE to Docker run commands in launch scripts" - - "Set ep:4 for all tp:4 entries, ep:8 for all tp:8 entries" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/204 - -- config-keys: - - gptoss-fp4-mi355x-vllm - - gptoss-fp4-b200-vllm - description: - - "Extend concurrency to 128 for gptoss mi355x/b200 vllm configurations" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/209 - -- config-keys: - - gptoss-fp4-b200-trt - description: - - "Extend concurrency to 128 for gptoss b200 TRT configurations" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/233 - -- config-keys: - - gptoss-fp4-b200-trt - description: - - "Add benchmark script for GPTOSS FP4 B200 TRT-LLM" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/256 - -- config-keys: - - "*gb200-dynamo-sglang" - description: - - "Introduce improvements in GB200 SGLang DSR1 submission" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/257 - -- config-keys: - - dsr1-fp8-h200-trt - description: - - "Update TRT image from nvcr.io#nvidia/tensorrt-llm/release:1.2.0rc0.post1 to nvcr.io#nvidia/tensorrt-llm/release:1.2.0rc2" - - "Increase concurrency for some configurations" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/266 - -- config-keys: - - gptoss-fp4-b200-vllm - - gptoss-fp4-h100-vllm - - gptoss-fp4-h200-vllm - description: - - "Update vLLM image for NVIDIA configs from vLLM 0.11.0 to vLLM 0.11.2" - - "Add kv-cache-dtype: fp8 to benchmarks/gptoss_fp4_b200_docker.sh" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/273 - -- config-keys: - - dsr1-fp4-b200-sglang - - dsr1-fp8-b200-sglang - - dsr1-fp8-h200-sglang - description: - - "Update NVIDIA DeepSeek sglang Docker image from v0.5.5 to v0.5.6" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/276 - -- config-keys: - - gptoss-fp4-b200-vllm - - gptoss-fp4-h100-vllm - - gptoss-fp4-h200-vllm - description: - - "Update vLLM image from v0.11.2 to v0.13.0" - - "Add VLLM_MXFP4_USE_MARLIN=1 to H100 and H200 benchmark scripts" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/327 - -- config-keys: - - dsr1-fp4-mi355x-sglang - description: - - "Update MI355x Deepseek-R1 FP4 SGLang Image to upstream v0.5.6.post1" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/330 - -- config-keys: - - dsr1-fp8-mi300x-sglang - - dsr1-fp8-mi325x-sglang - - dsr1-fp8-mi355x-sglang - description: - - Use upstream SGLang images on mi300, mi325 and mi355 for dsr1fp8 - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/332 - -- config-keys: - - dsr1-fp4-gb200-dynamo-trt - - dsr1-fp4-gb200-dynamo-sglang - - dsr1-fp8-gb200-dynamo-sglang - description: - - "Add more configurations for GB200 SGLang DSR1" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/335 - -- config-keys: - - dsr1-fp4-gb200-dynamo-sglang - - dsr1-fp8-gb200-dynamo-sglang - description: - - "fix: Pruning unnecessary concurrencies " - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/358 - -- config-keys: - - dsr1-fp4-mi355x-sglang - description: - - "Updating MI355x Deepseek-R1 FP4 SGLang Image to upstream v0.5.6.post2" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/369 - -- config-keys: - - gptoss-fp4-gb200-dynamo-trt - - gptoss-fp4-b200-trt - description: - - Explicitly add EP=TP for DP attention configs for B200 AGG nvidia-master file. Multinode Refactor inadvertently changed default EP=1 - - Add GPTOSS DISAGG configurations for GB200 1k1k and 8k1k. - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/387 - -- config-keys: - - dsr1-fp4-b200-trt-mtp - - dsr1-fp8-b200-trt-mtp - - dsr1-fp8-h200-trt-mtp - description: - - Add MTP (Multi-Token Prediction) support for single-node TRT configs - - Add spec-decoding field to config entries and update launch scripts to select MTP benchmark scripts - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/392 - -- config-keys: - - dsr1-fp4-mi355x-sglang - description: - - "Updating MI355x Deepseek-R1 FP4 SGLang Image to upstream v0.5.7" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/395 - -- config-keys: - - dsr1-fp8-mi355x-sglang-disagg - description: - - "Add PD disaggregation (1P2D) for Mi355X" - - "Includes with and without speculative decoding" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/409 - -- config-keys: - - dsr1-fp8-b200-sglang - description: - - "Adds TP4 configurations to DSR1-FP8 B200 SGLang deployment experiments" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/411 - -- config-keys: - - dsr1-fp8-mi355x-atom - - dsr1-fp4-mi355x-atom - - gptoss-fp4-mi355x-atom - description: - - Add internal AMD ATOM inference engine for DeepSeek R1 FP8, FP4 and GPTOSS FP4 Mi355X - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/419 - -- config-keys: - - gptoss-fp4-mi300x-vllm - - gptoss-fp4-mi325x-vllm - description: - - "Update AMD MI300X and MI325X GPT-OSS 120B vLLM to use upstream ROCm image vllm/vllm-openai-rocm:v0.14.0" - - "Remove deprecated --async-scheduling flag (now enabled by default in vLLM v0.14.0)" - - "Remove deprecated --max-seq-len-to-capture flag" - - "Add HIP_VISIBLE_DEVICES env var for Ray compatibility in vLLM 0.14+" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/496 - -- config-keys: - - dsr1-fp4-gb200-dynamo-trt - description: - - "Update Dynamo TRT image from 0.5.1-rc0.pre3 to 0.8.1.post2" - - "Update TRT configurations" - - "Refactor configurations to use CONFIG_FILE-based recipes instead of inline parameter settings" - - "Introduce srt-slurm workflow for launching Dynamo jobs" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/510 - -- config-keys: - - gptoss-fp4-mi300x-vllm - - gptoss-fp4-mi325x-vllm - description: - - "Fix AITER env vars for vLLM v0.14.0 on AMD MI300X and MI325X" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/535 - -- config-keys: - - dsr1-fp8-h200-sglang - description: - - "Update H200 DeepSeek R1 FP8 SGLang image from v0.5.6 to v0.5.7" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/538 - -- config-keys: - - dsr1-fp8-mi300x-sglang - description: - - "Update MI300X DeepSeek R1 FP8 SGLang image from v0.5.5.post3 to v0.5.7" - - "Add SGLANG_AITER_MLA_PERSIST=1 for persistent MLA kernel optimization" - - "Set --kv-cache-dtype fp8_e4m3 for fp8 KV cache" - - "Set --attention-backend aiter for AMD aiter attention backend" - - "Update chunked-prefill-size and max-prefill-tokens from 196608 to 131072" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/544 - -- config-keys: - - dsr1-fp8-mi325x-sglang - description: - - "Update MI325X DeepSeek R1 FP8 SGLang image from v0.5.5.post3 to v0.5.7" - - "Add SGLANG_AITER_MLA_PERSIST=1 for persistent MLA kernel with fp8 KV cache" - - "Add --kv-cache-dtype fp8_e4m3 for explicit FP8 KV cache" - - "Add --attention-backend aiter for AMD aiter attention backend" - - "Reduce chunked-prefill-size from 196608 to 131072" - - "Reduce max-prefill-tokens from 196608 to 131072" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/545 - -- config-keys: - - dsr1-fp8-h200-dynamo-trt - description: - - "Add DSR1 FP8 H200 Dynamo TRT-LLM disaggregated multinode configuration" - - "Image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post1" - - "Runner: h200-dgxc with multinode and disagg enabled" - - "Includes MTP and STP configurations for 1k1k and 8k1k sequence lengths" - - "Concurrency levels: 4, 8, 16, 32, 64, 128, 256, 512" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/570 - -- config-keys: - - dsr1-fp8-mi355x-sglang - description: - - "Update MI355X DeepSeek R1 FP8 SGLang image from v0.5.5.post3 to v0.5.8" - - "Key fix: Disables mla persistent kernel when not using fp8 kv_cache (https://github.com/sgl-project/sglang/pull/17327)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/572 - -- config-keys: - - dsr1-fp8-h200-dynamo-sglang - description: - - "Add DSR1 FP8 H200 Dynamo SGLang disaggregated multinode configuration" - - "Image: lmsysorg/sglang:v0.5.8-cu130-runtime" - - "Runner: h200-multinode-slurm with multinode and disagg enabled" - - "Recipes sourced from srtslurm repo (recipes/h200/)" - - "1k1k configs: aggregated, low-latency (1P9D), high-throughput TEP (1P6D), DEP (1P6D)" - - "8k1k configs: aggregated, TEP configs (1P7D, 1P6D, 1P3D, 2P3D), DEP (1P1D)" - - "Concurrency levels range from 1 to 2048 depending on configuration" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/582 - -- config-keys: - - dsr1-fp4-b300-dynamo-trt - description: - - "Add DSR1 FP4 B300 Dynamo TRT configurations" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/585 - -- config-keys: - # NVIDIA single-node - - dsr1-fp4-b200-sglang - - dsr1-fp4-b200-trt - - dsr1-fp4-b200-trt-mtp - - dsr1-fp8-b200-sglang - - dsr1-fp8-b200-trt - - dsr1-fp8-b200-trt-mtp - - dsr1-fp8-h200-sglang - - dsr1-fp8-h200-trt - - dsr1-fp8-h200-trt-mtp - - gptoss-fp4-b200-trt - - gptoss-fp4-b200-vllm - - gptoss-fp4-h100-vllm - - gptoss-fp4-h200-trt - - gptoss-fp4-h200-vllm - # AMD single-node - - dsr1-fp4-mi355x-sglang - - dsr1-fp4-mi355x-atom - - dsr1-fp8-mi300x-sglang - - dsr1-fp8-mi325x-sglang - - dsr1-fp8-mi355x-sglang - - dsr1-fp8-mi355x-atom - - gptoss-fp4-mi300x-vllm - - gptoss-fp4-mi325x-vllm - - gptoss-fp4-mi355x-vllm - - gptoss-fp4-mi355x-atom - description: - - Add official GSM8k eval results to GPT-OSS and DeepSeek R1 scenarios - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/587 - evals-only: true - -- config-keys: - - dsr1-fp4-b200-dynamo-trt - description: - - "Update DSR1 FP4 B200 Dynamo TRT configurations" - - "Update TRTLLM version to 1.2.0rc6.post2" - - "Transform to use srt-slurm recipes" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/588 - -- config-keys: - - dsr1-fp8-b200-trt - description: - - "Update TensorRT-LLM container from release:1.1.0rc2.post2 to release:1.2.0rc6.post2" - - "Change default MOE backend from DEEPGEMM to TRTLLM" - - "Add dynamic piecewise CUDA graphs for 1k1k (CONC≥64) and 8k1k (CONC≥64) workloads" - - "Add delay batching (batch_wait_timeout_iters/batch_wait_max_tokens_ratio) for 1k1k high-concurrency" - - "Add dynamic KV cache memory fraction tuning (0.7-0.8) based on ISL/OSL/TP configuration" - - "Update search space: remove EP=TP constraint, add TP=4 configurations, extend concurrency ranges" - - "Add TLLM_OVERRIDE_LAYER_NUM=61 to avoid OOM errors" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/594 - -- config-keys: - - dsr1-fp4-mi355x-sglang - description: - - "Update SGLang image from v0.5.7 to v0.5.8 for DeepSeek-R1 FP4 on MI355x" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/595 - -- config-keys: - - dsr1-fp8-mi355x-sglang - description: - - "Disable torch.compile for MI355X DeepSeek-R1 FP8 SGLang" - - "set cuda-graph-max-bs to CONC" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/613 - -- config-keys: - - dsr1-fp8-b200-dynamo-trt - description: - - "Introduce new DSR1 FP8 B200 Dynamo TRT configurations for 8k1k and 1k1k" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/616 - -- config-keys: - - dsr1-fp8-gb200-dynamo-trt - description: - - "Add DeepSeek R1 FP8 GB200 Dynamo TRT-LLM disaggregated multinode configurations" - - "Image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post2" - - "1k1k: 14 scenarios (7 MTP, 7 STP) with varying DP attention/TEP modes" - - "1k8k: 10 scenarios (5 MTP, 5 STP) for long output generation" - - "8k1k: 14 scenarios (7 MTP, 7 STP) for long context workloads" - - "Prefill workers: 1-5P, Decode workers: 1-4D, TP/EP: 8/16/32" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/617 - -- config-keys: - - dsr1-fp4-gb300-dynamo-trt - description: - - "Add DeepSeek-R1 FP4 GB300 Dynamo TRT disaggregated multinode configurations" - - "Image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post2" - - "Includes MTP and STP configs for 1k1k, 1k8k, and 8k1k sequence lengths" - - "Add gb300-nv runner and launch script for srt-slurm integration" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/618 - -- config-keys: - - dsr1-fp4-b200-trt - description: - - "Update TensorRT-LLM container from release:1.1.0rc2.post2 to release:1.2.0rc6.post2" - - "Change default MOE backend from DEEPGEMM to TRTLLM" - - "Add dynamic piecewise CUDA graphs for 1k1k (TEP8 and CONC64)" - - "Update search space: remove EP=TP constraint, add TP=4 configurations, extend concurrency ranges" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/620 - -- config-keys: - - dsr1-fp4-mi355x-sglang-disagg - description: - - "enable PD/D for both MTP and non-MTP MI355X DeepSeek-R1 FP4 SGLang" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/622 - -- config-keys: - - dsr1-fp8-b200-sglang-mtp - description: - - "Add MTP (Multi-Token Prediction) support for DeepSeek R1 FP8 B200 SGLang using EAGLE speculative decoding" - - "Image: lmsysorg/sglang:v0.5.8-cu130-amd64" - - "Add benchmark script dsr1_fp8_b200_mtp.sh with EAGLE speculative decoding (num-steps=2, draft-tokens=3, topk=1)" - - "Update launch_b200-dgxc.sh to support SPEC_SUFFIX for MTP script selection" - - "Configurations: TP=8, EP=1, concurrency 4-64 for 1k1k, 1k8k, and 8k1k sequence lengths" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/626 - -- config-keys: - - dsr1-fp8-gb300-dynamo-trt - description: - - "Add DeepSeek R1 FP8 GB300 Dynamo TRT-LLM disaggregated multinode configurations for 8k1k and 1k1k" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/627 - -- config-keys: - - dsr1-fp8-b200-trt-mtp - description: - - Update to the latest TRTLLM 1.2 release container version - - Fine-tune the choice of parallelism in nvidia-master file, mainly going to TP for most points - - Enable piecewise CUDA graphs under most conditions - - fine-tune max batch sizes and other optimizations - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/632 - -- config-keys: - - dsr1-fp4-gb200-dynamo-sglang - description: - - "Update SGLang image from v0.5.5.post2 to v0.5.8-cu130" - - "Add FP4 model path separation via SRT_SLURM_MODEL_PREFIX in launch script" - - "Refactor to use CONFIG_FILE-based srt-slurm recipes instead of inline parameters" - - "Add 1k1k configurations: low-latency (1P2D), mid-curve (4P8D), max-tpt (4P12D)" - - "Add 8k1k configurations: low-latency (1P4D), mid-curve (6P12D), max-tpt (10P8D)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/633 - -- config-keys: - - dsr1-fp8-gb200-dynamo-sglang - - dsr1-fp8-gb300-dynamo-sglang - description: - - "Update GB200 and GB300 configs for DSR1 FP8 SGLANG STP mode" - - "Image: lmsysorg/sglang:v0.5.8-cu130" - - "Update prefill/decode worker counts, TP/EP parallelism, and dp-attn settings for 1k1k and 8k1k" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/635 - -- config-keys: - - dsr1-fp4-gb300-dynamo-sglang - description: - - "Add GB300 FP4 Dynamo SGLang disaggregated multinode configuration" - - "Image: lmsysorg/sglang:v0.5.8.post1-cu130-runtime" - - "Recipes sourced from srt-slurm repo (recipes/gb300-fp4/ folder)" - - "Add 1k1k configurations: low-latency (1P2D), mid-curve (4P8D), max-tpt (4P12D)" - - "Add 8k1k configurations: low-latency (1P4D), mid-curve (6P12D), max-tpt (10P8D)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/636 - -- config-keys: - - dsr1-fp8-b300-dynamo-trt - description: - - "New B300 FP8 Dynamo TRT configurations" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/638 - -- config-keys: - - gptoss-fp4-b200-trt - description: - - "Update GPT-OSS FP4 B200 TRT pareto configurations and new container image" - - "Extend maximum concurrency to 256 across all sequence lengths" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/639 - -- config-keys: - - dsr1-fp8-h200-dynamo-sglang - description: - - "Add MTP (EAGLE speculative decoding) configs alongside STP" - - "Update container to lmsysorg/sglang:v0.5.8.post1-cu130" - - "Remove aggregated configs, keep only disaggregated" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/640 - -- config-keys: - - dsr1-fp4-b200-trt-mtp - description: - - "Upgrade TensorRT-LLM container from release:1.1.0rc2.post2 to release:1.2.0rc6.post3" - - "Enable dynamic piecewise CUDA graphs for several conditions" - - "Adjust TP8/TP4 search space to reduce overlapping points" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/642 - -- config-keys: - - dsr1-fp8-h100-dynamo-sglang - description: - - "Add DeepSeek-R1 FP8 H100 Dynamo SGLang STP disaggregated multinode configurations" - - "Image: lmsysorg/sglang:v0.5.8-cu130" - - "1k1k, 1k8k, 8k1k sequence lengths" - - "Two modes per seq-len: Max throughput TEP (1P2D) and Max throughput DEP (1P1D with dp-attention)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/643 - -- config-keys: - - dsr1-fp8-h100-dynamo-sglang - description: - - "Add DeepSeek-R1 FP8 H100 Dynamo SGLang MTP disaggregated multinode configurations" - - "Image: lmsysorg/sglang:v0.5.8-cu130" - - "1k1k, 1k8k, 8k1k sequence lengths with MTP speculative decoding" - - "Two modes per seq-len: Max throughput TEP (1P2D) and Max throughput DEP (1P1D with dp-attention)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/644 - -- config-keys: - - dsr1-fp8-gb200-dynamo-trt - description: - - "Fix model_prefix argument in yaml configs" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/646 - -- config-keys: - - dsr1-fp8-mi355x-sglang-disagg - description: - - "Add --use-chat-template argument to benchmark_serving script" - - "Without this arg, MTP acceptance rates are artificially high for DeepSeek with MTP" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/649 - -- config-keys: - - dsr1-fp8-h100-dynamo-trt - description: - - "Add DeepSeek R1 FP8 H100 Dynamo TRT-LLM disaggregated multinode configurations" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/651 - -- config-keys: - - dsr1-fp8-gb300-dynamo-trt - description: - - "Add DeepSeek R1 FP8 GB300 Dynamo TRT-LLM disaggregated multinode configurations for 1k8k" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/654 - -- config-keys: - - dsr1-fp8-b200-dynamo-sglang - description: - - "Add DSR1 FP8 B200 disaggregated SGLang multinode configuration" - - "Image: lmsysorg/sglang:v0.5.8.post1-cu130-amd64" - - "9 recipes: 4x 1k1k + 5x 8k1k, low-latency and max-throughput profiles" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/658 - -- config-keys: - - dsr1-fp8-h100-dynamo-trt - description: - - "Add DeepSeek R1 FP8 H100 Dynamo TRT-LLM disaggregated multinode configurations" - - "fix model_prefix bug from https://github.com/SemiAnalysisAI/InferenceX/pull/651" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/663 - -- config-keys: - - dsr1-fp8-b200-dynamo-sglang-mtp - description: - - "Add DSR1 FP8 B200 disaggregated SGLang MTP multinode configuration" - - "Image: lmsysorg/sglang:v0.5.8.post1-cu130-amd64" - - "9 recipes: 4x 1k1k + 5x 8k1k, low-latency and max-throughput with EAGLE speculative decoding" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/667 - -- config-keys: - - dsr1-fp4-b200-dynamo-sglang - description: - - "Add DSR1 FP4 B200 Dynamo SGLang STP mode" - - "Image: lmsysorg/sglang:v0.5.8.post1-cu130-runtime" - - "1k1k configs: low-latency DEP (1P5D, 1P6D), max-throughput DEP (1P1D, 1P2D)" - - "8k1k configs: low-latency DEP/TEP (1P1D, 1P5D, 2P5D), TEP (1P1D), max-throughput DEP (7P2D)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/672 - -- config-keys: - - dsr1-fp8-mi355x-atom-mtp - - dsr1-fp4-mi355x-atom-mtp - description: - - "Add DSR1 FP8/FP4 MI355X ATOM with MTP configuration" - - "Image: rocm/atom:rocm7.2.0-ubuntu24.04-pytorch2.9-atom0.1.1" - - "Deepseek R1 with speculative decoding: 1k1k, 1k8k, 8k1k" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/673 - -- config-keys: - - dsr1-fp8-mi355x-sglang-disagg - - dsr1-fp4-mi355x-sglang-disagg - description: - - "Bump MI355X MORI FP8/FP4 image to latest rocm/sgl-dev:sglang-0.5.8-rocm700-mi35x-mori-0210" - - "Bump mi355x sglang disagg recipe to sa-260211" - - "Add conc 4/8/16" - - "Use Pure TP with MTP=2 for 1k1k conc smaller than 128 and reduce MTP to 1 for DEP configs" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/674 - -- config-keys: - - dsr1-fp4-b200-dynamo-sglang-mtp - description: - - "Add B200 configs for DSR1 FP4 SGLANG MTP mode for 1k1k and 8k1k" - - "Image: lmsysorg/sglang:v0.5.8.post1-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/683 - -- config-keys: - - dsr1-fp8-b200-dynamo-trt - description: - - "Update max_num_tokens and max_batch_size for min-latency decode workers" - - "See srt-slurm recipe changes: https://github.com/ishandhanani/srt-slurm/pull/173" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/686 - -- config-keys: - - dsr1-fp8-mi355x-sglang-disagg - description: - - "Add more sweep points for DSR1 FP8 both MTP and non-MTP 1k1k, 8k1k" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/689 - -- config-keys: - - dsr1-fp8-b300-dynamo-trt - description: - - "Update max_num_tokens and max_batch_size for min-latency decode workers" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/690 - -- config-keys: - - dsr1-fp8-b200-dynamo-sglang-mtp - description: - - "Patches one missing concurrency point for " - - "DSR1 FP8 B200 disaggregated SGLang MTP multinode configuration. " - - "Image: lmsysorg/sglang:v0.5.8.post1-cu130-amd64" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/691 - -- config-keys: - - dsr1-fp4-mi355x-sglang-disagg - description: - - "Add more sweep points for DSR1 FP4 both MTP and non-MTP 1k1k, 8k1k" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/692 - -- config-keys: - - dsr1-fp8-mi325x-sglang - description: - - "Update MI325X DeepSeek R1 FP8 SGLang image from v0.5.7 to v0.5.8" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/695 - -- config-keys: - - dsr1-fp8-mi300x-sglang - description: - - "Update MI300X DeepSeek R1 FP8 SGLang image from v0.5.7 to v0.5.8" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/696 - -- config-keys: - - dsr1-fp8-b200-dynamo-sglang-mtp - description: - - "Add new 1P2D max-throughput MTP config for 1k1k" - - "MTP settings: speculative-num-steps=1, speculative-num-draft-tokens=2" - - "Image: lmsysorg/sglang:v0.5.8.post1-cu130-amd64" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/697 - -- config-keys: - - dsr1-fp8-h200-dynamo-trt - description: - - "Add min-latency configurations for H200 Dynamo TRT" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/698 - -- config-keys: - - dsr1-fp8-mi355x-sglang-disagg - description: - - "Bump MI355X disagg FP8 recipe commit to 953f7c5 (bug fixes in sglang_disagg fork)" - - "1k1k: Switch prefill workers from DEP (tp:1, ep:8, dp-attn) to Pure TP (tp:8, ep:1) for MTP and non-MTP middle-of-curve 1P2D configs" - - "1k1k: Extend middle-of-curve concurrency range by adding conc=128 for both MTP and non-MTP 1P2D configs" - - "8k1k: Add new 2P1D (2-prefill, 1-decode) configs at conc [512, 1024] for both MTP (DECODE_MTP_SIZE=0) and non-MTP, with Pure TP prefill and DEP decode" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/700 - -- config-keys: - - dsr1-fp8-mi355x-sglang-disagg - description: - - "Bump MI355X disagg FP8 recipe commit to fix perf regression on 8k1k DEP8" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/701 - -- config-keys: - - qwen3.5-bf16-b200-sglang - description: - - "Add Qwen3.5-397B-A17B BF16 B200 SGLang benchmark" - - "Image: lmsysorg/sglang:nightly-dev-20260216-d3bae71e" - - "TP=8, concurrency 4-64 for 1k1k, 1k8k, and 8k1k sequence lengths" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/704 - -- config-keys: - - qwen3.5-bf16-mi355x-sglang - description: - - "Add Qwen3.5-397B-A17B BF16 SGLang benchmark for MI355X" - - "Image: rocm/sgl-dev:v0.5.8.post1-rocm720-mi35x-20260215" - - "Uses triton attention backend, TP=8, concurrency 4-64" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/705 - -- config-keys: - - kimik2.5-int4-mi355x-vllm - description: - - "Add Kimi-K2.5 INT4 vLLM benchmark for MI355X" - - "Model: moonshotai/Kimi-K2.5 with --mm-encoder-tp-mode data" - - "Image: vllm/vllm-openai-rocm:v0.15.1" - - "TP=8, concurrency 4-64 for 1k1k, 1k8k, and 8k1k sequence lengths" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/734 - -- config-keys: - - kimik2.5-int4-b200-vllm - description: - - "Add Kimi-K2.5 INT4 vLLM benchmark for B200" - - "Model: moonshotai/Kimi-K2.5 with --mm-encoder-tp-mode data and --trust-remote-code" - - "Image: vllm/vllm-openai:v0.15.1" - - "TP=8, concurrency 4-64 for 1k1k, 1k8k, and 8k1k sequence lengths" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/735 - -- config-keys: - - minimaxm2.5-fp8-mi355x-vllm - description: - - "Add MiniMax-M2.5 FP8 vLLM benchmark for MI355X" - - "Model: MiniMaxAI/MiniMax-M2.5 with --trust-remote-code" - - "Image: vllm/vllm-openai-rocm:v0.15.1" - - "Environment: VLLM_ROCM_USE_AITER=1" - - "TP=2 and TP=4, concurrency 4-64 for 1k1k, 1k8k, and 8k1k sequence lengths" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/755 - -- config-keys: - - minimaxm2.5-fp8-b200-vllm - description: - - "Add MiniMax-M2.5 FP8 vLLM benchmark for B200" - - "Model: MiniMaxAI/MiniMax-M2.5 with --trust-remote-code" - - "Image: vllm/vllm-openai:v0.17.0" - - "TP=2 and TP=4, concurrency 4-64 for 1k1k, 1k8k, and 8k1k sequence lengths" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/757 - -- config-keys: - - qwen3.5-bf16-b200-sglang - description: - - "Update Qwen3.5-397B-A17B BF16 SGLang B200 benchmark launch config" - - "Image: lmsysorg/sglang:nightly-dev-20260216-d3bae71e" - - "Add trtllm_mha attention backend, flashinfer_trtllm MOE runner" - - "Add context-length, tokenizer-worker-num, env tuning (NCCL_NVLS_ENABLE, SGLANG_ENABLE_FLASHINFER_GEMM)" - - "Set cuda-graph-max-bs to match concurrency, scheduler-recv-interval based on concurrency" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/758 - -- config-keys: - - glm5-fp8-mi355x-sglang - description: - - "Add GLM-5 FP8 SGLang benchmark for MI355X" - - "Model: zai-org/GLM-5-FP8 with NSA tilelang backends" - - "Image: rocm/sgl-dev:v0.5.8.post1-rocm720-mi35x-20260219" - - "TP=8, concurrency 4-64 for 1k1k, 1k8k, and 8k1k sequence lengths" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/762 - -- config-keys: - - qwen3.5-fp8-mi355x-sglang - description: - - "Add Qwen3.5-397B-A17B-FP8 SGLang benchmark configuration for MI355X" - - "Image: rocm/sgl-dev:v0.5.8.post1-rocm720-mi35x-20260218" - - "Uses triton attention backend, TP=8, concurrency 4-64" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/768 - -- config-keys: - - dsr1-fp8-mi355x-sglang-disagg - description: - - "Add more configs for MI355X FP8 Disagg" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/770 - -- config-keys: - - gptoss-fp4-mi300x-vllm - - gptoss-fp4-mi325x-vllm - description: - - "Update vLLM ROCm image from v0.14.0 to v0.15.1 for MI300X and MI325X GPT-OSS" - - "Gains: ROCm skinny GEMM dispatch fix, MoRI EP all2all backend, KV cache shuffle + paged attention for AITER" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/781 - -- config-keys: - - gptoss-fp4-b200-vllm - - gptoss-fp4-h100-vllm - - gptoss-fp4-h200-vllm - description: - - "Update vLLM image from v0.13.0 to v0.15.1 for NVIDIA GPT-OSS configs" - - "Remove deprecated async-scheduling flag (now enabled by default since v0.14.0)" - - "Gains: CUTLASS MoE optimizations (~8% throughput), FP4 kernel improvements (~4% E2E on B200), torch.compile cold-start fix" - - "v0.15.1 includes fix for prefix cache hit rate of 0% on GPT-OSS hybrid attention models" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/789 - -- config-keys: - - dsr1-fp4-mi355x-atom - - dsr1-fp4-mi355x-atom-mtp - description: - - "Update search-space configurations for DSR1 FP4 MI355X ATOM and ATOM-MTP" - - "Comment out TP=4 configs, consolidate to TP=8 only" - - "Extend concurrency range to conc-end: 256 across all sequence lengths (1k1k, 1k8k, 8k1k)" - - "Fix MTP 1k8k conc-start from 256 to 4 to enable full concurrency sweep" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/792 - -- config-keys: - - qwen3.5-fp8-b200-sglang - description: - - "Add Qwen3.5-397B-A17B-FP8 SGLang benchmark configuration for B200" - - "Image: lmsysorg/sglang:v0.5.9-cu129-amd64" - - "Uses trtllm_mha attention backend and flashinfer_trtllm MOE runner" - - "Enable SGLANG_ENABLE_FLASHINFER_GEMM=true, NCCL_NVLS_ENABLE=1" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/804 - -- config-keys: - - gptoss-fp4-mi300x-vllm - - gptoss-fp4-mi325x-vllm - - gptoss-fp4-mi355x-vllm - description: - - "Update AMD GPT-OSS vLLM images to v0.16.0 (MI300X/MI325X from v0.15.1, MI355X from custom v0.10.1)" - - "MI355X: Fix env vars (VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION), add VLLM_ROCM_USE_AITER=1, remove deprecated flags" - - "MI355X: Simplify compilation config to cudagraph_mode FULL_AND_PIECEWISE, add HIP_VISIBLE_DEVICES Ray fix" - - "Gains: fused add+rmsnorm+pad for GPT-OSS (automatic via PassManager), AITER attention block size fix" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/806 - -- config-keys: - - dsr1-fp4-mi355x-sglang - - dsr1-fp8-mi300x-sglang - - dsr1-fp8-mi325x-sglang - - dsr1-fp8-mi355x-sglang - description: - - "Update SGLang image from v0.5.8 to v0.5.9 for AMD single-node DeepSeek R1 configs" - - "Key changes: AITER v0.1.10.post3 with FP8 Prefill/Decode/KV Cache, FP8 prefill attention kernel, MORI EP two-batch overlapping, OOM fix for DeepSeek weight loading" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/816 - -- config-keys: - - qwen3.5-fp4-b200-sglang - description: - - "Add Qwen3.5-397B-A17B NVFP4 B200 SGLang benchmark config and launch script" - - "Image: lmsysorg/sglang:v0.5.9-cu129-amd64" - - "Model: nvidia/Qwen3.5-397B-A17B-NVFP4" - - "Configs: 1k1k (TP4 conc 4-128), 8k1k (TP4 conc 4-128)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/820 - -- config-keys: - - dsr1-fp8-mi355x-sglang-disagg - - dsr1-fp8-mi355x-sglang-disagg-mtp - - dsr1-fp4-mi355x-sglang-disagg - - dsr1-fp4-mi355x-sglang-disagg-mtp - description: - - "Add more sweep configs for MI355X FP8/FP4 Disagg" - - "Add TP/DP/EP size < 8 support " - - "Support DSR1-0528 MTP Disagg" - - "Bump SGL mori image to Feb 27" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/823 - -- config-keys: - - kimik2.5-fp4-mi355x-vllm - description: - - "Add Kimi-K2.5 MXFP4 vLLM benchmark for MI355X" - - "Model: amd/Kimi-K2.5-MXFP4 with --mm-encoder-tp-mode data and --trust-remote-code" - - "Image: vllm/vllm-openai-rocm:v0.15.1" - - "TP=8, concurrency 4-64 for 1k1k, 1k8k, and 8k1k sequence lengths" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/825 - -- config-keys: - - minimaxm2.5-fp4-mi355x-vllm - description: - - "Add MiniMax M2.5 MXFP4 vLLM benchmark for MI355X" - - "Model: amd/MiniMax-M2.5-MXFP4 with --trust-remote-code and --block-size=32" - - "Image: vllm/vllm-openai-rocm:v0.19.1" - - "Environment: VLLM_ROCM_USE_AITER=1" - - "Tp=1, TP=2 and TP=4, concurrency 4-64 for 1k1k, 1k8k, and 8k1k sequence lengths" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/827 - -- config-keys: - - minimaxm2.5-fp8-h200-vllm - description: - - "Add MiniMax M2.5 FP8 single-node config for H200 with vLLM v0.16.0 (TP4)" - - "New benchmark script with --trust-remote-code for MiniMaxAI/MiniMax-M2.5" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/831 - -- config-keys: - - minimaxm2.5-fp8-h100-vllm - description: - - "Add MiniMax-M2.5 FP8 vLLM benchmark for H100" - - "Model: MiniMaxAI/MiniMax-M2.5 with --trust-remote-code" - - "Image: vllm/vllm-openai:v0.16.0" - - "Switch from TP=8/EP=8 to TP=4/EP=4, concurrency 4-64 for 1k1k, 1k8k, and 8k1k" - - "Script uses conditional --enable-expert-parallel based on EP_SIZE env var" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/832 - -- config-keys: - - minimaxm2.5-fp8-mi325x-vllm - description: - - "Add MiniMax-M2.5 FP8 vLLM benchmark for MI325X" - - "Model: MiniMaxAI/MiniMax-M2.5 with --trust-remote-code" - - "Image: vllm/vllm-openai-rocm:v0.16.0" - - "Environment: VLLM_ROCM_USE_AITER=1" - - "TP=2 and TP=4, concurrency 4-64 for 1k1k, 1k8k, and 8k1k sequence lengths" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/836 - -- config-keys: - - minimaxm2.5-fp8-mi300x-vllm - description: - - "Add MiniMax-M2.5 FP8 vLLM benchmark for MI300X" - - "Model: MiniMaxAI/MiniMax-M2.5 with --trust-remote-code" - - "Image: vllm/vllm-openai-rocm:v0.16.0" - - "Environment: VLLM_ROCM_USE_AITER=1" - - "TP=2 and TP=4, concurrency 4-64 for 1k1k, 1k8k, and 8k1k sequence lengths" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/837 - -- config-keys: - - qwen3.5-bf16-mi325x-sglang - description: - - "Add Qwen3.5-397B-A17B BF16 SGLang benchmark for MI325X" - - "Image: lmsysorg/sglang:v0.5.9-rocm720-mi30x" - - "Uses triton attention backend, TP=8, concurrency 4-64" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/842 - -- config-keys: - - qwen3.5-bf16-mi300x-sglang - description: - - "Add Qwen3.5-397B-A17B BF16 SGLang benchmark for MI300X" - - "Image: lmsysorg/sglang:v0.5.9-rocm720-mi30x" - - "Uses triton attention backend, TP=8, concurrency 4-64" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/843 - -- config-keys: - - kimik2.5-int4-h200-vllm - description: - - "Add Kimi-K2.5 INT4 vLLM benchmark for H200" - - "Model: moonshotai/Kimi-K2.5 with --reasoning-parser kimi_k2 and --trust-remote-code" - - "Image: vllm/vllm-openai:v0.16.0" - - "TP=8, concurrency 4-64 for 1k1k, 1k8k, and 8k1k sequence lengths" - - "following https://docs.vllm.ai/projects/recipes/en/latest/moonshotai/Kimi-K2.5.html" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/847 - -- config-keys: - - qwen3.5-fp8-mi300x-sglang - description: - - "Add Qwen3.5-397B-A17B-FP8 SGLang benchmark for MI300X" - - "Image: lmsysorg/sglang:v0.5.9-rocm720-mi30x" - - "Uses triton attention backend, TP=8, concurrency 4-64" - - "Following AMD Andy Luo's recipe" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/850 - -- config-keys: - - qwen3.5-fp8-mi325x-sglang - description: - - "Add Qwen3.5-397B-A17B-FP8 SGLang benchmark for MI325X" - - "Image: lmsysorg/sglang:v0.5.9-rocm720-mi30x" - - "Following AMD Andy Luo's recipe with triton attention backend" - - "TP=8, concurrency 4-64 for 1k1k, 1k8k, and 8k1k sequence lengths" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/852 - -- config-keys: - - gptoss-fp4-h200-trt - description: - - "Upgrade TensorRT-LLM container from release:gpt-oss-dev to release:v1.3.0rc5" - - "Remove sed hack for TensorRT bug (fixed upstream in v1.3.0rc5)" - - "Remove enable_block_reuse: false from kv_cache_config (default true is now recommended)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/854 - -- config-keys: - - qwen3.5-fp8-h200-sglang - description: - - "Add Qwen 3.5 FP8 H200 SGLang configuration" - - "Model: Qwen/Qwen3.5-397B-A17B-FP8, runner: h200, image: lmsysorg/sglang:v0.5.8-cu130-amd64" - - "Benchmark script: benchmarks/single_node/qwen3.5_fp8_h200.sh" - - "Server: reasoning-parser qwen3, tool-call-parser qwen3_coder, enable-flashinfer-allreduce-fusion, mem-fraction-static 0.8" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/855 - -- config-keys: - - kimik2.5-fp4-b200-vllm - description: - - "Add Kimi K2.5 FP4 B200 vLLM benchmark configuration" - - "Image: vllm/vllm-openai:v0.17.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/862 - -- config-keys: - - dsr1-fp8-mi355x-sglang - description: - - "Expanding TP search space" - - "Adding kv-cache-fp8" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/865 - -- config-keys: - - minimaxm2.5-fp8-h200-vllm - description: - - "Extend MiniMax M2.5 FP8 single-node config for H200 with vLLM v0.16.0 (TP8)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/869 - -- config-keys: - - dsr1-fp8-h200-sglang - description: - - "Update H200 DeepSeek R1 FP8 SGLang image from v0.5.7 to v0.5.9" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/887 - -- config-keys: - - gptoss-fp4-mi300x-vllm - - gptoss-fp4-mi325x-vllm - - gptoss-fp4-mi355x-vllm - description: - - "Update AMD GPT-OSS vLLM image from v0.16.0 to v0.17.0 for MI300X, MI325X, and MI355X" - - "MI355X: Switch model to amd/gpt-oss-120b-w-mxfp4-a-fp8 (MXFP4 weights + FP8 activations)" - - "MI355X: Add VLLM_ROCM_USE_AITER_TRITON_ROPE=1 for AITER triton RoPE kernel" - - "Add AMDGCN_USE_BUFFER_OPS=0 and VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 env vars" - - "Switch to --attention-backend ROCM_AITER_UNIFIED_ATTN and add fuse_rope_kvcache compilation pass" - - "Remove deprecated VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION/VLLM_ROCM_USE_AITER_MHA env vars and compilation-config cudagraph_mode" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/889 - -- config-keys: - - qwen3.5-bf16-b200-sglang - - qwen3.5-bf16-mi300x-sglang - - qwen3.5-bf16-mi325x-sglang - - qwen3.5-bf16-mi355x-sglang - - qwen3.5-fp8-b200-sglang - - qwen3.5-fp8-h200-sglang - - qwen3.5-fp8-mi300x-sglang - - qwen3.5-fp8-mi325x-sglang - - qwen3.5-fp8-mi355x-sglang - description: - - "Redo qwen eval" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/892 - evals-only: true - -- config-keys: - - qwen3.5-fp8-b200-sglang-mtp - description: - - "Add Single Node Agg FP8 MTP config for Qwen3.5 B200 SGLang" - - "EAGLE speculative decoding: num-steps 3, draft-tokens 4, topk 1" - - "New script: benchmarks/single_node/qwen3.5_fp8_b200_mtp.sh" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/898 - -- config-keys: - - dsr1-fp4-mi355x-sglang-disagg - - dsr1-fp4-mi355x-sglang-disagg-mtp - description: - - "Add more sweep configs for MI355X FP4 Disagg" - - "Bump FP4 image to Feb 27 latest" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/899 - -- config-keys: - - kimik2.5-int4-mi325x-vllm - description: - - "Add Kimi K2.5 INT4 single-node MI325X vLLM benchmark (TP8)" - - "Uses vLLM ROCm v0.16.0 image following AMD Andy Luo's recipe" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/901 - -- config-keys: - - dsr1-fp8-b200-dynamo-sglang - - dsr1-fp8-b200-dynamo-sglang-mtp - description: - - "Update B200 FP8 DSR1 8k1k dynamo-sglang recipes to new pareto configs" - - "Replace old per-file recipes with resolved variants from consolidated 8k1k.yaml" - - "14 variants: STP/MTP x low-latency/max-throughput with updated concurrencies and scale points" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/907 - -- config-keys: - # NVIDIA single-node - - dsr1-fp4-b200-sglang - - dsr1-fp4-b200-trt - - dsr1-fp4-b200-trt-mtp - - dsr1-fp8-b200-sglang - - dsr1-fp8-b200-sglang-mtp - - dsr1-fp8-b200-trt - - dsr1-fp8-b200-trt-mtp - - dsr1-fp8-h200-sglang - - dsr1-fp8-h200-trt - - dsr1-fp8-h200-trt-mtp - - glm5-fp8-b200-sglang - - glm5-fp8-h200-sglang - - gptoss-fp4-b200-trt - - gptoss-fp4-b200-vllm - - gptoss-fp4-h100-vllm - - gptoss-fp4-h200-trt - - gptoss-fp4-h200-vllm - - kimik2.5-fp4-b200-vllm - - kimik2.5-int4-b200-vllm - - kimik2.5-int4-h200-vllm - - minimaxm2.5-fp8-b200-vllm - - minimaxm2.5-fp8-h100-vllm - - minimaxm2.5-fp8-h200-vllm - - qwen3.5-bf16-b200-sglang - - qwen3.5-fp8-b200-sglang - - qwen3.5-fp8-b200-sglang-mtp - - qwen3.5-fp8-h200-sglang - # AMD single-node - - dsr1-fp4-mi355x-atom - - dsr1-fp4-mi355x-atom-mtp - - dsr1-fp4-mi355x-sglang - - dsr1-fp8-mi325x-sglang - - dsr1-fp8-mi300x-sglang - - dsr1-fp8-mi355x-atom - - dsr1-fp8-mi355x-atom-mtp - - dsr1-fp8-mi355x-sglang - - glm5-fp8-mi355x-sglang - - gptoss-fp4-mi300x-vllm - - gptoss-fp4-mi325x-vllm - - gptoss-fp4-mi355x-atom - - gptoss-fp4-mi355x-vllm - - kimik2.5-fp4-mi355x-vllm - - kimik2.5-int4-mi325x-vllm - - kimik2.5-int4-mi355x-vllm - - minimaxm2.5-fp8-mi300x-vllm - - minimaxm2.5-fp8-mi325x-vllm - - minimaxm2.5-fp8-mi355x-vllm - - qwen3.5-bf16-mi300x-sglang - - qwen3.5-bf16-mi325x-sglang - - qwen3.5-bf16-mi355x-sglang - - qwen3.5-fp8-mi300x-sglang - - qwen3.5-fp8-mi325x-sglang - - qwen3.5-fp8-mi355x-sglang - description: - - "Separate evals, change to 8k1k, fail loudly, 5-shot, top of curve & middle of curve" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/911 - evals-only: true - -- config-keys: - - glm5-fp8-h200-sglang - description: - - "Add GLM-5 FP8 SGLang H200 single-node benchmark" - - "Model: zai-org/GLM-5-FP8, image: lmsysorg/sglang:glm5-hopper" - - "Benchmark script: benchmarks/single_node/glm5_fp8_h200.sh" - - "Tool-call-parser glm47, reasoning-parser glm45, mem-fraction-static 0.85" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/914 - -- config-keys: - - glm5-fp8-b200-sglang - description: - - "Add GLM-5 FP8 SGLang benchmark for B200" - - "Supports TP8 (low latency) and DEP8 (high throughput) modes with NSA attention backend" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/915 - -- config-keys: - - qwen3.5-fp8-b200-sglang - description: - - "Replace FP8 with combination of TP4 and TP8 config" - - "Add --enable-flashinfer-allreduce-fusion to TP8" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/918 - -- config-keys: - - dsr1-fp8-b200-dynamo-trt - - dsr1-fp8-h200-dynamo-trt - - dsr1-fp4-gb200-dynamo-trt - description: - - "Fix metadata inconsistencies in nvidia-master.yaml - TP/EP/DP-attn values now match actual recipe files" - - "B200 FP8 TRT 8K/1K: prefill_ep 8→1 (15 entries), prefill_dp_attn true→false (1 entry)" - - "H200 FP8 TRT 1K/1K: prefill_dp_attn false→true (9 entries)" - - "H200 FP8 TRT 8K/1K: prefill_dp_attn true→false (8 entries)" - - "GB200 FP4 TRT 8K/1K: decode_dp_attn true→false (2 entries)" - - "All fixes are metadata-only; no recipe files were modified" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/919 - -- config-keys: - - kimik2.5-int4-mi325x-vllm - - kimik2.5-int4-mi355x-vllm - - kimik2.5-int4-h200-vllm - - kimik2.5-fp4-mi355x-vllm - - kimik2.5-fp4-b200-vllm - description: - - "Disable prefix caching (--no-enable-prefix-caching) for all Kimi K2.5 benchmarks using random datasets" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/926 - -- config-keys: - - minimaxm2.5-fp8-mi355x-vllm - description: - - "ADD minimax TP=8 with EP, in config of 1k1k, 1k8k, and 8k1k sequence lengths" - - "Config concurrency: 32-256" - - "update image to vllm/vllm-openai-rocm:v0.18.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/927 - -- config-keys: - - gptoss-fp4-mi325x-vllm - - minimaxm2.5-fp8-mi325x-vllm - description: - - "Document --exclusive flag on MI325X salloc prevents node sharing during benchmarks" - - "Only non-TP8 configs are affected; TP8 already uses all GPUs on the node" - - "May yield performance improvements" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/931 - -- config-keys: - - dsr1-fp4-mi355x-atom - - dsr1-fp4-mi355x-atom-mtp - - dsr1-fp4-mi355x-sglang - - dsr1-fp4-mi355x-sglang-disagg - - dsr1-fp4-mi355x-sglang-disagg-mtp - - dsr1-fp8-mi355x-sglang - - dsr1-fp8-mi355x-sglang-disagg - - dsr1-fp8-mi355x-sglang-disagg-mtp - - gptoss-fp4-mi355x-atom - - gptoss-fp4-mi355x-vllm - - minimaxm2.5-fp8-mi355x-vllm - description: - - "Add --exclusive flag to MI355X single-node salloc and multi-node sbatch to prevent node sharing during benchmarks" - - "Only non-TP8 configs listed; TP8 already uses all GPUs on the node" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/934 - -- config-keys: - - kimik2.5-int4-b200-vllm - description: - - "Enable VLLM_USE_FLASHINFER_MOE_INT4=1 for Kimi K2.5 INT4 B200 benchmark" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/935 - -- config-keys: - - kimik2.5-fp4-mi355x-vllm - description: - - "Upgrade vLLM ROCm image from v0.16.0 to v0.18.0" - - "Enable AITER with INT4 quick reduce; disable AITER RMSNorm for TP < 8 (accuracy)" - - "Add expert parallel, TP4, and TP4/EP4 search spaces" - - "Switch block-size 64 to 1 gpu-memory-utilization 0.95 to 0.90" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/936 - -- config-keys: - - dsr1-fp4-b200-sglang - - dsr1-fp8-b200-sglang - - dsr1-fp8-b200-sglang-mtp - - dsr1-fp8-h200-sglang - description: - - "Update SGLang image to v0.5.9-cu130 for all DSR1 SGLang configs" - - "dsr1-fp4-b200-sglang: v0.5.6-cu129-amd64 → v0.5.9-cu130" - - "dsr1-fp8-b200-sglang: v0.5.6-cu129-amd64 → v0.5.9-cu130" - - "dsr1-fp8-b200-sglang-mtp: v0.5.8-cu130-amd64 → v0.5.9-cu130" - - "dsr1-fp8-h200-sglang: v0.5.9-cu129-amd64 → v0.5.9-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/943 - -- config-keys: - - minimaxm2.5-fp8-b200-vllm - description: - - "Update vLLM image from v0.17.0 to v0.19.0 for MiniMax-M2.5 FP8 B200" - - "Add tp4 ep4 search-space entries (conc 32-256) for all seq-len configs" - - "Remove ISL 1024 / OSL 8192 seq-len config" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/947 - -- config-keys: - - kimik2.5-int4-mi355x-vllm - description: - - "Upgrade vLLM ROCm image from v0.15.1 to v0.18.0" - - "Enable AITER MLA, export VLLM_ROCM_USE_AITER=1, https://github.com/vllm-project/vllm/issues/35641" - - "Triton Fused Moe Tuning https://github.com/vllm-project/vllm/pull/35093" - - "Add --max-num-seqs 256, remove --disable-log-requests" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/950 - -- config-keys: - - minimaxm2.5-fp8-mi325x-vllm - description: - - "Upgrade vLLM ROCm image from v0.16.0 to v0.18.0" - - "Replace TP4 with TP8/EP8, add conc range 4-256" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/953 - -- config-keys: - - kimik2.5-int4-mi325x-vllm - description: - - "Upgrade vLLM ROCm image from v0.16.0 to v0.18.0" - - "Enable AITER MLA, export VLLM_ROCM_USE_AITER=1, https://github.com/vllm-project/vllm/issues/35641" - - "Triton Fused Moe Tuning https://github.com/vllm-project/vllm/pull/35093" - - "Add --max-num-seqs 256, remove --disable-log-requests" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/957 - -- config-keys: - - minimaxm2.5-fp8-h100-vllm - - minimaxm2.5-fp8-h200-vllm - description: - - "Update vLLM image from v0.16.0 to v0.18.0 for minimax h100 and h200 configs" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/958 - -- config-keys: - - gptoss-fp4-h100-vllm - - gptoss-fp4-h200-vllm - description: - - "Update vLLM image from v0.15.1 to v0.18.0 for gptoss H100 and H200 configs" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/960 - -- config-keys: - - minimaxm2.5-fp8-b200-vllm - - minimaxm2.5-fp8-h100-vllm - - minimaxm2.5-fp8-h200-vllm - - minimaxm2.5-fp8-mi300x-vllm - - minimaxm2.5-fp8-mi325x-vllm - - minimaxm2.5-fp8-mi355x-vllm - description: - - "Disable prefix caching (--no-enable-prefix-caching) for all MiniMax benchmarks using random datasets" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/966 - -- config-keys: - - qwen3.5-bf16-mi300x-sglang - - qwen3.5-bf16-mi325x-sglang - - qwen3.5-fp8-mi300x-sglang - - qwen3.5-fp8-mi325x-sglang - description: - - "Add --disable-radix-cache to SGLang server launch command for qwen3.5 MI300X and MI325X benchmark scripts" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/970 - -- config-keys: - - glm5-nvfp4-b200-sglang - description: - - "Add GLM-5 NVFP4 single-node B200 SGLang benchmark (TP8, conc 4-64)" - - "Uses nvidia/GLM-5-NVFP4 model with modelopt_fp4 quantization" - - "Image: lmsysorg/sglang:nightly-dev-cu13-20260328-a27651d5" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/973 - -- config-keys: - - kimik2.5-int4-mi300x-vllm - description: - - "Add Kimi K2.5 INT4 single-node MI300X vLLM benchmark (TP8)" - - "Uses vLLM ROCm v0.18.0 image following AMD Andy Luo's recipe" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/975 - -- config-keys: - - dsr1-fp8-mi355x-atom-mtp - description: - - "DeepSeek R1 MI355X FP8 ATOM-MTP config to support MTP 3 tokens" - - "Image: rocm/atom:rocm7.2.1-ubuntu24.04-pytorch2.9.1-atom0.1.2" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/984 - -- config-keys: - - kimik2.5-fp4-mi355x-atom - - minimaxm2.5-fp8-mi355x-atom - description: - - "New model support on ATOM framework" - - "Kimi-K2.5 FP4, and MiniMax-M2.5 FP8 configs added for MI355X ATOM" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/992 - -- config-keys: - - minimaxm2.5-fp4-b200-vllm - description: - - "Optimize MiniMax-M2.5 NVFP4 B200 vLLM search-space" - - "Expand from tp2/tp4 to tp1/tp2/tp4/tp8 with expert parallel and dp-attn variants" - - "Add ep2, ep4, and dp-attn configurations for higher concurrency sweeps" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/996 - -- config-keys: - - dsr1-fp4-b200-dynamo-trt - - dsr1-fp8-b200-dynamo-trt - - dsr1-fp4-b200-dynamo-sglang - - dsr1-fp8-b200-dynamo-sglang - - dsr1-fp8-b200-dynamo-sglang-mtp - - dsr1-fp4-b200-dynamo-sglang-mtp - - dsr1-fp4-b300-dynamo-trt - - dsr1-fp8-b300-dynamo-trt - - dsr1-fp4-gb300-dynamo-trt - - dsr1-fp8-gb300-dynamo-trt - - dsr1-fp4-gb300-dynamo-sglang - - dsr1-fp8-gb300-dynamo-sglang - - dsr1-fp8-mi355x-sglang-disagg - - dsr1-fp8-mi355x-sglang-disagg-mtp - - dsr1-fp4-mi355x-sglang-disagg - - dsr1-fp4-mi355x-sglang-disagg-mtp - description: - - "Add multi-node lm-eval accuracy runs" - - "Eval picks the config with highest max eligible concurrency per (model, runner, framework, precision, spec-decoding, prefill-dp-attn, decode-dp-attn) group on 8k1k" - - "Eval concurrency set to the median eligible conc (>= MIN_EVAL_CONC=16) of the selected config to avoid OOM" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1000 - evals-only: true - -- config-keys: - - qwen3.5-fp8-h200-sglang-mtp - description: - - "Add Qwen3.5-397B-A17B-FP8 H200 SGLang MTP (EAGLE speculative decoding)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1001 - -- config-keys: - - minimaxm2.5-fp8-mi355x-vllm - description: - - "Optimize MiniMax-M2.5 FP8 MI355X vLLM search-space" - - "Add tp2 ep2 search-space entries (conc 2-256) for all seq-len configs" - - "Upgrade vLLM image to v0.19.0" - - "Enable FP8 KV cache + AITER FA for minimaxm2.5-fp8-mi355x-vllm" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1003 - -- config-keys: - - qwen3.5-fp4-mi355x-sglang - description: - - "Qwen3.5 fp4 support on SGL" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1006 - -- config-keys: - - kimik2.5-fp4-gb200-dynamo-vllm - description: - - "Add Kimi K2.5 NVFP4 GB200 disaggregated multinode vLLM benchmark via Dynamo frontend" - - "Image: vllm/vllm-openai:v0.18.0-cu130" - - "Model: nvidia/Kimi-K2.5-NVFP4 with NixlConnector KV transfer, FLASHINFER_MLA attention" - - "1k1k configs: high-throughput DEP (1P1D dep4/dep16), low-latency TEP (1P4D dep4/tep4)" - - "8k1k configs: low-latency TEP (1P4D), mid-curve DEP (3P1D dep16), high-throughput (5P1D dep8, 6P1D dep16)" - - "Recipes sourced from NVIDIA/srt-slurm branch sa-submission-q2-2026" - - "New framework: dynamo-vllm (Dynamo frontend + vLLM backend)" - - "Runner script updated to clone NVIDIA/srt-slurm and map vLLM container image" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1008 - -- config-keys: - - glm5-fp8-mi355x-atom - description: - - "GLM5 FP8 configs added for MI355X ATOM" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1009 - -- config-keys: - - minimaxm2.5-fp8-b200-vllm - description: - - "Update MiniMax-M2.5 FP8 B200 config with new search spaces" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1010 - -- config-keys: - - glm5-fp4-b200-sglang - description: - - "Update GLM-5 NVFP4 B200 SGLang benchmark script with optimized launch parameters" - - "Add TP4 search space with higher concurrency (128-256) for 1k1k and 8k1k configs" - - "Enable FP8 E4M3 KV cache, NSA backends (trtllm), flashinfer allreduce fusion, MoE flashinfer_trtllm runner" - - "Tune mem-fraction-static to 0.9, chunked-prefill-size to 32768, add tokenizer-worker-num 6" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1011 - -- config-keys: - - glm5-fp8-b200-sglang - description: - - "Bump GLM-5 FP8 B200 SGLang concurrency from 128 to 256" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1012 - -- config-keys: - - qwen3.5-fp8-h200-sglang-mtp - description: - - "Enable SGLANG_ENABLE_SPEC_V2=1 for Qwen3.5 FP8 H200 SGLang MTP" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1017 - -- config-keys: - - qwen3.5-fp4-mi355x-sglang - description: - - "TP2/TP4 seach space exploration for Qwen3.5 fp4 on SGL" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1022 - -- config-keys: - - glm5-fp8-mi355x-sglang - description: - - "Upgrade GLM5 FP8 MI355X SGLang image to v0.5.10rc0-rocm720-mi35x-20260413" - - "Set --kv-cache-dtype fp8_e4m3 and --disable-radix-cache" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1023 - -- config-keys: - - kimik2.5-fp4-gb200-dynamo-trt - description: - - "Add Kimi K2.5 NVFP4 GB200 disaggregated TRT-LLM benchmarks via Dynamo (14 STP configs)" - - "New framework: dynamo-trt (Dynamo frontend + TensorRT-LLM backend)" - - "Container: nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:1.1.0-dev.2" - - "Recipes sourced from NVIDIA/srt-slurm branch sa-submission-q2-2026" - - "Runner script updated to support kimik2.5 model prefix with dynamo-trt framework" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1026 - -- config-keys: - - glm5-fp4-b200-sglang - description: - - "Update SGLang image from nightly-dev-cu13-20260328-a27651d5 to v0.5.10.post1-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1031 - -- config-keys: - - qwen3.5-fp8-b300-sglang-mtp - description: - - "Add Qwen3.5-397B-A17B-FP8 B300 SGLang MTP benchmark" - - "Image: lmsysorg/sglang:v0.5.10.post1-cu130" - - "EAGLE speculative decoding with MTP, TP=4, concurrency 4-256 for 1k1k and 8k1k" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1035 - -- config-keys: - - qwen3.5-fp8-mi355x-sglang - - qwen3.5-bf16-mi355x-sglang - description: - - "Update cli args of Qwen3.5 FP8 and BF16 SGLang benchmarks for MI355X to achieve better performance" - - "Use lmsysorg/sglang-rocm:v0.5.10rc0-rocm720-mi35x-20260415 for BF16 benchmark" - - "Use lmsysorg/sglang-rocm:v0.5.10rc0-rocm720-mi35x-20260414 for FP8 benchmark" - - "Image includes upstream SGLang PRs: https://github.com/sgl-project/sglang/pull/21188, https://github.com/sgl-project/sglang/pull/21421, https://github.com/sgl-project/sglang/pull/20736" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1036 - -- config-keys: - - qwen3.5-fp4-mi355x-sglang - description: - - "Update SGLang image from 'lmsysorg/sglang:v0.5.10-rocm720-mi35x' to 'rocm/sgl-dev:v0.5.10rc0-rocm720-mi35x-20260413'" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1041 - -- config-keys: - - glm5.1-fp4-mi355x-atom - description: - - "Add GLM-5.1 MXFP4 single-node MI355X ATOM benchmark" - - "Image: rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post" - - "TP=2 and TP=4, concurrency 4-256 for 1k1k and 8k1k sequence lengths" - - "Add --max-num-seqs and --gpu-memory-utilization 0.9 to server launch" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1043 - -- config-keys: - - kimik2.5-fp4-b200-vllm - description: - - "Add kv-cache-dtype fp8, max-cudagraph-capture-size 2048, max-num-batched-tokens, and stream-interval 20 to server launch args" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1047 - -- config-keys: - - qwen3.5-fp8-b300-sglang - description: - - "Add Qwen3.5-397B-A17B-FP8 B300 SGLang benchmark (non-MTP)" - - "Image: lmsysorg/sglang:v0.5.10.post1-cu130" - - "TP=4, concurrency 4-256 for 1k1k and 8k1k" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1048 - -- config-keys: - - dsr1-fp4-b300-sglang - description: - - "Add DeepSeek-R1-0528 FP4 B300 SGLang benchmark (non-MTP)" - - "Image: lmsysorg/sglang:v0.5.10.post1-cu130" - - "At the time of submission, https://cookbook.sglang.io/autoregressive/DeepSeek/DeepSeek-R1 does not have a B300-specific recipe, so this reuses the existing DSR1 FP4 B200 SGLang recipe as-is" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1049 - -- config-keys: - - dsr1-fp8-b300-sglang - description: - - "Add DeepSeek-R1-0528 FP8 B300 SGLang benchmark (non-MTP)" - - "Image: lmsysorg/sglang:v0.5.10.post1-cu130" - - "At the time of submission, https://cookbook.sglang.io/autoregressive/DeepSeek/DeepSeek-R1 does not have a B300-specific recipe, so this reuses the existing DSR1 FP8 B200 SGLang recipe as-is" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1050 - -- config-keys: - - glm5-fp8-b300-sglang - description: - - "Add GLM-5 FP8 B300 SGLang benchmark" - - "Image: lmsysorg/sglang:v0.5.10.post1-cu130" - - "At the time of submission, https://cookbook.sglang.io/autoregressive/GLM/GLM-5.1 does not have a B300-specific recipe, so this reuses the existing GLM5 FP8 B200 SGLang recipe as-is" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1051 - -- config-keys: - - minimaxm2.5-fp8-b300-vllm - description: - - "Add MiniMax-M2.5 FP8 B300 vLLM benchmark" - - "Image: vllm/vllm-openai:v0.19.0-cu130" - - "At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/MiniMax/MiniMax-M2.html does not have a B300-specific recipe, so this reuses the existing MiniMax-M2.5 FP8 B200 vLLM recipe as-is" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1054 - -- config-keys: - - minimaxm2.5-fp4-b300-vllm - description: - - "Add MiniMax-M2.5 FP4 (NVFP4) B300 vLLM benchmark" - - "Image: vllm/vllm-openai:v0.19.0-cu130" - - "At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/MiniMax/MiniMax-M2.html does not have a B300-specific recipe, so this reuses the existing MiniMax-M2.5 FP4 B200 vLLM recipe as-is" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1055 - -- config-keys: - - glm5-fp4-b300-sglang - description: - - "Add GLM-5 FP4 (NVFP4) B300 SGLang benchmark" - - "Image: lmsysorg/sglang:v0.5.10.post1-cu130" - - "At the time of submission, https://cookbook.sglang.io/autoregressive/GLM/GLM-5 does not have a B300-specific recipe, so this reuses the existing GLM-5 FP4 B200 SGLang recipe as-is" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1058 - -- config-keys: - - dsr1-fp8-b300-sglang-mtp - description: - - "Add DeepSeek-R1-0528 FP8 B300 SGLang MTP benchmark" - - "Image: lmsysorg/sglang:v0.5.10.post1-cu130" - - "EAGLE speculative decoding with MTP, TP=8, concurrency 4-512 for 1k1k and 8k1k" - - "At the time of submission, https://cookbook.sglang.io/autoregressive/DeepSeek/DeepSeek-R1 does not have a B300-specific recipe, so this reuses the existing DSR1 FP8 B200 SGLang MTP recipe as-is" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1059 - -- config-keys: - - gptoss-fp4-mi300x-vllm - description: - - "Expand GPT-OSS 120B FP4 MI300X TP=1 concurrency from 64 to 256 for 1k1k" - - "Higher concurrency improves MoE weight amortization: 8552 total TPS at conc=256 vs 4016 at conc=64 (2.1x)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1061 - -- config-keys: - - qwen3.5-bf16-mi300x-sglang - - qwen3.5-bf16-mi325x-sglang - - qwen3.5-fp8-mi300x-sglang - - qwen3.5-fp8-mi325x-sglang - description: - - "Update cli args of Qwen3.5 FP8 and BF16 SGLang benchmarks for MI300X and MI325X to achieve better performance" - - "Use lmsysorg/sglang:v0.5.10-rocm720-mi30x" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1063 - -- config-keys: - - minimaxm2.5-fp8-b200-vllm - description: - - "Add VLLM_FLOAT32_MATMUL_PRECISION=high, remove VLLM_FLASHINFER_ALLREDUCE_BACKEND=mnnvl" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1068 - -- config-keys: - - minimaxm2.5-fp4-b200-vllm - description: - - "Add VLLM_FLOAT32_MATMUL_PRECISION=high" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1069 - -- config-keys: - - qwen3.5-fp4-b300-sglang - description: - - "Add Qwen3.5-397B-A17B NVFP4 B300 SGLang benchmark" - - "Image: lmsysorg/sglang:v0.5.10.post1-cu130" - - "Model: nvidia/Qwen3.5-397B-A17B-NVFP4" - - "Follows the SGLang cookbook recipe at https://cookbook.sglang.io/autoregressive/Qwen/Qwen3.5 as of 2026-04-17" - - "Mirrors the B200 FP4 recipe with mem-fraction-static lowered to 0.8 and an extra TP2/EP2 search-space entry" - - "Configs: 1k1k and 8k1k, TP4/EP1 conc 4-128 + TP2/EP2 conc 4-128" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1072 - -- config-keys: - - qwen3.5-bf16-b200-sglang-mtp - description: - - "Add Qwen3.5-397B-A17B BF16 B200 SGLang MTP benchmark" - - "Image: lmsysorg/sglang:nightly-dev-20260216-d3bae71e" - - "Model: Qwen/Qwen3.5-397B-A17B" - - "Mirrors the qwen3.5-bf16-b200-sglang non-MTP recipe and adds EAGLE speculative decoding (num-steps=3, eagle-topk=1, num-draft-tokens=4)" - - "Configs: 1k1k and 8k1k, TP=8/EP=1 conc 4-64 with spec-decoding=mtp" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1074 - -- config-keys: - - qwen3.5-fp4-b200-sglang-mtp - description: - - "Add Qwen3.5-397B-A17B NVFP4 B200 SGLang MTP benchmark" - - "Image: lmsysorg/sglang:nightly-dev-20260402-d7256eb6" - - "Model: nvidia/Qwen3.5-397B-A17B-NVFP4" - - "Mirrors the qwen3.5-fp4-b200-sglang non-MTP recipe and adds EAGLE speculative decoding (num-steps=3, eagle-topk=1, num-draft-tokens=4)" - - "Configs: 1k1k and 8k1k, TP=4/EP=1 conc 4-128 with spec-decoding=mtp" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1075 - -- config-keys: - - qwen3.5-fp8-mi355x-sglang-mtp - description: - - "Add Qwen3.5-397B-A17B FP8 MI355X SGLang MTP benchmark" - - "Image: lmsysorg/sglang-rocm:v0.5.10rc0-rocm720-mi35x-20260414" - - "Model: Qwen/Qwen3.5-397B-A17B-FP8" - - "Mirrors the qwen3.5-fp8-mi355x-sglang non-MTP recipe and adds EAGLE speculative decoding (num-steps=3, eagle-topk=1, num-draft-tokens=4)" - - "Configs: 1k1k (TP8/EP1, TP8/EP8, TP2/EP2) and 8k1k (TP2/EP2, TP4/EP1) with spec-decoding=mtp" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1076 - -- config-keys: - - qwen3.5-bf16-mi355x-sglang-mtp - description: - - "Add Qwen3.5-397B-A17B BF16 MI355X SGLang MTP benchmark" - - "Image: lmsysorg/sglang-rocm:v0.5.10rc0-rocm720-mi35x-20260415" - - "Model: Qwen/Qwen3.5-397B-A17B" - - "Mirrors the qwen3.5-bf16-mi355x-sglang non-MTP recipe and adds EAGLE speculative decoding (num-steps=3, eagle-topk=1, num-draft-tokens=4)" - - "Configs: 1k1k and 8k1k, TP=8/EP=1 conc 4-256 with spec-decoding=mtp" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1077 - -- config-keys: - - qwen3.5-bf16-b300-sglang - description: - - "Add Qwen3.5-397B-A17B BF16 B300 SGLang benchmark" - - "Image: lmsysorg/sglang:v0.5.10.post1-cu130" - - "Model: Qwen/Qwen3.5-397B-A17B" - - "Mirrors the B200 BF16 recipe with an extra TP4/EP1 search-space entry alongside the existing TP8/EP1 sweep" - - "Configs: 1k1k and 8k1k, TP8/EP1 conc 4-64 + TP4/EP1 conc 4-64" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1081 - -- config-keys: - - qwen3.5-bf16-b300-sglang-mtp - description: - - "Add Qwen3.5-397B-A17B BF16 B300 SGLang MTP benchmark" - - "Image: lmsysorg/sglang:v0.5.10.post1-cu130" - - "Model: Qwen/Qwen3.5-397B-A17B" - - "Mirrors the qwen3.5-bf16-b300-sglang non-MTP recipe and adds EAGLE speculative decoding (num-steps=3, eagle-topk=1, num-draft-tokens=4)" - - "Configs: 1k1k and 8k1k, TP8/EP1 conc 4-64 + TP4/EP1 conc 4-64, spec-decoding=mtp" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1082 - -- config-keys: - - qwen3.5-fp4-b300-sglang-mtp - description: - - "Add Qwen3.5-397B-A17B NVFP4 B300 SGLang MTP benchmark" - - "Image: lmsysorg/sglang:v0.5.10.post1-cu130" - - "Model: nvidia/Qwen3.5-397B-A17B-NVFP4" - - "Mirrors the qwen3.5-fp4-b300-sglang non-MTP recipe and adds EAGLE speculative decoding (num-steps=3, eagle-topk=1, num-draft-tokens=4)" - - "Configs: 1k1k and 8k1k, TP4/EP1 conc 4-128 + TP2/EP2 conc 4-128, spec-decoding=mtp" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1083 - -- config-keys: - - glm5-fp8-b300-sglang-mtp - description: - - "Add GLM-5 FP8 B300 SGLang MTP benchmark" - - "Image: lmsysorg/sglang:v0.5.10.post1-cu130" - - "Model: zai-org/GLM-5-FP8" - - "Mirrors the glm5-fp8-b300-sglang non-MTP recipe and adds EAGLE speculative decoding (num-steps=3, eagle-topk=1, num-draft-tokens=4) behind SGLANG_ENABLE_SPEC_V2=1" - - "Configs: 1k1k and 8k1k, TP=8/EP=1 conc 4-256 with spec-decoding=mtp" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1084 - -- config-keys: - - glm5-fp8-b200-sglang-mtp - description: - - "Add GLM-5 FP8 B200 SGLang MTP benchmark" - - "Image: lmsysorg/sglang:nightly-dev-cu13-20260317-1eea7448" - - "Model: zai-org/GLM-5-FP8" - - "Mirrors the glm5-fp8-b200-sglang non-MTP recipe and adds EAGLE speculative decoding (num-steps=3, eagle-topk=1, num-draft-tokens=4) behind SGLANG_ENABLE_SPEC_V2=1" - - "Configs: 1k1k and 8k1k, TP=8/EP=1 conc 4-256 with spec-decoding=mtp" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1085 - -- config-keys: - - glm5-fp8-mi355x-sglang-mtp - description: - - "Add GLM-5 FP8 MI355X SGLang MTP benchmark" - - "Image: lmsysorg/sglang-rocm:v0.5.10rc0-rocm720-mi35x-20260413" - - "Model: zai-org/GLM-5-FP8" - - "Mirrors the glm5-fp8-mi355x-sglang non-MTP recipe and adds EAGLE speculative decoding (num-steps=3, eagle-topk=1, num-draft-tokens=4) behind SGLANG_ENABLE_SPEC_V2=1" - - "Configs: 1k1k and 8k1k, TP=8 conc 4-64 with spec-decoding=mtp" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1086 - -- config-keys: - - glm5-fp4-b200-sglang-mtp - description: - - "Add GLM-5 NVFP4 B200 SGLang MTP benchmark (draft)" - - "Image: lmsysorg/sglang:v0.5.10.post1-cu130" - - "Model: nvidia/GLM-5-NVFP4" - - "Follows the glm5-fp8-b200-sglang launch recipe as requested, plus EAGLE speculative decoding (num-steps=3, eagle-topk=1, num-draft-tokens=4) behind SGLANG_ENABLE_SPEC_V2=1" - - "Configs: 1k1k and 8k1k, TP8/EP1 conc 4-4 + TP4/EP1 conc 4-256 with spec-decoding=mtp" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1087 - -- config-keys: - - glm5-fp4-b300-sglang-mtp - description: - - "Add GLM-5 NVFP4 B300 SGLang MTP benchmark (draft)" - - "Image: lmsysorg/sglang:v0.5.10.post1-cu130" - - "Model: nvidia/GLM-5-NVFP4" - - "Follows the glm5-fp8-b300-sglang launch recipe as requested, plus EAGLE speculative decoding (num-steps=3, eagle-topk=1, num-draft-tokens=4) behind SGLANG_ENABLE_SPEC_V2=1" - - "Configs: 1k1k and 8k1k, TP8/EP1 conc 4-4 + TP4/EP1 conc 4-256 with spec-decoding=mtp" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1088 - -- config-keys: - - gptoss-fp4-mi300x-vllm - description: - - "Extend GPT-OSS FP4 TP=8 search to conc=1" - - "low-latency endpoint for users prioritizing interactive single-user use cases (chat, copilot, agentic)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1092 - -- config-keys: - - dsr1-fp8-h200-dynamo-trt - - dsr1-fp8-h200-dynamo-sglang - description: - - "Add H200 multinode evals-only runs" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1094 - evals-only: true - -- config-keys: - - glm5.1-fp4-mi355x-sglang - description: - - "Add GLM5.1 MXFP4 (FP4) MI355X SGLang Support" - - "Container : lmsysorg/sglang-rocm:v0.5.10rc0-rocm720-mi35x-20260415" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1098 - -- config-keys: - - kimik2.5-fp4-b300-vllm - description: - - "Add Kimi-K2.5 FP4 (NVFP4) B300 vLLM benchmark" - - "Image: vllm/vllm-openai:v0.19.0-cu130" - - "At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/moonshotai/Kimi-K2.5.html does not have a B300-specific recipe, so this reuses the existing Kimi-K2.5 FP4 B200 vLLM recipe as-is" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1100 - -- config-keys: - - minimaxm2.5-fp8-b300-vllm - description: - - "Add VLLM_FLOAT32_MATMUL_PRECISION=high, remove VLLM_FLASHINFER_ALLREDUCE_BACKEND=mnnvl" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1106 - -- config-keys: - - minimaxm2.5-fp4-b300-vllm - description: - - "Add VLLM_FLOAT32_MATMUL_PRECISION=high, update search space concurrency ranges" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1107 - -- config-keys: - - dsr1-fp8-h100-dynamo-trt - - dsr1-fp8-h100-dynamo-sglang - description: - - "Trigger H100 multinode evals after dist-timeout and health-check timeout fixes" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1119 - -- config-keys: - - dsr1-fp8-h100-dynamo-trt - - dsr1-fp8-h100-dynamo-sglang - description: - - "Trigger H100 multinode evals after NVSHEMM fixes" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1120 - evals-only: true - -- config-keys: - - dsv4-fp4-gb200-dynamo-vllm - description: - - "Add DeepSeek-V4-Pro FP4 GB200 disaggregated vLLM benchmarks via Dynamo (1k/1k sweep; 8k/1k currently commented out)" - - "Container: vllm/vllm-openai:deepseekv4-cu130; model from /mnt/numa1/models/deepseek-v4-pro/ (compute-node-local NVMe)" - - "Topologies: low-conc 1p1d-dep8-tep8 (4 nodes, mirrored from NVIDIA srt-slurm PR #71 with offload kept and numa-bind dropped); mid 1p1d-dep8-dep16 (6 nodes) and high 3p1d-dep8-dep16 (10 nodes) hand-rolled, structurally derived from the kimi-k2.5 1k/1k pattern" - - "Recipes stored under benchmarks/multi_node/srt-slurm-recipes/ and overlaid onto the upstream srt-slurm checkout at runtime" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1129 - -- config-keys: - - qwen3.5-fp4-mi355x-atom - description: - - "Add Qwen3.5-397B-A17B MXFP4 single-node MI355X ATOM benchmark" - - "Image: rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post" - - "TP=2 and TP=4, concurrency 4-256 for 1k1k and 8k1k sequence lengths" - - "Add --gpu-memory-utilization 0.9 to server launch" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1133 - - -- config-keys: - - dsv4-fp8-h200-vllm - description: - - "Add DeepSeek-V4-Pro vLLM H200 benchmark per https://vllm.ai/blog/deepseek-v4" - - "Image: vllm/vllm-openai:deepseekv4-cu129" - - "Model: deepseek-ai/DeepSeek-V4-Pro" - - "EP + DP=8, FP8 KV cache, block size 256, max-model-len 800000, prefix caching disabled" - - "H200 has no FP4 path, so --attention_config.use_fp4_indexer_cache is omitted" - - "VLLM_ENGINE_READY_TIMEOUT_S=3600 to accommodate large weight loading" - - "Configs: 1k1k conc 4-64, 8k1k conc 4-64" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1130 - -- config-keys: - - dsv4-fp4-b200-sglang - description: - - "Add DeepSeek-V4-Pro single-node B200 SGLang benchmark (TP8, EP8, dp-attention)" - - "Container: lmsysorg/sglang:deepseek-v4-blackwell" - - "Recipe from https://docs.sglang.io/cookbook/autoregressive/DeepSeek/DeepSeek-V4" - - "Parallelism and sweep conc ranges match the dsv4-fp4-b200-vllm config" - - "Prefix caching and speculative decoding disabled for baseline numbers" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1131 - -- config-keys: - - dsv4-fp8-mi355x-sglang - description: - - "Day 0 DeepSeek-V4-Pro FP8 MI355X SGLang benchmark" - - "Image: rocm/sgl-dev:deepseek-v4-mi35x (from sgl-project/sglang#23608)" - - "Model: sgl-project/DeepSeek-V4-Pro-FP8" - - "https://github.com/sgl-project/sglang/pull/23608#issuecomment-4311952977" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1134 - -- config-keys: - - dsv4-fp4-b300-sglang - description: - - "Add DeepSeek-V4-Pro FP4 B300 SGLang benchmark (low-latency fallback)" - - "Image: lmsysorg/sglang:deepseek-v4-b300" - - "Model: deepseek-ai/DeepSeek-V4-Pro" - - "Low-latency only (TP=8, EP=1, no DP-attn, no DeepEP) — DeepEP FP8 weight-postprocess path is broken for this checkpoint on B300" - - "Prefix caching disabled, no speculative decoding" - - "Configs: 1k1k conc 4-1024, 8k1k conc 4-512" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1143 - -- config-keys: - - dsv4-fp4-b300-vllm - description: - - "Add DeepSeek-V4-Pro single-node B300 vLLM aggregate benchmark" - - "Image: vllm/vllm-openai:deepseekv4-cu130" - - "Model: deepseek-ai/DeepSeek-V4-Pro" - - "Uses the submitted B300 pareto schedule for both 1k1k and 8k1k, excluding conc 1: TP8 at conc 4/128, TP4 at conc 4/8/16/32/64/128, DP4 at conc 256/512" - - "Launch args match the provided vllm serve command, including FP4 indexer cache, FULL_AND_PIECEWISE cudagraph config, and max-num-batched-tokens 2048" - - "1k1k uses --max-model-len 4096; 8k1k uses the workflow-provided benchmark context length" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1144 - -- config-keys: - - dsv4-fp8-mi355x-sglang - description: - - "Bump MI355X SLURM allocation from --time=180 to --time=300 in runners/launch_mi355x-amds.sh" - - "DSv4-Pro on MI355X exceeded the 3h cap (STEP CANCELLED DUE TO TIME LIMIT) due to ~30min MoE JIT compile plus slow torch-fallback kernels (SGLANG_HACK_FLASHMLA_BACKEND=torch et al.) from sgl-project/sglang#23608" - - "300 minutes matches the GH Actions outer timeout-minutes cap in benchmark-tmpl.yml" - - "Retriggering dsv4-fp8-mi355x-sglang" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1148 - -- config-keys: - - dsv4-fp8-mi355x-sglang - description: - - "Drop --mem-fraction-static 0.88 and --max-total-tokens from dsv4_fp8_mi355x.sh" - - "Bump --chunked-prefill-size from 4096 to 8192" - - "Retrigger dsv4-fp8-mi355x-sglang" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1160 - -- config-keys: - - dsv4-fp4-mi355x-atom - description: - - "Add DeepSeek-V4-Pro FP4 MI355X ATOM Day-0 marker (single-sequence, TP=8, conc=1)" - - "Image: rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post (matches qwen3.5-fp8-mi355x-atom base); ROCm/ATOM#650 overlaid at runtime via pip install --no-deps -e . from a pinned PR SHA (cdbff35) inside the benchmark script" - - "triton_kernels is missing from the release image (build-stage path /triton-test/python/triton_kernels/ is cleaned up); the script falls back to ROCm/triton@e491726 (RI3.5.x), which has matmul_ogs.py and routing.py (PR #650 imports both — upstream triton-lang/triton refactored matmul_ogs into matmul.py and removed routing) plus CDNA4MXScaleLayout and a target_info.py compatible with the image's bundled triton" - - "Model: deepseek-ai/DeepSeek-V4-Pro (same canonical checkpoint used by dsv4-fp4-b300-vllm); compatibility with PR #650's WeightsMapper not yet verified — first run will tell us" - - "Pinned to PR1 limitations: single-sequence kv_cache hardcode, --enforce-eager required, ATOM_USE_TRITON_MOE=1 (aiter fused_moe broken on gfx950)" - - "Sweep will expand to TP=4/8 conc 4–256 once ROCm/ATOM PR3 (multi-request) and PR4 (CUDAGraph) land" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1165 - -- config-keys: - - dsv4-fp4-mi355x-atom - description: - - "Add DeepSeek-V4-Pro FP4 MI355X ATOM Day-0 marker (single-sequence, TP=8, conc=1)" - - "Image: rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post (matches qwen3.5-fp8-mi355x-atom base); ROCm/ATOM#650 overlaid at runtime via pip install --no-deps -e . from a pinned PR SHA (cdbff35) inside the benchmark script" - - "triton_kernels is missing from the release image (build-stage path /triton-test/python/triton_kernels/ is cleaned up); the script falls back to ROCm/triton@e491726 (RI3.5.x), which has matmul_ogs.py and routing.py (PR #650 imports both — upstream triton-lang/triton refactored matmul_ogs into matmul.py and removed routing) plus CDNA4MXScaleLayout and a target_info.py compatible with the image's bundled triton" - - "Model: deepseek-ai/DeepSeek-V4-Pro (same canonical checkpoint used by dsv4-fp4-b300-vllm); compatibility with PR #650's WeightsMapper not yet verified — first run will tell us" - - "Pinned to PR1 limitations: single-sequence kv_cache hardcode, --enforce-eager required, ATOM_USE_TRITON_MOE=1 (aiter fused_moe broken on gfx950)" - - "Sweep will expand to TP=4/8 conc 4–256 once ROCm/ATOM PR3 (multi-request) and PR4 (CUDAGraph) land" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1170 - -- config-keys: - - dsv4-fp4-b300-sglang-mtp - description: - - "Add DeepSeek-V4-Pro FP4 B300 SGLang benchmark with EAGLE/MTP speculative decoding" - - "Image: lmsysorg/sglang:deepseek-v4-b300@sha256:26e116bd211e300dbb76924d56c5cbe6cc3ee5ee2fe314859cb8774f5bc070f3 (pinned for deep_gemm transform_weights_for_mega_moe support; same digest as PR #1158)" - - "Model: deepseek-ai/DeepSeek-V4-Pro" - - "EAGLE/MTP flags hardcoded in script: num-steps=3, eagle-topk=1, num-draft-tokens=4" - - "Recipe (MoE backend, chunked-prefill) selected in script by dp-attn: TP-only + flashinfer_mxfp4 (small batch) vs DP-attn + deepep mega_moe (large batch)" - - "Three CONC bands: A=TP8 (1-8), B=TP4 (16-128), C=DP4 dp-attn (64-512); B/C overlap at conc 64,128" - - "Configs: 1k1k and 8k1k, no validation.py / launcher / yaml-field changes (knob-free)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1166 - -- config-keys: - - dsv4-fp4-b300-vllm - description: - - "Update search space based on B300 pareto sweep results" - - "ISL=1024: TP4 conc 4-128; DP4 (dp-attn) conc 256-4096; DP8 (dp-attn) conc 2048-8192" - - "ISL=8192: TP4 conc 4-64; DP4 (dp-attn) conc 128-1024; DP8 (dp-attn) conc 1024-8192" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1155 - -- config-keys: - - dsv4-fp4-b300-sglang - description: - - "Recipe-per-CONC split for DeepSeek-V4-Pro on B300: low-latency (TP=8, EP=1), balanced (TP=4, EP=1) at conc=32, max-throughput (TP=4, EP=4, DP-attn + DeepEP) at conc=512, for both 1k1k and 8k1k" - - "Recipes from https://docs.sglang.io/cookbook/autoregressive/DeepSeek/DeepSeek-V4" - - "Image pinned to lmsysorg/sglang:deepseek-v4-b300@sha256:26e116bd211e300dbb76924d56c5cbe6cc3ee5ee2fe314859cb8774f5bc070f3" - - "DP-attention path enables SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN=1 for better SWA eviction behavior" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1185 - - -- config-keys: - - dsv4-fp4-b200-sglang - description: - - "Two-recipe dispatch for DeepSeek-V4-Pro on B200, selected by DP_ATTENTION knob: low-latency (TP=8, EP=1, flashinfer_mxfp4) for conc 1-32, DP-attention (TP=8, EP=8, DP-attn + DeepEP + mega_moe) for conc 64-{512,1024}. The DP-attention recipe uses identical flags across balanced and max-throughput CONC ranges; only --max-running-requests scales with CONC." - - "Recipes from https://docs.sglang.io/cookbook/autoregressive/DeepSeek/DeepSeek-V4" - - "Image pinned to lmsysorg/sglang:deepseek-v4-blackwell@sha256:df18bfc4aa9ecf59451002b49ba00cae58042de9e2a96378bbd21b404dd62c7b" - - "Adds SGLANG_OPT_* env knobs (SWA_SPLIT_LEAF_ON_INSERT, USE_JIT_NORM, USE_JIT_INDEXER_METADATA, USE_TOPK_V2, USE_CUSTOM_ALL_REDUCE_V2)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1187 - -- config-keys: - - dsv4-fp4-b300-sglang-mtp - description: - - "Pass --dsv4 to run_benchmark_serving so MTP benchmarks use the DSv4 chat template (PR #1153)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1182 - -- config-keys: - - gptoss-fp4-mi355x-atom - description: - - "Update GPTOSS-120B FP4 MI355X Atom benchmark (rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1195 - - -- config-keys: - - dsv4-fp4-b300-vllm - description: - - Add low-latency configs and remove non-pareto configs - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1193 - -- config-keys: - - dsv4-fp4-b200-vllm - description: - - "Add DeepSeek-V4-Pro single-node B200 vLLM benchmark derived from B200 pareto sweep" - - "ISL=1024: TP8 conc 4-128; DP8 (dp-attn) conc 256-4096" - - "ISL=8192: TP8 conc 4-32; DP8 (dp-attn) conc 64-1024" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1156 - -- config-keys: - - dsv4-fp4-b300-sglang-mtp - description: - - "Add DeepSeek-V4-Pro FP4 B300 SGLang benchmark with EAGLE/MTP speculative decoding" - - "Image: lmsysorg/sglang:deepseek-v4-b300@sha256:26e116bd211e300dbb76924d56c5cbe6cc3ee5ee2fe314859cb8774f5bc070f3 (pinned for deep_gemm transform_weights_for_mega_moe support; same digest as PR #1158)" - - "Model: deepseek-ai/DeepSeek-V4-Pro" - - "EAGLE/MTP flags hardcoded in script: num-steps=3, eagle-topk=1, num-draft-tokens=4" - - "Recipe (MoE backend, chunked-prefill) selected in script by dp-attn: TP-only + flashinfer_mxfp4 (small batch) vs DP-attn + deepep mega_moe (large batch)" - - "Three CONC bands: A=TP8 (1-8), B=TP4 (16-128), C=DP4 dp-attn (64-512); B/C overlap at conc 64,128" - - "Configs: 1k1k and 8k1k, no validation.py / launcher / yaml-field changes (knob-free)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1180 - -- config-keys: - - dsv4-fp8-mi355x-vllm - description: - - "Add vLLM DeepSeek-V4-Pro FP8 benchmark for MI355X with AITER-accelerated MLA decode (vllm-project/vllm#40889, stacked on #40871)" - - "Base image rocm/atom:rocm7.2.2 (MI355X ROCm 7.2.2, aiter with MLA decode); vLLM rebuilt from PR branch at pinned SHA b3a4a44 at runtime via --no-deps overlay" - - "Key flags: --enforce-eager, --moe-backend triton_unfused, --kv-cache-dtype fp8, VLLM_ROCM_USE_AITER=1" - - "Search space: TP=8, concurrency 4-64, 1k1k and 8k1k" - - "MI355X runner updated to resolve framework-specific script names (dsv4_fp8_mi355x_vllm.sh) with fallback to generic names" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1188 - -- config-keys: - - dsv4-fp4-b300-sglang - description: - - "conc=2048/4096: mega_moe deepep backend; conc=2048 cuda-graph-max-bs 288, mem 0.87; conc=4096 cuda-graph-max-bs 544, mem 0.835, swa-ratio 0.075, tokenizer-workers 8" - - "1k1k conc=512/1024: add mega_moe deepep backend with cuda-graph-max-bs 550, chunked-prefill 16384, max-running-requests 768" - - "ep=8 naming convention in yaml distinguishes mega_moe from existing flashinfer_mxfp4 ep=4 entries" - - "Recipes from https://docs.sglang.io/cookbook/autoregressive/DeepSeek/DeepSeek-V4" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1179 - -- config-keys: - - dsv4-fp4-mi355x-atom - description: - - "Use ROCm/aiter#2916 mhc_pre device-allocation fix instead of disabling ATOM mhc_pre" - - "Patch installed aiter/ops/mhc.py at runtime to allocate mhc_pre intermediates on residual.device, preserving the aiter MHC fast path without rebuilding aiter" - - "Remove the ATOM deepseek_v4.py sed workaround that forced mhc_pre to torch fallback" - - "Keep dsv4-fp4-mi355x-atom at CONC=1 only; run 24953107645 showed high-concurrency DSv4 ATOM OOMs in PR #650 torch sparse-attention fallbacks before upstream AITER sparse-attention support lands" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1202 - -- config-keys: - - dsv4-fp4-b300-vllm-mtp - description: - - "Add preliminary vLLM MTP configs for DeepSeek-V4-Pro on B300" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1210 - -- config-keys: - - dsv4-fp4-b200-vllm - description: - - "Pin image to vllm/vllm-openai:v0.20.0-cu130 (was floating deepseekv4-cu130 tag); DeepGEMM is preinstalled in this image" - - "Use --attention_config.use_fp4_indexer_cache=True and --compilation-config {\"cudagraph_mode\": \"FULL_AND_PIECEWISE\", \"custom_ops\": [\"all\"]} for all configs" - - "Gate --moe-backend deep_gemm_mega_moe and --gpu-memory-utilization 0.85 on DP_ATTENTION=true per the v0.20.0 recipe" - - "Drop --pipeline-parallel-size 1; keep --no-enable-prefix-caching and --max-cudagraph-capture-size 2048" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1204 - -- config-keys: - - minimaxm2.5-fp4-mi355x-atom - description: - - "Add MiniMax-M2.5 MXFP4 MI355X Atom benchmark (rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post)" - - "Single-node sweep: TP1–TP8, 1k/1k and 8k/1k ISL/OSL" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1042 - -- config-keys: - - dsv4-fp4-gb200-dynamo-vllm - description: - - "DSV4-Pro FP4 GB200 dynamo-vLLM disagg against srt-slurm aflowers/vllm-gb200-v0.20.0" - - "Keeps the three validated 8k/1k points: low-latency 1P/1D TP8 conc=1, mid-curve 1P/1D DEP8 conc=256, and max-tpt 3P/1D DEP8 conc=4096" - - "All three recipes run NATS/etcd on a dedicated infra node and use compute-node local NVMe model weights via /mnt/numa1/models/deepseek-v4-pro/" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1163 - -- config-keys: - - dsv4-fp4-gb200-dynamo-vllm - description: - - "Add GB200 Dynamo vLLM MegaMOE max-throughput recipe at conc=4096" - - "Topology matches max-tpt: 3 prefill DEP8 workers and 1 decode DEP8 worker with dedicated NATS/etcd" - - "Uses deep_gemm_mega_moe on prefill/decode, TORCH_SYMMMEM=NVSHMEM, and no offload" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1218 - -- config-keys: - - dsv4-fp4-gb200-dynamo-vllm - description: - - "Add GB200 Dynamo vLLM low-middle curve recipe at conc=256/512" - - "Topology: 1 prefill DEP8 worker and 4 decode TP8 workers with dedicated NATS/etcd" - - "Mirrors the historical 1P4D DEP8/TP8 offload point from srt-slurm aflowers/vllm-gb200-v0.20.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1218 - -- config-keys: - - dsv4-fp4-b300-sglang - description: - - "Add conc=8192 recipe for 1k1k: deepep mega_moe backend with cuda-graph-max-bs 1088, max-running-requests 8192, mem-fraction-static 0.80, swa-full-tokens-ratio 0.3, tokenizer-worker-num 16" - - "conc=8192 enables SGLANG_OPT_USE_ONLINE_COMPRESS=1 and --stream-interval 30" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1209 - - -- config-keys: - - dsv4-fp4-b300-vllm - description: - - "Change image to vllm/vllm-openai:v0.20.0-cu130" - - "Use Mega MoE for DEP configs" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1221 - -- config-keys: - - dsv4-fp4-b200-vllm-mtp - description: - - "Add preliminary vLLM MTP configs for DeepSeek-V4-Pro on B200" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1230 - -- config-keys: - - dsv4-fp4-gb200-dynamo-vllm - description: - - "Keep the GB200 Dynamo vLLM MegaMOE max-throughput recipe at 3P/1D DEP8 conc=4096" - - "Add GB200 Dynamo vLLM MegaMOE high-throughput recipe at 2P/1D DEP8 conc=4096" - - "Add GB200 Dynamo vLLM MegaMOE mid-curve recipe at 1P/1D DEP8 conc=256/512/1024" - - "Remove stale offload recipe copies and the old no-MegaMOE mid/max-throughput points from the GB200 Dynamo vLLM matrix" - - "Disable FlashInfer autotune on GB200 decode workers for accuracy stability, matching the srt-slurm recipe fix" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1223 - -- config-keys: - - dsv4-fp4-gb300-dynamo-sglang - description: - - "Add DeepSeek-V4-Pro FP4 GB300 disaggregated SGLang benchmarks via Dynamo (1k/1k sweep; 8k/1k recipes shipped but commented out)" - - "Container: lmsysorg/sglang:deepseek-v4-grace-blackwell (linux/arm64); model from /mnt/numa1/models/deepseek-v4-pro/ (compute-node-local NVMe)" - - "Topologies mirror the dsv4-fp4-gb300-dynamo-vllm sibling: low-conc 1p1d-dep8-tep8 (4 nodes), mid 1p1d-dep8-dep16 (6 nodes), high 3p1d-dep8-dep16 (10 nodes). 4096 overlap between mid and high gives a topology-crossover A/B" - - "No upstream GB300 DSV4 sglang disagg recipe exists. Per-worker sglang_config (env vars + flashinfer_mxfp4 + chunked-prefill-size 4096 + disable-flashinfer-autotune + mem-fraction-static 0.82) is mirrored from NVIDIA/srt-slurm PR #69 (recipes/gb300-fp4/1k1k-dsv4/agg-2n-low-latency.yaml — GB300 DSV4 SGLang aggregated). Disagg flag set (nixl transfer backend, enable-dp-attention + moe-a2a-backend deepep) cross-checked against PR #75 (recipes/gb300-fp4/1k1k-dsv4/disagg-1p1d-tp4-mxfp4.yaml — GB300 DSV4 SGLang disagg) and the SGLang DeepSeek-V4 cookbook. Stored under benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/ and overlaid onto the upstream srt-slurm checkout at runtime" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1157 - -- config-keys: - - dsv4-fp4-gb300-dynamo-vllm - description: - - "Add DeepSeek-V4-Pro FP4 GB300 disaggregated Dynamo vLLM benchmarks at 8k/1k" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1238 - -- config-keys: - - qwen3.5-fp8-b200-sglang - description: - - updated sglang container image - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1027 - -- config-keys: - - glm5-fp8-mi355x-atom - description: - - "Update GLM-5 FP8 MI355X ATOM benchmark: new image, add TP=4, set gpu-memory-utilization" - - "Image: rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post" - - "Add TP=4, concurrency 4-256 for 1k1k and 8k1k sequence lengths" - - "Add --gpu-memory-utilization 0.9 to server launch" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1126 - -- config-keys: - - dsr1-fp8-mi355x-sglang - description: - - "Tune --num-continuous-decode-steps 4 → 8 (+4.7% avg output throughput gain)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1243 - -- config-keys: - - dsv4-fp4-gb200-dynamo-vllm-mtp2 - description: - - "Add final DeepSeek-V4-Pro FP4 GB200 Dynamo vLLM MTP2 Pareto recipes using vLLM nightly image" - - "Recipes cover 8k/1k aggregate TP8 low-latency conc=1, low-latency bridge 1P DEP8 + 4D TP8 no-offload conc=16/32/64, mid 1P/1D DEP8 MegaMOE conc=128, and high-throughput 2P/1D DEP8 MegaMOE conc=1024" - - "All recipes enable FP4 indexer cache and speculative-config mtp with num_speculative_tokens=2" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1242 - -- config-keys: - - glm5-fp8-mi355x-sglang-mtp - description: - - "Updated the Image for glm5-fp8-mi355x-sglang-mtp" - - "Optimized the search space" - - "Removed redundant transformer installation" - - "Optimization model serves configs" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1252 - -- config-keys: - - qwen3.5-fp4-b200-sglang - description: - - "Update image to lmsysorg/sglang:nightly-dev-20260422-de962f32" - - "Fix tp:2 search-space: ep 2 -> 1" - - "Dynamic scheduler-recv-interval: 30 for CONC>4, 10 otherwise" - - "Remove --max-running-requests, reduce prefill/chunked from 81920 to 16384" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1018 - -- config-keys: - - minimaxm2.5-fp8-mi355x-atom - description: - - "Update Atom image to rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post" - - "Search-space: expand conc-end to 256 for tp2/tp4 (1k/1k and 8k/1k); remove tp8/ep8 configs" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1194 - -- config-keys: - - dsv4-fp4-mi355x-sglang - description: - - "Add MI355X DSv4 FP4 SGLang benchmark on rocm/sgl-dev:rocm720-mi35x-583b1b6-20260501-DSv4" - - "Use deepseek-ai/DeepSeek-V4-Pro with SGLANG_DSV4_FP4_EXPERTS=True and SGLANG_FORCE_TRITON_MOE_FP8=0" - - "Overlay SGLang from amd/deepseek_v4 SHA a8410de6 at runtime" - - "Sweep DP attention on/off and EP=8 via DP_ATTENTION and EP_SIZE matrix env vars" - - "Ship a DeepSeek-V4 thinking-mode chat template so eval /v1/chat/completions works (the canonical checkpoint ships no chat_template)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1244 - -- config-keys: - - dsv4-fp8-h200-sglang - description: - - "Add DeepSeek-V4-Pro FP8 H200 single-node SGLang recipe (Marlin MoE backend, TP=8, EP=1)" - - "Image: lmsysorg/sglang:deepseek-v4-hopper pinned by digest" - - "Server flags: --moe-runner-backend marlin, --chunked-prefill-size 4096, --disable-flashinfer-autotune, --disable-radix-cache, --mem-fraction-static 0.88" - - "Search space: TP=8 EP=1, conc 1 and 4-64 for both 1k1k and 8k1k" - - "Pinned to the h200-dgxc runner pool (new runners.yaml group); launch_h200-dgxc-slurm.sh extended to support framework-tagged script names and mount /ix instead of /workspace for the deepseek-v4-hopper image" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1264 - -- config-keys: - - dsv4-fp8-h200-sglang-mtp - description: - - "Add DeepSeek-V4-Pro FP8 H200 single-node SGLang MTP variant (mirrors dsv4-fp8-h200-sglang)" - - "EAGLE speculative decoding chain: --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4" - - "run_benchmark_serving uses --dsv4 (chat-formatted prompts) per the AGENTS.md MTP rule, since EAGLE acceptance regresses on raw random tokens" - - "Search space mirrors the non-MTP H200 SGLang entry: TP=8 EP=1, conc 1 and 4-64 for both 1k1k and 8k1k, with spec-decoding: mtp" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1265 - -- config-keys: - - kimik2.5-int4-b300-vllm - description: - - "Add Kimi-K2.5 INT4 B300 vLLM benchmark" - - "Image: vllm/vllm-openai:v0.20.0-cu130" - - "Search-space: tp=8 and tp=4/ep=1 over conc 4-64, on both 1024/1024 and 8192/1024 ISL/OSL" - - "At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/moonshotai/Kimi-K2.5.html does not have a B300-specific recipe, so this reuses the existing Kimi-K2.5 INT4 B200 vLLM recipe as-is until B300-specific tuning is available" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1057 - -- config-keys: - - kimik2.5-int4-b300-vllm - description: - - "Add Kimi-K2.5 INT4 B300 vLLM benchmark" - - "Image: vllm/vllm-openai:v0.20.0-cu130" - - "Search-space: tp=8 and tp=4/ep=1 over conc 4-64, on both 1024/1024 and 8192/1024 ISL/OSL" - - "At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/moonshotai/Kimi-K2.5.html does not have a B300-specific recipe, so this reuses the existing Kimi-K2.5 INT4 B200 vLLM recipe as-is until B300-specific tuning is available" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1057 - -- config-keys: - - dsv4-fp4-b300-trt - description: - - "Add B300 TensorRT-LLM DeepSeek-V4-Pro eval coverage using the feat/deepseek_v4 image" - - "Disable TRTLLM fused MHC hyper-connection for eval servers via TRTLLM_MHC_ENABLE_FUSED_HC=0 because the current fused kernel corrupts DeepSeek-V4-Pro hidden size 7168 generations" - - "Keep this as eval-only PR validation until the TensorRT-LLM fused MHC kernel is guarded or supports hidden size 7168" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1233 - -- config-keys: - - dsr1-fp4-mi355x-sglang-disagg - - dsr1-fp4-mi355x-sglang-disagg-mtp - description: - - "Bump SGL mori image to lmsysorg/sglang-rocm" - - "Add more high tput / low latency sweep configs" - - "Enable v2 mxfp4 DSR1 0528 model" - - "Enable fp4 disp / fp8 combine feature on mori" - - "Enable Mori SDMA + two batch overlapping feature" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1236 - -- config-keys: - - dsv4-fp4-mi355x-sglang - description: - - "Tune DSv4 FP4 MI355X SGLang runtime envs: enable fused compress and TileLang MHC post, and drop the Torch fallback env overrides for top-k transform and FP8 paged MQA logits" - - "Drive SGLang --max-running-requests and --cuda-graph-max-bs from the matrix CONC value instead of hard-coding 256, so each sweep point launches with matching serving capacity and graph capture size" - - "Adjust the sweep coverage: start 1k1k dp-attn=true at conc=16, add 1k1k dp-attn=false conc=1-16, and extend 8k1k dp-attn=false coverage to conc=16" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1272 - -- config-keys: - - dsv4-fp4-mi355x-sglang - description: - - "Bump image to rocm/sgl-dev:rocm720-mi35x-a8410de-20260502-DSv4 (one commit forward on amd/deepseek_v4: sglang PR #24249, fuse-compress-decode 0501)" - - "Drop the runtime sglang clone+pip overlay from benchmarks/single_node/dsv4_fp4_mi355x_sglang.sh — the new image bakes the same a8410de6 SHA the overlay was pinning, so the overlay is redundant. Future sglang bumps now go through an image tag bump" - - "Context: ep=8 dp-attn=true entries failed gsm8k eval after #1244 merged. PR sweep (run 25246535693) reported gsm8k strict-match=0.9318 because the launcher silently dropped --ep-size and sglang ran with ep_size=1 regardless of the matrix label; post-merge sweep (run 25262278289) ran with ep_size=8 and gsm8k strict-match dropped to 0.0000. The image bump is the candidate fix to verify on rerun" - - "ep=1 entries (dp-attn true and false) are unaffected by the EP=8 regression" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1266 - -- config-keys: - - dsv4-fp4-gb200-dynamo-vllm-mtp2 - description: - - "Fix result file parsing for non-disagg multinode benchmarks (decode num-worker=0)" - - "sa-bench result files without _ctx_/_gen_ suffix were silently dropped from the summary table" - - "Use vllm/vllm-openai:v0.20.1-ubuntu2404 directly for GB200 MTP2 instead of upgrading vLLM inside the v0.20.0 container" - - "Fix applies to all 7 multinode launch scripts, the benchmark-multinode-tmpl workflow, and process_result.py" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1256 - -- config-keys: - - qwen3.5-fp8-b200-sglang-mtp - description: - - "Update image to lmsysorg/sglang:nightly-dev-20260422-de962f32" - - "Add TP8 search-space point (conc 4) for 1k1k and 8k1k" - - "Dynamic scheduler-recv-interval: 30 for CONC>4, 10 otherwise" - - "Align B200 flags with B300: SGLANG_ENABLE_SPEC_V2=1, --enable-symm-mem, --expert-parallel-size" - - "Reduce prefill tokens from 32768 to 16384, drop flashinfer_allreduce_fusion" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1065 - -- config-keys: - - dsv4-fp8-h200-vllm-mtp - description: - - "Add DeepSeek-V4-Pro FP8 H200 vLLM MTP variant (mirrors dsv4-fp8-h200-vllm with --speculative-config {\"method\":\"mtp\",\"num_speculative_tokens\":1})" - - "Image: vllm/vllm-openai:v0.20.1" - - "Set VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 to skip the cudagraph-memory estimator (it overshoots the H200 + MTP memory budget at profile time even though actual cudagraph capture works fine)" - - "run_benchmark_serving uses --dsv4 (chat-formatted prompts) per the AGENTS.md MTP rule, since EAGLE-style speculative decoding regresses acceptance on raw random tokens" - - "Search space mirrors the non-MTP H200 entry: TP=8, EP=8, DP-attn=true, CONC 4-64 for both 1k1k and 8k1k, with spec-decoding: mtp" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1222 - -- config-keys: - - dsv4-fp8-h200-vllm-mtp - description: - - "Bump --speculative-config num_speculative_tokens from 1 to 2 (`{\"method\":\"mtp\",\"num_speculative_tokens\":2}`)" - - "Re-test whether H200 MTP kernels accept 2 draft tokens — Blackwell MTP runs at 2 (per @wzhao18's vLLM Blackwell MTP submission); checking if H200 has parity now" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1279 - -- config-keys: - - dsv4-fp4-b300-trt - description: - - "Update the TensorRT-LLM DeepSeek-V4-Pro image to ghcr.io/semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-9aa3715" - - "Enable TRTLLM fused MHC by default with the DeepSeek-V4 feature image" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1270 - -- config-keys: - - qwen3.5-fp4-b200-sglang-mtp - description: - - "Update image to lmsysorg/sglang:nightly-dev-20260422-de962f32" - - "Add tp:2 ep:1 conc 4-128 search-space for 1k1k and 8k1k" - - "Align server flags with FP4 B200 STP: --enable-symm-mem, --expert-parallel-size, dynamic scheduler-recv-interval" - - "Add MTP flags: SGLANG_ENABLE_SPEC_V2=1, EAGLE speculative decoding (steps=3, topk=1, draft=4)" - - "Reduce prefill/chunked from 32768 to 16384" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1257 - -- config-keys: - - dsv4-fp4-b200-trt - description: - - "Add B200 TensorRT-LLM DeepSeek-V4-Pro single-node coverage using the feat/deepseek_v4 image" - - "Mirror the B300 TRT launch path with OpenAI chat serving, FP8 KV cache, TRTLLM MoE, NCCL NVLS disabled by default, and fused MHC disabled for hidden size 7168 correctness" - - "Update the B200 DGXC Slurm partition from removed gpu to gpu-2 so single-node B200 jobs can allocate" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1277 - -- config-keys: - - dsv4-fp8-h200-vllm - - dsv4-fp8-h200-vllm-mtp - description: - - "Add pure TP (tp:8, ep:1) search-space row alongside the existing DP+EP row" - - "Raise conc-end from 64 to 256 on both 1k1k and 8k1k sweeps" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1287 - -- config-keys: - - dsv4-fp4-b300-trt-mtp - description: - - "Add DeepSeek-V4-Pro FP4 B300 TensorRT-LLM MTP coverage using ghcr.io/semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-9aa3715" - - "Mirror the B300 TRT STP search space with spec-decoding: mtp and TensorRT-LLM MTP num_nextn_predict_layers=2" - - "Benchmark serving uses the DeepSeek-V4 chat template for MTP acceptance-rate correctness" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1291 - -- config-keys: - - qwen3.5-fp8-b200-sglang-mtp - description: - - "Re-run qwen3.5-fp8-b200-sglang-mtp sweep after the B200 DGXC Slurm partition change (gpu → gpu-2)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1292 - -- config-keys: - - minimaxm2.5-fp8-mi355x-vllm - description: - - "Tune MiniMax-M2.5 FP8 MI355X vLLM scheduling thresholds for better throughput and stability across the 1k/1k and 8k/1k sweep points" - - "Default path: block-size=32, shuffled KV cache disabled (VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=0), async scheduling enabled" - - "1k/1k TP8/EP8: keep block-size=32 and shuffled KV cache disabled; disable async scheduling (--no-async-scheduling)" - - "1k/1k non-TP8/EP8: block-size=16 with shuffled KV cache enabled; disable async scheduling through c128" - - "8k/1k TP8/EP8: keep block-size=32 and shuffled KV cache disabled; disable AITER MoE (VLLM_ROCM_USE_AITER_MOE=0); disable async scheduling" - - "8k/1k non-TP8/EP8: disable async scheduling through c64; switch to block-size=16 with shuffled KV cache at c64 and above" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1276 - -- config-keys: - - dsv4-fp4-b200-trt-mtp - description: - - "Add DeepSeek-V4-Pro FP4 B200 TensorRT-LLM MTP coverage using ghcr.io/semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-9aa3715" - - "Mirror the B200 TRT STP search space with spec-decoding: mtp and TensorRT-LLM MTP num_nextn_predict_layers=2" - - "Benchmark serving uses the DeepSeek-V4 chat template for MTP acceptance-rate correctness" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1294 - -- config-keys: - - dsv4-fp4-gb300-dynamo-sglang - description: - - "Overhaul the 8k/1k DSv4-Pro GB300 SGLang search space: switch decode to WideEP TP=16 across most concurrency points (TP=12 at the 21504 max-concurrency point), and scale concurrencies to 1024/1024/4096/8192/21504; keep a 1p1d TP=4 baseline at conc=1" - - "Rename and consolidate the per-concurrency recipe files to `disagg-gb300-{N}p1d-{topo}-{nodes}-c{conc}.yaml`" - - "Re-enable lm-eval scoring for dsv4-fp4-gb300-dynamo-sglang now that the srt-slurm pin includes the lm-eval orchestrator path" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1295 - -- config-keys: - - dsv4-fp4-gb300-dynamo-sglang-mtp - description: - - "Add DeepSeek-V4-Pro FP4 GB300 disaggregated SGLang MTP coverage using lmsysorg/sglang-staging:deepseek-v4-grace-blackwell-dev" - - "Mirror the base 8k/1k disagg search space (1p1d-tp4, 1p6d-dep4-tp4, 1p1d-dep4-dep8, 1p1d-dep4-dep16, 2p1d-dep4-dep8, 4p1d-dep4-dep8) with spec-decoding: mtp and EAGLE on the decode side (num-steps=3, eagle-topk=1, num-draft-tokens=4)" - - "Recipes adapted from elvischenv/srt-slurm@dsv4-gb300-disagg-8k1k-mtp:recipes/dsv4-pro/sglang/gb300-fp4/8k1k/disagg/mtp/, repointed at the public sglang-staging container and the deepseek-v4-pro model alias" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1297 - -- config-keys: - - dsv4-fp4-mi355x-sglang - description: - - "Bump image to rocm/sgl-dev:rocm720-mi35x-bfd32b6-20260507-DSv4." - - "Tune DSv4 FP4 MI355X SGLang runtime envs: enable aiter MHC pre/post, and enable triton swa prepare kernel." - - "Add --context-length. Add --enable-prefill-delayer for dp config" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1300 - -- config-keys: - - dsv4-fp4-b200-dynamo-vllm - description: - - "Add DeepSeek-V4-Pro FP4 B200 disaggregated multi-node coverage using Dynamo vLLM" - - "9eff9734a30b6713a8566217d36f8277630fd2d31cec7f0a0292835901a23aa4" - - "Adapt the existing DSV4 GB200 vLLM disagg recipes to B200 by mapping each worker to one full 8-GPU B200 node" - - "Update the B200 DGXC Slurm launcher to support dsv4/fp4 with dynamo-vllm and local recipe overlays" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1303 - -- config-keys: - - dsv4-fp4-b300-dynamo-vllm - description: - - "Turn off explicit deep_gemm_mega_moe backend selection in B300 Dynamo vLLM throughput recipes" - - "Let vLLM choose the MoE backend automatically to avoid CUDA symmetric-memory rendezvous failures during startup" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1304 - -- config-keys: - - dsv4-fp4-b200-dynamo-vllm - description: - - "Fix B200 Dynamo vLLM recipe benchmark concurrencies to match the nvidia-master.yaml search space" - - "Propagate low-latency concurrencies 1/64/128, high-throughput concurrencies 1024/2048/4096/8192, and max-throughput concurrency 8192 into the srt-slurm recipe files" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1305 - -- config-keys: - - dsv4-fp4-mi355x-sglang - description: - - "Bump image to rocm/sgl-dev:rocm720-mi35x-0363e6c-20260509-DSv4." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1308 - -- config-keys: - - qwen3.5-fp8-mi355x-atom - - qwen3.5-fp8-mi355x-atom-mtp - description: - - "Add Qwen3.5-397B-A17B FP8 MI355X ATOM benchmark configs with and without MTP" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1310 - -- config-keys: - - minimaxm2.5-fp8-mi300x-vllm - description: - - "Update vLLM ROCm image from v0.16.0 to v0.21.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1405 - -- config-keys: - - kimik2.5-int4-b200-vllm - description: - - "Update vLLM image from v0.15.1 to v0.20.2" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1337 - -- config-keys: - - gptoss-fp4-h200-vllm - description: - - "Update vLLM image from v0.18.0 to v0.20.2" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1335 - -- config-keys: - - gptoss-fp4-h100-vllm - description: - - "Update vLLM image from v0.18.0 to v0.20.2" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1334 - -- config-keys: - - dsr1-fp4-b300-sglang - description: - - "Update SGLang image from v0.5.10.post1-cu130 to v0.5.11-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1321 - -- config-keys: - - dsr1-fp4-b300-sglang - description: - - "Update SGLang image from v0.5.10.post1-cu130 to v0.5.11-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1321 - -- config-keys: - - qwen3.5-fp8-h200-sglang-mtp - description: - - "Update SGLang image from v0.5.10.post1 to v0.5.11" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1347 - -- config-keys: - - qwen3.5-fp8-h200-sglang-mtp - description: - - "Rerun update SGLang image from v0.5.10.post1 to v0.5.11" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1347 - -- config-keys: - - dsr1-fp8-h200-sglang - description: - - "Update SGLang image from v0.5.9-cu130 to v0.5.11-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1326 - -- config-keys: - - glm5-fp8-b300-sglang-mtp - description: - - "Update SGLang image from v0.5.10.post1-cu130 to v0.5.11-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1332 - -- config-keys: - - glm5-fp4-b300-sglang-mtp - description: - - "Update SGLang image from v0.5.10.post1-cu130 to v0.5.11-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1330 - -- config-keys: - - glm5-fp8-b300-sglang - description: - - "Update SGLang image from v0.5.10.post1-cu130 to v0.5.11-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1331 - -- config-keys: - - dsr1-fp8-b300-sglang - description: - - "Update SGLang image from v0.5.10.post1-cu130 to v0.5.11-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1324 - -- config-keys: - - minimaxm2.5-fp8-h100-vllm - description: - - "Update vLLM image from v0.18.0 to v0.20.2" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1339 - -- config-keys: - - minimaxm2.5-fp8-h200-vllm - description: - - "Update vLLM image from v0.18.0 to v0.20.2" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1340 - -- config-keys: - - qwen3.5-bf16-b300-sglang - description: - - "Update SGLang image from v0.5.10.post1-cu130 to v0.5.11-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1341 - -- config-keys: - - qwen3.5-bf16-b300-sglang-mtp - description: - - "Update SGLang image from v0.5.10.post1-cu130 to v0.5.11-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1342 - -- config-keys: - - qwen3.5-fp4-b300-sglang - description: - - "Update SGLang image from v0.5.10.post1-cu130 to v0.5.11-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1343 - -- config-keys: - - qwen3.5-fp4-b300-sglang-mtp - description: - - "Update SGLang image from v0.5.10.post1-cu130 to v0.5.11-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1344 - -- config-keys: - - qwen3.5-fp8-b300-sglang-mtp - description: - - "Update SGLang image from v0.5.10.post1-cu130 to v0.5.11-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1346 - -- config-keys: - - dsr1-fp4-mi355x-sglang-disagg - - dsr1-fp4-mi355x-sglang-disagg-mtp - description: - - "Fix the eval result of dsr1 fp4 with fp8 blockwise combine" - - "Bump the image to May 19" - - "Add conc 512 new sweep point" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1566 - -- config-keys: - - kimik2.5-int4-h200-vllm - description: - - "Update vLLM image from v0.16.0 to v0.20.2" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1338 - -- config-keys: - - glm5-fp4-b300-sglang - description: - - "Update SGLang image from v0.5.10.post1-cu130 to v0.5.11-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1329 - -- config-keys: - - glm5-fp4-b200-sglang-mtp - description: - - "Update SGLang image from v0.5.10.post1-cu130 to v0.5.11-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1328 - -- config-keys: - - dsv4-fp4-mi355x-atom - description: - - "Add DeepSeek-V4-Pro FP4 MI355X ATOM benchmark config; bump image to rocm/atom-dev:nightly_202605101539, expand concurrency range (conc 4–1024), and simplify runtime script" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1311 - -- config-keys: - - glm5-fp8-mi355x-sglang - description: - - "Image: lmsysorg/sglang-rocm:v0.5.11-rocm720-mi35x-20260513" - - "Add --cuda-graph-max-bs to meet conc" - - "Turn to tp=4 for best perf" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1375 - -- config-keys: - - dsv4-fp4-gb300-dynamo-sglang - description: - - "Enable W4A4 (MXFP4) megamoe by appending w4a4 related environ flags when megamoe is enabled" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1382 - -- config-keys: - - dsr1-fp8-b200-sglang-mtp - description: - - "Update SGLang image from v0.5.9-cu130 to v0.5.11-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1323 - -- config-keys: - - dsr1-fp8-b200-sglang - description: - - "Update SGLang image from v0.5.9-cu130 to v0.5.11-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1322 - -- config-keys: - - kimik2.5-fp4-mi355x-atom - description: - - "Bump ATOM image to rocm/atom:rocm7.2.3_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom20260511" - - "Improves tput/GPU by up to +31% at low concurrency (tp=4, isl=1024, c=4-16)" - - "Ref ATOM upstream benchmark run https://github.com/ROCm/ATOM/actions/runs/25686894636" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1387 - - -- config-keys: - - dsv4-fp4-b300-vllm-mtp - description: - - "Update image tag to vllm/vllm-openai:v0.20.2" - - "Add DEP configs for B300 vLLM MTP" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1271 - -- config-keys: - - minimaxm2.5-fp8-mi355x-vllm - description: - - "Update vLLM ROCm image from v0.19.0 to v0.21.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1410 - -- config-keys: - - gptoss-fp4-h200-vllm - description: - - "Update vLLM image from v0.20.2 to v0.21.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1400 - -- config-keys: - - dsv4-fp4-b300-vllm-mtp - description: - - "Update vLLM image from v0.20.2 to v0.21.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1397 - -- config-keys: - - kimik2.5-int4-b200-vllm - description: - - "Update vLLM image from v0.20.2 to v0.21.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1396 - -- config-keys: - - gptoss-fp4-mi355x-vllm - description: - - "Update vLLM ROCm image from v0.17.0 to v0.21.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1406 - -- config-keys: - - kimik2.5-fp4-mi355x-vllm - description: - - "Update vLLM ROCm image from v0.18.0 to v0.21.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1407 - -- config-keys: - - kimik2.5-int4-mi355x-vllm - description: - - "Update vLLM ROCm image from v0.18.0 to v0.21.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1408 - -- config-keys: - - minimaxm2.5-fp4-mi355x-vllm - description: - - "Update vLLM ROCm image from v0.19.1 to v0.21.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1409 - -- config-keys: - - minimaxm2.5-fp8-h200-vllm - description: - - "Update vLLM image from v0.20.2 to v0.21.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1402 - -- config-keys: - - dsr1-fp4-b300-sglang - description: - - "Update SGLang image from v0.5.11-cu130 to v0.5.12-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1418 - -- config-keys: - - kimik2.5-int4-h200-vllm - description: - - "Update vLLM image from v0.20.2 to v0.21.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1401 - -- config-keys: - - dsr1-fp8-b300-sglang - - dsr1-fp8-b300-sglang-mtp - description: - - "Update SGLang image from v0.5.11-cu130 to v0.5.12-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1419 - -- config-keys: - - glm5-fp4-b200-sglang - - glm5-fp4-b200-sglang-mtp - description: - - "Update SGLang image from v0.5.10.post1-cu130 to v0.5.12-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1417 - -- config-keys: - - qwen3.5-fp8-h200-sglang-mtp - description: - - "Update SGLang image from v0.5.11 to v0.5.12" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1424 - -- config-keys: - - gptoss-fp4-h100-vllm - description: - - "Update vLLM image from v0.20.2 to v0.21.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1398 - -- config-keys: - - qwen3.5-fp8-mi325x-sglang - description: - - "Update SGLang image from v0.5.10-rocm720-mi30x to v0.5.12-rocm720-mi30x" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1430 - -- config-keys: - - minimaxm2.5-fp8-h100-vllm - description: - - "Update vLLM image from v0.20.2 to v0.21.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1399 - -- config-keys: - - dsr1-fp8-h200-sglang - description: - - "Update SGLang image from v0.5.11-cu130 to v0.5.12-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1423 - -- config-keys: - - qwen3.5-bf16-mi325x-sglang - description: - - "Update SGLang image from v0.5.10-rocm720-mi30x to v0.5.12-rocm720-mi30x" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1429 - -- config-keys: - - dsr1-fp8-b200-sglang - - dsr1-fp8-b200-sglang-mtp - description: - - "Update SGLang image from v0.5.11-cu130 to v0.5.12-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1416 - -- config-keys: - - gptoss-fp4-b200-vllm - description: - - "Update vLLM image from v0.15.1 to v0.20.2" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1394 - -- config-keys: - - kimik2.5-int4-mi300x-vllm - description: - - "Update vLLM ROCm image from v0.18.0 to v0.21.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1404 - -- config-keys: - - qwen3.5-fp8-mi300x-sglang - description: - - "Update SGLang image from v0.5.10-rocm720-mi30x to v0.5.12-rocm720-mi30x" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1427 - -- config-keys: - - glm5-fp8-h200-sglang - description: - - "Update SGLang image from custom glm5-hopper tag (59d old) to v0.5.12-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1459 - -- config-keys: - - qwen3.5-fp8-h200-sglang - description: - - "Update SGLang image from v0.5.9-cu129-amd64 (74d old) to v0.5.12-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1458 - -- config-keys: - - dsv4-fp8-h200-vllm - - dsv4-fp8-h200-vllm-mtp - description: - - "Update vLLM image to v0.21.0 (from custom deepseekv4-cu129 / v0.20.1@sha256-pinned)" - - "Lower --gpu-memory-utilization from 0.95 to 0.90 in dsv4_fp8_h200.sh and dsv4_fp8_h200_mtp.sh — v0.21.0 uses more memory at load time, OOM'd on GPU 2 at 0.95" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1461 - -- config-keys: - - dsr1-fp8-mi325x-sglang - description: - - "Update SGLang image from v0.5.9-rocm700-mi30x to v0.5.12-rocm700-mi30x" - - "Workaround LlamaTokenizer.all_special_tokens_extended removal in newer transformers: prefer backend_request_func.get_tokenizer over vLLM's" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1428 - -- config-keys: - - dsr1-fp4-b200-sglang - description: - - "Update SGLang image from v0.5.11-cu130 to v0.5.12-cu130" - - "Temporarily disable agentic-coding scenario (blocked by e2e-tests.yml artifact-name mismatch)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1415 - -- config-keys: - - qwen3.5-bf16-b200-sglang - - qwen3.5-bf16-b200-sglang-mtp - description: - - "Update SGLang image from nightly-dev-20260216-d3bae71e (86d old) to v0.5.12-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1446 - -- config-keys: - - gptoss-fp4-mi325x-vllm - description: - - "Update vLLM ROCm image from v0.17.0 (70d old) to v0.21.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1467 - -- config-keys: - - kimik2.5-int4-mi325x-vllm - description: - - "Update vLLM ROCm image from v0.18.0 (52d old) to v0.21.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1468 - -- config-keys: - - minimaxm2.5-fp8-mi325x-vllm - description: - - "Update vLLM ROCm image from v0.18.0 (50d old) to v0.21.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1469 - -- config-keys: - - minimaxm2.5-fp8-b200-vllm - description: - - "Update vLLM image from v0.19.0-cu130 (25d old) to v0.21.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1449 - -- config-keys: - - minimaxm2.5-fp4-b200-vllm - description: - - "Update vLLM image from v0.19.0-cu130 (25d old) to v0.21.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1448 - -- config-keys: - - kimik2.5-fp4-b200-vllm - description: - - "Update vLLM image from v0.20.2 to v0.21.0" - - "Add VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 to disable aggressive CUDA-graph memory profiler that OOMs the KV cache" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1395 - -- config-keys: - - qwen3.5-bf16-mi300x-sglang - description: - - "Update SGLang image from v0.5.10-rocm720-mi30x to v0.5.12-rocm720-mi30x" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1426 - -- config-keys: - - glm5-fp8-h200-sglang-mtp - description: - - "Add MTP/EAGLE speculative-decoding sibling for glm5-fp8-h200-sglang on lmsysorg/sglang:v0.5.12-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1480 - -- config-keys: - - qwen3.5-bf16-mi325x-sglang-mtp - description: - - "Add MTP/EAGLE speculative-decoding sibling of qwen3.5-bf16-mi325x-sglang" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1483 - -- config-keys: - - kimik2.5-fp4-b300-vllm - description: - - "Update vLLM image from v0.19.0-cu130 (27d old) to v0.21.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1452 - -- config-keys: - - minimaxm2.5-fp8-b300-vllm - description: - - "Update vLLM image from v0.19.0-cu130 (26d old) to v0.21.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1454 - -- config-keys: - - dsr1-fp8-mi300x-sglang - description: - - "Update SGLang image from v0.5.9-rocm700-mi30x to v0.5.12-rocm700-mi30x" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1425 - -- config-keys: - - glm5-fp8-b300-sglang - - glm5-fp8-b300-sglang-mtp - description: - - "Update SGLang image from v0.5.11-cu130 to v0.5.12-cu130" - - "Disable JIT DeepGemm (SGL_ENABLE_JIT_DEEPGEMM=0) to bypass v0.5.12 DeepGemm TMA-descriptor regression on B300 — see sgl-project/sglang#25551" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1421 - -- config-keys: - - dsr1-fp8-b200-trt - - dsr1-fp8-b200-trt-mtp - description: - - "Update TensorRT-LLM image (off: v1.2.0rc6.post2 109d / mtp: v1.2.0rc6.post3 102d) to v1.3.0rc14 (latest pre-release)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1488 - -- config-keys: - - dsr1-fp8-h200-trt - - dsr1-fp8-h200-trt-mtp - description: - - "Update TensorRT-LLM image from v1.1.0rc2.post2 (154d/124d old) to v1.3.0rc14 (latest pre-release)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1487 - -- config-keys: - - dsv4-fp4-b200-vllm - - dsv4-fp4-b200-vllm-mtp - description: - - "Update vLLM image from v0.20.0-cu130 (20d/18d old) to v0.21.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1476 - -- config-keys: - - qwen3.5-fp4-b200-sglang - - qwen3.5-fp4-b200-sglang-mtp - description: - - "Update SGLang image from nightly-dev-20260422-de962f32 (17d/13d old) to v0.5.12-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1474 - -- config-keys: - - qwen3.5-fp8-b200-sglang - - qwen3.5-fp8-b200-sglang-mtp - description: - - "Update SGLang image from nightly-dev-20260422-de962f32 (18d/12d old) to v0.5.12-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1473 - -- config-keys: - - gptoss-fp4-b200-vllm - description: - - "Update vLLM image from v0.20.2 (1d old) to v0.21.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1466 - -- config-keys: - - kimik2.5-int4-b300-vllm - description: - - "Update vLLM image from v0.20.0-cu130 (14d old) to v0.21.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1457 - -- config-keys: - - dsv4-fp4-b300-vllm - description: - - "Update vLLM image from v0.20.0-cu130 (18d old) to v0.21.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1456 - -- config-keys: - - minimaxm2.5-fp4-b300-vllm - description: - - "Update vLLM image from v0.19.0-cu130 (26d old) to v0.21.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1453 - -- config-keys: - - gptoss-fp4-h200-trt - description: - - "Update TensorRT-LLM image from v1.3.0rc11 (34d old) to v1.3.0rc14 (latest pre-release)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1491 - -- config-keys: - - gptoss-fp4-b200-trt - description: - - "Update TensorRT-LLM image from v1.2.0rc2.post2 (102d old) to v1.3.0rc14 (latest pre-release)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1490 - -- config-keys: - - qwen3.5-bf16-b300-sglang - - qwen3.5-bf16-b300-sglang-mtp - description: - - "Update SGLang image from v0.5.11-cu130 to v0.5.12-cu130" - - "Add --mm-attention-backend triton_attn to bypass flash-attn cute sm_103 assertion (see sgl-project/sglang#25564)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1422 - -- config-keys: - - qwen3.5-fp8-h100-sglang - - qwen3.5-fp8-h100-sglang-mtp - description: - - "Add Qwen-3.5-397B-A17B FP8 sglang recipes (off + MTP/EAGLE) for H100 on lmsysorg/sglang:v0.5.12-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1509 - -- config-keys: - - dsr1-fp8-mi325x-sglang-mtp - description: - - "Add MTP/EAGLE speculative-decoding sibling of dsr1-fp8-mi325x-sglang on lmsysorg/sglang:v0.5.12-rocm700-mi30x" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1500 - -- config-keys: - - glm5-fp8-b200-sglang - - glm5-fp8-b200-sglang-mtp - description: - - "Update SGLang image from nightly-dev-cu13-20260317-1eea7448 (33d/29d old) to v0.5.12-cu130" - - "Add --fp8-gemm-runner-backend cutlass to bypass DeepGemm illegal-memory-access on CUDA-graph capture (KLAUD_DEBUG §4a, B200 sm_100 variant)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1447 - -- config-keys: - - qwen3.5-fp8-mi325x-sglang-mtp - description: - - "Add MTP/EAGLE speculative-decoding sibling of qwen3.5-fp8-mi325x-sglang" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1484 - -- config-keys: - - glm5-fp8-b200-dynamo-sglang - description: - - "Add GLM-5 FP8 B200 Dynamo SGLang disaggregated multi-node coverage using lmsysorg/sglang:v0.5.11-cu130" - - "1k1k and 8k1k STP low-latency and max-throughput srt-slurm recipes under benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/b200-fp8/" - - "Use the B200 local model alias glm5-fp8 mapped to /scratch/fsw/models/GLM-5-FP8" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1372 - -- config-keys: - - dsv4-fp4-mi355x-sglang - description: - - "Bump image to rocm/sgl-dev:rocm720-mi35x-b19052c-20260518-DSv4." - - "Enabled Triton attention backend, FlyDSL MoE, and fused hash topk." - - "Concurrency sweep up to 1024." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1355 - -- config-keys: - - dsr1-fp4-b200-trt - - dsr1-fp4-b200-trt-mtp - description: - - "Update TensorRT-LLM image (off: v1.2.0rc6.post2 104d / mtp: v1.2.0rc6.post3 101d) to v1.3.0rc14 (latest pre-release)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1489 - -- config-keys: - - dsr1-fp8-mi355x-sglang - description: - - "Update SGLang image to v0.5.12-rocm700-mi35x" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1432 - -- config-keys: - - kimik2.5-fp4-b200-vllm - description: - - "Update vLLM image from v0.20.2 to v0.21.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1504 - -- config-keys: - - dsv4-fp4-gb200-dynamo-vllm-mtp2 - description: - - "Upgrade GB200 DSV4 MTP2 1P/1D DEP8 MegaMOE mid-curve recipe to vLLM v0.21.0" - - "Extend mid-curve concurrency sweep to also benchmark conc=256, conc=512, and conc=1024" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1390 - -- config-keys: - - qwen3.5-fp4-mi355x-sglang-mtp - description: - - "Add MI355X config: qwen3.5-fp4-sglang-mtp using lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260517" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1445 - -- config-keys: - - dsr1-fp4-mi355x-sglang - description: - - "Update SGLang image to v0.5.12-rocm700-mi35x" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1431 - -- config-keys: - - qwen3.5-bf16-mi355x-sglang - - qwen3.5-bf16-mi355x-sglang-mtp - description: - - "Update SGLang ROCm image from v0.5.10rc0-rocm720-mi35x-20260415 to v0.5.12-rocm720-mi35x-20260517" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1443 - -- config-keys: - - qwen3.5-fp8-mi355x-sglang - - qwen3.5-fp8-mi355x-sglang-mtp - description: - - "Update SGLang ROCm image from v0.5.10rc0-rocm720-mi35x-20260414 to v0.5.12-rocm720-mi35x-20260517" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1444 - -- config-keys: - - qwen3.5-fp4-mi355x-sglang - description: - - "Update SGLang ROCm image from rocm/sgl-dev:v0.5.10rc0-rocm720-mi35x-20260413 (32d old, deprecated rocm/sgl-dev repo) to lmsysorg/sglang:v0.5.12-rocm720-mi35x (clean stable tag on the active lmsysorg/sglang repo)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1497 - -- config-keys: - - glm5-fp8-mi355x-sglang - - glm5-fp8-mi355x-sglang-mtp - description: - - "Update SGLang ROCm image from v0.5.11/v0.5.10rc0 to v0.5.12-rocm720-mi35x-20260517" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1440 - -- config-keys: - - kimik2.5-fp4-mi355x-vllm-disagg - description: - - "Add Kimi-K2.5-MXFP4 FP4 vLLM disagg PD recipe (1P2D, MoRI-EP + MoRI-IO) for MI355X on vllm/vllm-openai-rocm:nightly" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1569 - -- config-keys: - - minimaxm2.5-fp8-mi355x-vllm-disagg - description: - - "Add MiniMax-M2.5 FP8 vLLM disagg PD recipe (1P2D, MoRI-EP + MoRI-IO) for MI355X on vllm/vllm-openai-rocm:nightly" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1569 - -- config-keys: - - dsv4-fp4-mi355x-vllm - description: - - "Following recipe from https://github.com/vllm-project/recipes/pull/433" - - "Add DEP8 dp-attn=true validation probes at conc=64 for 1k1k and 8k1k" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1374 - -- config-keys: - - glm5-fp8-h200-sglang - description: - - "Update SGLang image from glm5-hopper to v0.5.10.post1-cu130" - - "Add --enable-flashinfer-allreduce-fusion to server launch" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1033 - -- config-keys: - - dsr1-fp8-mi355x-sglang-mtp - description: - - "Add MTP/EAGLE speculative-decoding sibling for dsr1-fp8-mi355x-sglang (model: deepseek-ai/DeepSeek-R1-0528) on lmsysorg/sglang:v0.5.12-rocm700-mi35x" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1521 - -- config-keys: - - dsr1-fp4-mi355x-atom - - dsr1-fp4-mi355x-atom-mtp - description: - - "Update Atom ROCm image (off: rocm7.1.1-...-atom0.1.1-MI350x 125d / mtp: rocm7.2.0-...-atom0.1.1 83d) to rocm7.2.3_..._atom20260511" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1518 - -- config-keys: - - glm5-fp8-mi325x-sglang - - glm5-fp8-mi325x-sglang-mtp - description: - - "Add GLM-5 FP8 SGLang ROCm recipes (off + MTP/EAGLE) for MI325X on lmsysorg/sglang:v0.5.12-rocm720-mi30x" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1485 - -- config-keys: - - dsr1-fp8-mi355x-atom - - dsr1-fp8-mi355x-atom-mtp - description: - - "Update Atom ROCm image (off: rocm7.1.1-...-atom0.1.1-MI350x 125d / mtp: rocm7.2.1-...-atom0.1.2) to rocm7.2.3_..._atom20260511" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1519 - -- config-keys: - - dsr1-fp4-mi355x-sglang-mtp - description: - - "Add MTP/EAGLE speculative-decoding sibling for dsr1-fp4-mi355x-sglang (model: amd/DeepSeek-R1-0528-MXFP4) on lmsysorg/sglang:v0.5.12-rocm700-mi35x" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1520 - -- config-keys: - - dsr1-fp8-h200-sglang-mtp - description: - - "Add MTP/EAGLE speculative-decoding sibling for dsr1-fp8-h200-sglang (model: deepseek-ai/DeepSeek-R1-0528) on lmsysorg/sglang:v0.5.12-cu130 — TP=8, EP=1, search-space conc 4..64 on 1k1k + 8k1k" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1523 - -- config-keys: - - dsr1-fp8-b200-sglang - - dsr1-fp8-b300-sglang - - dsr1-fp4-b200-sglang - - dsr1-fp4-b300-sglang - - dsr1-fp8-b200-sglang-mtp - - dsr1-fp8-b300-sglang-mtp - description: - - "Re-run DSR1 SGLang agg configs (B200/B300, FP8/FP4, no-MTP/MTP) — picks up tokenizer fix from #1381" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1502 - -- config-keys: - - dsv4-fp4-gb300-dynamo-sglang - description: - - "Update SGLang container image from `lmsysorg/sglang-staging:deepseek-v4-grace-blackwell-dev` to `lmsysorg/sglang:nightly-dev-cu13-20260519-dbac4647` for all non-MTP disagg configs" - - "Switch moe-a2a-backend from `deepep` to `megamoe` for wideep configs" - - "Remove obsolete/redundant environment variables and replace deprecated ones" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1492 - -- config-keys: - - dsv4-fp4-gb300-dynamo-sglang - description: - - "Update SGLang image from nightly-dev-cu13-20260518-c67b2870 to nightly-dev-cu13-20260519-dbac4647" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1492 - -- config-keys: - - dsr1-fp4-b200-sglang-mtp - description: - - "Add MTP/EAGLE speculative-decoding sibling for dsr1-fp4-b200-sglang (model: nvidia/DeepSeek-R1-0528-FP4-V2) on lmsysorg/sglang:v0.5.12-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1522 - -- config-keys: - - qwen3.5-fp8-mi355x-atom - description: - - "Bump ATOM image to rocm/atom:rocm7.2.3_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom20260511" - - "TP=4 shows +3.2% to +16.3% throughput improvement across 1k1k and 8k1k workloads (concurrency 4-256)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1411 - -- config-keys: - - dsv4-fp4-gb300-dynamo-sglang - description: - - "Update SGLang image from nightly-dev-cu13-20260519-dbac4647 to nightly-dev-cu13-20260520-425dffbd for all non-MTP disagg configs" - - "Remove SGLANG_OPT_FP8_WO_A_GEMM=0 workaround (topk_v2 crash fixed upstream in sgl-project/sglang#25805)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1528 - -- config-keys: - - qwen3.5-fp4-b300-sglang - - qwen3.5-fp4-b300-sglang-mtp - description: - - "Update SGLang image from v0.5.11-cu130 (5d old) to v0.5.12-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1475 - -- config-keys: - - dsv4-fp4-mi355x-vllm - description: - - "Bump vLLM ROCm image from nightly-b50646e5effd7cb5884cd96fdff4c53c18521198 to nightly-4f940896a32c9e2a0eba7f50d521bf5f6b4de458" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1546 - -- config-keys: - - dsv4-fp4-mi355x-sglang - description: - - "Bump image to rocm/sgl-dev:rocm720-mi35x-8c3b5aa-20260521-DSv4" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1548 - -- config-keys: - - qwen3.5-fp8-b300-sglang - - qwen3.5-fp8-b300-sglang-mtp - description: - - "Update SGLang image from v0.5.10.post1-cu130 / v0.5.11-cu130 (30d old) to v0.5.12-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1451 - -- config-keys: - - dsv4-fp4-gb300-dynamo-sglang-mtp - description: - - "Enable W4A4 (MXFP4) megamoe by setting SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS=1 and SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND=1 in the megamoe environment blocks (not the low-latency 1p1d-tp4-tp4 or the 4p1d recipe)" - - "Update SGLang image from nightly-dev-cu13-20260510-2473659e to nightly-dev-20260527-14f81a67" - - "Switch moe-a2a-backend from deepep to megamoe and drop the deepep-config override" - - "Add two high concurrency configs" - - "Clean up obsolete environs in the 8k1k disagg recipes: drop SGLANG_OPT_USE_JIT_NORM / SGLANG_OPT_USE_JIT_INDEXER_METADATA / SGLANG_OPT_USE_TOPK_V2 (now default-on); drop the auto-set MegaMoE companions (SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE, SGLANG_OPT_FIX_HASH_MEGA_MOE, SGLANG_OPT_FIX_MEGA_MOE_MEMORY, SGLANG_OPT_FIX_NEXTN_MEGA_MOE, SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK); drop SGLANG_OPT_USE_FAST_MASK_EP which no longer exists in sglang environ.py (SGLANG_RADIX_DISABLE_REUSE is kept)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1378 - -- config-keys: - - dsr1-fp8-b200-sglang - - dsr1-fp8-b300-sglang - - dsr1-fp4-b200-sglang - - dsr1-fp4-b300-sglang - - dsr1-fp8-b200-sglang-mtp - - dsr1-fp8-b300-sglang-mtp - description: - - "Truncate sweep to conc=1 and conc=2 only: set conc-start=1, conc-end=2 in every search-space across all six DSR1 SGLang agg configs" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1534 - -- config-keys: - - qwen3.5-fp4-mi355x-sglang-mtp - description: - - "Add --use-chat-template to run_benchmark_serving so prompts are formatted with the Qwen chat template (matching the other Qwen MTP recipes)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1554 - -- config-keys: - - glm5-fp4-gb300-dynamo-sglang - description: - - "Add GLM-5 FP4 GB300 Dynamo SGLang disaggregated multi-node coverage using lmsysorg/sglang:v0.5.11-cu130" - - "1k1k and 8k1k STP low-latency and max-throughput srt-slurm recipes under benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp4/ (ported from upstream srt-slurm PR #152)" - - "Wire glm5/fp4 model + dynamo-sglang framework branches into runners/launch_gb300-nv.sh" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1514 - -- config-keys: - - minimaxm2.5-fp8-h100-vllm - description: - - "Update minimaxm2.5-fp8-h100-vllm recipe (v0.19.1)" - - "Image: vllm/vllm-openai:v0.21.0 -> v0.19.1-cu130" - - "Replace recipe flags: drop PIECEWISE/0.90 mem util/256 max-num-seqs/no-prefix-caching/explicit max-model-len; add --enable-auto-tool-choice, --tool-call-parser minimax_m2, --reasoning-parser minimax_m2_append_think, --compilation-config mode:3+fuse_minimax_qk_norm" - - "Search-space: tp:8 ep:8 (TEP=8), conc-end 128 chosen at saturation per local sweep" - - "Local bench: TEP=8 peaks at C=128 with 26923 tot tps (+178% vs TEP=4 peak at C=32 in May 6 j11600242 sweep)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1516 - -- config-keys: - - qwen3.5-fp8-mi355x-atom-mtp - description: - - "Add --use-chat-template to run_benchmark_serving so prompts are formatted with the Qwen chat template (matching the other Qwen MTP recipes)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1555 - -- config-keys: - - minimaxm2.5-fp8-h200-vllm - description: - - "Update MiniMax-M2.5 FP8 H200 vLLM to vllm/vllm-openai:v0.20.1-ubuntu2404" - - "Set vLLM serving knobs in benchmarks/single_node/minimaxm2.5_fp8_h200.sh: generated benchmark max-model-len, previous eval max-model-len handling, fp8 KV cache, FlashInfer attention/autotune, Triton MoE, and MiniMax QK norm fusion" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1354 - -- config-keys: - - dsv4-fp4-mi355x-sglang - description: - - "Bump image to rocm/sgl-dev:rocm720-mi35x-f96ac98-20260526-DSv4" - - "Add args to avoid kvcache pool full issue on high conc" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1568 - -- config-keys: - - qwen3.5-fp8-h200-sglang - - dsr1-fp8-mi355x-sglang - description: - - "Validates measured-power aggregation pipeline (PR #1558) on both NVIDIA (H200) and AMD (MI355X) hardware — different SMI tools (nvidia-smi vs amd-smi), different CSV schemas (power.draw [W] vs socket_power), same aggregator. No config change. Entry intentionally kept past merge so run-sweep produces canonical agg JSONs with avg_power_w + joules_per_output_token on main for both vendors, seeding the dashboard's day-zero data." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1558 - -- config-keys: - - qwen3.5-fp8-mi355x-sglang-disagg - description: - - "Add Qwen3.5-397B-A17B-FP8 MI355X SGLang disaggregated prefill-decode benchmark" - - "Image: lmsysorg/sglang-rocm:v0.5.11-rocm700-mi35x-20260511" - - "1P+1D TP8/EP1 smoke sweep for 1k1k and 8k1k (conc 8-512); MoRI transfer backend" - - "Add models.yaml server flags and multinode launch script qwen3.5_fp8_mi355x_sglang-disagg.sh" - - "8k1k row uses dp-attn=false (matches 1k1k): with --enable-dp-attention + --moe-a2a-backend mori, sglang auto-promotes moe_ep_size=tp_size=8, but is_deepep_class_backend() excludes MoRI, so num_shared_slots stays at the global value (1) and the (num_experts - num_shared_slots) % moe_ep_size assertion in fused_moe_triton/layer.py fires for Qwen3.5 (512 routed + 1 shared). Track upstream sglang; flip back to dp-attn=true once MoRI is added to is_deepep_class_backend() or shared-slot accounting is reconciled." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1570 - -- config-keys: - - glm5-fp8-mi355x-sglang-disagg - description: - - "Add GLM-5-FP8 MI355X SGLang disaggregated prefill-decode benchmark" - - "Image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523 (bumped from .v0.5.12-...-20260517 to unlock the PD-disagg MoRI overlay; matches chun-chang/sglang-disagg-qwen3.5)" - - "Adds patches/mori_conn.py overlay (bind-mounted via job.slurm) to fix sglang v0.5.12.post1 MoRI/PD startup crashes for hybrid-attention models (GLM-5 NSA, etc.): sender flatten, state_types plural fallback, consumer normalize, SWA/DSA rank/length normalize. Validated: GSM8K=0.971 strict/0.970 flexible on chun-chang. Auto-applied for v0.5.12.post1 images; opt-out via MORI_CONN_PATCH=skip." - - "1P+1D TP8/EP1 CI smoke sweep for 1k1k and 8k1k (conc 8-512)" - - "Add GLM-5-FP8 models.yaml flags, setup_deps.sh (aiter gluon + transformers glm_moe_dsa), GLM-5 env tuning in env.sh" - - "Add multinode launch script glm5_fp8_mi355x_sglang-disagg.sh; server.sh sources setup_deps.sh" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1572 - -- config-keys: - - qwen3.5-fp4-mi355x-sglang-disagg - description: - - "Add Qwen3.5-397B-A17B-MXFP4 MI355X SGLang PD-disaggregation" - - "Bump image to lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523, 1P1D TP8/EP1, dp-attn false, conc [8..512]" - - "MoRI conn.py overlay (48e459bd) via job.slurm; launcher qwen3.5_fp4_mi355x_sglang-disagg.sh" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1579 - -- config-keys: - - dsv4-fp4-b300-sglang - description: - - "Update sglang image to nightly-dev-cu13-20260529-a8cfae0b" - - "Refactor benchmark script to dispatch by CONC instead of nested DP_ATTENTION/CONC/EP_SIZE" - - "Switch CONC 2048/4096/8192 from --moe-a2a-backend deepep to megamoe" - - "Remove env vars deleted from sglang main (SGLANG_OPT_USE_JIT_NORM, SGLANG_OPT_USE_FAST_MASK_EP, SGLANG_OPT_FIX_NEXTN_MEGA_MOE, SGLANG_OPT_FIX_HASH_MEGA_MOE)" - - "Remove env vars redundant with sglang defaults (SGLANG_OPT_USE_JIT_INDEXER_METADATA, SGLANG_OPT_USE_TOPK_V2, SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2)" - - "Remove env vars auto-set by megamoe backend (SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE, SGLANG_OPT_FIX_MEGA_MOE_MEMORY)" - - "Remove --deepep-config and SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK (unused by megamoe/StandardDispatcher)" - - "Fix CONC=512 yaml ep from 4 to 1 (flashinfer_mxfp4 does not set ep=tp)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1506 - - -- config-keys: - - dsr1-fp4-mi355x-sglang-disagg-8k1k-mtp - description: - - "Bump the image to May 26" - - "Add conc 128/256 new sweep point" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1584 - -- config-keys: - - glm5-fp8-gb300-dynamo-sglang - description: - - "Add GLM-5 FP8 GB300 Dynamo SGLang disaggregated multi-node coverage using lmsysorg/sglang:v0.5.11-cu130" - - "1k1k and 8k1k STP hightpt and lowlat srt-slurm recipes under benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/ (resolved from upstream srt-slurm PR #160 via srtctl resolve-override)" - - "Wire glm5/fp8 model + dynamo-sglang framework branches into runners/launch_gb300-nv.sh with SA upstream defaults (SLURM_PARTITION=batch_1, SLURM_ACCOUNT=benchmark, SQUASH_FILE under /home/sa-shared/gharunners/squash/)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1557 - -- config-keys: - - dsv4-fp4-b200-vllm - description: - - "Update vLLM image tag to v0.22.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1384 - -- config-keys: - - glm5-fp4-gb300-dynamo-sglang - description: - - "Update GB300 FP4 GLM-5 8k1k low-latency sweep to mirror NVIDIA/srt-slurm#175: add a 5th 1p17d topology (decode_nodes/workers=17), and lower decode max-running-requests / cuda-graph-max-bs / benchmark concurrency per-zip-index from a flat 4096/1024 to 128/64/32/16/1 (mrr & cuda-graph) and 128/64/32/16/12 (concurrency)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1583 - -- config-keys: - - glm5.1-fp4-mi355x-sglang - description: - - "Bump SGLang ROCm image from v0.5.10rc0-rocm720-mi35x-20260415 to v0.5.12.post1-rocm720-mi35x-20260529" - - "Picks up the fix for the GSM8K accuracy regression reported in sgl-project/sglang#25742 (v0.5.12-20260517 collapsed to ~0.32 at TP=2)" - - "Local eval-only runs on MI355X recover to gsm8k strict-match 0.975 at TP=2/conc=64 and 0.974 at TP=4/conc=16, well above the 0.92 upstream gate added in sgl-project/sglang#26396" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1593 - -- config-keys: - - minimaxm2.5-fp8-mi325x-vllm - description: - - "Pin AITER FA attention backend in benchmarks/single_node/minimaxm2.5_fp8_mi325x.sh to recover the ~38% MI325X throughput regression introduced when vLLM PR #36702 (between v0.18.0 and v0.21.0) flipped the dense full-attention default on ROCm from ROCM_AITER_FA to ROCM_ATTN (vllm-project/vllm#43029)" - - "Export VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=1 to enable the AITER asm/hip paged-attention auto-dispatch" - - "Pass --attention-backend ROCM_AITER_FA to vllm serve, aligning with the merged upstream MiniMax ROCm recipe (vllm-project/recipes#481)" - - "Pin image vllm/vllm-openai-rocm:v0.20.2 — the version the upstream recipe explicitly validates (`min_vllm_version: 0.20.2`). v0.21.0 separately crashes during AITER MoE CUDA-graph capture on MiniMax-M2.5 (silent worker death, `Engine core initialization failed`) reproducible via the recipe's exact flags; v0.20.2 + recipe completes a 100-prompt vllm bench serve cleanly at 2030 tok/s total throughput on MI325X (TP=4)" - - "Add --compilation-config '{\"mode\":3,\"cudagraph_mode\":\"PIECEWISE\"}' to vllm serve, mirroring `model.base_args` from the upstream recipe. `pass_config.fuse_minimax_qk_norm` from the recipe is intentionally omitted — it triggers an upstream NameError on ROCm because vllm/compilation/passes/pass_manager.py imports MiniMaxQKNormPass under `is_cuda()` (NVIDIA-only) while using it unconditionally" - - "Conditionally enable VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=1 per (TP, EP, CONC) — on for shapes where the AITER ASM paged-attention kernel exists in the gfx942 heuristic table (TP=2 EP=1 CONC<=16, TP=8 EP=8 CONC<=64), off otherwise. Above the thresholds vllm/v1/attention/backends/rocm_aiter_fa.py routes decode through aiter pa_fwd_asm and crashes with `RuntimeError: get_heuristic_kernel: cannot get heuristic kernel!` for MiniMax-M2.5's attention shape (gqa=6 block_size=32 qTile=0); below them the ASM auto-dispatch is the perf win the recipe wants. Thresholds confirmed across 17 bench cells + 3 eval cells in PR #1594 sweep run 26692603804. Mirrors the per-shape toggle pattern in benchmarks/single_node/minimaxm2.5_fp8_mi355x.sh; can collapse to unconditional SHUFFLE=1 once AITER registers the missing kernel on gfx942" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1594 - -- config-keys: - - gptoss-fp4-mi355x-vllm - description: - - "Update vLLM ROCm image from v0.21.0 to v0.22.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1623 - -- config-keys: - - minimaxm2.5-fp8-h200-vllm - description: - - "Update vLLM image from v0.21.0 to v0.22.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1610 - -- config-keys: - - gptoss-fp4-h200-vllm - description: - - "Update vLLM image from v0.21.0 to v0.22.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1606 - -- config-keys: - - kimik2.5-int4-h200-vllm - description: - - "Update vLLM image from v0.21.0 to v0.22.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1600 - -- config-keys: - - minimaxm2.5-fp8-h100-vllm - description: - - "Update vLLM image from v0.19.1-cu130 to v0.22.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1609 - -- config-keys: - - minimaxm2.5-fp8-mi325x-vllm - description: - - "Update vLLM ROCm image from v0.20.2 to v0.22.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1619 - -- config-keys: - - gptoss-fp4-mi325x-vllm - description: - - "Update vLLM ROCm image from v0.21.0 to v0.22.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1622 - -- config-keys: - - minimaxm2.5-fp4-mi355x-vllm - description: - - "Update vLLM ROCm image from v0.21.0 to v0.22.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1620 - -- config-keys: - - minimaxm2.5-fp8-mi355x-vllm - description: - - "Update vLLM ROCm image from v0.21.0 to v0.22.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1617 - -- config-keys: - - minimaxm2.5-fp4-b200-vllm - description: - - "Update vLLM image from v0.21.0 to v0.22.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1611 - -- config-keys: - - gptoss-fp4-b200-vllm - description: - - "Update vLLM image from v0.21.0 to v0.22.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1604 - -- config-keys: - - kimik2.5-int4-b200-vllm - description: - - "Update vLLM image from v0.21.0 to v0.22.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1598 - -- config-keys: - - minimaxm2.5-fp8-b200-vllm - description: - - "Update vLLM image from v0.21.0 to v0.22.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1607 - -- config-keys: - - kimik2.5-fp4-b200-vllm - description: - - "Update vLLM image from v0.21.0 to v0.22.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1602 - -- config-keys: - - dsv4-fp4-mi355x-vllm - description: - - "Update vLLM ROCm image from nightly-4f940896a32c9e2a0eba7f50d521bf5f6b4de458 to v0.22.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1624 - -- config-keys: - - dsr1-fp8-mi355x-atom-mtp - description: - - "Update ATOM image to rocm/atom:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.3" - - "isl=1024/osl=1024: +47% to +116% improvement across conc 4-256 vs prior InferenceX numbers" - - "isl=8192/osl=1024: +47% to +131% improvement across conc 4-256" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1628 - -- config-keys: - - kimik2.5-fp4-mi355x-vllm - description: - - "Update vLLM ROCm image from v0.21.0 to v0.22.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1616 - -- config-keys: - - dsv4-fp4-mi355x-atom - description: - - "Add DeepSeek-V4-Pro FP4 MI355X ATOM DP-attention benchmark; image rocm/atom:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.3" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1626 - -- config-keys: - - dsv4-fp4-mi355x-vllm-mtp - description: - - "Add MTP speculative-decoding sibling for dsv4-fp4-mi355x-vllm (model: deepseek-ai/DeepSeek-V4-Pro) on vllm/vllm-openai-rocm:v0.22.0, per vllm-project/vllm#43385" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1630 - -- config-keys: - - dsv4-fp4-mi355x-sglang-mtp - description: - - "Add MTP/EAGLE speculative-decoding sibling for dsv4-fp4-mi355x-sglang (model: deepseek-ai/DeepSeek-V4-Pro) on v0.5.12.post1-rocm720-mi35x-20260601 (mainline ROCm nightly carrying sgl-project/sglang#26383); routes around the absent deep_gemm via SGLANG_OPT_FP8_WO_A_GEMM=0 + SGLANG_TOPK_TRANSFORM_512_TORCH=1." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1631 - -- config-keys: - - dsv4-fp4-b300-vllm - description: - - "Update DSv4 FP4 B300 vLLM image tag to v0.22.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1588 - -- config-keys: - - dsv4-fp4-mi355x-atom-mtp - description: - - "Add DeepSeek-V4-Pro FP4 MI355X ATOM MTP3 benchmark; image rocm/atom:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.3" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1627 - -- config-keys: - - dsv4-fp4-gb300-dynamo-sglang-mtp - description: - - "Update SGLang image from nightly-dev-cu13-20260509-9ee83034 to nightly-dev-20260527-14f81a67" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1637 - -- config-keys: - - minimaxm2.5-fp4-gb200-dynamo-vllm - description: - - "Add MiniMax-M2.5 NVFP4 GB200 disaggregated multinode vLLM benchmarks via Dynamo" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1642 - -- config-keys: - - minimaxm2.5-fp8-b200-dynamo-vllm - description: - - "Add MiniMax-M2.5 FP8 B200 disaggregated multinode vLLM benchmarks via Dynamo" - - "Add 1k1k/8k1k FP8 recipe set under benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m2.5-b200-fp8/" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1649 - -- config-keys: - - minimaxm2.5-fp4-b200-dynamo-vllm - description: - - "Add MiniMax-M2.5 NVFP4 B200 disaggregated multinode vLLM benchmarks via Dynamo" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1643 - -- config-keys: - - minimaxm2.5-fp4-b300-dynamo-vllm - description: - - "Add MiniMax-M2.5 NVFP4 B300 disaggregated multinode vLLM benchmarks via Dynamo" - - "Image: vllm/vllm-openai:v0.20.1" - - "Same 1k/1k and 8k/1k search space as gb300, plus a new tp8-1p1d at low concurrencies for both ISLs" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1652 - -- config-keys: - - minimaxm2.5-fp4-gb300-dynamo-vllm - description: - - "Add MiniMax-M2.5 NVFP4 GB300 disaggregated multinode vLLM benchmarks via Dynamo" - - "Add 1k1k/8k1k minimax recipe set under benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m2.5/" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1641 - -- config-keys: - - dsv4-fp4-b200-vllm - description: - - "Enable EPLB for DEP configs" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1655 - -- config-keys: - - minimaxm2.5-fp8-gb300-dynamo-vllm - description: - - "Add MiniMax-M2.5 FP8 GB300 disaggregated multinode vLLM benchmarks via Dynamo" - - "Add 1k1k/8k1k FP8 recipe set under benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m2.5-fp8/" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1647 - -- config-keys: - - minimaxm2.5-fp8-gb200-dynamo-vllm - description: - - "Add MiniMax-M2.5 FP8 GB200 disaggregated multinode vLLM benchmarks via Dynamo" - - "Add 1k1k/8k1k FP8 recipe set under benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m2.5-gb200-fp8/" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1648 - -- config-keys: - - minimaxm2.5-fp8-b300-dynamo-vllm - description: - - "Add MiniMax-M2.5 FP8 B300 disaggregated multinode vLLM benchmarks via Dynamo" - - "Add 1k1k/8k1k FP8 recipe set under benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m2.5-b300-fp8/" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1663 - -- config-keys: - - qwen3.5-fp8-h100-sglang - description: - - "Tune Qwen3.5-397B-A17B-FP8 H100 SGLang aggregate recipe for 1k/1k and 8k/1k sweeps" - - "Use TP8/EP1 for conc 1-8 and TP8/EP8 for conc 16-256" - - "Use scheduler-recv-interval values 2/60/30/1200/600/1920 for conc 1-4/8/16/32/64/128-256" - - "Set max-running-requests=256, chunked-prefill-size=16384, mem-fraction-static=0.8, cuda-graph-max-bs=CONC, and enable symm-mem" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1544 - -- config-keys: - - qwen3.5-fp8-mi355x-sglang - - qwen3.5-fp8-mi355x-sglang-mtp - description: - - "Bump image to lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260528." - - "Update script for aiter attention backend from triton." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1669 - -- config-keys: - - minimaxm2.5-fp8-h200-vllm - description: - - "Switch attention backend from FLASHINFER to FLASH_ATTN for the 8k/1k cell of MiniMax-M2.5 FP8 H200 vLLM." - - "1k/1k cell not changed in this PR: at 1k/1k all three measured configs." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1668 - -- config-keys: - - gptoss-fp4-mi355x-vllm - description: - - "Update GPT-OSS model for MI355X vLLM from amd/gpt-oss-120b-w-mxfp4-a-fp8 to openai/gpt-oss-120b" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1670 - -- config-keys: - - kimik2.5-fp4-b300-vllm - description: - - "Update vLLM image from v0.21.0 to v0.22.0" - - "Expand concurrency sweep for the 1k/1k and 8k/1k cells: TP4/EP1 conc 4-64 -> 1-128, TP8/EP1 conc-start 4 -> 1 (conc-end 4 unchanged)." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1674 - -- config-keys: - - kimik2.5-fp4-b200-vllm - description: - - "Expand concurrency sweep for the 1k/1k and 8k/1k cells: TP4/EP1 conc 4-64 -> 1-128, TP8/EP1 conc-start 4 -> 1 (conc-end 4 unchanged)." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1677 - -- config-keys: - - dsv4-fp4-gb300-dynamo-sglang - description: - - "Add wide-EP sweep configs (EP=12/16/24/32/40) matching srt-slurm PR#173 topology (18 nodes total)" - - "EP=12 15P+3D conc=12000, EP=16 14P+4D conc=8192, EP=24 12P+6D conc=3000, EP=32 10P+8D conc=2500, EP=40 8P+10D conc=2048" - - "Aligned decode params with Weiliang config: swa-full-tokens-ratio=0.20, max-running-requests=18432, moe-dense-tp-size=1; added prefill enable-dp-lm-head and cuda-graph-max-bs=512" - - "Remove 4 dominated old configs (4p-dep16-8n, 8p-dep16-12n, 10p-dep16-14n, 12p-dep12-15n) superseded by wide-EP frontier" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1586 - -- config-keys: - - qwen3.5-fp4-mi355x-sglang - - qwen3.5-fp4-mi355x-sglang-mtp - description: - - "Bump image to lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260604." - - "Update script for aiter attention backend from triton." - - "Enable aiter unfied attention." - - "Enable aiter allreduce fusion." - - "Remove sweep config mtp+cc256+tp2, which may OOM." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1680 - -- config-keys: - - minimaxm2.5-fp4-mi355x-vllm - description: - - "Enable vLLM Rust request frontend by exporting VLLM_USE_RUST_FRONTEND=1 in benchmarks/single_node/minimaxm2.5_fp4_mi355x.sh (v0.22.0 ROCm image ships the vllm-rs binary, so the flag engages it). Environment-only change; serve flags, TP/EP, attention/kernel settings unchanged" - - "The Rust frontend replaces only the Python serving/API layer (HTTP, tokenization, scheduling glue, detokenization) and spawns the same Python EngineCore, so GPU kernels/attention/MoE GEMM/KV cache are untouched" - - "A/B sweep (28 single-node points, 1k1k + 8k1k, TP 1/2/4) vs the Python-frontend baseline (run 26696260751): throughput Pareto-neutral (peak tok/s/GPU within <1.5%, frontiers coincident) and TPOT flat (+-0.5%); TTFT improves ~8% at 1k1k and ~22% at 8k1k (every point), the expected signature of lower frontend CPU latency before first token, scaling with input length" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1634 - -- config-keys: - - dsv4-fp4-b200-trt - - dsv4-fp4-b200-trt-mtp - description: - - "Update the B200 TensorRT-LLM DeepSeek-V4-Pro image to ghcr.io/semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066" - - "Sync the dsv4-fp4-b200-trt and dsv4-fp4-b200-trt-mtp recipes with the B200 aggregated frontier config (worker GC off, NCCL graph mixing off, mimalloc/PyTorch alloc tweaks, higher KV cache fractions by DP path, stream_interval 100, use_low_precision_moe_combine, DP batching_wait_iters 30, max_num_tokens drops the OSL term)" - - "MTP recipe uses max_draft_len with a variable default draft length, enable_lm_head_tp_in_adp on the DP-attn path, and removes timeout_iters from the DP config" - - "Raise dsv4-fp4-b200-trt-mtp DP-attn conc-end (1k ISL: 512->1024; 8k ISL: 128->256) to cover the new high-concurrency regime" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1699 - -- config-keys: - - dsv4-fp4-mi355x-atom-disagg - description: - - "Add multi-node ATOM disaggregated prefill/decode support for DeepSeek-V4-Pro on MI355X" - - "Add server_atom.sh: multi-node launcher using atom.entrypoints.openai_server with mooncake RDMA KV transfer and atomesh router" - - "Add env_atom.sh: ATOM/mooncake-specific environment (mooncake LD_LIBRARY_PATH, ATOM_MOE_GU_ITLV=1, AITER_BF16_FP8_MOE_BOUND=0)" - - "Add models_atom.yaml: per-model configs for ATOM engine" - - "Add atom-disagg sweep: 2P1D DPA+TP8 (conc 256-2048) and 1P1D TP8 (conc 4-256) at isl=8192/osl=1024" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1683 - - -- config-keys: - - dsv4-fp4-b300-trt - - dsv4-fp4-b300-trt-mtp - description: - - "Update the B300 TensorRT-LLM DeepSeek-V4-Pro image to ghcr.io/semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066" - - "B300 analog of PR #1699 (B200): sync the dsv4-fp4-b300-trt and dsv4-fp4-b300-trt-mtp recipes with the agg frontier config (worker GC off, NCCL graph mixing off, mimalloc/PyTorch alloc tweaks, higher KV cache fractions by DP path, stream_interval 100, use_low_precision_moe_combine, DP batching_wait_iters 30, max_num_tokens drops the OSL term)" - - "MTP recipe uses max_draft_len with a variable default draft length, enable_lm_head_tp_in_adp on the DP-attn path, and removes timeout_iters from the DP config" - - "Cap cuda_graph_config.max_batch_size at 1024 on both recipes: TRTLLM_MLA_EXTRA_OVERLAP hands MLA prologue tensors across streams without record_stream(), so CUDA-graph warmup at decode batch >1024 (repros at 1088, e.g. tp8/ep8 dp-attn conc-2048 on B300) use-after-frees into CUDA_ERROR_ILLEGAL_ADDRESS; workaround until NVIDIA/TensorRT-LLM#15265 ships in the image. Runtime --max_batch_size stays = CONC, so batches >1024 run eager" - - "B300-specific bits preserved (MODEL_PATH download block, TRTLLM_MHC_ENABLE_FUSED_HC=1, trtllm-serve MODEL_PATH); drop the 1k1k conc-2048 point on the tp8/ep8 DP-attn row (both recipes), the batch regime that triggers the MLA-overlap crash above; rest of the search space unchanged" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1703 - -- config-keys: - - dsr1-fp4-b200-dynamo-sglang-mtp - description: - - "Move both the 1k1k and 8k1k scenarios of the DeepSeek-R1 FP4 B200 dynamo-sglang MTP disagg sweep to local split recipes (one flat recipe YAML per topology under benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/{1k1k,8k1k}/disagg/mtp/), rather than referencing recipes from the srt-slurm repo" - - "1k1k: 4 MTP variants, behavior unchanged from the previous srt-slurm 1k1k recipe — 2 low-latency (dep4-1p prefill / tep8 decode at 5 and 6 decode nodes, conc up to 512) + 2 max-throughput (dep4-1p prefill / dep8 decode at 1 and 2 decode nodes, conc up to 1024)" - - "8k1k: 6-variant sweep — 3 low-latency (1p5d / 1p3d / 1p1d, TP4 prefill / TP8 decode, conc up to 512) + 3 MTP2 high-throughput (2p1d / 3p1d / 5p1d, DEP4 prefill / DEP8 decode, single concurrency 768 / 1024 / 2048); MTP2 recipes use scheduler-recv-interval=1, enable-dp-lm-head, spec 2 steps / 3 draft tokens, and UCX_TLS in the prefill/decode environments" - - "Bump container image to lmsysorg/sglang:v0.5.12.post1 (from v0.5.8.post1-cu130); the 1k1k recipes keep the dynamo-sglang container alias and follow the config image" - - "Clone srt-slurm at NVIDIA/srt-slurm@main for the srtctl/dynamo tooling and copy the local b200-fp4 recipes into the checkout" - - "Pin the 1k1k MTP recipes to the same dynamo source hash as the 8k1k recipes (dynamo.hash=5b4bc1dd70965017a737c71b19db5a0aeaa88727, install: true), so both scenarios build dynamo from an identical revision instead of relying on whatever is baked into the dynamo-sglang container alias" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1688 - -- config-keys: - - dsv4-fp4-mi355x-sglang - description: - - "Switch back to the main branch on InferenceX benchmarking. Since the AMD v4 model code has been merged into main, we now use the main branch instead of the previously used amd/deepseek_v4 branch. The image is bumped to the main branch image rocm/sgl-dev:v0.5.12.post1-rocm720-mi35x-20260610." - - "Refresh the script's env vars. The main branch introduced several env var renames and new default settings, so we refreshed the env vars in the script accordingly." - - "Enable the unified KV attention kernel. Set export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton for better performance." - - "Specify the prefill delay explicitly. Set --prefill-delayer-max-delay-ms 5000 to define a concrete prefill delay, allowing DP to batch more requests together per execution." - pr-link: - https://github.com/SemiAnalysisAI/InferenceX/pull/1701 - -- config-keys: - - dsv4-fp4-b300-sglang-mtp - description: - - "Align MTP env vars to GB300: replace PRECOMPILE=0 with FAST_WARMUP=1, add RADIX_FORCE_MISS, DEFAULT_THINKING, DSV4_REASONING_EFFORT=max" - - "Replace DP-attn env vars with shared GB300 block: MEGA_MOE_USE_FP4_ACTS, USE_MXF4_KIND, NUM_MAX_TOKENS_PER_RANK=8192" - - "Unify EAGLE spec-decoding to (3,1,4) for both DP-attn and TP-only paths, add --enable-deepseek-v4-fp4-indexer" - - "Bump image to nightly-dev-cu13-20260610-f332e526" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1700 - -- config-keys: - - dsr1-fp4-mi355x-sglang - description: - - "MI355x DSR1-FP4: Include TP4 configurations for 8k1k" - - "Expand the TP sweep (included TP=4) for 8k/1k configuration for conc=4 to 64" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1692 - -- config-keys: - - qwen3.5-fp4-mi355x-sglang - - qwen3.5-fp4-mi355x-sglang-mtp - description: - - "Bump image from lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260604 to lmsysorg/sglang-rocm:v0.5.13-rocm720-mi35x-20260612" - - "Enable AITER_FLYDSL_FORCE=1 in both non-MTP and MTP benchmark scripts" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1716 - -- config-keys: - - dsv4-fp4-mi355x-sglang - description: - - "Bump image to lmsysorg/sglang-rocm:v0.5.13-rocm720-mi35x-20260612." - - "Fix the intermediate_pad setting in the MoE computation in sglang PR#27858. This avoids the unnecessary overhead of computing useless padding." - - "Correct the chunk prefill setting size under tp8/dp8 config." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1715 - - -- config-keys: - - dsv4-fp4-gb200-dynamo-sglang - description: - - "Initial submission: DSv4-Pro FP4 disagg on GB200 with SGLang (8k/1k)." - - "Image: lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" - - "8 topologies sweeping low-latency (1p1d-tp8-tp8) through max throughput (6p1d-dep8-dep12)." - - "Updated 1p1d and 2p1d configs to match https://github.com/shyeh25/srt-slurm/commit/ede724d7cc9a780be5b84659f599733bf9fd0097" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1675 - -- config-keys: - - glm5-fp4-b200-sglang - - glm5-fp4-b200-sglang-mtp - - glm5-fp8-b200-sglang - - glm5-fp8-b200-sglang-mtp - description: - - "Update SGLang image from v0.5.12-cu130 to nightly-dev-cu13-20260605-7dc73766" - - "glm5-fp4-b200-sglang-mtp: override --mem-fraction-static from 0.85 to 0.8 when ISL=8192, OSL=1024, and CONC>128 (default 0.85 elsewhere)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1567 - -- config-keys: - - minimaxm2.5-fp4-b300-trt - description: - - "Add MiniMax-M2.5 FP4 (NVFP4) B300 TensorRT-LLM benchmark (model: nvidia/MiniMax-M2.5-NVFP4)" - - "Image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1712 - -- config-keys: - - minimaxm3-fp8-b300-vllm - description: - - "Initial submission: MiniMax-M3 MXFP8 day-zero single-node vLLM benchmark on B300 (model: MiniMaxAI/MiniMax-M3-MXFP8, 427B total / 26B active MoE with MSA sparse attention)" - - "Image: vllm/vllm-openai:minimax-m3 (already the cu130 build; M3 support is unmerged upstream — vllm-project/vllm#45381)" - - "--block-size 128 is mandatory (MSA sparse/index cache alignment); --language-model-only skips the vision encoder for text-only throughput; conc-scaled --max-cudagraph-capture-size" - - "Layouts: TP8 and TP4 (latency), TP4+EP4 / TP8+EP8 (TEP throughput), tp2-ep2, TP8+EP8 dp-attn (DEP) across 1k1k and 8k1k" - - "Serves from the launch_b300-nv.sh MODEL/MODEL_PATH split (model not in the SRE-staged /scratch/models list -> writable /data/models)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1724 - -- config-keys: - - minimaxm3-fp8-b200-vllm - description: - - "Initial submission: MiniMax-M3 MXFP8 day-zero single-node vLLM benchmark on B200 (model: MiniMaxAI/MiniMax-M3-MXFP8, 427B total / 26B active MoE with MSA sparse attention)" - - "Image: vllm/vllm-openai:minimax-m3 (M3 support is unmerged upstream; the dedicated image is built from the m3_release branch, vllm-project/vllm#45381)" - - "--block-size 128 is mandatory (MSA sparse/index cache alignment); --language-model-only skips the vision encoder for text-only throughput; conc-scaled --max-cudagraph-capture-size" - - "Layouts: TP8 and TP4 (latency), TP4+EP4 / TP8+EP8 (TEP throughput), TP8+EP8 dp-attn (DEP) across 1k1k and 8k1k" - - "Weights are not SRE-staged: launch_b200-dgxc.sh adds a MODEL_PATH case for /lustre/fsw/gharunners/models/MiniMax-M3-MXFP8; runners.yaml adds a b200-dgxc runner-type group" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1723 - -- config-keys: - - minimaxm3-fp8-b300-vllm-mtp - description: - - "Initial submission: MiniMax-M3 MXFP8 B300 vLLM benchmark with EAGLE3 speculative decoding (target: MiniMaxAI/MiniMax-M3-MXFP8, draft: Inferact/MiniMax-M3-EAGLE3, 3 speculative tokens)" - - "Image: vllm/vllm-openai:minimax-m3 (same m3_release-branch build as the non-MTP entry)" - - "Serve shape follows minimaxm3-fp8-b300-vllm (--block-size 128, --language-model-only); cudagraph capture scaled to CONC * (1 + spec tokens); prompts routed through the chat template for realistic acceptance" - - "Drafter pinned to FLASH_ATTN via speculative-config attention_backend: the EAGLE3 head is MHA and FlashInfer only supports the mandatory page size 128 through its GQA-only trtllm-gen kernel" - - "Layouts: TP8 / TP4 (latency), TP8+EP8 / TP4+EP4 (TEP), TP8+EP8 dp-attn (DEP) across 1k1k and 8k1k — non-MTP search space trimmed at the extreme-concurrency end, tp2-ep2 dropped (draft weights + draft KV headroom)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1733 - -- config-keys: - - minimaxm3-fp8-h200-vllm - - minimaxm3-fp8-h100-vllm - description: - - "Day-zero MiniMax-M3 MXFP8 single-node recipes for H200 and H100 (vLLM)." - - "Image: vllm/vllm-openai:minimax-m3 (dedicated day-zero image; M3 not in a stable release yet)." - - "Sweeps TP4/TP8, TP+EP (TEP), and DP-attention+EP (DEP) per https://recipes.vllm.ai/MiniMaxAI/MiniMax-M3; H100 is TP8-only (MXFP8 weights ~427 GB)." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1731 - -- config-keys: - - minimaxm2.5-fp4-b200-trt - description: - - "Add MiniMax-M2.5 NVFP4 B200 TensorRT-LLM single-node benchmark (1k1k and 8k1k)" - - "Image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc18" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1722 - -- config-keys: - - minimaxm3-fp8-h200-vllm-mtp - - minimaxm3-fp8-h100-vllm-mtp - description: - - "Initial submission: MiniMax-M3 MXFP8 H200 + H100 vLLM benchmarks with EAGLE3 speculative decoding (target: MiniMaxAI/MiniMax-M3-MXFP8, draft: Inferact/MiniMax-M3-EAGLE3, 3 speculative tokens) — spec-decoding=mtp variants of the day-zero H200/H100 recipes (PR #1731)" - - "Image: vllm/vllm-openai:minimax-m3 (same m3_release-branch build as the non-MTP entries)" - - "Serve shape follows the non-MTP H200/H100 scripts (--block-size 128, --language-model-only, HF_HUB_OFFLINE serve to dodge the shared-FS download-lock race); cudagraph capture scaled to CONC * (1 + spec tokens); prompts routed through the chat template for realistic acceptance" - - "Drafter pinned to FLASH_ATTN via speculative-config attention_backend: the EAGLE3 head is MHA and FlashInfer only supports the mandatory page size 128 through its GQA-only trtllm-gen kernel" - - "H200 layouts: TP4 / TP8 (latency), TP4+EP4 / TP8+EP8 (TEP), TP8+EP8 dp-attn (DEP) across 1k1k and 8k1k — non-MTP search space trimmed at the extreme-concurrency end. H100 is TP8-only (no room below TP8 at ~56 GB weights/GPU); DEP omitted (KV-cache init fails at high conc, draft only tightens it)" - - "Adds SPEC_SUFFIX to the three H100 launchers (cw, cr, dgxc-slurm) so spec-decoding=mtp routes to the _mtp script — they hardcoded _h100.sh and never gained the _mtp routing the H200 launchers have had since #392" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1739 - -- config-keys: - - minimaxm3-fp8-b200-vllm-mtp - description: - - "Initial submission: MiniMax-M3 MXFP8 B200 vLLM benchmark with EAGLE3 speculative decoding (target: MiniMaxAI/MiniMax-M3-MXFP8, draft: Inferact/MiniMax-M3-EAGLE3, 3 speculative tokens) — spec-decoding=mtp variant of the day-zero B200 recipe" - - "Image: vllm/vllm-openai:minimax-m3 (same m3_release-branch build as the non-MTP entry)" - - "Serve shape follows minimaxm3-fp8-b200-vllm (--block-size 128, --language-model-only); cudagraph capture scaled to CONC * (1 + spec tokens); prompts routed through the chat template for realistic acceptance" - - "Drafter pinned to FLASH_ATTN via speculative-config attention_backend: the EAGLE3 head is MHA and FlashInfer only supports the mandatory page size 128 through its GQA-only trtllm-gen kernel" - - "Layouts: TP8 / TP4 (latency), TP8+EP8 / TP4+EP4 (TEP), TP8+EP8 dp-attn (DEP) across 1k1k and 8k1k — non-MTP search space trimmed at the extreme-concurrency end, mirroring the minimaxm3-fp8-b300-vllm-mtp search space" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1741 - -- config-keys: - - minimaxm3-fp8-mi355x-vllm - description: - - "Initial submission: MiniMax-M3 MXFP8 day-zero single-node vLLM benchmark on MI355X / gfx950 (model: MiniMaxAI/MiniMax-M3-MXFP8, 427B total / 26B active MoE with MSA sparse attention)" - - "Image: vllm/vllm-openai-rocm:minimax-m3" - - "Matches vllm-project/recipes@2a3728ed MI355X guidance with --block-size 128, --attention-backend TRITON_ATTN, --kv-cache-dtype fp8, MiniMax-M3 tool/reasoning parsers, and automatic tool choice" - - "MI355X deviations required for fixed-sequence text benchmarking: --language-model-only, scenario-specific --max-model-len, and --enforce-eager to bypass the M3 decode CUDA-graph assertion" - - "B300-parity layouts and concurrency ranges: TP8, TP8+EP8, TP4, TP4+EP4, TP2+EP2, and TP8+EP8 dp-attn (DEP) across 1k1k and 8k1k" - - "launch_mi355x-amds.sh routes M3 weights to NFS /it-share/hf-hub-cache instead of node-local /var/lib NVMe" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1725 - -- config-keys: - - minimaxm3-fp8-h200-vllm-mtp - - minimaxm3-fp8-h100-vllm-mtp - description: - - "Start the TP-only latency rows of the MiniMax-M3 EAGLE3 MTP sweeps (H200, H100) at concurrency 1 instead of 4, matching the conc-1 start used on the non-MTP day-zero recipes — captures the single-request latency point. TEP/DEP rows keep their higher concurrency starts." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1743 - -- config-keys: - - minimaxm3-fp8-mi355x-vllm-mtp - description: - - "Initial submission: MiniMax-M3 MXFP8 MI355X (gfx950) vLLM benchmark with EAGLE3 speculative decoding (target: MiniMaxAI/MiniMax-M3-MXFP8, draft: Inferact/MiniMax-M3-EAGLE3, 3 speculative tokens) — spec-decoding=mtp variant of the MI355X day-zero recipe" - - "Image: vllm/vllm-openai-rocm:minimax-m3 (same day-zero ROCm build as the non-MTP entry)" - - "Serve shape follows minimaxm3-fp8-mi355x-vllm (--block-size 128, --language-model-only, --kv-cache-dtype fp8, --attention-backend TRITON_ATTN, --enforce-eager, minimax_m3 parsers); prompts routed through the chat template for realistic acceptance" - - "No attention_backend override on the drafter: the server runs on TRITON_ATTN, so the FlashInfer page-128/MHA limitation that forced FLASH_ATTN on the CUDA recipes does not apply on ROCm" - - "Layouts: TP8 / TP4 (latency), TP8+EP8 / TP4+EP4 (TEP), TP8+EP8 dp-attn (DEP) across 1k1k and 8k1k — non-MTP search space trimmed at the extreme-concurrency end, tp2-ep2 dropped, mirroring the minimaxm3-fp8-b300-vllm-mtp search space" - - "[AI generated draft test] The shipped ROCm image's AMD MiniMax-M3 model lacks SupportsEagle3, so the recipe patches it in-place at runtime (functionstackx/vllm#1, ported from nvidia/model.py) before serving — validates EAGLE3 on MI355X ahead of an image rebuild" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1745 - -- config-keys: - - minimaxm3-fp8-mi300x-vllm - description: - - "Initial submission: MiniMax-M3 MXFP8 day-zero single-node vLLM benchmark on MI300X / gfx942" - - "Image and serving shape reuse the MI355X recipe: vllm/vllm-openai-rocm:minimax-m3, block size 128, prefix caching disabled, TRITON_ATTN, language-model-only, and eager execution" - - "Use the default BF16 KV cache on MI300X because MiniMax-M3-MXFP8 lacks calibrated q/prob scales for ROCm FP8 attention; vLLM warns that its fallback scale of 1.0 may cause accuracy issues" - - "H100-aligned layouts and concurrency ranges: TP8 and TP8+EP8 across 1k1k and 8k1k" - - "Fix launch_mi300x-amds.sh node exclusion to use the current short Slurm node name" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1746 - -- config-keys: - - minimaxm3-fp8-mi300x-vllm-mtp - description: - - "Initial submission: MiniMax-M3 MXFP8 MI300X (gfx942) vLLM benchmark with EAGLE3 speculative decoding (target: MiniMaxAI/MiniMax-M3-MXFP8, draft: Inferact/MiniMax-M3-EAGLE3, 3 speculative tokens) — spec-decoding=mtp variant of the MI300X day-zero recipe" - - "Image: vllm/vllm-openai-rocm:minimax-m3 (same day-zero ROCm build as the non-MTP entry)" - - "Serve shape follows minimaxm3-fp8-mi300x-vllm: --block-size 128, --no-enable-prefix-caching, --language-model-only, --attention-backend TRITON_ATTN, --enforce-eager, minimax_m3 parsers, and the default BF16 KV cache (gfx942 lacks calibrated ROCm FP8 attention scales); prompts routed through the chat template for realistic acceptance" - - "TP8-only search space (gfx942 192 GB is memory-tight, like H100): TP8 latency rows started at conc 1, TP8+EP8 (TEP) at high concurrency, across 1k1k and 8k1k" - - "[AI generated draft test] The shipped ROCm image's AMD MiniMax-M3 model lacks SupportsEagle3, so the recipe patches it in-place at runtime (functionstackx/vllm#1, upstream vllm-project/vllm#45546; validated green on MI355X) before serving — validates EAGLE3 on MI300X ahead of an image rebuild" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1749 - -- config-keys: - - minimaxm3-fp8-mi300x-vllm - description: - - "Enable CUDA graphs for MiniMax-M3 MXFP8 on MI300X and set VLLM_USE_BREAKABLE_CUDAGRAPH=0 per AMD guidance" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1750 - -- config-keys: - - minimaxm3-fp8-mi300x-vllm-mtp - description: - - "Run the MiniMax-M3 MXFP8 MI300X EAGLE3 MTP recipe with CUDA graphs instead of --enforce-eager" - - "Drop --enforce-eager and set VLLM_USE_BREAKABLE_CUDAGRAPH=0 (matching the non-MTP MI300X recipe, #1750), which avoids the M3-decode breakable-cudagraph path that previously forced eager execution" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1756 - -- config-keys: - - minimaxm3-fp8-mi355x-vllm - description: - - "Enable CUDA graphs for MiniMax-M3 MXFP8 on MI355X, set VLLM_USE_BREAKABLE_CUDAGRAPH=0, and disable prefix caching" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1754 - -- config-keys: - - minimaxm3-fp8-mi325x-vllm - description: - - "Initial submission: MiniMax-M3 MXFP8 day-zero single-node vLLM benchmark on MI325X / gfx942" - - "Follows the official MI325X MXFP8 recipe with vllm/vllm-openai-rocm:minimax-m3, block size 128, TRITON_ATTN, and MiniMax-M3 tool/reasoning parsers; fixed-sequence text benchmarking adds language-model-only, the default BF16 KV cache, disabled prefix caching, and a scenario-specific max model length" - - "H200-aligned layouts and concurrency ranges: TP4 and TP8 latency, TP4/TP8 expert parallelism, and TP8 data-parallel attention across 1k1k and 8k1k" - - "Route the MI325X Hugging Face cache and runtime compiler caches to node-local storage, and mount ROCm GPU devices explicitly" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1748 - -- config-keys: - - minimaxm3-fp8-mi355x-vllm-mtp - description: - - "Run the MiniMax-M3 MXFP8 MI355X EAGLE3 MTP recipe with CUDA graphs instead of --enforce-eager" - - "Drop --enforce-eager and set VLLM_USE_BREAKABLE_CUDAGRAPH=0, which avoids the M3-decode breakable-cudagraph path that previously forced eager execution (the non-MTP MI355X recipe already got this in #1754)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1755 - -- config-keys: - - minimaxm3-fp8-mi325x-vllm-mtp - description: - - "Initial submission: MiniMax-M3 MXFP8 MI325X (gfx942) vLLM benchmark with EAGLE3 speculative decoding (target: MiniMaxAI/MiniMax-M3-MXFP8, draft: Inferact/MiniMax-M3-EAGLE3, 3 speculative tokens) — spec-decoding=mtp variant of the MI325X day-zero recipe (#1748)" - - "Image: vllm/vllm-openai-rocm:minimax-m3 (same day-zero ROCm build as the non-MTP entry)" - - "Serve shape follows minimaxm3-fp8-mi325x-vllm: --block-size 128, --no-enable-prefix-caching, --language-model-only, --attention-backend TRITON_ATTN, minimax_m3 parsers, default BF16 KV cache (gfx942 lacks calibrated ROCm FP8 attention scales). Runs with CUDA graphs (no --enforce-eager, VLLM_USE_BREAKABLE_CUDAGRAPH=0); prompts via chat template for realistic acceptance" - - "H200-style search space (TP4/TP8 latency, TP4+EP4/TP8+EP8 TEP, TP8+EP8 dp-attn DEP) trimmed at the extreme-concurrency end with TP-only latency rows started at conc 1" - - "[AI generated draft test] The shipped ROCm image's AMD MiniMax-M3 model lacks SupportsEagle3, so the recipe patches it in-place at runtime (functionstackx/vllm#1, upstream vllm-project/vllm#45546; validated green on MI355X/MI300X) before serving; also adds SPEC_SUFFIX to launch_mi325x-amds.sh so spec-decoding=mtp routes to the _mtp script" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1759 - -- config-keys: - - minimaxm3-fp8-mi300x-vllm - - minimaxm3-fp8-mi325x-vllm - description: - - "Extend the MiniMax-M3 MXFP8 MI300X and MI325X non-MTP sweeps down to concurrency 1 on the TP-only latency rows (was conc 4), to capture the single-request latency point; TEP/DEP rows keep their higher concurrency starts" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1760 - -- config-keys: - - minimaxm3-fp8-h100-vllm - - minimaxm3-fp8-h200-vllm - description: - - "Extend MiniMax-M3 MXFP8 H100/H200 non-MTP sweeps to concurrency 1 on the latency rows (H100: TP8; H200: TP4 and TP8) and add full TEP coverage from conc 1 to 256 (H100: TP8+EP8; H200: TP4+EP4 and TP8+EP8, incl. a new TP4+EP4 row for 8k1k). H200 TP8+EP8 upper bound moves 512->256 (high concurrency stays covered by the TP8+EP8 dp-attn DEP rows). DEP rows unchanged" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1761 - -- config-keys: - - dsv4-fp4-mi355x-sglang - description: - - "Switch fixed-seq-len search space from TP8 to TP4 for both isl=1024 and isl=8192 scenarios" - - "Expand isl=8192 coverage: add TP4 dp-attn sweep (conc 32–2048) and TP4 TP-only sweep (conc 1–32)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1762 - - -- config-keys: - - dsv4-fp4-gb300-dynamo-trt - - dsv4-fp4-gb300-dynamo-trt-mtp - description: - - "Add DeepSeek-V4-Pro MXFP4 GB300 disaggregated TRT-LLM benchmarks via Dynamo (27 STP + 27 MTP configs)" - - "New configs: dsv4-fp4-gb300-dynamo-trt (STP) and dsv4-fp4-gb300-dynamo-trt-mtp (MTP)" - - "Covers ISL 1024/OSL 1024 (14 STP + 14 MTP) and ISL 8192/OSL 1024 (13 STP + 13 MTP)" - - "Container: nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-deepseek-v4-dev.1" - - "Recipes sourced from NVIDIA/srt-slurm branch sa-submission-q2-2026" - - "Runner script updated to support dsv4 model prefix with dynamo-trt framework on GB300" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1689 - -- config-keys: - - minimaxm3-fp8-b200-vllm - description: - - "Align MiniMax-M3 B200 vLLM fixed-sequence serving with MiniMax-M2.5 FP8 B200 settings by setting VLLM_FLOAT32_MATMUL_PRECISION=high and restoring max cudagraph capture size 2048." - - "Add TP4+EP4 coverage for MiniMax-M3 B200: DP-attention rows for 1k1k/8k1k and the missing non-DP-attention row for 8k1k." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1779 - -- config-keys: - - minimaxm3-fp8-b200-vllm - description: - - "Align MiniMax-M3 B200 vLLM fixed-sequence serving with MiniMax-M2.5 FP8 B200 settings by setting VLLM_FLOAT32_MATMUL_PRECISION=high and restoring max cudagraph capture size 2048." - - "Add TP4+EP4 coverage for MiniMax-M3 B200: DP-attention rows for 1k1k/8k1k and the missing non-DP-attention row for 8k1k." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1779 - -- config-keys: - - dsv4-fp4-mi355x-sglang - description: - - "Switch fixed-seq-len search space from TP8 to TP4 for both isl=1024 and isl=8192 scenarios" - - "Expand isl=8192 coverage: add TP4 dp-attn sweep (conc 32–2048) and TP4 TP-only sweep (conc 1–32)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1762 - -- config-keys: - - kimik2.5-int4-mi355x-vllm - description: - - "Replace triton w4a16 MoE with FlyDSL w4a16 MoE" - - "Image: vllm/vllm-openai-rocm:nightly" - - "Add more sweep points" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1785 - -- config-keys: - - minimaxm3-fp8-b200-vllm-mtp - - minimaxm3-fp8-b300-vllm-mtp - description: - - "Align MiniMax-M3 B200/B300 EAGLE3 MTP serving with the MiniMax-M2.5 FP8 serving settings by setting VLLM_FLOAT32_MATMUL_PRECISION=high and using max cudagraph capture size 2048." - - "Add TP4+EP4 MTP coverage: DP-attention rows for 1k1k/8k1k and the missing non-DP-attention row for 8k1k." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1784 - -- config-keys: - - minimaxm3-fp8-b300-vllm - description: - - "Align MiniMax-M3 B300 vLLM fixed-sequence serving with MiniMax-M2.5 FP8 settings by setting VLLM_FLOAT32_MATMUL_PRECISION=high and restoring max cudagraph capture size 2048." - - "Add TP4+EP4 coverage for MiniMax-M3 B300: DP-attention rows for 1k1k/8k1k and the missing non-DP-attention row for 8k1k." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1781 - -- config-keys: - - qwen3.5-fp4-b200-trt - description: - - "Add Qwen3.5-397B-A17B-NVFP4 B200 single-node TensorRT-LLM benchmark (1k/1k and 8k/1k) with a TP/TEP/DEP parallelism sweep" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1711 - -- config-keys: - - minimaxm3-fp8-b200-vllm - - minimaxm3-fp8-b300-vllm - - minimaxm3-fp8-b200-vllm-mtp - - minimaxm3-fp8-b300-vllm-mtp - description: - - "Use the Marlin MoE backend for MiniMax-M3 B200/B300 TP-only vLLM configurations by adding --moe-backend marlin when expert parallelism is disabled." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1809 - -- config-keys: - - dsr1-fp8-gb300-dynamo-trt - description: - - "Fix gsm8k accuracy at 88% instead of 95% for a single point." - - "In previous submission, there was an numeric issue causing accuracy degradation and performance anomaly in some MTP points at certain concurrency." - - "This issue is now fixed in the latest TRTLLM release." - - "Also update all configs for DSR1 TRTLLM FP8 to reflect latest released image usage" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1767 - -- config-keys: - - minimaxm3-fp4-mi355x-atom - description: - - "Add day-zero MiniMax-M3 MXFP4 (amd/MiniMax-M3-MXFP4) single-node atom benchmark on MI355X, following the ROCm/ATOM MiniMax-M3 recipe (TP4, block size 128 for MSA, default KV cache dtype)." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1813 - - -- config-keys: - - glm5-fp4-gb300-dynamo-trt - description: - - "Add GLM-5 NVFP4 GB300 disaggregated TRT-LLM (STP, non-MTP) benchmarks via Dynamo (22 STP configs: 13 for 1K/1K, 9 for 8K/1K)" - - "Container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13" - - "Recipes sourced from NVIDIA/srt-slurm branch sa-submission-q2-2026 (gb300_nvfp4 STP recipes)" - - "Runner script launch_gb300-nv.sh: added dynamo-trt-specific glm5-fp4 case with SERVED_MODEL_NAME and SRT_SLURM_MODEL_PREFIX=nvidia/GLM-5-NVFP4" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1798 - -- config-keys: - - dsv4-fp4-mi355x-atom - description: - - "Update image to rocm/atom:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.4_20260612" - - "Update ISL=8192 search-space: TP8-only from conc=4-64, DPA from conc=128-1024 (previously conc=1-64 and DPA conc=64-512)" - - "Update Applied TBO on high concurrencies" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1717 - -- config-keys: - - dsv4-fp4-mi355x-atom - description: - - "Update image to rocm/atom:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.4_20260612" - - "Update ISL=8192 search-space: TP8-only from conc=4-64, DPA from conc=128-1024 (previously conc=1-64 and DPA conc=64-512)" - - "Update Applied TBO on high concurrencies" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1717 - -- config-keys: - - minimaxm3-fp8-b200-vllm-mtp - description: - - "Update the MiniMax-M3 B200 single-node image to vllm/vllm-openai:minimax-m3-0618-x86_64-cu130." - - "Enable FlashInfer TRT-LLM attention with FP8 indexer KV and KV cache; use TRITON_ATTN for the EAGLE3 drafter." - - "Switch TP-only configurations from explicit Marlin MoE to the new image's default FlashInfer TRT-LLM MoE backend." - - "Patch the image's MiniMax M3 MSA prefill path to materialize sliced top-k indices before CSR construction." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1832 - -- config-keys: - - minimaxm3-fp8-b200-vllm - description: - - "Update the MiniMax-M3 B200 single-node image to vllm/vllm-openai:minimax-m3-0618-x86_64-cu130." - - "Enable FlashInfer TRT-LLM attention with FP8 indexer KV and KV cache." - - "Switch TP-only configurations from explicit Marlin MoE to the new image's default FlashInfer TRT-LLM MoE backend." - - "Patch the image's MiniMax M3 MSA prefill path to materialize sliced top-k indices before CSR construction." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1833 - -- config-keys: - - minimaxm3-fp8-b300-vllm - description: - - "Update the MiniMax-M3 B300 single-node image to vllm/vllm-openai:minimax-m3-0618-x86_64-cu130." - - "Enable FlashInfer TRT-LLM attention with FP8 indexer KV and KV cache." - - "Switch TP-only configurations from explicit Marlin MoE to the new image's default FlashInfer TRT-LLM MoE backend." - - "Patch the image's MiniMax M3 MSA prefill path to materialize sliced top-k indices before CSR construction." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1834 - -- config-keys: - - minimaxm3-fp4-mi355x-atom - description: - - "Expand search space for minimaxm3-fp4-mi355x-atom: add TP2 and TP8 configurations, extend concurrency range to 256 for ISL1024 and ISL8192, and add TP8 conc=1-2 for ISL8192." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1825 - -- config-keys: - - minimaxm3-fp4-mi355x-atom - - minimaxm3-fp4-mi355x-atom-mtp - description: - - "Add minimaxm3-fp4-mi355x-atom-mtp: MiniMax-M3 MXFP4 on MI355X with EAGLE3 speculative decoding (3 draft tokens)" - - "Bump image to rocm/atom-dev:MiniMax-M3-20260623 for both fp4 atom entries" - - "Search space: TP2/TP4, ISL=1024,8192, OSL=1024, conc 1–256" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1917 - -- config-keys: - - minimaxm3-fp8-b300-vllm-mtp - description: - - "Update the MiniMax-M3 B300 single-node image to vllm/vllm-openai:minimax-m3-0618-x86_64-cu130." - - "Enable FlashInfer TRT-LLM attention with FP8 indexer KV and KV cache; use TRITON_ATTN for the EAGLE3 drafter." - - "Switch TP-only configurations from explicit Marlin MoE to the new image's default FlashInfer TRT-LLM MoE backend." - - "Patch the image's MiniMax M3 MSA prefill path to materialize sliced top-k indices before CSR construction." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1835 - -- config-keys: - - minimaxm3-fp8-mi300x-vllm-mtp - description: - - "Update the MI300X MiniMax-M3 EAGLE3 vLLM image to vllm/vllm-openai-rocm:nightly-b53b1c7ffe7aebdafd0876350f30e51d1226c92a" - - "Use FP8 KV cache" - - "Remove the runtime SupportsEagle3 source patch now included in the pinned nightly" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1843 - -- config-keys: - - minimaxm3-fp8-mi355x-atom - - minimaxm3-fp8-mi355x-atom-mtp - description: - - "Add minimaxm3-fp8-mi355x-atom: MiniMax-M3 MXFP8 single-node benchmark on MI355X using ATOM framework" - - "Uses rocm/atom-dev:MiniMax-M3-20260623; TP4, block size 128, ISL=1024,8192 OSL=1024" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1916 - -- config-keys: - - minimaxm3-fp8-mi355x-atom-mtp - description: - - "Add minimaxm3-fp8-mi355x-atom-mtp: same with EAGLE3 speculative decoding (3 draft tokens)" - - "Both use rocm/atom-dev:MiniMax-M3-20260619; search space mirrors FP4 atom variants (ISL=1024,8192 OSL=1024 TP2/TP4/TP8)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1867 - -- config-keys: - - minimaxm3-fp8-gb300-dynamo-vllm - description: - - "Switch to vllm/vllm-openai:nightly-aarch64 — contains upstream head_ratio fix (vllm-project/vllm#45879), avoids gemm1_alpha crash in minimax-m3-0618" - - "Add --moe-backend marlin for TP-only prefill/decode workers (no EP, no DP-attention) per PR #1809 pattern" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1735 - -- config-keys: - - kimik2.5-fp4-gb200-dynamo-trt - description: - - "Update Kimi K2.5 NVFP4 GB200 TRT-LLM Dynamo disaggregated configurations" - - "Bump image from nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.1.0-dev.2 to 1.3.0-dev.1-cuda13" - - "Replace allconc-style recipes with 25 per-concurrency STP recipes (13 ISL1K/OSL1K + 12 ISL8K/OSL1K)" - - "Recipes from srt-slurm sa-submission-q2-2026 branch (srt-slurm PR#89)" - - "ISL1K/OSL1K: conc 8–8192 across ctx1dep4 and ctx2dep4 topologies" - - "ISL8K/OSL1K: conc 5–4301 across ctx1dep4 to ctx9dep4 topologies" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1797 - -- config-keys: - - kimik2.5-fp4-gb300-dynamo-vllm - description: - - "Add Kimi-K2.5 NVFP4 GB300 disaggregated multinode vLLM benchmarks via Dynamo" - - "1k/1k topologies (5 shapes): 1p1d-dep4-dep16 (conc 2048,4096), 1p1d-dep4-dep24 (conc 1024), 1p2d-dep4-dep4 (conc 6144), 1p7d-tep4-tp4 (conc 8-128), 2p3d-dep4-dep8 (conc 8192)" - - "8k/1k topologies (6 shapes): 1p4d-dep4-tp8 (conc 4), 1p8d-dep4-tp4 (conc 32,128), 2p1d-dep4-dep24 (conc 2048), 3p1d-dep4-dep16 (conc 2048), 4p1d-dep4-dep8 (conc 3072, multi-frontend), 8p1d-dep4-dep24 (conc 15360, multi-frontend)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1861 - -- config-keys: - - kimik2.5-fp4-gb200-dynamo-trt - description: - - "Recover the skipped official ingest for PR #1797 from validated sweep run 27591355916 (attempt 6)" - - "No benchmark configuration change; reuse the exact 25-point fixed-sequence matrix and 2 eval jobs" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1869 - -- config-keys: - - glm5.1-fp4-mi355x-sglang - description: - - "Bump SGLang ROCm image from v0.5.12.post1-rocm720-mi35x-20260529 to v0.5.13.post1-rocm720-mi35x-20260622" - - "Enable aiter allreduce fusion via --enable-aiter-allreduce-fusion in benchmarks/single_node/fixed_seq_len/glm5.1_fp4_mi355x.sh" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1905 - -- config-keys: - - minimaxm3-fp8-b300-dynamo-vllm - description: - - "Add MiniMax-M3 MXFP8 B300 disaggregated vLLM benchmarks via Dynamo for 1k1k and 8k1k STP." - - "Add local srt-slurm recipes under benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8 and wire the B300 launcher to overlay them into srt-slurm." - - "Add right-sized TP4 decode variants with expert parallelism disabled and the Marlin MoE backend for selected low-concurrency 1k1k and 8k1k shapes." - - "Colocate the six-GPU TP4 prefill/decode pairs on one B300 node and enable CUDA IPC for NIXL KV transfer." - - "Patch the 0618 image's MiniMax M3 MSA prefill top-k slice to be contiguous before CSR construction." - - "Align 8k1k expert-parallel settings with the 1k1k recipes and correct the decode CUDA graph capture limit." - - "Backport NVIDIA/srt-slurm#38 to sanitize Slurm node-IP discovery output on the pinned submission branch." - - "Backport vllm-project/vllm#45879 so NIXL validates heterogeneous-TP KV block lengths using the GQA KV-head ratio." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1863 - -- config-keys: - - kimik2.5-fp4-gb300-dynamo-trt - description: - - "Add Kimi K2.5 NVFP4 GB300 TRT-LLM Dynamo disaggregated multinode configurations" - - "Image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13" - - "Recipes from srt-slurm sa-submission-q2-2026 branch (srt-slurm PR#214)" - - "STP-only: 14 configs for ISL1K/OSL1K (conc 8–8192), 11 configs for ISL8K/OSL1K (conc 4–2253)" - - "Topology range: 1–8 prefill workers (TP4/EP4), decode 1d-dep8 to 1d-dep32 and 4d/5d-tep8/tep4" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1796 - -- config-keys: - - kimik2.5-fp4-gb200-dynamo-vllm - description: - - "Refresh GB200 dynamo-vllm disagg sweep: bump image to vllm/vllm-openai:v0.21.0, dynamo 1.2.1, switch to checked-in kimi-k2.5-fp4 recipes" - - "1k/1k: 1p1d-dep4-dep8 (conc 4096,12288), 1p4d-dep4-tp8 (conc 4-128), 1p1d-dep4-dep16 (conc 4096,6144)" - - "8k/1k: 1p4d-dep4-tep4 (conc 128), 1p4d-dep4-tp8 (conc 4-256), 3p1d-dep4-dep16 (conc 1024), 6p1d-dep4-dep16 (conc 3072), 8p1d-dep4-dep16 (conc 6144)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1862 - -- config-keys: - - minimaxm3-fp4-mi355x-atom-disagg - description: - - "Add minimaxm3-fp4-mi355x-atom-disagg CI script: multi-node disaggregated PD on MI355X via ATOM for MiniMax-M3-MXFP4" - - "No MTP, KV_CACHE_DTYPE=auto (MXFP4 native, no fp8 override), MAX_MODEL_LEN=32768, MAX_NUM_BATCHED_TOKENS=32768" - - "server_atom.sh: conditional --kv_cache_dtype, MAX_MODEL_LEN/MAX_NUM_BATCHED_TOKENS/CUDAGRAPH_OPT support, syntax fixes" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1856 - -- config-keys: - - dsv4-fp4-mi355x-sglang - description: - - "Bump image to lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260618." - - "Enable SGLANG_DP_USE_GATHERV=1. Change to use all_gatherv + reduce_scatterv for tp + dp config." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1824 - -- config-keys: - - kimik2.5-fp4-gb300-dynamo-trt - description: - - "Recover the failed official ingest for PR #1796 from validated sweep run 27663808752 (attempt 2)" - - "Artifact-only recovery: reuse 23 fixed-sequence rows and 2 eval rows without rerunning benchmarks" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1884 - -- config-keys: - - qwen3.5-fp8-gb200-dynamo-sglang - description: - - "Add Qwen3.5-397B-A17B-FP8 GB200 disaggregated multinode SGLang benchmarks via Dynamo" - - "Image: lmsysorg/sglang:nightly-dev-cu13-20260608-303757cc" - - "6 topologies across 1k/1k and 8k/1k: 1P1D TP4 STP + wide-EP (DEP4 prefill / DEP16 decode) from 1P1D up to 8P1D, recipes under benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1810 - -- config-keys: - - minimaxm3-fp8-b300-dynamo-vllm - description: - - "Add MiniMax-M3 MXFP8 B300 disaggregated vLLM benchmarks via Dynamo for 1k1k and 8k1k STP." - - "Add local srt-slurm recipes under benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8 and wire the B300 launcher to overlay them into srt-slurm." - - "Add right-sized TP4 decode variants with expert parallelism disabled and the Marlin MoE backend for selected low-concurrency 1k1k and 8k1k shapes." - - "Colocate the six-GPU TP4 prefill/decode pairs on one B300 node and enable CUDA IPC for NIXL KV transfer." - - "Patch the 0618 image's MiniMax M3 MSA prefill top-k slice to be contiguous before CSR construction." - - "Align 8k1k expert-parallel settings with the 1k1k recipes and correct the decode CUDA graph capture limit." - - "Backport NVIDIA/srt-slurm#38 to sanitize Slurm node-IP discovery output on the pinned submission branch." - - "Backport vllm-project/vllm#45879 so NIXL validates heterogeneous-TP KV block lengths using the GQA KV-head ratio." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1893 - -- config-keys: - - minimaxm3-fp8-gb200-dynamo-vllm - description: - - "Initial addition of MiniMax-M3 FP8 GB200 disaggregated Dynamo-vLLM benchmark using vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - - "Allocate FlashInfer MNNVL workspace for one-shot all-reduce and materialize the MSA prefill top-k slice before CSR construction" - - "Preserve current Qwen3.5 and Kimi-K2.5 GB200 launcher paths while adding MiniMax-M3 shared-FS staging and atomic image import" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1938 - -- config-keys: - - minimaxm3-fp8-gb300-dynamo-vllm - description: - - "Update the GB300 MiniMax-M3 Dynamo-vLLM image to vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - - "Use the dedicated ARM64 MiniMax-M3 performance image; benchmark settings unchanged" - - "Allocate FlashInfer MNNVL workspace for one-shot TP8 all-reduce during CUDA graph capture" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1888 - -- config-keys: - - minimaxm3-fp8-gb300-dynamo-vllm - description: - - "Update the GB300 MiniMax-M3 Dynamo-vLLM image to vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - - "Use the dedicated ARM64 MiniMax-M3 performance image; benchmark settings unchanged" - - "Allocate FlashInfer MNNVL workspace for one-shot TP8 all-reduce during CUDA graph capture" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1897 - -- config-keys: - - qwen3.5-fp4-b200-trt-mtp - description: - - "Add Qwen3.5-397B-A17B-NVFP4 B200 single-node TensorRT-LLM benchmark with MTP speculative decode (1k/1k and 8k/1k) across a TP/TEP/DEP parallelism sweep" - - "Image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1894 - -- config-keys: - - dsr1-fp4-b200-sglang - description: - - "Update B200 FP4 SGLang image from lmsysorg/sglang:v0.5.12-cu130 to lmsysorg/sglang:v0.5.12.post1." - - "Update search space: 1k/1k TP4/EP4 conc 1-256 and TP8/EP8 conc 1; 8k/1k TP4/EP4 conc 1-128, add TP4/EP4 DP-attention conc 64-256, and keep TP8/EP8 conc 1." - - "For DP-attention runs, use TP-sized data parallelism with DP attention local-control broadcast, DP LM head, prefill delayer, scheduler recv interval 1, chunked prefill size 32768, and schedule conservativeness 3.33." - - "Set SGLANG_RADIX_FORCE_MISS=1, remove --disable-radix-cache, and explicitly pass --disable-piecewise-cuda-graph." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1792 - -- config-keys: - - dsv4-fp4-gb200-dynamo-sglang-mtp - description: - - "Initial submission: DSv4-Pro FP4 disagg on GB200 with SGLang + MTP speculative decoding (8k/1k)." - - "Image: lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" - - "8 topologies: low-latency 1p1d-tp8-tp8 + 1p6d-dep8-tp8; mid-curve 1p1d through 6p1d-dep8-dep16." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1676 - -- config-keys: - - minimaxm3-fp8-b300-dynamo-vllm - description: - - "Run the PR #1891 MiniMax-M3 MXFP8 B300 Dynamo-vLLM recipe set on top of current main." - - "Uses the vllm/vllm-openai:minimax-m3-0618-x86_64-cu130 image and the TEP4/TEP8 8k1k topologies not covered by PR #1890." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1891 - -- config-keys: - - dsv4-fp4-b300-sglang - description: - - "Update B300 FP4 SGLang (non-MTP) image to latest nightly: lmsysorg/sglang:nightly-dev-cu13-20260624-b2c8f7a2 (was nightly-dev-cu13-20260529-a8cfae0b)." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1913 - -- config-keys: - - minimaxm3-fp8-mi355x-atom-disagg - description: - - "Add minimaxm3-fp8-mi355x-atom-disagg CI recipe: multi-node disaggregated PD on MI355X via ATOM for MiniMax-M3-MXFP8" - - "Settings aligned with slurm reference: MEM_FRAC_STATIC=0.8, MAX_NUM_SEQS=128, BLOCK_SIZE=128, MAX_MODEL_LEN=32768, KV_CACHE_DTYPE=auto" - - "server_atom.sh: fix _MAX_CONC assignment before cudagraph size check; gate ATOM_MOE_GU_ITLV/AITER_BF16_FP8_MOE_BOUND on DeepSeek-V4-Pro only" - - "Search space: ISL=8192 and ISL=1024, 1P1D TP4, conc 1-512" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1865 - -- config-keys: - - minimaxm3-fp8-mi355x-vllm-disagg - description: - - "Initial submission: MiniMax-M3 MXFP8 MI355X vLLM disaggregated (prefill/decode) smoke test on the day-zero ROCm image (vllm/vllm-openai-rocm:minimax-m3) — 1 prefill (TP8) + 1 decode (TP8) across conc 1,2,4,8,16, validating the MoRI-IO KV-transfer disagg pipeline end-to-end for M3" - - "Layered on the MoRI-IO patch-removal infra (#1585): uses benchmarks/multi_node/amd_utils with the runtime MoRI patches removed" - - "Per-worker serve flags (models_vllm.yaml MiniMax-M3-MXFP8): --block-size 128 (MSA), --language-model-only, --kv-cache-dtype fp8, --attention-backend TRITON_ATTN, minimax_m3 parsers; no EP (TP8, MoE experts TP-sharded)" - - "M3 disagg script points MODEL_PATH at the cluster's shared HF cache (/it-share/hf-hub-cache) where the ~414 GB MiniMax-M3-MXFP8 checkpoint is pre-staged, instead of the launcher default /it-share/data; scoped to M3 only (other disagg models keep /it-share/data)" - - "Sweeps conc 1,2,4,8,16,32,64,128,256,512,1024 at both 1k1k and 8k1k (1P TP8 + 1D TP8). The 8k1k point makes the multi-node eval policy (8k1k + conc >= 16) mark one lm-eval on the highest-max-conc layout (eval-conc=median), validating the disagg pipeline's correctness; run with non-canary-full-sweep-enabled so the eval entry actually runs" - - "Adds two asymmetric prefill/decode layouts at both 1k1k and 8k1k alongside the TP8+TP8 sweep: 1P TP4 + 1D TP8 (smaller prefill, full-node decode) at conc 1,2,4,8,16,32,64,128,256; and balanced 1P TP4 + 1D TP4 at conc 64,128,256,512,1024. Per-worker TP comes from the master-config prefill/decode tp (server_vllm.sh rewrites the models_vllm.yaml --tensor-parallel-size placeholder); no EP, dp-attn off, PREFILL_NODES=1/DECODE_NODES=1 (TP4 uses half an 8-GPU node)" - - "Adds a 2P TP4 + 1D TP8 layout at both 1k1k and 8k1k for high conc 256,512,768,1024: two TP4 prefill workers (num-worker 2, PREFILL_NODES=2, each TP4 on half an 8-GPU node) feeding one TP8 decode (DECODE_NODES=1); 3 nodes total" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1762 - -- config-keys: - - minimaxm3-fp8-mi355x-atom-disagg - description: - - "Add minimaxm3-fp8-mi355x-atom-disagg CI recipe: multi-node disaggregated PD on MI355X via ATOM for MiniMax-M3-MXFP8" - - "Settings aligned with slurm reference: MEM_FRAC_STATIC=0.8, MAX_NUM_SEQS=128, BLOCK_SIZE=128, MAX_MODEL_LEN=32768, KV_CACHE_DTYPE=auto" - - "server_atom.sh: fix _MAX_CONC assignment before cudagraph size check; gate ATOM_MOE_GU_ITLV/AITER_BF16_FP8_MOE_BOUND on DeepSeek-V4-Pro only" - - "Search space: ISL=8192 and ISL=1024, 1P1D TP4, conc 1-512" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1927 - -- config-keys: - - dsv4-fp4-b200-dynamo-vllm - description: - - "Update the DeepSeek-V4-Pro B200 disaggregated Dynamo-vLLM benchmark to the vllm/vllm-openai:v0.23.0 image" - - "Lower max-num-batched-tokens to 16384 and gpu-memory-utilization to 0.9 on the high-throughput and max-throughput recipes to avoid OOM" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1899 - -- config-keys: - - minimaxm3-fp4-mi355x-vllm-disagg - description: - - "Initial submission: MiniMax-M3 MXFP4 disagg (prefill/decode) on MI355X with vLLM over the MoRI-IO KV connector (8k/1k)." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1914 - -- config-keys: - - minimaxm3-fp4-b200-vllm - description: - - "Add MiniMax-M3 NVFP4 (nvidia/MiniMax-M3-NVFP4) B200 single-node aggregated vLLM benchmark (no spec decode), runner b200-dgxc" - - "Image vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41 (bakes in MiniMax-M3 modelopt NVFP4 support, vllm-project/vllm PR #46380; no runtime patch needed)" - - "Weights pre-staged at /scratch/fsw/models/MiniMax-M3-NVFP4 (added minimaxm3-fp4 MODEL_PATH branch to launch_b200-dgxc.sh); --block-size 128 (MSA), --language-model-only" - - "Sweeps tp 4/8 with and without EP and dp-attn at 1k1k and 8k1k, conc 1-1024" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1932 - -- config-keys: - - minimaxm3-fp8-gb300-dynamo-vllm - description: - - "Refresh GB300 MiniMax-M3 disagg recipe set: replace disagg-gb300-* files with new naming convention; drop TP4+Marlin variants." - - "1k/1k topologies (6 shapes): 1p1d-dep2-dep4 (conc 8192), 1p2d-dep2-tep8 (conc 4,16,64,128,256), 2p2d-dep2-tep8 (conc 32), 2p3d-dep2-dep4 (conc 8192), 2p4d-dep2-dep4 (conc 8192), 4p2d-dep2-dep8 (conc 1024,4096)." - - "8k/1k topologies (9 shapes): 1p1d-dep2-dep8 (conc 256), 1p1d-dep2-tep8 (conc 128), 1p2d-dep2-tep8 (conc 32,64,128), 2p1d-dep2-dep8 (conc 512), 2p2d-dep2-tep8 (conc 16), 2p4d-dep2-tep4 (conc 4), 3p1d-dep2-dep8 (conc 1024), 3p2d-dep2-dep8 (conc 512), 6p1d-dep2-dep8 (conc 2048)." - - "Image unchanged: vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1925 - -- config-keys: - - minimaxm3-fp4-b300-vllm - description: - - "Add MiniMax-M3 NVFP4 (nvidia/MiniMax-M3-NVFP4) B300 single-node aggregated vLLM benchmark (no spec decode)" - - "Image vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41 (bakes in MiniMax-M3 modelopt NVFP4 support, vllm-project/vllm PR #46380; no runtime patch needed)" - - "Weights pre-staged read-only at /scratch/models/MiniMax-M3-NVFP4 (added MiniMax-M3-NVFP4 to launch_b300-nv.sh STAGED_MODELS); --block-size 128 (MSA), --language-model-only" - - "Sweeps tp 2/4/8 with and without EP and dp-attn at 1k1k and 8k1k, conc 1-1024" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1928 - -- config-keys: - - minimaxm3-fp4-b300-vllm-mtp - description: - - "Add MiniMax-M3 NVFP4 (nvidia/MiniMax-M3-NVFP4) B300 single-node aggregated vLLM benchmark with EAGLE3 speculative decoding (spec-decoding: mtp, 3 draft tokens via Inferact/MiniMax-M3-EAGLE3)" - - "Image vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41 (bakes in MiniMax-M3 modelopt NVFP4 support, vllm-project/vllm PR #46380; no runtime patch needed); prompts routed through the chat template" - - "Target weights pre-staged read-only at /scratch/models/MiniMax-M3-NVFP4 (added MiniMax-M3-NVFP4 to launch_b300-nv.sh STAGED_MODELS); EAGLE3 draft downloaded to the writable /data/models; --block-size 128 (MSA), --language-model-only" - - "Sweeps tp 4/8 with and without EP and dp-attn at 1k1k and 8k1k, conc 1-512" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1929 - -- config-keys: - - minimaxm3-fp4-mi355x-vllm - description: - - "Add a MiniMax-M3 MXFP4 single-node vLLM benchmark on MI355X using amd/MiniMax-M3-MXFP4." - - "Pin vllm/vllm-openai-rocm:nightly-3f5a1e1733200760169ff31ebe60a271072b199e, which includes upstream Quark MXFP4 support from vllm-project/vllm#45794." - - "Serve through the text-only language-model path with block size 128, TRITON_ATTN, MiniMax-M3 tool/reasoning parsers, automatic tool choice, and VLLM_USE_BREAKABLE_CUDAGRAPH=0; let vLLM select the MoE backend and retain the default KV-cache dtype." - - "Mirror the MiniMax-M3 MXFP8 MI355X TP/EP/DP-attention search space at 1k1k and 8k1k for a direct precision comparison." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1935 - -- config-keys: - - qwen3.5-fp4-gb300-dynamo-sglang - description: - - "Add Qwen3.5-397B-A17B-NVFP4 GB300 disaggregated multinode SGLang benchmarks via Dynamo." - - "Image: lmsysorg/sglang:nightly-dev-cu13-20260608-303757cc." - - "8k/1k STP recipes: 1P1D TP4 (conc 1-256), 5P1D DEP4+1D DEP16 (conc 2048, NIXL), 6P1D and 7P1D DEP4+1D DEP16 (conc 5120, Mooncake)." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1921 - -- config-keys: - - minimaxm3-fp4-mi355x-vllm-mtp - description: - - "Add a MiniMax-M3 MXFP4 MI355X vLLM benchmark with EAGLE3 speculative decoding using amd/MiniMax-M3-MXFP4 and Inferact/MiniMax-M3-EAGLE3 with three speculative tokens." - - "Reuse the pinned vllm/vllm-openai-rocm:nightly-3f5a1e1733200760169ff31ebe60a271072b199e image, text-only target path, TRITON_ATTN, automatic tool choice, MiniMax-M3 parsers, VLLM_USE_BREAKABLE_CUDAGRAPH=0, default KV-cache dtype, and automatic MoE backend selection." - - "Pass --use-chat-template for MTP acceptance and mirror the existing MiniMax-M3 MXFP8 MI355X MTP TP/EP/DP-attention search space at 1k1k and 8k1k." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1939 - -- config-keys: - - minimaxm3-fp8-mi355x-vllm-mtp - description: - - "Update the MiniMax-M3 MXFP8 MI355X vLLM EAGLE3 benchmark image from vllm/vllm-openai-rocm:minimax-m3 to vllm/vllm-openai-rocm:nightly-3f5a1e1733200760169ff31ebe60a271072b199e." - - "Benchmark configuration, EAGLE3 draft model, serving flags, and search space are unchanged." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1941 - -- config-keys: - - minimaxm3-fp8-mi355x-vllm - description: - - "Update the MiniMax-M3 MXFP8 MI355X vLLM image from vllm/vllm-openai-rocm:minimax-m3 to vllm/vllm-openai-rocm:nightly-3f5a1e1733200760169ff31ebe60a271072b199e." - - "Benchmark serving flags and TP/EP/DP-attention search space are unchanged." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1942 - -- config-keys: - - dsv4-fp4-b300-dynamo-vllm - description: - - "Update the DeepSeek-V4-Pro B300 disaggregated Dynamo-vLLM benchmark to the vllm/vllm-openai:v0.23.0 image" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1952 - -- config-keys: - - minimaxm3-fp4-mi355x-vllm - description: - - "Enable AITER MoE on MiniMax-M3 MXFP4 MI355X single-node vLLM STP: export VLLM_ROCM_USE_AITER=1, VLLM_ROCM_USE_AITER_MOE=1, and VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1; pass --moe-backend aiter." - - "Pin vllm/vllm-openai-rocm:nightly-4559c43a9526597c00cbcc4f59979496500268d1 (from nightly-3f5a1e1733200760169ff31ebe60a271072b199e) for AITER MoE and shared-expert fusion support (vllm-project/vllm#46419, vllm-project/vllm#46545)." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1954 - -- config-keys: - - minimaxm3-fp8-mi300x-vllm - description: - - "Update the MiniMax-M3 MXFP8 MI300X vLLM image from vllm/vllm-openai-rocm:minimax-m3 to vllm/vllm-openai-rocm:nightly-4559c43a9526597c00cbcc4f59979496500268d1." - - "Enable AITER kernels for MiniMax-M3 MXFP8 on MI300X/gfx942 via the single master toggle VLLM_ROCM_USE_AITER=1: the stock image left it unset, so the hot decode GEMMs and fused MoE ran on the generic kernels. The per-component AITER flags (MoE, linear, RMSNorm, FP8 batched-GEMM) default to True and are gated behind the master flag, so they are left at their defaults. Keep attention on TRITON_ATTN (VLLM_ROCM_USE_AITER_MHA=0, which defaults to True) because the MXFP8 checkpoint lacks calibrated q/prob scales for ROCm FP8 attention." - - "Add AMD-recommended, numerically-inert MI300X runtime knobs: TORCH_BLAS_PREFER_HIPBLASLT=1, NCCL_MIN_NCHANNELS=112 (raises RCCL channels above the ~32-64 default for TP8), GPU_MAX_HW_QUEUES=2 (caps HIP streams below the default of 4)." - - "Stack two accuracy-safe scheduling levers: --async-scheduling (overlaps CPU input-prep with GPU decode) and --max-num-batched-tokens 16384 (amortizes the per-step BF16-emulated MoE weight read of ~95 GB/rank over more prompt tokens, halving prefill weight-reads vs the 8192 default). Both are token-for-token identical (scheduling only); GSM8K exact-match holds at 0.959." - - "Switch the 1k1k conc256 search-space row from TP8/EP8 to TP8/EP1: the EP8 topology regressed high-concurrency throughput (434 vs 905 tok/s/gpu @ conc256, EP8 vs EP1) and EP1 matches the topology the prior AITER uplift was measured against." - - "Measured uplift on 8xMI300X, 1k1k random sweep (total tok/s/gpu): conc256 782.7->905 (EP8->EP1 + scheduling levers), conc128 598.9->628 (+4.9%), conc64 365.1->429 (+17.5%), conc32 295.6->327.4 (+10.8%), conc16 203.1->216.5 (+6.6%), conc8 127.6->136.6 (+7.1%), conc4 80.1->84.6 (+5.6%); conc1-2 unchanged (latency-bound). GSM8K exact-match holds at 0.959." - - "Drop the remaining TP8/EP8 row from the 8k1k search space (conc128-256), same fix as the 1k1k row above: the CI eval-only job at 8k1k TP8/EP8 conc256 caught GSM8K collapsing to ~0 (0.0000 strict / ~0.01 flexible). Reproduced locally on 8xMI300X against this image: --enable-expert-parallel (EP8) returns garbled/incoherent tokens on both /v1/completions and /v1/chat/completions for trivial prompts (e.g. 'The capital of France is' -> gibberish), while the same request against plain TP8 (EP1) answers correctly and scores 0.92 exact-match on a 50-question GSM8K sample. Root cause is EP8 itself (garbage generation), not --async-scheduling, which was tried and ruled out first. 8k1k is now TP8/EP1 across the full conc1-256 range." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1951 - -- config-keys: - - minimaxm3-fp8-mi355x-vllm - - minimaxm3-fp8-mi355x-vllm-mtp - description: - - "Update the MiniMax-M3 MXFP8 MI355X vLLM benchmark image from vllm/vllm-openai-rocm:minimax-m3 to vllm/vllm-openai-rocm:nightly-4559c43a9526597c00cbcc4f59979496500268d1, which includes the gfx950 mxfp8 MoE/linear tuning (vllm-project/vllm#45725), fused shared-experts MoE for the mxfp8 model (#46545), and the AITER flydsl MoE backend (#46184)." - - "Align the standard and EAGLE3 (MTP) bench scripts with vllm-project/recipes#581: gate VLLM_ROCM_USE_AITER on expert parallelism (on for EP/DP-attention runs, where the AITER fused MoE is the auto-selected backend; off for TP-only runs, which fall back to the native MXFP8 path since the master switch otherwise yields degenerate MiniMax-M3 output), export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 unconditionally (the router-append shared-experts fusion is independent of the master switch and self-disables under EP), and export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT6 (INT6 quick all-reduce)." - - "Retune the TP/EP search space to the best layout per concurrency band and drop redundant points (full TP8/EP8, TP2/EP2, DP-attention)." - - "minimaxm3-fp8-mi355x-vllm: 1k1k sweeps TP8 (conc 1-32), TP4 (conc 4-32), TP4/EP4 (conc 64-512); 8k1k sweeps TP8 (conc 1-2), TP4 (conc 2-128)." - - "minimaxm3-fp8-mi355x-vllm-mtp: 1k1k sweeps TP8 (conc 4-32), TP8/EP8 (conc 1-256), TP4 (conc 1-2 and 32-64), TP4/EP4 (conc 128-256); 8k1k sweeps TP8 (conc 1 and 4-16), TP4 (conc 16-128)." - - "Serving flags are otherwise unchanged." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1946 - -- config-keys: - - minimaxm3-fp4-b300-dynamo-vllm - description: - - "Add MiniMax-M3 NVFP4 B300 disaggregated vLLM benchmarks via Dynamo for 1k1k and 8k1k STP (no MTP)" - - "Use nvidia/MiniMax-M3-NVFP4 from /scratch/models/MiniMax-M3-NVFP4 with vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41, which includes vllm-project/vllm PR #46380; no runtime patch needed" - - "Reuse the existing MXFP8 B300 topology and concurrency matrix across 15 srt-slurm recipes, while dropping the FP8-only Marlin override from TP4 decode" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1931 - -- config-keys: - - minimaxm3-fp4-mi355x-vllm-mtp - description: - - "Enable AITER MoE on MiniMax-M3 MXFP4 MI355X single-node vLLM MTP (EAGLE3), mirroring the STP recipe: export VLLM_ROCM_USE_AITER=1, VLLM_ROCM_USE_AITER_MOE=1, VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1; pass --moe-backend aiter unconditionally (including expert parallelism)." - - "Fixes the ~8h engine-core startup hang on EP configs: with moe_backend=auto, EP fell back to Mxfp4MoeBackend.EMULATION, which deadlocked all expert-parallel workers building the Quark hw-emulation C++ kernel into a shared torch_extensions dir. Forcing --moe-backend aiter selects AITER_MXFP4_MXFP4 (no emulation build)." - - "Pin vllm/vllm-openai-rocm:nightly-4559c43a9526597c00cbcc4f59979496500268d1 (from nightly-3f5a1e1733200760169ff31ebe60a271072b199e), matching the STP recipe." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1964 - -- config-keys: - - minimaxm3-fp4-b300-dynamo-vllm - description: - - "Add MiniMax-M3 NVFP4 B300 disaggregated vLLM benchmarks via Dynamo for 1k1k and 8k1k STP (no MTP)" - - "Use nvidia/MiniMax-M3-NVFP4 from /scratch/models/MiniMax-M3-NVFP4 with vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41, which includes vllm-project/vllm PR #46380; no runtime patch needed" - - "Reuse the existing MXFP8 B300 topology and concurrency matrix across 15 srt-slurm recipes, while dropping the FP8-only Marlin override from TP4 decode" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1966 - -- config-keys: - - dsv4-fp4-b200-dynamo-vllm - description: - - "Add point-specific DeepSeek-V4-Pro FP4 B200 Dynamo-vLLM disaggregated recipes for the 8k/1k concurrency points c=1, 32, 64, 128, 256, 512." - - "Wire those rows in nvidia-master.yaml; c=16, 1024, 8192, and 12345 are not part of this update." - - "Recipes use vllm/vllm-openai:v0.23.0, DSV4 revision 0366e4e064385807ea86b088a5c6c878ff23343b, max-model-len 9280, with per-point P1/D1 (1 prefill node DEP=8 / 1 decode node) serving knobs." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1963 - -- config-keys: - - minimaxm3-fp4-b200-vllm-mtp - description: - - "Add MiniMax-M3 NVFP4 (nvidia/MiniMax-M3-NVFP4) B200 single-node aggregated vLLM benchmark with EAGLE3 speculative decoding (spec-decoding: mtp, 3 draft tokens via Inferact/MiniMax-M3-EAGLE3), runner b200-dgxc" - - "Image vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41 (bakes in MiniMax-M3 modelopt NVFP4 support, vllm-project/vllm PR #46380; no runtime patch needed); prompts routed through the chat template" - - "Target weights pre-staged at /scratch/fsw/models/MiniMax-M3-NVFP4 (added minimaxm3-fp4 MODEL_PATH branch to launch_b200-dgxc.sh); EAGLE3 draft fetched next to the target weights; --block-size 128 (MSA), --language-model-only" - - "Sweeps tp 4/8 with and without EP at 1k1k and 8k1k, conc 1-256" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1933 - -- config-keys: - - dsv4-fp4-b200-sglang - description: - - "Bump SGLang image from lmsysorg/sglang:deepseek-v4-blackwell (digest sha256:df18bfc4...) to mainline nightly lmsysorg/sglang:nightly-dev-cu13-20260628-da802ddc." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1923 - -- config-keys: - - dsr1-fp4-b200-sglang-mtp - description: - - "Update the B200 FP4 SGLang MTP image to lmsysorg/sglang:v0.5.12.post1." - - "Update the MTP search space to include the low-latency TP4/EP1 lane with MTP speculative decoding enabled." - - "Use 1k/1k TP4/EP1 c1-c32 and TP4/EP4 c64-c256; use 8k/1k TP4/EP1 c1-c32 and TP4/EP4 DP-attention c64-c256." - - "Enable the MTP DP-attention path with local-control broadcast, DP LM head, prefill delayer, chunked prefill size 32768, scheduler recv interval 1, and schedule conservativeness 3.33." - - "Use SGLANG_RADIX_FORCE_MISS=1, remove --disable-radix-cache and --enable-symm-mem, and explicitly pass --disable-piecewise-cuda-graph." - - "Align MTP runtime settings with the non-MTP aggregate: cuda graph max batch size 256, max running requests 256, mem fraction static 0.85, stream interval 10, and no explicit max-prefill-tokens." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1962 - -- config-keys: - - dsr1-fp4-b200-sglang - description: - - "Update the B200 FP4 SGLang aggregate search space to low-latency lanes: TP4/EP1 for the low-concurrency range and TP4/EP4 for the higher-concurrency range." - - "Use 1k/1k TP4/EP1 c1-c32 and TP4/EP4 c64-c256; use 8k/1k TP4/EP1 c1-c32 and TP4/EP4 DP-attention c64-c256." - - "Drop the TP8/EP8 single-concurrency points." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1975 - -- config-keys: - - minimaxm3-fp4-b300-vllm - description: - - "Update Minimax M3 b300 vllm image tag" - - "Update search space to cover more configs" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1990 - -- config-keys: - - minimaxm3-fp4-b200-vllm - description: - - "Update Minimax M3 b200 vllm image tag" - - "Update search space to cover more configs" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1978 - -- config-keys: - - minimaxm3-fp4-mi355x-atom - - minimaxm3-fp4-mi355x-atom-mtp - - minimaxm3-fp8-mi355x-atom - - minimaxm3-fp8-mi355x-atom-mtp - description: - - "Bump ATOM image from rocm/atom-dev:MiniMax-M3-20260623 to rocm/atom-dev:nightly_202607011530 for all single-node MiniMax-M3 ATOM recipes." - - "Add --online_quant_config with ptpc_fp8 and MoE layer exclusions (*block_sparse_moe) to all scripts." - - "Replace deprecated AITER_QUICK_REDUCE_CAST_BF16_TO_FP16=0 and ATOM_M3_SPARSE_USE_ASM_PA=1 with ATOM_FORCE_ATTN_TRITON=1." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2001 - -- config-keys: - - minimaxm3-fp8-mi355x-vllm - description: - - "Bump the MiniMax-M3 MXFP8 MI355X vLLM image to nightly-09663abde0f50944a8d5ea30120666024b503faa" - - "Use --linear-backend emulation for the MXFP8 dense-linear path (beats the stock nightly native MXFP8 linear: ~+26% tput / -21% TPOT at 8k1k conc1, ~+2-3% at high concurrency)" - - "Add --max-num-batched-tokens 32768 (env MAX_NUM_BATCHED_TOKENS) to enlarge the per-step prefill budget and improve TP4 throughput at high concurrency" - - "Enable the AITER master switch for TP-only (no-EP) runs via --moe-backend aiter: the earlier degenerate-output issue that forced it off for TP-only is fixed by vllm-project/vllm#47158, so TP4 uses the AITER_MXFP8 MoE path (verified GSM8K 0.9613 flex / 0.9621 strict on this nightly)" - - "Simplify both search spaces to a single TP4 conc 1-512 sweep for 1k1k and 8k1k (drop TP8 and TP4/EP4: TP8 has poor throughput/GPU and plain TP4 matches or beats TP4/EP4 at high concurrency)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2003 - -- config-keys: - - minimaxm3-fp8-mi355x-vllm-mtp - description: - - "Bump the MiniMax-M3 MXFP8 MI355X vLLM MTP (EAGLE3) image to nightly-09663abde0f50944a8d5ea30120666024b503faa, which natively supports SupportsEagle3 (the in-place EAGLE3 patch is now a no-op) and carries vllm-project/vllm#47158" - - "Port the non-MTP serve-command tuning to the MTP recipe: --moe-backend aiter, --linear-backend emulation, --max-num-batched-tokens 32768, and the AITER master switch on for TP-only runs (kept --speculative-config eagle3 with 3 draft tokens)" - - "Simplify both search spaces to a single TP4 conc 1-512 sweep for 1k1k and 8k1k (drop TP8 and TP4/EP4, matching the non-MTP entry; verified locally on this nightly at TP4 conc512, 5120/5120 completed)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2003 - -- config-keys: - - dsv4-fp4-mi355x-atom - description: - - "Remove --hf-overrides use_index_cache/index_topk_freq indexer-skipping override from the ATOM serve command (not allowed: reduces model architecture FLOPs per PR_REVIEW_CHECKLIST.md)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2037 - -- config-keys: - - dsv4-fp4-mi355x-vllm - description: - - "Bump DeepSeek-V4-Pro FP4 MI355X single-node vLLM STP image from vllm/vllm-openai-rocm:v0.22.0 to the latest nightly vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa." - - "The nightly enables two-stage attention kernels (split-KV decode), which reduce decode attention latency across all concurrency levels." - - "Employ the AITER MLA attention backend for the DeepSeek-V4 MLA path." - - "Switch the MoE backend from triton_unfused to AITER MoE (VLLM_ROCM_USE_AITER_MOE=1 + --moe-backend aiter) for the FP4 experts." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1980 - -- config-keys: - - qwen3.5-fp8-h200-sglang - - qwen3.5-fp8-h200-sglang-mtp - description: - - "Update SGLang image from qwen3.5-fp8-h200-sglang: lmsysorg/sglang:v0.5.12-cu130 / qwen3.5-fp8-h200-sglang-mtp: lmsysorg/sglang:v0.5.12 to lmsysorg/sglang:v0.5.14-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2061 - -- config-keys: - - kimik2.5-int4-b300-vllm - description: - - "Update vLLM image from vllm/vllm-openai:v0.21.0 to vllm/vllm-openai:v0.24.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2070 - -- config-keys: - - qwen3.5-fp4-b300-sglang - - qwen3.5-fp4-b300-sglang-mtp - description: - - "Update SGLang image from lmsysorg/sglang:v0.5.12-cu130 to lmsysorg/sglang:v0.5.14-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2057 - -- config-keys: - - qwen3.5-bf16-b200-sglang - - qwen3.5-bf16-b200-sglang-mtp - description: - - "Update SGLang image from lmsysorg/sglang:v0.5.12-cu130 to lmsysorg/sglang:v0.5.14-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2054 - -- config-keys: - - dsv4-fp4-mi355x-vllm-mtp - description: - - "Bump DeepSeek-V4-Pro FP4 MI355X single-node vLLM MTP image from vllm/vllm-openai-rocm:v0.22.0 to the latest nightly vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa." - - "The nightly enables two-stage attention kernels (split-KV decode), which reduce decode attention latency across all concurrency levels." - - "Employ the AITER MLA attention backend for the DeepSeek-V4 MLA path." - - "Switch the MoE backend from triton_unfused to AITER MoE (VLLM_ROCM_USE_AITER_MOE=1 + --moe-backend aiter) for the FP4 experts." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1981 - -- config-keys: - - qwen3.5-fp8-h100-sglang - - qwen3.5-fp8-h100-sglang-mtp - description: - - "Update SGLang image from lmsysorg/sglang:v0.5.12-cu130 to lmsysorg/sglang:v0.5.14-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2060 - -- config-keys: - - qwen3.5-fp8-b200-sglang - - qwen3.5-fp8-b200-sglang-mtp - description: - - "Update SGLang image from lmsysorg/sglang:v0.5.12-cu130 to lmsysorg/sglang:v0.5.14-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2058 - -- config-keys: - - kimik2.5-int4-b200-vllm - description: - - "Update vLLM image from vllm/vllm-openai:v0.22.0 to vllm/vllm-openai:v0.24.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2073 - -- config-keys: - - qwen3.5-fp4-b200-sglang - - qwen3.5-fp4-b200-sglang-mtp - description: - - "Update SGLang image from lmsysorg/sglang:v0.5.12-cu130 to lmsysorg/sglang:v0.5.14-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2056 - -- config-keys: - - qwen3.5-fp8-mi355x-sglang - - qwen3.5-fp8-mi355x-sglang-mtp - description: - - "Update SGLang ROCm image from lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260528 to lmsysorg/sglang:v0.5.14-rocm720-mi35x" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2067 - -- config-keys: - - kimik2.5-fp4-mi355x-vllm - description: - - "Update vLLM ROCm image from vllm/vllm-openai-rocm:v0.22.0 to vllm/vllm-openai-rocm:v0.24.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2074 - -- config-keys: - - kimik2.5-int4-mi355x-vllm - description: - - "Update vLLM ROCm image from vllm/vllm-openai-rocm:nightly-b8336c3c7c298e0878f22a7bf70f4e295b2f4e01 to vllm/vllm-openai-rocm:v0.24.0" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2077 - -- config-keys: - - dsv4-fp4-gb300-dynamo-vllm - description: - - "Refresh DSV4 8k/1k vLLM GB300 recipes with new w4a4 container and updated configs" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2010 - -- config-keys: - - minimaxm3-fp4-b200-vllm-mtp - description: - - "Update image tag to nightly" - - "Update B200 MTP search space" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2007 - -- config-keys: - - minimaxm3-fp4-b300-vllm-mtp - description: - - "Update image tag to nightly" - - "Update B300 MTP search space" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2006 - -- config-keys: - - minimaxm3-fp4-b300-vllm - description: - - "Add high concurrency configs" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1994 - -- config-keys: - - minimaxm3-fp8-mi355x-vllm-mtp - description: - - "Pin the EAGLE3 drafter to TRITON_ATTN via the speculative-config attention_backend override; without it the drafter falls back to a slow default backend." - - "Speeds up the draft forward measured on MI355X TP4 MXFP8, 8k/1k throughput gains growing with concurrency vs no override." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2107 - -- config-keys: - - glm5-fp8-gb200-dynamo-sglang - description: - - "Add GLM-5.1-FP8 GB200 disaggregated multinode SGLang benchmarks via Dynamo" - - "Image: lmsysorg/sglang:v0.5.12" - - "14 topologies across 1k/1k and 8k/1k: prefill TP8 STP + decode wide-EP (DEP16/DEP32 high-throughput) and per-node TP8 low-latency, recipes under benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1895 - -- config-keys: - - glm5-fp4-gb200-dynamo-sglang - description: - - "Initial submission: GLM-5 FP4 disagg on GB200 with Dynamo SGLang (8k1k + 1k1k, STP / no-MTP)." - - "Model: nvidia/GLM-5.1-NVFP4 (GB200 Lustre path: /mnt/lustre01/models/GLM-5.1-NVFP4)." - - "Image: lmsysorg/sglang:v0.5.11-cu130" - - "Recipes ported from srt-slurm PR #211 (recipes/gb200-fp4/glm5.yaml), split per topology under recipes/sglang/glm5/gb200-fp4/." - - "8k1k: 4 wide-EP (TP=32) max-throughput + 6 per-node (TP=4) low-latency topologies." - - "1k1k: 1 wide-EP max-throughput + 6 low-latency (DEP16/DEP32 + TP=4) topologies." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1780 - -- config-keys: - - dsr1-fp4-b200-dynamo-sglang-mtp - description: - - "Prune the 8k/1k MTP disagg low-latency points: 1p5d-tp8 to conc 4-32, 1p3d-tp8 to conc 32-64, and 1p1d-tp8 to conc 32." - - "Add 8k/1k MTP disagg high-throughput points: 1p1d and 4p1d (DEP4 prefill / DEP8 decode) at conc 512." - - "Add an 8k/1k MTP disagg 4p1d (DEP4 prefill / DEP8 decode) high-throughput point at conc 2048 with a dedicated engine tune (max-running-requests 2048, decode scheduler-recv-interval 1, prefill/decode stream-interval 100/34, SGLANG_HACK_SEQ_BOOTSTRAP_ROOM)." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2113 - -- config-keys: - - glm5-fp4-gb200-dynamo-sglang-mtp - description: - - "Add GLM-5.1 NVFP4 GB200 disaggregated dynamo-sglang MTP config covering 8k1k and 1k1k, high-throughput and low-latency topologies" - - "Image: lmsysorg/sglang:v0.5.13.post1-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2115 - -- config-keys: - - glm5-fp4-gb300-dynamo-sglang-mtp - description: - - "Add GLM-5.1 NVFP4 GB300 disaggregated dynamo-sglang MTP config covering 8k1k and 1k1k, high-throughput and low-latency topologies" - - "Image: lmsysorg/sglang:v0.5.13.post1-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2114 From 416fc3d42039502c077f131121297ed8679ee98c Mon Sep 17 00:00:00 2001 From: Hyukjoon Lee Date: Thu, 9 Jul 2026 15:29:39 +0900 Subject: [PATCH 5/9] agentx: restore perf-changelog with kimik2.7 entry Previous commit accidentally dropped perf-changelog.yaml (blob too large for inline arg). Restore it with the kimik2.7-fp4-mi355x-vllm-agentic entry appended. --- perf-changelog.yaml | 4644 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 4644 insertions(+) create mode 100644 perf-changelog.yaml diff --git a/perf-changelog.yaml b/perf-changelog.yaml new file mode 100644 index 0000000000..7c684c4c88 --- /dev/null +++ b/perf-changelog.yaml @@ -0,0 +1,4644 @@ +- config-keys: + - 70b-fp8-*-vllm + description: + - 'Add compilation-config ''{"custom_ops": ["-rms_norm", "-quant_fp8", "-silu_and_mul"]}'' as extra config to all benchmarks/70b_fp8_mi*.sh scripts' + - "6-7% uplift for llama for 6/8 configs" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/95 + +- config-keys: + - gptoss-fp4-*-trt + description: + - "Upgrade GPT-OSS TRT images from 'release:1.1.0rc2.post2' to '1.2.0rc0.post1'" + - "Add NCCL_GRAPH_REGISTER=0 to benchmarks/gptoss_fp4_b200_trt_slurm.sh" + - "Change kv_cache_config.dtype from 'auto' to 'fp8' in benchmarks/gptoss_fp4_b200_trt_slurm.sh" + - "Remove MOE_BACKEND=CUTLASS, now just defaults to TRTLLM" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/110 + +- config-keys: + - gptoss* + - dsr1* + description: + - "Remove Llama 70B runs to make room for multi-node disagg prefill+wideEP on h100/h200/b200/mi300/mi325/mi355" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/149 + +- config-keys: + - gptoss-fp4-b200-vllm + - gptoss-fp4-h100-vllm + - gptoss-fp4-h200-vllm + description: + - "Upgrade vLLM from 0.10.2 to 0.11.0 for GPT-OSS NVIDIA single-node configs" + - 'Add compilation-config ''{"cudagraph_mode":"PIECEWISE"}'' since vLLM 0.11.0 now defaults to FULL_AND_PIECEWISE' + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/159 + +- config-keys: + - dsr1* + description: + - "Fix bug where 1k8k and 8k1k full sweeps had incorrect max-model-len for DeepSeek" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/163 + +- config-keys: + - dsr1-fp4-b200-sglang + - dsr1-fp8-b200-sglang + - dsr1-fp8-h200-sglang + description: + - "Consolidate H200 and B200 SGLang configurations to use unified v0.5.5-cu129-amd64 image tag" + - "Update deprecated SGLang server arguments to current equivalents" + - "Replace --enable-ep-moe with --ep-size $EP_SIZE" + - "Replace --enable-flashinfer-trtllm-moe with --moe-runner-backend flashinfer_trtllm" + - "Add -e EP_SIZE to Docker run commands in launch scripts" + - "Set ep:4 for all tp:4 entries, ep:8 for all tp:8 entries" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/204 + +- config-keys: + - gptoss-fp4-mi355x-vllm + - gptoss-fp4-b200-vllm + description: + - "Extend concurrency to 128 for gptoss mi355x/b200 vllm configurations" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/209 + +- config-keys: + - gptoss-fp4-b200-trt + description: + - "Extend concurrency to 128 for gptoss b200 TRT configurations" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/233 + +- config-keys: + - gptoss-fp4-b200-trt + description: + - "Add benchmark script for GPTOSS FP4 B200 TRT-LLM" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/256 + +- config-keys: + - "*gb200-dynamo-sglang" + description: + - "Introduce improvements in GB200 SGLang DSR1 submission" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/257 + +- config-keys: + - dsr1-fp8-h200-trt + description: + - "Update TRT image from nvcr.io#nvidia/tensorrt-llm/release:1.2.0rc0.post1 to nvcr.io#nvidia/tensorrt-llm/release:1.2.0rc2" + - "Increase concurrency for some configurations" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/266 + +- config-keys: + - gptoss-fp4-b200-vllm + - gptoss-fp4-h100-vllm + - gptoss-fp4-h200-vllm + description: + - "Update vLLM image for NVIDIA configs from vLLM 0.11.0 to vLLM 0.11.2" + - "Add kv-cache-dtype: fp8 to benchmarks/gptoss_fp4_b200_docker.sh" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/273 + +- config-keys: + - dsr1-fp4-b200-sglang + - dsr1-fp8-b200-sglang + - dsr1-fp8-h200-sglang + description: + - "Update NVIDIA DeepSeek sglang Docker image from v0.5.5 to v0.5.6" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/276 + +- config-keys: + - gptoss-fp4-b200-vllm + - gptoss-fp4-h100-vllm + - gptoss-fp4-h200-vllm + description: + - "Update vLLM image from v0.11.2 to v0.13.0" + - "Add VLLM_MXFP4_USE_MARLIN=1 to H100 and H200 benchmark scripts" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/327 + +- config-keys: + - dsr1-fp4-mi355x-sglang + description: + - "Update MI355x Deepseek-R1 FP4 SGLang Image to upstream v0.5.6.post1" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/330 + +- config-keys: + - dsr1-fp8-mi300x-sglang + - dsr1-fp8-mi325x-sglang + - dsr1-fp8-mi355x-sglang + description: + - Use upstream SGLang images on mi300, mi325 and mi355 for dsr1fp8 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/332 + +- config-keys: + - dsr1-fp4-gb200-dynamo-trt + - dsr1-fp4-gb200-dynamo-sglang + - dsr1-fp8-gb200-dynamo-sglang + description: + - "Add more configurations for GB200 SGLang DSR1" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/335 + +- config-keys: + - dsr1-fp4-gb200-dynamo-sglang + - dsr1-fp8-gb200-dynamo-sglang + description: + - "fix: Pruning unnecessary concurrencies " + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/358 + +- config-keys: + - dsr1-fp4-mi355x-sglang + description: + - "Updating MI355x Deepseek-R1 FP4 SGLang Image to upstream v0.5.6.post2" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/369 + +- config-keys: + - gptoss-fp4-gb200-dynamo-trt + - gptoss-fp4-b200-trt + description: + - Explicitly add EP=TP for DP attention configs for B200 AGG nvidia-master file. Multinode Refactor inadvertently changed default EP=1 + - Add GPTOSS DISAGG configurations for GB200 1k1k and 8k1k. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/387 + +- config-keys: + - dsr1-fp4-b200-trt-mtp + - dsr1-fp8-b200-trt-mtp + - dsr1-fp8-h200-trt-mtp + description: + - Add MTP (Multi-Token Prediction) support for single-node TRT configs + - Add spec-decoding field to config entries and update launch scripts to select MTP benchmark scripts + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/392 + +- config-keys: + - dsr1-fp4-mi355x-sglang + description: + - "Updating MI355x Deepseek-R1 FP4 SGLang Image to upstream v0.5.7" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/395 + +- config-keys: + - dsr1-fp8-mi355x-sglang-disagg + description: + - "Add PD disaggregation (1P2D) for Mi355X" + - "Includes with and without speculative decoding" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/409 + +- config-keys: + - dsr1-fp8-b200-sglang + description: + - "Adds TP4 configurations to DSR1-FP8 B200 SGLang deployment experiments" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/411 + +- config-keys: + - dsr1-fp8-mi355x-atom + - dsr1-fp4-mi355x-atom + - gptoss-fp4-mi355x-atom + description: + - Add internal AMD ATOM inference engine for DeepSeek R1 FP8, FP4 and GPTOSS FP4 Mi355X + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/419 + +- config-keys: + - gptoss-fp4-mi300x-vllm + - gptoss-fp4-mi325x-vllm + description: + - "Update AMD MI300X and MI325X GPT-OSS 120B vLLM to use upstream ROCm image vllm/vllm-openai-rocm:v0.14.0" + - "Remove deprecated --async-scheduling flag (now enabled by default in vLLM v0.14.0)" + - "Remove deprecated --max-seq-len-to-capture flag" + - "Add HIP_VISIBLE_DEVICES env var for Ray compatibility in vLLM 0.14+" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/496 + +- config-keys: + - dsr1-fp4-gb200-dynamo-trt + description: + - "Update Dynamo TRT image from 0.5.1-rc0.pre3 to 0.8.1.post2" + - "Update TRT configurations" + - "Refactor configurations to use CONFIG_FILE-based recipes instead of inline parameter settings" + - "Introduce srt-slurm workflow for launching Dynamo jobs" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/510 + +- config-keys: + - gptoss-fp4-mi300x-vllm + - gptoss-fp4-mi325x-vllm + description: + - "Fix AITER env vars for vLLM v0.14.0 on AMD MI300X and MI325X" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/535 + +- config-keys: + - dsr1-fp8-h200-sglang + description: + - "Update H200 DeepSeek R1 FP8 SGLang image from v0.5.6 to v0.5.7" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/538 + +- config-keys: + - dsr1-fp8-mi300x-sglang + description: + - "Update MI300X DeepSeek R1 FP8 SGLang image from v0.5.5.post3 to v0.5.7" + - "Add SGLANG_AITER_MLA_PERSIST=1 for persistent MLA kernel optimization" + - "Set --kv-cache-dtype fp8_e4m3 for fp8 KV cache" + - "Set --attention-backend aiter for AMD aiter attention backend" + - "Update chunked-prefill-size and max-prefill-tokens from 196608 to 131072" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/544 + +- config-keys: + - dsr1-fp8-mi325x-sglang + description: + - "Update MI325X DeepSeek R1 FP8 SGLang image from v0.5.5.post3 to v0.5.7" + - "Add SGLANG_AITER_MLA_PERSIST=1 for persistent MLA kernel with fp8 KV cache" + - "Add --kv-cache-dtype fp8_e4m3 for explicit FP8 KV cache" + - "Add --attention-backend aiter for AMD aiter attention backend" + - "Reduce chunked-prefill-size from 196608 to 131072" + - "Reduce max-prefill-tokens from 196608 to 131072" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/545 + +- config-keys: + - dsr1-fp8-h200-dynamo-trt + description: + - "Add DSR1 FP8 H200 Dynamo TRT-LLM disaggregated multinode configuration" + - "Image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post1" + - "Runner: h200-dgxc with multinode and disagg enabled" + - "Includes MTP and STP configurations for 1k1k and 8k1k sequence lengths" + - "Concurrency levels: 4, 8, 16, 32, 64, 128, 256, 512" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/570 + +- config-keys: + - dsr1-fp8-mi355x-sglang + description: + - "Update MI355X DeepSeek R1 FP8 SGLang image from v0.5.5.post3 to v0.5.8" + - "Key fix: Disables mla persistent kernel when not using fp8 kv_cache (https://github.com/sgl-project/sglang/pull/17327)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/572 + +- config-keys: + - dsr1-fp8-h200-dynamo-sglang + description: + - "Add DSR1 FP8 H200 Dynamo SGLang disaggregated multinode configuration" + - "Image: lmsysorg/sglang:v0.5.8-cu130-runtime" + - "Runner: h200-multinode-slurm with multinode and disagg enabled" + - "Recipes sourced from srtslurm repo (recipes/h200/)" + - "1k1k configs: aggregated, low-latency (1P9D), high-throughput TEP (1P6D), DEP (1P6D)" + - "8k1k configs: aggregated, TEP configs (1P7D, 1P6D, 1P3D, 2P3D), DEP (1P1D)" + - "Concurrency levels range from 1 to 2048 depending on configuration" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/582 + +- config-keys: + - dsr1-fp4-b300-dynamo-trt + description: + - "Add DSR1 FP4 B300 Dynamo TRT configurations" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/585 + +- config-keys: + # NVIDIA single-node + - dsr1-fp4-b200-sglang + - dsr1-fp4-b200-trt + - dsr1-fp4-b200-trt-mtp + - dsr1-fp8-b200-sglang + - dsr1-fp8-b200-trt + - dsr1-fp8-b200-trt-mtp + - dsr1-fp8-h200-sglang + - dsr1-fp8-h200-trt + - dsr1-fp8-h200-trt-mtp + - gptoss-fp4-b200-trt + - gptoss-fp4-b200-vllm + - gptoss-fp4-h100-vllm + - gptoss-fp4-h200-trt + - gptoss-fp4-h200-vllm + # AMD single-node + - dsr1-fp4-mi355x-sglang + - dsr1-fp4-mi355x-atom + - dsr1-fp8-mi300x-sglang + - dsr1-fp8-mi325x-sglang + - dsr1-fp8-mi355x-sglang + - dsr1-fp8-mi355x-atom + - gptoss-fp4-mi300x-vllm + - gptoss-fp4-mi325x-vllm + - gptoss-fp4-mi355x-vllm + - gptoss-fp4-mi355x-atom + description: + - Add official GSM8k eval results to GPT-OSS and DeepSeek R1 scenarios + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/587 + evals-only: true + +- config-keys: + - dsr1-fp4-b200-dynamo-trt + description: + - "Update DSR1 FP4 B200 Dynamo TRT configurations" + - "Update TRTLLM version to 1.2.0rc6.post2" + - "Transform to use srt-slurm recipes" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/588 + +- config-keys: + - dsr1-fp8-b200-trt + description: + - "Update TensorRT-LLM container from release:1.1.0rc2.post2 to release:1.2.0rc6.post2" + - "Change default MOE backend from DEEPGEMM to TRTLLM" + - "Add dynamic piecewise CUDA graphs for 1k1k (CONC≥64) and 8k1k (CONC≥64) workloads" + - "Add delay batching (batch_wait_timeout_iters/batch_wait_max_tokens_ratio) for 1k1k high-concurrency" + - "Add dynamic KV cache memory fraction tuning (0.7-0.8) based on ISL/OSL/TP configuration" + - "Update search space: remove EP=TP constraint, add TP=4 configurations, extend concurrency ranges" + - "Add TLLM_OVERRIDE_LAYER_NUM=61 to avoid OOM errors" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/594 + +- config-keys: + - dsr1-fp4-mi355x-sglang + description: + - "Update SGLang image from v0.5.7 to v0.5.8 for DeepSeek-R1 FP4 on MI355x" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/595 + +- config-keys: + - dsr1-fp8-mi355x-sglang + description: + - "Disable torch.compile for MI355X DeepSeek-R1 FP8 SGLang" + - "set cuda-graph-max-bs to CONC" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/613 + +- config-keys: + - dsr1-fp8-b200-dynamo-trt + description: + - "Introduce new DSR1 FP8 B200 Dynamo TRT configurations for 8k1k and 1k1k" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/616 + +- config-keys: + - dsr1-fp8-gb200-dynamo-trt + description: + - "Add DeepSeek R1 FP8 GB200 Dynamo TRT-LLM disaggregated multinode configurations" + - "Image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post2" + - "1k1k: 14 scenarios (7 MTP, 7 STP) with varying DP attention/TEP modes" + - "1k8k: 10 scenarios (5 MTP, 5 STP) for long output generation" + - "8k1k: 14 scenarios (7 MTP, 7 STP) for long context workloads" + - "Prefill workers: 1-5P, Decode workers: 1-4D, TP/EP: 8/16/32" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/617 + +- config-keys: + - dsr1-fp4-gb300-dynamo-trt + description: + - "Add DeepSeek-R1 FP4 GB300 Dynamo TRT disaggregated multinode configurations" + - "Image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post2" + - "Includes MTP and STP configs for 1k1k, 1k8k, and 8k1k sequence lengths" + - "Add gb300-nv runner and launch script for srt-slurm integration" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/618 + +- config-keys: + - dsr1-fp4-b200-trt + description: + - "Update TensorRT-LLM container from release:1.1.0rc2.post2 to release:1.2.0rc6.post2" + - "Change default MOE backend from DEEPGEMM to TRTLLM" + - "Add dynamic piecewise CUDA graphs for 1k1k (TEP8 and CONC64)" + - "Update search space: remove EP=TP constraint, add TP=4 configurations, extend concurrency ranges" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/620 + +- config-keys: + - dsr1-fp4-mi355x-sglang-disagg + description: + - "enable PD/D for both MTP and non-MTP MI355X DeepSeek-R1 FP4 SGLang" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/622 + +- config-keys: + - dsr1-fp8-b200-sglang-mtp + description: + - "Add MTP (Multi-Token Prediction) support for DeepSeek R1 FP8 B200 SGLang using EAGLE speculative decoding" + - "Image: lmsysorg/sglang:v0.5.8-cu130-amd64" + - "Add benchmark script dsr1_fp8_b200_mtp.sh with EAGLE speculative decoding (num-steps=2, draft-tokens=3, topk=1)" + - "Update launch_b200-dgxc.sh to support SPEC_SUFFIX for MTP script selection" + - "Configurations: TP=8, EP=1, concurrency 4-64 for 1k1k, 1k8k, and 8k1k sequence lengths" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/626 + +- config-keys: + - dsr1-fp8-gb300-dynamo-trt + description: + - "Add DeepSeek R1 FP8 GB300 Dynamo TRT-LLM disaggregated multinode configurations for 8k1k and 1k1k" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/627 + +- config-keys: + - dsr1-fp8-b200-trt-mtp + description: + - Update to the latest TRTLLM 1.2 release container version + - Fine-tune the choice of parallelism in nvidia-master file, mainly going to TP for most points + - Enable piecewise CUDA graphs under most conditions + - fine-tune max batch sizes and other optimizations + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/632 + +- config-keys: + - dsr1-fp4-gb200-dynamo-sglang + description: + - "Update SGLang image from v0.5.5.post2 to v0.5.8-cu130" + - "Add FP4 model path separation via SRT_SLURM_MODEL_PREFIX in launch script" + - "Refactor to use CONFIG_FILE-based srt-slurm recipes instead of inline parameters" + - "Add 1k1k configurations: low-latency (1P2D), mid-curve (4P8D), max-tpt (4P12D)" + - "Add 8k1k configurations: low-latency (1P4D), mid-curve (6P12D), max-tpt (10P8D)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/633 + +- config-keys: + - dsr1-fp8-gb200-dynamo-sglang + - dsr1-fp8-gb300-dynamo-sglang + description: + - "Update GB200 and GB300 configs for DSR1 FP8 SGLANG STP mode" + - "Image: lmsysorg/sglang:v0.5.8-cu130" + - "Update prefill/decode worker counts, TP/EP parallelism, and dp-attn settings for 1k1k and 8k1k" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/635 + +- config-keys: + - dsr1-fp4-gb300-dynamo-sglang + description: + - "Add GB300 FP4 Dynamo SGLang disaggregated multinode configuration" + - "Image: lmsysorg/sglang:v0.5.8.post1-cu130-runtime" + - "Recipes sourced from srt-slurm repo (recipes/gb300-fp4/ folder)" + - "Add 1k1k configurations: low-latency (1P2D), mid-curve (4P8D), max-tpt (4P12D)" + - "Add 8k1k configurations: low-latency (1P4D), mid-curve (6P12D), max-tpt (10P8D)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/636 + +- config-keys: + - dsr1-fp8-b300-dynamo-trt + description: + - "New B300 FP8 Dynamo TRT configurations" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/638 + +- config-keys: + - gptoss-fp4-b200-trt + description: + - "Update GPT-OSS FP4 B200 TRT pareto configurations and new container image" + - "Extend maximum concurrency to 256 across all sequence lengths" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/639 + +- config-keys: + - dsr1-fp8-h200-dynamo-sglang + description: + - "Add MTP (EAGLE speculative decoding) configs alongside STP" + - "Update container to lmsysorg/sglang:v0.5.8.post1-cu130" + - "Remove aggregated configs, keep only disaggregated" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/640 + +- config-keys: + - dsr1-fp4-b200-trt-mtp + description: + - "Upgrade TensorRT-LLM container from release:1.1.0rc2.post2 to release:1.2.0rc6.post3" + - "Enable dynamic piecewise CUDA graphs for several conditions" + - "Adjust TP8/TP4 search space to reduce overlapping points" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/642 + +- config-keys: + - dsr1-fp8-h100-dynamo-sglang + description: + - "Add DeepSeek-R1 FP8 H100 Dynamo SGLang STP disaggregated multinode configurations" + - "Image: lmsysorg/sglang:v0.5.8-cu130" + - "1k1k, 1k8k, 8k1k sequence lengths" + - "Two modes per seq-len: Max throughput TEP (1P2D) and Max throughput DEP (1P1D with dp-attention)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/643 + +- config-keys: + - dsr1-fp8-h100-dynamo-sglang + description: + - "Add DeepSeek-R1 FP8 H100 Dynamo SGLang MTP disaggregated multinode configurations" + - "Image: lmsysorg/sglang:v0.5.8-cu130" + - "1k1k, 1k8k, 8k1k sequence lengths with MTP speculative decoding" + - "Two modes per seq-len: Max throughput TEP (1P2D) and Max throughput DEP (1P1D with dp-attention)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/644 + +- config-keys: + - dsr1-fp8-gb200-dynamo-trt + description: + - "Fix model_prefix argument in yaml configs" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/646 + +- config-keys: + - dsr1-fp8-mi355x-sglang-disagg + description: + - "Add --use-chat-template argument to benchmark_serving script" + - "Without this arg, MTP acceptance rates are artificially high for DeepSeek with MTP" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/649 + +- config-keys: + - dsr1-fp8-h100-dynamo-trt + description: + - "Add DeepSeek R1 FP8 H100 Dynamo TRT-LLM disaggregated multinode configurations" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/651 + +- config-keys: + - dsr1-fp8-gb300-dynamo-trt + description: + - "Add DeepSeek R1 FP8 GB300 Dynamo TRT-LLM disaggregated multinode configurations for 1k8k" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/654 + +- config-keys: + - dsr1-fp8-b200-dynamo-sglang + description: + - "Add DSR1 FP8 B200 disaggregated SGLang multinode configuration" + - "Image: lmsysorg/sglang:v0.5.8.post1-cu130-amd64" + - "9 recipes: 4x 1k1k + 5x 8k1k, low-latency and max-throughput profiles" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/658 + +- config-keys: + - dsr1-fp8-h100-dynamo-trt + description: + - "Add DeepSeek R1 FP8 H100 Dynamo TRT-LLM disaggregated multinode configurations" + - "fix model_prefix bug from https://github.com/SemiAnalysisAI/InferenceX/pull/651" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/663 + +- config-keys: + - dsr1-fp8-b200-dynamo-sglang-mtp + description: + - "Add DSR1 FP8 B200 disaggregated SGLang MTP multinode configuration" + - "Image: lmsysorg/sglang:v0.5.8.post1-cu130-amd64" + - "9 recipes: 4x 1k1k + 5x 8k1k, low-latency and max-throughput with EAGLE speculative decoding" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/667 + +- config-keys: + - dsr1-fp4-b200-dynamo-sglang + description: + - "Add DSR1 FP4 B200 Dynamo SGLang STP mode" + - "Image: lmsysorg/sglang:v0.5.8.post1-cu130-runtime" + - "1k1k configs: low-latency DEP (1P5D, 1P6D), max-throughput DEP (1P1D, 1P2D)" + - "8k1k configs: low-latency DEP/TEP (1P1D, 1P5D, 2P5D), TEP (1P1D), max-throughput DEP (7P2D)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/672 + +- config-keys: + - dsr1-fp8-mi355x-atom-mtp + - dsr1-fp4-mi355x-atom-mtp + description: + - "Add DSR1 FP8/FP4 MI355X ATOM with MTP configuration" + - "Image: rocm/atom:rocm7.2.0-ubuntu24.04-pytorch2.9-atom0.1.1" + - "Deepseek R1 with speculative decoding: 1k1k, 1k8k, 8k1k" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/673 + +- config-keys: + - dsr1-fp8-mi355x-sglang-disagg + - dsr1-fp4-mi355x-sglang-disagg + description: + - "Bump MI355X MORI FP8/FP4 image to latest rocm/sgl-dev:sglang-0.5.8-rocm700-mi35x-mori-0210" + - "Bump mi355x sglang disagg recipe to sa-260211" + - "Add conc 4/8/16" + - "Use Pure TP with MTP=2 for 1k1k conc smaller than 128 and reduce MTP to 1 for DEP configs" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/674 + +- config-keys: + - dsr1-fp4-b200-dynamo-sglang-mtp + description: + - "Add B200 configs for DSR1 FP4 SGLANG MTP mode for 1k1k and 8k1k" + - "Image: lmsysorg/sglang:v0.5.8.post1-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/683 + +- config-keys: + - dsr1-fp8-b200-dynamo-trt + description: + - "Update max_num_tokens and max_batch_size for min-latency decode workers" + - "See srt-slurm recipe changes: https://github.com/ishandhanani/srt-slurm/pull/173" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/686 + +- config-keys: + - dsr1-fp8-mi355x-sglang-disagg + description: + - "Add more sweep points for DSR1 FP8 both MTP and non-MTP 1k1k, 8k1k" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/689 + +- config-keys: + - dsr1-fp8-b300-dynamo-trt + description: + - "Update max_num_tokens and max_batch_size for min-latency decode workers" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/690 + +- config-keys: + - dsr1-fp8-b200-dynamo-sglang-mtp + description: + - "Patches one missing concurrency point for " + - "DSR1 FP8 B200 disaggregated SGLang MTP multinode configuration. " + - "Image: lmsysorg/sglang:v0.5.8.post1-cu130-amd64" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/691 + +- config-keys: + - dsr1-fp4-mi355x-sglang-disagg + description: + - "Add more sweep points for DSR1 FP4 both MTP and non-MTP 1k1k, 8k1k" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/692 + +- config-keys: + - dsr1-fp8-mi325x-sglang + description: + - "Update MI325X DeepSeek R1 FP8 SGLang image from v0.5.7 to v0.5.8" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/695 + +- config-keys: + - dsr1-fp8-mi300x-sglang + description: + - "Update MI300X DeepSeek R1 FP8 SGLang image from v0.5.7 to v0.5.8" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/696 + +- config-keys: + - dsr1-fp8-b200-dynamo-sglang-mtp + description: + - "Add new 1P2D max-throughput MTP config for 1k1k" + - "MTP settings: speculative-num-steps=1, speculative-num-draft-tokens=2" + - "Image: lmsysorg/sglang:v0.5.8.post1-cu130-amd64" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/697 + +- config-keys: + - dsr1-fp8-h200-dynamo-trt + description: + - "Add min-latency configurations for H200 Dynamo TRT" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/698 + +- config-keys: + - dsr1-fp8-mi355x-sglang-disagg + description: + - "Bump MI355X disagg FP8 recipe commit to 953f7c5 (bug fixes in sglang_disagg fork)" + - "1k1k: Switch prefill workers from DEP (tp:1, ep:8, dp-attn) to Pure TP (tp:8, ep:1) for MTP and non-MTP middle-of-curve 1P2D configs" + - "1k1k: Extend middle-of-curve concurrency range by adding conc=128 for both MTP and non-MTP 1P2D configs" + - "8k1k: Add new 2P1D (2-prefill, 1-decode) configs at conc [512, 1024] for both MTP (DECODE_MTP_SIZE=0) and non-MTP, with Pure TP prefill and DEP decode" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/700 + +- config-keys: + - dsr1-fp8-mi355x-sglang-disagg + description: + - "Bump MI355X disagg FP8 recipe commit to fix perf regression on 8k1k DEP8" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/701 + +- config-keys: + - qwen3.5-bf16-b200-sglang + description: + - "Add Qwen3.5-397B-A17B BF16 B200 SGLang benchmark" + - "Image: lmsysorg/sglang:nightly-dev-20260216-d3bae71e" + - "TP=8, concurrency 4-64 for 1k1k, 1k8k, and 8k1k sequence lengths" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/704 + +- config-keys: + - qwen3.5-bf16-mi355x-sglang + description: + - "Add Qwen3.5-397B-A17B BF16 SGLang benchmark for MI355X" + - "Image: rocm/sgl-dev:v0.5.8.post1-rocm720-mi35x-20260215" + - "Uses triton attention backend, TP=8, concurrency 4-64" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/705 + +- config-keys: + - kimik2.5-int4-mi355x-vllm + description: + - "Add Kimi-K2.5 INT4 vLLM benchmark for MI355X" + - "Model: moonshotai/Kimi-K2.5 with --mm-encoder-tp-mode data" + - "Image: vllm/vllm-openai-rocm:v0.15.1" + - "TP=8, concurrency 4-64 for 1k1k, 1k8k, and 8k1k sequence lengths" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/734 + +- config-keys: + - kimik2.5-int4-b200-vllm + description: + - "Add Kimi-K2.5 INT4 vLLM benchmark for B200" + - "Model: moonshotai/Kimi-K2.5 with --mm-encoder-tp-mode data and --trust-remote-code" + - "Image: vllm/vllm-openai:v0.15.1" + - "TP=8, concurrency 4-64 for 1k1k, 1k8k, and 8k1k sequence lengths" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/735 + +- config-keys: + - minimaxm2.5-fp8-mi355x-vllm + description: + - "Add MiniMax-M2.5 FP8 vLLM benchmark for MI355X" + - "Model: MiniMaxAI/MiniMax-M2.5 with --trust-remote-code" + - "Image: vllm/vllm-openai-rocm:v0.15.1" + - "Environment: VLLM_ROCM_USE_AITER=1" + - "TP=2 and TP=4, concurrency 4-64 for 1k1k, 1k8k, and 8k1k sequence lengths" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/755 + +- config-keys: + - minimaxm2.5-fp8-b200-vllm + description: + - "Add MiniMax-M2.5 FP8 vLLM benchmark for B200" + - "Model: MiniMaxAI/MiniMax-M2.5 with --trust-remote-code" + - "Image: vllm/vllm-openai:v0.17.0" + - "TP=2 and TP=4, concurrency 4-64 for 1k1k, 1k8k, and 8k1k sequence lengths" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/757 + +- config-keys: + - qwen3.5-bf16-b200-sglang + description: + - "Update Qwen3.5-397B-A17B BF16 SGLang B200 benchmark launch config" + - "Image: lmsysorg/sglang:nightly-dev-20260216-d3bae71e" + - "Add trtllm_mha attention backend, flashinfer_trtllm MOE runner" + - "Add context-length, tokenizer-worker-num, env tuning (NCCL_NVLS_ENABLE, SGLANG_ENABLE_FLASHINFER_GEMM)" + - "Set cuda-graph-max-bs to match concurrency, scheduler-recv-interval based on concurrency" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/758 + +- config-keys: + - glm5-fp8-mi355x-sglang + description: + - "Add GLM-5 FP8 SGLang benchmark for MI355X" + - "Model: zai-org/GLM-5-FP8 with NSA tilelang backends" + - "Image: rocm/sgl-dev:v0.5.8.post1-rocm720-mi35x-20260219" + - "TP=8, concurrency 4-64 for 1k1k, 1k8k, and 8k1k sequence lengths" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/762 + +- config-keys: + - qwen3.5-fp8-mi355x-sglang + description: + - "Add Qwen3.5-397B-A17B-FP8 SGLang benchmark configuration for MI355X" + - "Image: rocm/sgl-dev:v0.5.8.post1-rocm720-mi35x-20260218" + - "Uses triton attention backend, TP=8, concurrency 4-64" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/768 + +- config-keys: + - dsr1-fp8-mi355x-sglang-disagg + description: + - "Add more configs for MI355X FP8 Disagg" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/770 + +- config-keys: + - gptoss-fp4-mi300x-vllm + - gptoss-fp4-mi325x-vllm + description: + - "Update vLLM ROCm image from v0.14.0 to v0.15.1 for MI300X and MI325X GPT-OSS" + - "Gains: ROCm skinny GEMM dispatch fix, MoRI EP all2all backend, KV cache shuffle + paged attention for AITER" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/781 + +- config-keys: + - gptoss-fp4-b200-vllm + - gptoss-fp4-h100-vllm + - gptoss-fp4-h200-vllm + description: + - "Update vLLM image from v0.13.0 to v0.15.1 for NVIDIA GPT-OSS configs" + - "Remove deprecated async-scheduling flag (now enabled by default since v0.14.0)" + - "Gains: CUTLASS MoE optimizations (~8% throughput), FP4 kernel improvements (~4% E2E on B200), torch.compile cold-start fix" + - "v0.15.1 includes fix for prefix cache hit rate of 0% on GPT-OSS hybrid attention models" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/789 + +- config-keys: + - dsr1-fp4-mi355x-atom + - dsr1-fp4-mi355x-atom-mtp + description: + - "Update search-space configurations for DSR1 FP4 MI355X ATOM and ATOM-MTP" + - "Comment out TP=4 configs, consolidate to TP=8 only" + - "Extend concurrency range to conc-end: 256 across all sequence lengths (1k1k, 1k8k, 8k1k)" + - "Fix MTP 1k8k conc-start from 256 to 4 to enable full concurrency sweep" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/792 + +- config-keys: + - qwen3.5-fp8-b200-sglang + description: + - "Add Qwen3.5-397B-A17B-FP8 SGLang benchmark configuration for B200" + - "Image: lmsysorg/sglang:v0.5.9-cu129-amd64" + - "Uses trtllm_mha attention backend and flashinfer_trtllm MOE runner" + - "Enable SGLANG_ENABLE_FLASHINFER_GEMM=true, NCCL_NVLS_ENABLE=1" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/804 + +- config-keys: + - gptoss-fp4-mi300x-vllm + - gptoss-fp4-mi325x-vllm + - gptoss-fp4-mi355x-vllm + description: + - "Update AMD GPT-OSS vLLM images to v0.16.0 (MI300X/MI325X from v0.15.1, MI355X from custom v0.10.1)" + - "MI355X: Fix env vars (VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION), add VLLM_ROCM_USE_AITER=1, remove deprecated flags" + - "MI355X: Simplify compilation config to cudagraph_mode FULL_AND_PIECEWISE, add HIP_VISIBLE_DEVICES Ray fix" + - "Gains: fused add+rmsnorm+pad for GPT-OSS (automatic via PassManager), AITER attention block size fix" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/806 + +- config-keys: + - dsr1-fp4-mi355x-sglang + - dsr1-fp8-mi300x-sglang + - dsr1-fp8-mi325x-sglang + - dsr1-fp8-mi355x-sglang + description: + - "Update SGLang image from v0.5.8 to v0.5.9 for AMD single-node DeepSeek R1 configs" + - "Key changes: AITER v0.1.10.post3 with FP8 Prefill/Decode/KV Cache, FP8 prefill attention kernel, MORI EP two-batch overlapping, OOM fix for DeepSeek weight loading" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/816 + +- config-keys: + - qwen3.5-fp4-b200-sglang + description: + - "Add Qwen3.5-397B-A17B NVFP4 B200 SGLang benchmark config and launch script" + - "Image: lmsysorg/sglang:v0.5.9-cu129-amd64" + - "Model: nvidia/Qwen3.5-397B-A17B-NVFP4" + - "Configs: 1k1k (TP4 conc 4-128), 8k1k (TP4 conc 4-128)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/820 + +- config-keys: + - dsr1-fp8-mi355x-sglang-disagg + - dsr1-fp8-mi355x-sglang-disagg-mtp + - dsr1-fp4-mi355x-sglang-disagg + - dsr1-fp4-mi355x-sglang-disagg-mtp + description: + - "Add more sweep configs for MI355X FP8/FP4 Disagg" + - "Add TP/DP/EP size < 8 support " + - "Support DSR1-0528 MTP Disagg" + - "Bump SGL mori image to Feb 27" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/823 + +- config-keys: + - kimik2.5-fp4-mi355x-vllm + description: + - "Add Kimi-K2.5 MXFP4 vLLM benchmark for MI355X" + - "Model: amd/Kimi-K2.5-MXFP4 with --mm-encoder-tp-mode data and --trust-remote-code" + - "Image: vllm/vllm-openai-rocm:v0.15.1" + - "TP=8, concurrency 4-64 for 1k1k, 1k8k, and 8k1k sequence lengths" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/825 + +- config-keys: + - minimaxm2.5-fp4-mi355x-vllm + description: + - "Add MiniMax M2.5 MXFP4 vLLM benchmark for MI355X" + - "Model: amd/MiniMax-M2.5-MXFP4 with --trust-remote-code and --block-size=32" + - "Image: vllm/vllm-openai-rocm:v0.19.1" + - "Environment: VLLM_ROCM_USE_AITER=1" + - "Tp=1, TP=2 and TP=4, concurrency 4-64 for 1k1k, 1k8k, and 8k1k sequence lengths" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/827 + +- config-keys: + - minimaxm2.5-fp8-h200-vllm + description: + - "Add MiniMax M2.5 FP8 single-node config for H200 with vLLM v0.16.0 (TP4)" + - "New benchmark script with --trust-remote-code for MiniMaxAI/MiniMax-M2.5" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/831 + +- config-keys: + - minimaxm2.5-fp8-h100-vllm + description: + - "Add MiniMax-M2.5 FP8 vLLM benchmark for H100" + - "Model: MiniMaxAI/MiniMax-M2.5 with --trust-remote-code" + - "Image: vllm/vllm-openai:v0.16.0" + - "Switch from TP=8/EP=8 to TP=4/EP=4, concurrency 4-64 for 1k1k, 1k8k, and 8k1k" + - "Script uses conditional --enable-expert-parallel based on EP_SIZE env var" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/832 + +- config-keys: + - minimaxm2.5-fp8-mi325x-vllm + description: + - "Add MiniMax-M2.5 FP8 vLLM benchmark for MI325X" + - "Model: MiniMaxAI/MiniMax-M2.5 with --trust-remote-code" + - "Image: vllm/vllm-openai-rocm:v0.16.0" + - "Environment: VLLM_ROCM_USE_AITER=1" + - "TP=2 and TP=4, concurrency 4-64 for 1k1k, 1k8k, and 8k1k sequence lengths" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/836 + +- config-keys: + - minimaxm2.5-fp8-mi300x-vllm + description: + - "Add MiniMax-M2.5 FP8 vLLM benchmark for MI300X" + - "Model: MiniMaxAI/MiniMax-M2.5 with --trust-remote-code" + - "Image: vllm/vllm-openai-rocm:v0.16.0" + - "Environment: VLLM_ROCM_USE_AITER=1" + - "TP=2 and TP=4, concurrency 4-64 for 1k1k, 1k8k, and 8k1k sequence lengths" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/837 + +- config-keys: + - qwen3.5-bf16-mi325x-sglang + description: + - "Add Qwen3.5-397B-A17B BF16 SGLang benchmark for MI325X" + - "Image: lmsysorg/sglang:v0.5.9-rocm720-mi30x" + - "Uses triton attention backend, TP=8, concurrency 4-64" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/842 + +- config-keys: + - qwen3.5-bf16-mi300x-sglang + description: + - "Add Qwen3.5-397B-A17B BF16 SGLang benchmark for MI300X" + - "Image: lmsysorg/sglang:v0.5.9-rocm720-mi30x" + - "Uses triton attention backend, TP=8, concurrency 4-64" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/843 + +- config-keys: + - kimik2.5-int4-h200-vllm + description: + - "Add Kimi-K2.5 INT4 vLLM benchmark for H200" + - "Model: moonshotai/Kimi-K2.5 with --reasoning-parser kimi_k2 and --trust-remote-code" + - "Image: vllm/vllm-openai:v0.16.0" + - "TP=8, concurrency 4-64 for 1k1k, 1k8k, and 8k1k sequence lengths" + - "following https://docs.vllm.ai/projects/recipes/en/latest/moonshotai/Kimi-K2.5.html" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/847 + +- config-keys: + - qwen3.5-fp8-mi300x-sglang + description: + - "Add Qwen3.5-397B-A17B-FP8 SGLang benchmark for MI300X" + - "Image: lmsysorg/sglang:v0.5.9-rocm720-mi30x" + - "Uses triton attention backend, TP=8, concurrency 4-64" + - "Following AMD Andy Luo's recipe" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/850 + +- config-keys: + - qwen3.5-fp8-mi325x-sglang + description: + - "Add Qwen3.5-397B-A17B-FP8 SGLang benchmark for MI325X" + - "Image: lmsysorg/sglang:v0.5.9-rocm720-mi30x" + - "Following AMD Andy Luo's recipe with triton attention backend" + - "TP=8, concurrency 4-64 for 1k1k, 1k8k, and 8k1k sequence lengths" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/852 + +- config-keys: + - gptoss-fp4-h200-trt + description: + - "Upgrade TensorRT-LLM container from release:gpt-oss-dev to release:v1.3.0rc5" + - "Remove sed hack for TensorRT bug (fixed upstream in v1.3.0rc5)" + - "Remove enable_block_reuse: false from kv_cache_config (default true is now recommended)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/854 + +- config-keys: + - qwen3.5-fp8-h200-sglang + description: + - "Add Qwen 3.5 FP8 H200 SGLang configuration" + - "Model: Qwen/Qwen3.5-397B-A17B-FP8, runner: h200, image: lmsysorg/sglang:v0.5.8-cu130-amd64" + - "Benchmark script: benchmarks/single_node/qwen3.5_fp8_h200.sh" + - "Server: reasoning-parser qwen3, tool-call-parser qwen3_coder, enable-flashinfer-allreduce-fusion, mem-fraction-static 0.8" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/855 + +- config-keys: + - kimik2.5-fp4-b200-vllm + description: + - "Add Kimi K2.5 FP4 B200 vLLM benchmark configuration" + - "Image: vllm/vllm-openai:v0.17.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/862 + +- config-keys: + - dsr1-fp8-mi355x-sglang + description: + - "Expanding TP search space" + - "Adding kv-cache-fp8" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/865 + +- config-keys: + - minimaxm2.5-fp8-h200-vllm + description: + - "Extend MiniMax M2.5 FP8 single-node config for H200 with vLLM v0.16.0 (TP8)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/869 + +- config-keys: + - dsr1-fp8-h200-sglang + description: + - "Update H200 DeepSeek R1 FP8 SGLang image from v0.5.7 to v0.5.9" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/887 + +- config-keys: + - gptoss-fp4-mi300x-vllm + - gptoss-fp4-mi325x-vllm + - gptoss-fp4-mi355x-vllm + description: + - "Update AMD GPT-OSS vLLM image from v0.16.0 to v0.17.0 for MI300X, MI325X, and MI355X" + - "MI355X: Switch model to amd/gpt-oss-120b-w-mxfp4-a-fp8 (MXFP4 weights + FP8 activations)" + - "MI355X: Add VLLM_ROCM_USE_AITER_TRITON_ROPE=1 for AITER triton RoPE kernel" + - "Add AMDGCN_USE_BUFFER_OPS=0 and VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 env vars" + - "Switch to --attention-backend ROCM_AITER_UNIFIED_ATTN and add fuse_rope_kvcache compilation pass" + - "Remove deprecated VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION/VLLM_ROCM_USE_AITER_MHA env vars and compilation-config cudagraph_mode" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/889 + +- config-keys: + - qwen3.5-bf16-b200-sglang + - qwen3.5-bf16-mi300x-sglang + - qwen3.5-bf16-mi325x-sglang + - qwen3.5-bf16-mi355x-sglang + - qwen3.5-fp8-b200-sglang + - qwen3.5-fp8-h200-sglang + - qwen3.5-fp8-mi300x-sglang + - qwen3.5-fp8-mi325x-sglang + - qwen3.5-fp8-mi355x-sglang + description: + - "Redo qwen eval" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/892 + evals-only: true + +- config-keys: + - qwen3.5-fp8-b200-sglang-mtp + description: + - "Add Single Node Agg FP8 MTP config for Qwen3.5 B200 SGLang" + - "EAGLE speculative decoding: num-steps 3, draft-tokens 4, topk 1" + - "New script: benchmarks/single_node/qwen3.5_fp8_b200_mtp.sh" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/898 + +- config-keys: + - dsr1-fp4-mi355x-sglang-disagg + - dsr1-fp4-mi355x-sglang-disagg-mtp + description: + - "Add more sweep configs for MI355X FP4 Disagg" + - "Bump FP4 image to Feb 27 latest" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/899 + +- config-keys: + - kimik2.5-int4-mi325x-vllm + description: + - "Add Kimi K2.5 INT4 single-node MI325X vLLM benchmark (TP8)" + - "Uses vLLM ROCm v0.16.0 image following AMD Andy Luo's recipe" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/901 + +- config-keys: + - dsr1-fp8-b200-dynamo-sglang + - dsr1-fp8-b200-dynamo-sglang-mtp + description: + - "Update B200 FP8 DSR1 8k1k dynamo-sglang recipes to new pareto configs" + - "Replace old per-file recipes with resolved variants from consolidated 8k1k.yaml" + - "14 variants: STP/MTP x low-latency/max-throughput with updated concurrencies and scale points" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/907 + +- config-keys: + # NVIDIA single-node + - dsr1-fp4-b200-sglang + - dsr1-fp4-b200-trt + - dsr1-fp4-b200-trt-mtp + - dsr1-fp8-b200-sglang + - dsr1-fp8-b200-sglang-mtp + - dsr1-fp8-b200-trt + - dsr1-fp8-b200-trt-mtp + - dsr1-fp8-h200-sglang + - dsr1-fp8-h200-trt + - dsr1-fp8-h200-trt-mtp + - glm5-fp8-b200-sglang + - glm5-fp8-h200-sglang + - gptoss-fp4-b200-trt + - gptoss-fp4-b200-vllm + - gptoss-fp4-h100-vllm + - gptoss-fp4-h200-trt + - gptoss-fp4-h200-vllm + - kimik2.5-fp4-b200-vllm + - kimik2.5-int4-b200-vllm + - kimik2.5-int4-h200-vllm + - minimaxm2.5-fp8-b200-vllm + - minimaxm2.5-fp8-h100-vllm + - minimaxm2.5-fp8-h200-vllm + - qwen3.5-bf16-b200-sglang + - qwen3.5-fp8-b200-sglang + - qwen3.5-fp8-b200-sglang-mtp + - qwen3.5-fp8-h200-sglang + # AMD single-node + - dsr1-fp4-mi355x-atom + - dsr1-fp4-mi355x-atom-mtp + - dsr1-fp4-mi355x-sglang + - dsr1-fp8-mi325x-sglang + - dsr1-fp8-mi300x-sglang + - dsr1-fp8-mi355x-atom + - dsr1-fp8-mi355x-atom-mtp + - dsr1-fp8-mi355x-sglang + - glm5-fp8-mi355x-sglang + - gptoss-fp4-mi300x-vllm + - gptoss-fp4-mi325x-vllm + - gptoss-fp4-mi355x-atom + - gptoss-fp4-mi355x-vllm + - kimik2.5-fp4-mi355x-vllm + - kimik2.5-int4-mi325x-vllm + - kimik2.5-int4-mi355x-vllm + - minimaxm2.5-fp8-mi300x-vllm + - minimaxm2.5-fp8-mi325x-vllm + - minimaxm2.5-fp8-mi355x-vllm + - qwen3.5-bf16-mi300x-sglang + - qwen3.5-bf16-mi325x-sglang + - qwen3.5-bf16-mi355x-sglang + - qwen3.5-fp8-mi300x-sglang + - qwen3.5-fp8-mi325x-sglang + - qwen3.5-fp8-mi355x-sglang + description: + - "Separate evals, change to 8k1k, fail loudly, 5-shot, top of curve & middle of curve" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/911 + evals-only: true + +- config-keys: + - glm5-fp8-h200-sglang + description: + - "Add GLM-5 FP8 SGLang H200 single-node benchmark" + - "Model: zai-org/GLM-5-FP8, image: lmsysorg/sglang:glm5-hopper" + - "Benchmark script: benchmarks/single_node/glm5_fp8_h200.sh" + - "Tool-call-parser glm47, reasoning-parser glm45, mem-fraction-static 0.85" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/914 + +- config-keys: + - glm5-fp8-b200-sglang + description: + - "Add GLM-5 FP8 SGLang benchmark for B200" + - "Supports TP8 (low latency) and DEP8 (high throughput) modes with NSA attention backend" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/915 + +- config-keys: + - qwen3.5-fp8-b200-sglang + description: + - "Replace FP8 with combination of TP4 and TP8 config" + - "Add --enable-flashinfer-allreduce-fusion to TP8" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/918 + +- config-keys: + - dsr1-fp8-b200-dynamo-trt + - dsr1-fp8-h200-dynamo-trt + - dsr1-fp4-gb200-dynamo-trt + description: + - "Fix metadata inconsistencies in nvidia-master.yaml - TP/EP/DP-attn values now match actual recipe files" + - "B200 FP8 TRT 8K/1K: prefill_ep 8→1 (15 entries), prefill_dp_attn true→false (1 entry)" + - "H200 FP8 TRT 1K/1K: prefill_dp_attn false→true (9 entries)" + - "H200 FP8 TRT 8K/1K: prefill_dp_attn true→false (8 entries)" + - "GB200 FP4 TRT 8K/1K: decode_dp_attn true→false (2 entries)" + - "All fixes are metadata-only; no recipe files were modified" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/919 + +- config-keys: + - kimik2.5-int4-mi325x-vllm + - kimik2.5-int4-mi355x-vllm + - kimik2.5-int4-h200-vllm + - kimik2.5-fp4-mi355x-vllm + - kimik2.5-fp4-b200-vllm + description: + - "Disable prefix caching (--no-enable-prefix-caching) for all Kimi K2.5 benchmarks using random datasets" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/926 + +- config-keys: + - minimaxm2.5-fp8-mi355x-vllm + description: + - "ADD minimax TP=8 with EP, in config of 1k1k, 1k8k, and 8k1k sequence lengths" + - "Config concurrency: 32-256" + - "update image to vllm/vllm-openai-rocm:v0.18.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/927 + +- config-keys: + - gptoss-fp4-mi325x-vllm + - minimaxm2.5-fp8-mi325x-vllm + description: + - "Document --exclusive flag on MI325X salloc prevents node sharing during benchmarks" + - "Only non-TP8 configs are affected; TP8 already uses all GPUs on the node" + - "May yield performance improvements" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/931 + +- config-keys: + - dsr1-fp4-mi355x-atom + - dsr1-fp4-mi355x-atom-mtp + - dsr1-fp4-mi355x-sglang + - dsr1-fp4-mi355x-sglang-disagg + - dsr1-fp4-mi355x-sglang-disagg-mtp + - dsr1-fp8-mi355x-sglang + - dsr1-fp8-mi355x-sglang-disagg + - dsr1-fp8-mi355x-sglang-disagg-mtp + - gptoss-fp4-mi355x-atom + - gptoss-fp4-mi355x-vllm + - minimaxm2.5-fp8-mi355x-vllm + description: + - "Add --exclusive flag to MI355X single-node salloc and multi-node sbatch to prevent node sharing during benchmarks" + - "Only non-TP8 configs listed; TP8 already uses all GPUs on the node" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/934 + +- config-keys: + - kimik2.5-int4-b200-vllm + description: + - "Enable VLLM_USE_FLASHINFER_MOE_INT4=1 for Kimi K2.5 INT4 B200 benchmark" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/935 + +- config-keys: + - kimik2.5-fp4-mi355x-vllm + description: + - "Upgrade vLLM ROCm image from v0.16.0 to v0.18.0" + - "Enable AITER with INT4 quick reduce; disable AITER RMSNorm for TP < 8 (accuracy)" + - "Add expert parallel, TP4, and TP4/EP4 search spaces" + - "Switch block-size 64 to 1 gpu-memory-utilization 0.95 to 0.90" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/936 + +- config-keys: + - dsr1-fp4-b200-sglang + - dsr1-fp8-b200-sglang + - dsr1-fp8-b200-sglang-mtp + - dsr1-fp8-h200-sglang + description: + - "Update SGLang image to v0.5.9-cu130 for all DSR1 SGLang configs" + - "dsr1-fp4-b200-sglang: v0.5.6-cu129-amd64 → v0.5.9-cu130" + - "dsr1-fp8-b200-sglang: v0.5.6-cu129-amd64 → v0.5.9-cu130" + - "dsr1-fp8-b200-sglang-mtp: v0.5.8-cu130-amd64 → v0.5.9-cu130" + - "dsr1-fp8-h200-sglang: v0.5.9-cu129-amd64 → v0.5.9-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/943 + +- config-keys: + - minimaxm2.5-fp8-b200-vllm + description: + - "Update vLLM image from v0.17.0 to v0.19.0 for MiniMax-M2.5 FP8 B200" + - "Add tp4 ep4 search-space entries (conc 32-256) for all seq-len configs" + - "Remove ISL 1024 / OSL 8192 seq-len config" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/947 + +- config-keys: + - kimik2.5-int4-mi355x-vllm + description: + - "Upgrade vLLM ROCm image from v0.15.1 to v0.18.0" + - "Enable AITER MLA, export VLLM_ROCM_USE_AITER=1, https://github.com/vllm-project/vllm/issues/35641" + - "Triton Fused Moe Tuning https://github.com/vllm-project/vllm/pull/35093" + - "Add --max-num-seqs 256, remove --disable-log-requests" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/950 + +- config-keys: + - minimaxm2.5-fp8-mi325x-vllm + description: + - "Upgrade vLLM ROCm image from v0.16.0 to v0.18.0" + - "Replace TP4 with TP8/EP8, add conc range 4-256" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/953 + +- config-keys: + - kimik2.5-int4-mi325x-vllm + description: + - "Upgrade vLLM ROCm image from v0.16.0 to v0.18.0" + - "Enable AITER MLA, export VLLM_ROCM_USE_AITER=1, https://github.com/vllm-project/vllm/issues/35641" + - "Triton Fused Moe Tuning https://github.com/vllm-project/vllm/pull/35093" + - "Add --max-num-seqs 256, remove --disable-log-requests" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/957 + +- config-keys: + - minimaxm2.5-fp8-h100-vllm + - minimaxm2.5-fp8-h200-vllm + description: + - "Update vLLM image from v0.16.0 to v0.18.0 for minimax h100 and h200 configs" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/958 + +- config-keys: + - gptoss-fp4-h100-vllm + - gptoss-fp4-h200-vllm + description: + - "Update vLLM image from v0.15.1 to v0.18.0 for gptoss H100 and H200 configs" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/960 + +- config-keys: + - minimaxm2.5-fp8-b200-vllm + - minimaxm2.5-fp8-h100-vllm + - minimaxm2.5-fp8-h200-vllm + - minimaxm2.5-fp8-mi300x-vllm + - minimaxm2.5-fp8-mi325x-vllm + - minimaxm2.5-fp8-mi355x-vllm + description: + - "Disable prefix caching (--no-enable-prefix-caching) for all MiniMax benchmarks using random datasets" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/966 + +- config-keys: + - qwen3.5-bf16-mi300x-sglang + - qwen3.5-bf16-mi325x-sglang + - qwen3.5-fp8-mi300x-sglang + - qwen3.5-fp8-mi325x-sglang + description: + - "Add --disable-radix-cache to SGLang server launch command for qwen3.5 MI300X and MI325X benchmark scripts" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/970 + +- config-keys: + - glm5-nvfp4-b200-sglang + description: + - "Add GLM-5 NVFP4 single-node B200 SGLang benchmark (TP8, conc 4-64)" + - "Uses nvidia/GLM-5-NVFP4 model with modelopt_fp4 quantization" + - "Image: lmsysorg/sglang:nightly-dev-cu13-20260328-a27651d5" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/973 + +- config-keys: + - kimik2.5-int4-mi300x-vllm + description: + - "Add Kimi K2.5 INT4 single-node MI300X vLLM benchmark (TP8)" + - "Uses vLLM ROCm v0.18.0 image following AMD Andy Luo's recipe" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/975 + +- config-keys: + - dsr1-fp8-mi355x-atom-mtp + description: + - "DeepSeek R1 MI355X FP8 ATOM-MTP config to support MTP 3 tokens" + - "Image: rocm/atom:rocm7.2.1-ubuntu24.04-pytorch2.9.1-atom0.1.2" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/984 + +- config-keys: + - kimik2.5-fp4-mi355x-atom + - minimaxm2.5-fp8-mi355x-atom + description: + - "New model support on ATOM framework" + - "Kimi-K2.5 FP4, and MiniMax-M2.5 FP8 configs added for MI355X ATOM" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/992 + +- config-keys: + - minimaxm2.5-fp4-b200-vllm + description: + - "Optimize MiniMax-M2.5 NVFP4 B200 vLLM search-space" + - "Expand from tp2/tp4 to tp1/tp2/tp4/tp8 with expert parallel and dp-attn variants" + - "Add ep2, ep4, and dp-attn configurations for higher concurrency sweeps" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/996 + +- config-keys: + - dsr1-fp4-b200-dynamo-trt + - dsr1-fp8-b200-dynamo-trt + - dsr1-fp4-b200-dynamo-sglang + - dsr1-fp8-b200-dynamo-sglang + - dsr1-fp8-b200-dynamo-sglang-mtp + - dsr1-fp4-b200-dynamo-sglang-mtp + - dsr1-fp4-b300-dynamo-trt + - dsr1-fp8-b300-dynamo-trt + - dsr1-fp4-gb300-dynamo-trt + - dsr1-fp8-gb300-dynamo-trt + - dsr1-fp4-gb300-dynamo-sglang + - dsr1-fp8-gb300-dynamo-sglang + - dsr1-fp8-mi355x-sglang-disagg + - dsr1-fp8-mi355x-sglang-disagg-mtp + - dsr1-fp4-mi355x-sglang-disagg + - dsr1-fp4-mi355x-sglang-disagg-mtp + description: + - "Add multi-node lm-eval accuracy runs" + - "Eval picks the config with highest max eligible concurrency per (model, runner, framework, precision, spec-decoding, prefill-dp-attn, decode-dp-attn) group on 8k1k" + - "Eval concurrency set to the median eligible conc (>= MIN_EVAL_CONC=16) of the selected config to avoid OOM" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1000 + evals-only: true + +- config-keys: + - qwen3.5-fp8-h200-sglang-mtp + description: + - "Add Qwen3.5-397B-A17B-FP8 H200 SGLang MTP (EAGLE speculative decoding)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1001 + +- config-keys: + - minimaxm2.5-fp8-mi355x-vllm + description: + - "Optimize MiniMax-M2.5 FP8 MI355X vLLM search-space" + - "Add tp2 ep2 search-space entries (conc 2-256) for all seq-len configs" + - "Upgrade vLLM image to v0.19.0" + - "Enable FP8 KV cache + AITER FA for minimaxm2.5-fp8-mi355x-vllm" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1003 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang + description: + - "Qwen3.5 fp4 support on SGL" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1006 + +- config-keys: + - kimik2.5-fp4-gb200-dynamo-vllm + description: + - "Add Kimi K2.5 NVFP4 GB200 disaggregated multinode vLLM benchmark via Dynamo frontend" + - "Image: vllm/vllm-openai:v0.18.0-cu130" + - "Model: nvidia/Kimi-K2.5-NVFP4 with NixlConnector KV transfer, FLASHINFER_MLA attention" + - "1k1k configs: high-throughput DEP (1P1D dep4/dep16), low-latency TEP (1P4D dep4/tep4)" + - "8k1k configs: low-latency TEP (1P4D), mid-curve DEP (3P1D dep16), high-throughput (5P1D dep8, 6P1D dep16)" + - "Recipes sourced from NVIDIA/srt-slurm branch sa-submission-q2-2026" + - "New framework: dynamo-vllm (Dynamo frontend + vLLM backend)" + - "Runner script updated to clone NVIDIA/srt-slurm and map vLLM container image" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1008 + +- config-keys: + - glm5-fp8-mi355x-atom + description: + - "GLM5 FP8 configs added for MI355X ATOM" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1009 + +- config-keys: + - minimaxm2.5-fp8-b200-vllm + description: + - "Update MiniMax-M2.5 FP8 B200 config with new search spaces" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1010 + +- config-keys: + - glm5-fp4-b200-sglang + description: + - "Update GLM-5 NVFP4 B200 SGLang benchmark script with optimized launch parameters" + - "Add TP4 search space with higher concurrency (128-256) for 1k1k and 8k1k configs" + - "Enable FP8 E4M3 KV cache, NSA backends (trtllm), flashinfer allreduce fusion, MoE flashinfer_trtllm runner" + - "Tune mem-fraction-static to 0.9, chunked-prefill-size to 32768, add tokenizer-worker-num 6" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1011 + +- config-keys: + - glm5-fp8-b200-sglang + description: + - "Bump GLM-5 FP8 B200 SGLang concurrency from 128 to 256" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1012 + +- config-keys: + - qwen3.5-fp8-h200-sglang-mtp + description: + - "Enable SGLANG_ENABLE_SPEC_V2=1 for Qwen3.5 FP8 H200 SGLang MTP" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1017 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang + description: + - "TP2/TP4 seach space exploration for Qwen3.5 fp4 on SGL" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1022 + +- config-keys: + - glm5-fp8-mi355x-sglang + description: + - "Upgrade GLM5 FP8 MI355X SGLang image to v0.5.10rc0-rocm720-mi35x-20260413" + - "Set --kv-cache-dtype fp8_e4m3 and --disable-radix-cache" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1023 + +- config-keys: + - kimik2.5-fp4-gb200-dynamo-trt + description: + - "Add Kimi K2.5 NVFP4 GB200 disaggregated TRT-LLM benchmarks via Dynamo (14 STP configs)" + - "New framework: dynamo-trt (Dynamo frontend + TensorRT-LLM backend)" + - "Container: nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:1.1.0-dev.2" + - "Recipes sourced from NVIDIA/srt-slurm branch sa-submission-q2-2026" + - "Runner script updated to support kimik2.5 model prefix with dynamo-trt framework" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1026 + +- config-keys: + - glm5-fp4-b200-sglang + description: + - "Update SGLang image from nightly-dev-cu13-20260328-a27651d5 to v0.5.10.post1-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1031 + +- config-keys: + - qwen3.5-fp8-b300-sglang-mtp + description: + - "Add Qwen3.5-397B-A17B-FP8 B300 SGLang MTP benchmark" + - "Image: lmsysorg/sglang:v0.5.10.post1-cu130" + - "EAGLE speculative decoding with MTP, TP=4, concurrency 4-256 for 1k1k and 8k1k" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1035 + +- config-keys: + - qwen3.5-fp8-mi355x-sglang + - qwen3.5-bf16-mi355x-sglang + description: + - "Update cli args of Qwen3.5 FP8 and BF16 SGLang benchmarks for MI355X to achieve better performance" + - "Use lmsysorg/sglang-rocm:v0.5.10rc0-rocm720-mi35x-20260415 for BF16 benchmark" + - "Use lmsysorg/sglang-rocm:v0.5.10rc0-rocm720-mi35x-20260414 for FP8 benchmark" + - "Image includes upstream SGLang PRs: https://github.com/sgl-project/sglang/pull/21188, https://github.com/sgl-project/sglang/pull/21421, https://github.com/sgl-project/sglang/pull/20736" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1036 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang + description: + - "Update SGLang image from 'lmsysorg/sglang:v0.5.10-rocm720-mi35x' to 'rocm/sgl-dev:v0.5.10rc0-rocm720-mi35x-20260413'" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1041 + +- config-keys: + - glm5.1-fp4-mi355x-atom + description: + - "Add GLM-5.1 MXFP4 single-node MI355X ATOM benchmark" + - "Image: rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post" + - "TP=2 and TP=4, concurrency 4-256 for 1k1k and 8k1k sequence lengths" + - "Add --max-num-seqs and --gpu-memory-utilization 0.9 to server launch" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1043 + +- config-keys: + - kimik2.5-fp4-b200-vllm + description: + - "Add kv-cache-dtype fp8, max-cudagraph-capture-size 2048, max-num-batched-tokens, and stream-interval 20 to server launch args" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1047 + +- config-keys: + - qwen3.5-fp8-b300-sglang + description: + - "Add Qwen3.5-397B-A17B-FP8 B300 SGLang benchmark (non-MTP)" + - "Image: lmsysorg/sglang:v0.5.10.post1-cu130" + - "TP=4, concurrency 4-256 for 1k1k and 8k1k" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1048 + +- config-keys: + - dsr1-fp4-b300-sglang + description: + - "Add DeepSeek-R1-0528 FP4 B300 SGLang benchmark (non-MTP)" + - "Image: lmsysorg/sglang:v0.5.10.post1-cu130" + - "At the time of submission, https://cookbook.sglang.io/autoregressive/DeepSeek/DeepSeek-R1 does not have a B300-specific recipe, so this reuses the existing DSR1 FP4 B200 SGLang recipe as-is" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1049 + +- config-keys: + - dsr1-fp8-b300-sglang + description: + - "Add DeepSeek-R1-0528 FP8 B300 SGLang benchmark (non-MTP)" + - "Image: lmsysorg/sglang:v0.5.10.post1-cu130" + - "At the time of submission, https://cookbook.sglang.io/autoregressive/DeepSeek/DeepSeek-R1 does not have a B300-specific recipe, so this reuses the existing DSR1 FP8 B200 SGLang recipe as-is" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1050 + +- config-keys: + - glm5-fp8-b300-sglang + description: + - "Add GLM-5 FP8 B300 SGLang benchmark" + - "Image: lmsysorg/sglang:v0.5.10.post1-cu130" + - "At the time of submission, https://cookbook.sglang.io/autoregressive/GLM/GLM-5.1 does not have a B300-specific recipe, so this reuses the existing GLM5 FP8 B200 SGLang recipe as-is" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1051 + +- config-keys: + - minimaxm2.5-fp8-b300-vllm + description: + - "Add MiniMax-M2.5 FP8 B300 vLLM benchmark" + - "Image: vllm/vllm-openai:v0.19.0-cu130" + - "At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/MiniMax/MiniMax-M2.html does not have a B300-specific recipe, so this reuses the existing MiniMax-M2.5 FP8 B200 vLLM recipe as-is" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1054 + +- config-keys: + - minimaxm2.5-fp4-b300-vllm + description: + - "Add MiniMax-M2.5 FP4 (NVFP4) B300 vLLM benchmark" + - "Image: vllm/vllm-openai:v0.19.0-cu130" + - "At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/MiniMax/MiniMax-M2.html does not have a B300-specific recipe, so this reuses the existing MiniMax-M2.5 FP4 B200 vLLM recipe as-is" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1055 + +- config-keys: + - glm5-fp4-b300-sglang + description: + - "Add GLM-5 FP4 (NVFP4) B300 SGLang benchmark" + - "Image: lmsysorg/sglang:v0.5.10.post1-cu130" + - "At the time of submission, https://cookbook.sglang.io/autoregressive/GLM/GLM-5 does not have a B300-specific recipe, so this reuses the existing GLM-5 FP4 B200 SGLang recipe as-is" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1058 + +- config-keys: + - dsr1-fp8-b300-sglang-mtp + description: + - "Add DeepSeek-R1-0528 FP8 B300 SGLang MTP benchmark" + - "Image: lmsysorg/sglang:v0.5.10.post1-cu130" + - "EAGLE speculative decoding with MTP, TP=8, concurrency 4-512 for 1k1k and 8k1k" + - "At the time of submission, https://cookbook.sglang.io/autoregressive/DeepSeek/DeepSeek-R1 does not have a B300-specific recipe, so this reuses the existing DSR1 FP8 B200 SGLang MTP recipe as-is" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1059 + +- config-keys: + - gptoss-fp4-mi300x-vllm + description: + - "Expand GPT-OSS 120B FP4 MI300X TP=1 concurrency from 64 to 256 for 1k1k" + - "Higher concurrency improves MoE weight amortization: 8552 total TPS at conc=256 vs 4016 at conc=64 (2.1x)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1061 + +- config-keys: + - qwen3.5-bf16-mi300x-sglang + - qwen3.5-bf16-mi325x-sglang + - qwen3.5-fp8-mi300x-sglang + - qwen3.5-fp8-mi325x-sglang + description: + - "Update cli args of Qwen3.5 FP8 and BF16 SGLang benchmarks for MI300X and MI325X to achieve better performance" + - "Use lmsysorg/sglang:v0.5.10-rocm720-mi30x" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1063 + +- config-keys: + - minimaxm2.5-fp8-b200-vllm + description: + - "Add VLLM_FLOAT32_MATMUL_PRECISION=high, remove VLLM_FLASHINFER_ALLREDUCE_BACKEND=mnnvl" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1068 + +- config-keys: + - minimaxm2.5-fp4-b200-vllm + description: + - "Add VLLM_FLOAT32_MATMUL_PRECISION=high" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1069 + +- config-keys: + - qwen3.5-fp4-b300-sglang + description: + - "Add Qwen3.5-397B-A17B NVFP4 B300 SGLang benchmark" + - "Image: lmsysorg/sglang:v0.5.10.post1-cu130" + - "Model: nvidia/Qwen3.5-397B-A17B-NVFP4" + - "Follows the SGLang cookbook recipe at https://cookbook.sglang.io/autoregressive/Qwen/Qwen3.5 as of 2026-04-17" + - "Mirrors the B200 FP4 recipe with mem-fraction-static lowered to 0.8 and an extra TP2/EP2 search-space entry" + - "Configs: 1k1k and 8k1k, TP4/EP1 conc 4-128 + TP2/EP2 conc 4-128" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1072 + +- config-keys: + - qwen3.5-bf16-b200-sglang-mtp + description: + - "Add Qwen3.5-397B-A17B BF16 B200 SGLang MTP benchmark" + - "Image: lmsysorg/sglang:nightly-dev-20260216-d3bae71e" + - "Model: Qwen/Qwen3.5-397B-A17B" + - "Mirrors the qwen3.5-bf16-b200-sglang non-MTP recipe and adds EAGLE speculative decoding (num-steps=3, eagle-topk=1, num-draft-tokens=4)" + - "Configs: 1k1k and 8k1k, TP=8/EP=1 conc 4-64 with spec-decoding=mtp" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1074 + +- config-keys: + - qwen3.5-fp4-b200-sglang-mtp + description: + - "Add Qwen3.5-397B-A17B NVFP4 B200 SGLang MTP benchmark" + - "Image: lmsysorg/sglang:nightly-dev-20260402-d7256eb6" + - "Model: nvidia/Qwen3.5-397B-A17B-NVFP4" + - "Mirrors the qwen3.5-fp4-b200-sglang non-MTP recipe and adds EAGLE speculative decoding (num-steps=3, eagle-topk=1, num-draft-tokens=4)" + - "Configs: 1k1k and 8k1k, TP=4/EP=1 conc 4-128 with spec-decoding=mtp" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1075 + +- config-keys: + - qwen3.5-fp8-mi355x-sglang-mtp + description: + - "Add Qwen3.5-397B-A17B FP8 MI355X SGLang MTP benchmark" + - "Image: lmsysorg/sglang-rocm:v0.5.10rc0-rocm720-mi35x-20260414" + - "Model: Qwen/Qwen3.5-397B-A17B-FP8" + - "Mirrors the qwen3.5-fp8-mi355x-sglang non-MTP recipe and adds EAGLE speculative decoding (num-steps=3, eagle-topk=1, num-draft-tokens=4)" + - "Configs: 1k1k (TP8/EP1, TP8/EP8, TP2/EP2) and 8k1k (TP2/EP2, TP4/EP1) with spec-decoding=mtp" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1076 + +- config-keys: + - qwen3.5-bf16-mi355x-sglang-mtp + description: + - "Add Qwen3.5-397B-A17B BF16 MI355X SGLang MTP benchmark" + - "Image: lmsysorg/sglang-rocm:v0.5.10rc0-rocm720-mi35x-20260415" + - "Model: Qwen/Qwen3.5-397B-A17B" + - "Mirrors the qwen3.5-bf16-mi355x-sglang non-MTP recipe and adds EAGLE speculative decoding (num-steps=3, eagle-topk=1, num-draft-tokens=4)" + - "Configs: 1k1k and 8k1k, TP=8/EP=1 conc 4-256 with spec-decoding=mtp" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1077 + +- config-keys: + - qwen3.5-bf16-b300-sglang + description: + - "Add Qwen3.5-397B-A17B BF16 B300 SGLang benchmark" + - "Image: lmsysorg/sglang:v0.5.10.post1-cu130" + - "Model: Qwen/Qwen3.5-397B-A17B" + - "Mirrors the B200 BF16 recipe with an extra TP4/EP1 search-space entry alongside the existing TP8/EP1 sweep" + - "Configs: 1k1k and 8k1k, TP8/EP1 conc 4-64 + TP4/EP1 conc 4-64" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1081 + +- config-keys: + - qwen3.5-bf16-b300-sglang-mtp + description: + - "Add Qwen3.5-397B-A17B BF16 B300 SGLang MTP benchmark" + - "Image: lmsysorg/sglang:v0.5.10.post1-cu130" + - "Model: Qwen/Qwen3.5-397B-A17B" + - "Mirrors the qwen3.5-bf16-b300-sglang non-MTP recipe and adds EAGLE speculative decoding (num-steps=3, eagle-topk=1, num-draft-tokens=4)" + - "Configs: 1k1k and 8k1k, TP8/EP1 conc 4-64 + TP4/EP1 conc 4-64, spec-decoding=mtp" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1082 + +- config-keys: + - qwen3.5-fp4-b300-sglang-mtp + description: + - "Add Qwen3.5-397B-A17B NVFP4 B300 SGLang MTP benchmark" + - "Image: lmsysorg/sglang:v0.5.10.post1-cu130" + - "Model: nvidia/Qwen3.5-397B-A17B-NVFP4" + - "Mirrors the qwen3.5-fp4-b300-sglang non-MTP recipe and adds EAGLE speculative decoding (num-steps=3, eagle-topk=1, num-draft-tokens=4)" + - "Configs: 1k1k and 8k1k, TP4/EP1 conc 4-128 + TP2/EP2 conc 4-128, spec-decoding=mtp" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1083 + +- config-keys: + - glm5-fp8-b300-sglang-mtp + description: + - "Add GLM-5 FP8 B300 SGLang MTP benchmark" + - "Image: lmsysorg/sglang:v0.5.10.post1-cu130" + - "Model: zai-org/GLM-5-FP8" + - "Mirrors the glm5-fp8-b300-sglang non-MTP recipe and adds EAGLE speculative decoding (num-steps=3, eagle-topk=1, num-draft-tokens=4) behind SGLANG_ENABLE_SPEC_V2=1" + - "Configs: 1k1k and 8k1k, TP=8/EP=1 conc 4-256 with spec-decoding=mtp" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1084 + +- config-keys: + - glm5-fp8-b200-sglang-mtp + description: + - "Add GLM-5 FP8 B200 SGLang MTP benchmark" + - "Image: lmsysorg/sglang:nightly-dev-cu13-20260317-1eea7448" + - "Model: zai-org/GLM-5-FP8" + - "Mirrors the glm5-fp8-b200-sglang non-MTP recipe and adds EAGLE speculative decoding (num-steps=3, eagle-topk=1, num-draft-tokens=4) behind SGLANG_ENABLE_SPEC_V2=1" + - "Configs: 1k1k and 8k1k, TP=8/EP=1 conc 4-256 with spec-decoding=mtp" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1085 + +- config-keys: + - glm5-fp8-mi355x-sglang-mtp + description: + - "Add GLM-5 FP8 MI355X SGLang MTP benchmark" + - "Image: lmsysorg/sglang-rocm:v0.5.10rc0-rocm720-mi35x-20260413" + - "Model: zai-org/GLM-5-FP8" + - "Mirrors the glm5-fp8-mi355x-sglang non-MTP recipe and adds EAGLE speculative decoding (num-steps=3, eagle-topk=1, num-draft-tokens=4) behind SGLANG_ENABLE_SPEC_V2=1" + - "Configs: 1k1k and 8k1k, TP=8 conc 4-64 with spec-decoding=mtp" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1086 + +- config-keys: + - glm5-fp4-b200-sglang-mtp + description: + - "Add GLM-5 NVFP4 B200 SGLang MTP benchmark (draft)" + - "Image: lmsysorg/sglang:v0.5.10.post1-cu130" + - "Model: nvidia/GLM-5-NVFP4" + - "Follows the glm5-fp8-b200-sglang launch recipe as requested, plus EAGLE speculative decoding (num-steps=3, eagle-topk=1, num-draft-tokens=4) behind SGLANG_ENABLE_SPEC_V2=1" + - "Configs: 1k1k and 8k1k, TP8/EP1 conc 4-4 + TP4/EP1 conc 4-256 with spec-decoding=mtp" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1087 + +- config-keys: + - glm5-fp4-b300-sglang-mtp + description: + - "Add GLM-5 NVFP4 B300 SGLang MTP benchmark (draft)" + - "Image: lmsysorg/sglang:v0.5.10.post1-cu130" + - "Model: nvidia/GLM-5-NVFP4" + - "Follows the glm5-fp8-b300-sglang launch recipe as requested, plus EAGLE speculative decoding (num-steps=3, eagle-topk=1, num-draft-tokens=4) behind SGLANG_ENABLE_SPEC_V2=1" + - "Configs: 1k1k and 8k1k, TP8/EP1 conc 4-4 + TP4/EP1 conc 4-256 with spec-decoding=mtp" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1088 + +- config-keys: + - gptoss-fp4-mi300x-vllm + description: + - "Extend GPT-OSS FP4 TP=8 search to conc=1" + - "low-latency endpoint for users prioritizing interactive single-user use cases (chat, copilot, agentic)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1092 + +- config-keys: + - dsr1-fp8-h200-dynamo-trt + - dsr1-fp8-h200-dynamo-sglang + description: + - "Add H200 multinode evals-only runs" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1094 + evals-only: true + +- config-keys: + - glm5.1-fp4-mi355x-sglang + description: + - "Add GLM5.1 MXFP4 (FP4) MI355X SGLang Support" + - "Container : lmsysorg/sglang-rocm:v0.5.10rc0-rocm720-mi35x-20260415" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1098 + +- config-keys: + - kimik2.5-fp4-b300-vllm + description: + - "Add Kimi-K2.5 FP4 (NVFP4) B300 vLLM benchmark" + - "Image: vllm/vllm-openai:v0.19.0-cu130" + - "At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/moonshotai/Kimi-K2.5.html does not have a B300-specific recipe, so this reuses the existing Kimi-K2.5 FP4 B200 vLLM recipe as-is" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1100 + +- config-keys: + - minimaxm2.5-fp8-b300-vllm + description: + - "Add VLLM_FLOAT32_MATMUL_PRECISION=high, remove VLLM_FLASHINFER_ALLREDUCE_BACKEND=mnnvl" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1106 + +- config-keys: + - minimaxm2.5-fp4-b300-vllm + description: + - "Add VLLM_FLOAT32_MATMUL_PRECISION=high, update search space concurrency ranges" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1107 + +- config-keys: + - dsr1-fp8-h100-dynamo-trt + - dsr1-fp8-h100-dynamo-sglang + description: + - "Trigger H100 multinode evals after dist-timeout and health-check timeout fixes" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1119 + +- config-keys: + - dsr1-fp8-h100-dynamo-trt + - dsr1-fp8-h100-dynamo-sglang + description: + - "Trigger H100 multinode evals after NVSHEMM fixes" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1120 + evals-only: true + +- config-keys: + - dsv4-fp4-gb200-dynamo-vllm + description: + - "Add DeepSeek-V4-Pro FP4 GB200 disaggregated vLLM benchmarks via Dynamo (1k/1k sweep; 8k/1k currently commented out)" + - "Container: vllm/vllm-openai:deepseekv4-cu130; model from /mnt/numa1/models/deepseek-v4-pro/ (compute-node-local NVMe)" + - "Topologies: low-conc 1p1d-dep8-tep8 (4 nodes, mirrored from NVIDIA srt-slurm PR #71 with offload kept and numa-bind dropped); mid 1p1d-dep8-dep16 (6 nodes) and high 3p1d-dep8-dep16 (10 nodes) hand-rolled, structurally derived from the kimi-k2.5 1k/1k pattern" + - "Recipes stored under benchmarks/multi_node/srt-slurm-recipes/ and overlaid onto the upstream srt-slurm checkout at runtime" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1129 + +- config-keys: + - qwen3.5-fp4-mi355x-atom + description: + - "Add Qwen3.5-397B-A17B MXFP4 single-node MI355X ATOM benchmark" + - "Image: rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post" + - "TP=2 and TP=4, concurrency 4-256 for 1k1k and 8k1k sequence lengths" + - "Add --gpu-memory-utilization 0.9 to server launch" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1133 + + +- config-keys: + - dsv4-fp8-h200-vllm + description: + - "Add DeepSeek-V4-Pro vLLM H200 benchmark per https://vllm.ai/blog/deepseek-v4" + - "Image: vllm/vllm-openai:deepseekv4-cu129" + - "Model: deepseek-ai/DeepSeek-V4-Pro" + - "EP + DP=8, FP8 KV cache, block size 256, max-model-len 800000, prefix caching disabled" + - "H200 has no FP4 path, so --attention_config.use_fp4_indexer_cache is omitted" + - "VLLM_ENGINE_READY_TIMEOUT_S=3600 to accommodate large weight loading" + - "Configs: 1k1k conc 4-64, 8k1k conc 4-64" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1130 + +- config-keys: + - dsv4-fp4-b200-sglang + description: + - "Add DeepSeek-V4-Pro single-node B200 SGLang benchmark (TP8, EP8, dp-attention)" + - "Container: lmsysorg/sglang:deepseek-v4-blackwell" + - "Recipe from https://docs.sglang.io/cookbook/autoregressive/DeepSeek/DeepSeek-V4" + - "Parallelism and sweep conc ranges match the dsv4-fp4-b200-vllm config" + - "Prefix caching and speculative decoding disabled for baseline numbers" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1131 + +- config-keys: + - dsv4-fp8-mi355x-sglang + description: + - "Day 0 DeepSeek-V4-Pro FP8 MI355X SGLang benchmark" + - "Image: rocm/sgl-dev:deepseek-v4-mi35x (from sgl-project/sglang#23608)" + - "Model: sgl-project/DeepSeek-V4-Pro-FP8" + - "https://github.com/sgl-project/sglang/pull/23608#issuecomment-4311952977" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1134 + +- config-keys: + - dsv4-fp4-b300-sglang + description: + - "Add DeepSeek-V4-Pro FP4 B300 SGLang benchmark (low-latency fallback)" + - "Image: lmsysorg/sglang:deepseek-v4-b300" + - "Model: deepseek-ai/DeepSeek-V4-Pro" + - "Low-latency only (TP=8, EP=1, no DP-attn, no DeepEP) — DeepEP FP8 weight-postprocess path is broken for this checkpoint on B300" + - "Prefix caching disabled, no speculative decoding" + - "Configs: 1k1k conc 4-1024, 8k1k conc 4-512" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1143 + +- config-keys: + - dsv4-fp4-b300-vllm + description: + - "Add DeepSeek-V4-Pro single-node B300 vLLM aggregate benchmark" + - "Image: vllm/vllm-openai:deepseekv4-cu130" + - "Model: deepseek-ai/DeepSeek-V4-Pro" + - "Uses the submitted B300 pareto schedule for both 1k1k and 8k1k, excluding conc 1: TP8 at conc 4/128, TP4 at conc 4/8/16/32/64/128, DP4 at conc 256/512" + - "Launch args match the provided vllm serve command, including FP4 indexer cache, FULL_AND_PIECEWISE cudagraph config, and max-num-batched-tokens 2048" + - "1k1k uses --max-model-len 4096; 8k1k uses the workflow-provided benchmark context length" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1144 + +- config-keys: + - dsv4-fp8-mi355x-sglang + description: + - "Bump MI355X SLURM allocation from --time=180 to --time=300 in runners/launch_mi355x-amds.sh" + - "DSv4-Pro on MI355X exceeded the 3h cap (STEP CANCELLED DUE TO TIME LIMIT) due to ~30min MoE JIT compile plus slow torch-fallback kernels (SGLANG_HACK_FLASHMLA_BACKEND=torch et al.) from sgl-project/sglang#23608" + - "300 minutes matches the GH Actions outer timeout-minutes cap in benchmark-tmpl.yml" + - "Retriggering dsv4-fp8-mi355x-sglang" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1148 + +- config-keys: + - dsv4-fp8-mi355x-sglang + description: + - "Drop --mem-fraction-static 0.88 and --max-total-tokens from dsv4_fp8_mi355x.sh" + - "Bump --chunked-prefill-size from 4096 to 8192" + - "Retrigger dsv4-fp8-mi355x-sglang" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1160 + +- config-keys: + - dsv4-fp4-mi355x-atom + description: + - "Add DeepSeek-V4-Pro FP4 MI355X ATOM Day-0 marker (single-sequence, TP=8, conc=1)" + - "Image: rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post (matches qwen3.5-fp8-mi355x-atom base); ROCm/ATOM#650 overlaid at runtime via pip install --no-deps -e . from a pinned PR SHA (cdbff35) inside the benchmark script" + - "triton_kernels is missing from the release image (build-stage path /triton-test/python/triton_kernels/ is cleaned up); the script falls back to ROCm/triton@e491726 (RI3.5.x), which has matmul_ogs.py and routing.py (PR #650 imports both — upstream triton-lang/triton refactored matmul_ogs into matmul.py and removed routing) plus CDNA4MXScaleLayout and a target_info.py compatible with the image's bundled triton" + - "Model: deepseek-ai/DeepSeek-V4-Pro (same canonical checkpoint used by dsv4-fp4-b300-vllm); compatibility with PR #650's WeightsMapper not yet verified — first run will tell us" + - "Pinned to PR1 limitations: single-sequence kv_cache hardcode, --enforce-eager required, ATOM_USE_TRITON_MOE=1 (aiter fused_moe broken on gfx950)" + - "Sweep will expand to TP=4/8 conc 4–256 once ROCm/ATOM PR3 (multi-request) and PR4 (CUDAGraph) land" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1165 + +- config-keys: + - dsv4-fp4-mi355x-atom + description: + - "Add DeepSeek-V4-Pro FP4 MI355X ATOM Day-0 marker (single-sequence, TP=8, conc=1)" + - "Image: rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post (matches qwen3.5-fp8-mi355x-atom base); ROCm/ATOM#650 overlaid at runtime via pip install --no-deps -e . from a pinned PR SHA (cdbff35) inside the benchmark script" + - "triton_kernels is missing from the release image (build-stage path /triton-test/python/triton_kernels/ is cleaned up); the script falls back to ROCm/triton@e491726 (RI3.5.x), which has matmul_ogs.py and routing.py (PR #650 imports both — upstream triton-lang/triton refactored matmul_ogs into matmul.py and removed routing) plus CDNA4MXScaleLayout and a target_info.py compatible with the image's bundled triton" + - "Model: deepseek-ai/DeepSeek-V4-Pro (same canonical checkpoint used by dsv4-fp4-b300-vllm); compatibility with PR #650's WeightsMapper not yet verified — first run will tell us" + - "Pinned to PR1 limitations: single-sequence kv_cache hardcode, --enforce-eager required, ATOM_USE_TRITON_MOE=1 (aiter fused_moe broken on gfx950)" + - "Sweep will expand to TP=4/8 conc 4–256 once ROCm/ATOM PR3 (multi-request) and PR4 (CUDAGraph) land" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1170 + +- config-keys: + - dsv4-fp4-b300-sglang-mtp + description: + - "Add DeepSeek-V4-Pro FP4 B300 SGLang benchmark with EAGLE/MTP speculative decoding" + - "Image: lmsysorg/sglang:deepseek-v4-b300@sha256:26e116bd211e300dbb76924d56c5cbe6cc3ee5ee2fe314859cb8774f5bc070f3 (pinned for deep_gemm transform_weights_for_mega_moe support; same digest as PR #1158)" + - "Model: deepseek-ai/DeepSeek-V4-Pro" + - "EAGLE/MTP flags hardcoded in script: num-steps=3, eagle-topk=1, num-draft-tokens=4" + - "Recipe (MoE backend, chunked-prefill) selected in script by dp-attn: TP-only + flashinfer_mxfp4 (small batch) vs DP-attn + deepep mega_moe (large batch)" + - "Three CONC bands: A=TP8 (1-8), B=TP4 (16-128), C=DP4 dp-attn (64-512); B/C overlap at conc 64,128" + - "Configs: 1k1k and 8k1k, no validation.py / launcher / yaml-field changes (knob-free)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1166 + +- config-keys: + - dsv4-fp4-b300-vllm + description: + - "Update search space based on B300 pareto sweep results" + - "ISL=1024: TP4 conc 4-128; DP4 (dp-attn) conc 256-4096; DP8 (dp-attn) conc 2048-8192" + - "ISL=8192: TP4 conc 4-64; DP4 (dp-attn) conc 128-1024; DP8 (dp-attn) conc 1024-8192" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1155 + +- config-keys: + - dsv4-fp4-b300-sglang + description: + - "Recipe-per-CONC split for DeepSeek-V4-Pro on B300: low-latency (TP=8, EP=1), balanced (TP=4, EP=1) at conc=32, max-throughput (TP=4, EP=4, DP-attn + DeepEP) at conc=512, for both 1k1k and 8k1k" + - "Recipes from https://docs.sglang.io/cookbook/autoregressive/DeepSeek/DeepSeek-V4" + - "Image pinned to lmsysorg/sglang:deepseek-v4-b300@sha256:26e116bd211e300dbb76924d56c5cbe6cc3ee5ee2fe314859cb8774f5bc070f3" + - "DP-attention path enables SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN=1 for better SWA eviction behavior" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1185 + + +- config-keys: + - dsv4-fp4-b200-sglang + description: + - "Two-recipe dispatch for DeepSeek-V4-Pro on B200, selected by DP_ATTENTION knob: low-latency (TP=8, EP=1, flashinfer_mxfp4) for conc 1-32, DP-attention (TP=8, EP=8, DP-attn + DeepEP + mega_moe) for conc 64-{512,1024}. The DP-attention recipe uses identical flags across balanced and max-throughput CONC ranges; only --max-running-requests scales with CONC." + - "Recipes from https://docs.sglang.io/cookbook/autoregressive/DeepSeek/DeepSeek-V4" + - "Image pinned to lmsysorg/sglang:deepseek-v4-blackwell@sha256:df18bfc4aa9ecf59451002b49ba00cae58042de9e2a96378bbd21b404dd62c7b" + - "Adds SGLANG_OPT_* env knobs (SWA_SPLIT_LEAF_ON_INSERT, USE_JIT_NORM, USE_JIT_INDEXER_METADATA, USE_TOPK_V2, USE_CUSTOM_ALL_REDUCE_V2)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1187 + +- config-keys: + - dsv4-fp4-b300-sglang-mtp + description: + - "Pass --dsv4 to run_benchmark_serving so MTP benchmarks use the DSv4 chat template (PR #1153)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1182 + +- config-keys: + - gptoss-fp4-mi355x-atom + description: + - "Update GPTOSS-120B FP4 MI355X Atom benchmark (rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1195 + + +- config-keys: + - dsv4-fp4-b300-vllm + description: + - Add low-latency configs and remove non-pareto configs + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1193 + +- config-keys: + - dsv4-fp4-b200-vllm + description: + - "Add DeepSeek-V4-Pro single-node B200 vLLM benchmark derived from B200 pareto sweep" + - "ISL=1024: TP8 conc 4-128; DP8 (dp-attn) conc 256-4096" + - "ISL=8192: TP8 conc 4-32; DP8 (dp-attn) conc 64-1024" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1156 + +- config-keys: + - dsv4-fp4-b300-sglang-mtp + description: + - "Add DeepSeek-V4-Pro FP4 B300 SGLang benchmark with EAGLE/MTP speculative decoding" + - "Image: lmsysorg/sglang:deepseek-v4-b300@sha256:26e116bd211e300dbb76924d56c5cbe6cc3ee5ee2fe314859cb8774f5bc070f3 (pinned for deep_gemm transform_weights_for_mega_moe support; same digest as PR #1158)" + - "Model: deepseek-ai/DeepSeek-V4-Pro" + - "EAGLE/MTP flags hardcoded in script: num-steps=3, eagle-topk=1, num-draft-tokens=4" + - "Recipe (MoE backend, chunked-prefill) selected in script by dp-attn: TP-only + flashinfer_mxfp4 (small batch) vs DP-attn + deepep mega_moe (large batch)" + - "Three CONC bands: A=TP8 (1-8), B=TP4 (16-128), C=DP4 dp-attn (64-512); B/C overlap at conc 64,128" + - "Configs: 1k1k and 8k1k, no validation.py / launcher / yaml-field changes (knob-free)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1180 + +- config-keys: + - dsv4-fp8-mi355x-vllm + description: + - "Add vLLM DeepSeek-V4-Pro FP8 benchmark for MI355X with AITER-accelerated MLA decode (vllm-project/vllm#40889, stacked on #40871)" + - "Base image rocm/atom:rocm7.2.2 (MI355X ROCm 7.2.2, aiter with MLA decode); vLLM rebuilt from PR branch at pinned SHA b3a4a44 at runtime via --no-deps overlay" + - "Key flags: --enforce-eager, --moe-backend triton_unfused, --kv-cache-dtype fp8, VLLM_ROCM_USE_AITER=1" + - "Search space: TP=8, concurrency 4-64, 1k1k and 8k1k" + - "MI355X runner updated to resolve framework-specific script names (dsv4_fp8_mi355x_vllm.sh) with fallback to generic names" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1188 + +- config-keys: + - dsv4-fp4-b300-sglang + description: + - "conc=2048/4096: mega_moe deepep backend; conc=2048 cuda-graph-max-bs 288, mem 0.87; conc=4096 cuda-graph-max-bs 544, mem 0.835, swa-ratio 0.075, tokenizer-workers 8" + - "1k1k conc=512/1024: add mega_moe deepep backend with cuda-graph-max-bs 550, chunked-prefill 16384, max-running-requests 768" + - "ep=8 naming convention in yaml distinguishes mega_moe from existing flashinfer_mxfp4 ep=4 entries" + - "Recipes from https://docs.sglang.io/cookbook/autoregressive/DeepSeek/DeepSeek-V4" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1179 + +- config-keys: + - dsv4-fp4-mi355x-atom + description: + - "Use ROCm/aiter#2916 mhc_pre device-allocation fix instead of disabling ATOM mhc_pre" + - "Patch installed aiter/ops/mhc.py at runtime to allocate mhc_pre intermediates on residual.device, preserving the aiter MHC fast path without rebuilding aiter" + - "Remove the ATOM deepseek_v4.py sed workaround that forced mhc_pre to torch fallback" + - "Keep dsv4-fp4-mi355x-atom at CONC=1 only; run 24953107645 showed high-concurrency DSv4 ATOM OOMs in PR #650 torch sparse-attention fallbacks before upstream AITER sparse-attention support lands" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1202 + +- config-keys: + - dsv4-fp4-b300-vllm-mtp + description: + - "Add preliminary vLLM MTP configs for DeepSeek-V4-Pro on B300" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1210 + +- config-keys: + - dsv4-fp4-b200-vllm + description: + - "Pin image to vllm/vllm-openai:v0.20.0-cu130 (was floating deepseekv4-cu130 tag); DeepGEMM is preinstalled in this image" + - "Use --attention_config.use_fp4_indexer_cache=True and --compilation-config {\"cudagraph_mode\": \"FULL_AND_PIECEWISE\", \"custom_ops\": [\"all\"]} for all configs" + - "Gate --moe-backend deep_gemm_mega_moe and --gpu-memory-utilization 0.85 on DP_ATTENTION=true per the v0.20.0 recipe" + - "Drop --pipeline-parallel-size 1; keep --no-enable-prefix-caching and --max-cudagraph-capture-size 2048" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1204 + +- config-keys: + - minimaxm2.5-fp4-mi355x-atom + description: + - "Add MiniMax-M2.5 MXFP4 MI355X Atom benchmark (rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post)" + - "Single-node sweep: TP1–TP8, 1k/1k and 8k/1k ISL/OSL" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1042 + +- config-keys: + - dsv4-fp4-gb200-dynamo-vllm + description: + - "DSV4-Pro FP4 GB200 dynamo-vLLM disagg against srt-slurm aflowers/vllm-gb200-v0.20.0" + - "Keeps the three validated 8k/1k points: low-latency 1P/1D TP8 conc=1, mid-curve 1P/1D DEP8 conc=256, and max-tpt 3P/1D DEP8 conc=4096" + - "All three recipes run NATS/etcd on a dedicated infra node and use compute-node local NVMe model weights via /mnt/numa1/models/deepseek-v4-pro/" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1163 + +- config-keys: + - dsv4-fp4-gb200-dynamo-vllm + description: + - "Add GB200 Dynamo vLLM MegaMOE max-throughput recipe at conc=4096" + - "Topology matches max-tpt: 3 prefill DEP8 workers and 1 decode DEP8 worker with dedicated NATS/etcd" + - "Uses deep_gemm_mega_moe on prefill/decode, TORCH_SYMMMEM=NVSHMEM, and no offload" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1218 + +- config-keys: + - dsv4-fp4-gb200-dynamo-vllm + description: + - "Add GB200 Dynamo vLLM low-middle curve recipe at conc=256/512" + - "Topology: 1 prefill DEP8 worker and 4 decode TP8 workers with dedicated NATS/etcd" + - "Mirrors the historical 1P4D DEP8/TP8 offload point from srt-slurm aflowers/vllm-gb200-v0.20.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1218 + +- config-keys: + - dsv4-fp4-b300-sglang + description: + - "Add conc=8192 recipe for 1k1k: deepep mega_moe backend with cuda-graph-max-bs 1088, max-running-requests 8192, mem-fraction-static 0.80, swa-full-tokens-ratio 0.3, tokenizer-worker-num 16" + - "conc=8192 enables SGLANG_OPT_USE_ONLINE_COMPRESS=1 and --stream-interval 30" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1209 + + +- config-keys: + - dsv4-fp4-b300-vllm + description: + - "Change image to vllm/vllm-openai:v0.20.0-cu130" + - "Use Mega MoE for DEP configs" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1221 + +- config-keys: + - dsv4-fp4-b200-vllm-mtp + description: + - "Add preliminary vLLM MTP configs for DeepSeek-V4-Pro on B200" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1230 + +- config-keys: + - dsv4-fp4-gb200-dynamo-vllm + description: + - "Keep the GB200 Dynamo vLLM MegaMOE max-throughput recipe at 3P/1D DEP8 conc=4096" + - "Add GB200 Dynamo vLLM MegaMOE high-throughput recipe at 2P/1D DEP8 conc=4096" + - "Add GB200 Dynamo vLLM MegaMOE mid-curve recipe at 1P/1D DEP8 conc=256/512/1024" + - "Remove stale offload recipe copies and the old no-MegaMOE mid/max-throughput points from the GB200 Dynamo vLLM matrix" + - "Disable FlashInfer autotune on GB200 decode workers for accuracy stability, matching the srt-slurm recipe fix" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1223 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang + description: + - "Add DeepSeek-V4-Pro FP4 GB300 disaggregated SGLang benchmarks via Dynamo (1k/1k sweep; 8k/1k recipes shipped but commented out)" + - "Container: lmsysorg/sglang:deepseek-v4-grace-blackwell (linux/arm64); model from /mnt/numa1/models/deepseek-v4-pro/ (compute-node-local NVMe)" + - "Topologies mirror the dsv4-fp4-gb300-dynamo-vllm sibling: low-conc 1p1d-dep8-tep8 (4 nodes), mid 1p1d-dep8-dep16 (6 nodes), high 3p1d-dep8-dep16 (10 nodes). 4096 overlap between mid and high gives a topology-crossover A/B" + - "No upstream GB300 DSV4 sglang disagg recipe exists. Per-worker sglang_config (env vars + flashinfer_mxfp4 + chunked-prefill-size 4096 + disable-flashinfer-autotune + mem-fraction-static 0.82) is mirrored from NVIDIA/srt-slurm PR #69 (recipes/gb300-fp4/1k1k-dsv4/agg-2n-low-latency.yaml — GB300 DSV4 SGLang aggregated). Disagg flag set (nixl transfer backend, enable-dp-attention + moe-a2a-backend deepep) cross-checked against PR #75 (recipes/gb300-fp4/1k1k-dsv4/disagg-1p1d-tp4-mxfp4.yaml — GB300 DSV4 SGLang disagg) and the SGLang DeepSeek-V4 cookbook. Stored under benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/ and overlaid onto the upstream srt-slurm checkout at runtime" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1157 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm + description: + - "Add DeepSeek-V4-Pro FP4 GB300 disaggregated Dynamo vLLM benchmarks at 8k/1k" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1238 + +- config-keys: + - qwen3.5-fp8-b200-sglang + description: + - updated sglang container image + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1027 + +- config-keys: + - glm5-fp8-mi355x-atom + description: + - "Update GLM-5 FP8 MI355X ATOM benchmark: new image, add TP=4, set gpu-memory-utilization" + - "Image: rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post" + - "Add TP=4, concurrency 4-256 for 1k1k and 8k1k sequence lengths" + - "Add --gpu-memory-utilization 0.9 to server launch" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1126 + +- config-keys: + - dsr1-fp8-mi355x-sglang + description: + - "Tune --num-continuous-decode-steps 4 → 8 (+4.7% avg output throughput gain)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1243 + +- config-keys: + - dsv4-fp4-gb200-dynamo-vllm-mtp2 + description: + - "Add final DeepSeek-V4-Pro FP4 GB200 Dynamo vLLM MTP2 Pareto recipes using vLLM nightly image" + - "Recipes cover 8k/1k aggregate TP8 low-latency conc=1, low-latency bridge 1P DEP8 + 4D TP8 no-offload conc=16/32/64, mid 1P/1D DEP8 MegaMOE conc=128, and high-throughput 2P/1D DEP8 MegaMOE conc=1024" + - "All recipes enable FP4 indexer cache and speculative-config mtp with num_speculative_tokens=2" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1242 + +- config-keys: + - glm5-fp8-mi355x-sglang-mtp + description: + - "Updated the Image for glm5-fp8-mi355x-sglang-mtp" + - "Optimized the search space" + - "Removed redundant transformer installation" + - "Optimization model serves configs" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1252 + +- config-keys: + - qwen3.5-fp4-b200-sglang + description: + - "Update image to lmsysorg/sglang:nightly-dev-20260422-de962f32" + - "Fix tp:2 search-space: ep 2 -> 1" + - "Dynamic scheduler-recv-interval: 30 for CONC>4, 10 otherwise" + - "Remove --max-running-requests, reduce prefill/chunked from 81920 to 16384" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1018 + +- config-keys: + - minimaxm2.5-fp8-mi355x-atom + description: + - "Update Atom image to rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post" + - "Search-space: expand conc-end to 256 for tp2/tp4 (1k/1k and 8k/1k); remove tp8/ep8 configs" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1194 + +- config-keys: + - dsv4-fp4-mi355x-sglang + description: + - "Add MI355X DSv4 FP4 SGLang benchmark on rocm/sgl-dev:rocm720-mi35x-583b1b6-20260501-DSv4" + - "Use deepseek-ai/DeepSeek-V4-Pro with SGLANG_DSV4_FP4_EXPERTS=True and SGLANG_FORCE_TRITON_MOE_FP8=0" + - "Overlay SGLang from amd/deepseek_v4 SHA a8410de6 at runtime" + - "Sweep DP attention on/off and EP=8 via DP_ATTENTION and EP_SIZE matrix env vars" + - "Ship a DeepSeek-V4 thinking-mode chat template so eval /v1/chat/completions works (the canonical checkpoint ships no chat_template)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1244 + +- config-keys: + - dsv4-fp8-h200-sglang + description: + - "Add DeepSeek-V4-Pro FP8 H200 single-node SGLang recipe (Marlin MoE backend, TP=8, EP=1)" + - "Image: lmsysorg/sglang:deepseek-v4-hopper pinned by digest" + - "Server flags: --moe-runner-backend marlin, --chunked-prefill-size 4096, --disable-flashinfer-autotune, --disable-radix-cache, --mem-fraction-static 0.88" + - "Search space: TP=8 EP=1, conc 1 and 4-64 for both 1k1k and 8k1k" + - "Pinned to the h200-dgxc runner pool (new runners.yaml group); launch_h200-dgxc-slurm.sh extended to support framework-tagged script names and mount /ix instead of /workspace for the deepseek-v4-hopper image" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1264 + +- config-keys: + - dsv4-fp8-h200-sglang-mtp + description: + - "Add DeepSeek-V4-Pro FP8 H200 single-node SGLang MTP variant (mirrors dsv4-fp8-h200-sglang)" + - "EAGLE speculative decoding chain: --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4" + - "run_benchmark_serving uses --dsv4 (chat-formatted prompts) per the AGENTS.md MTP rule, since EAGLE acceptance regresses on raw random tokens" + - "Search space mirrors the non-MTP H200 SGLang entry: TP=8 EP=1, conc 1 and 4-64 for both 1k1k and 8k1k, with spec-decoding: mtp" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1265 + +- config-keys: + - kimik2.5-int4-b300-vllm + description: + - "Add Kimi-K2.5 INT4 B300 vLLM benchmark" + - "Image: vllm/vllm-openai:v0.20.0-cu130" + - "Search-space: tp=8 and tp=4/ep=1 over conc 4-64, on both 1024/1024 and 8192/1024 ISL/OSL" + - "At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/moonshotai/Kimi-K2.5.html does not have a B300-specific recipe, so this reuses the existing Kimi-K2.5 INT4 B200 vLLM recipe as-is until B300-specific tuning is available" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1057 + +- config-keys: + - kimik2.5-int4-b300-vllm + description: + - "Add Kimi-K2.5 INT4 B300 vLLM benchmark" + - "Image: vllm/vllm-openai:v0.20.0-cu130" + - "Search-space: tp=8 and tp=4/ep=1 over conc 4-64, on both 1024/1024 and 8192/1024 ISL/OSL" + - "At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/moonshotai/Kimi-K2.5.html does not have a B300-specific recipe, so this reuses the existing Kimi-K2.5 INT4 B200 vLLM recipe as-is until B300-specific tuning is available" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1057 + +- config-keys: + - dsv4-fp4-b300-trt + description: + - "Add B300 TensorRT-LLM DeepSeek-V4-Pro eval coverage using the feat/deepseek_v4 image" + - "Disable TRTLLM fused MHC hyper-connection for eval servers via TRTLLM_MHC_ENABLE_FUSED_HC=0 because the current fused kernel corrupts DeepSeek-V4-Pro hidden size 7168 generations" + - "Keep this as eval-only PR validation until the TensorRT-LLM fused MHC kernel is guarded or supports hidden size 7168" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1233 + +- config-keys: + - dsr1-fp4-mi355x-sglang-disagg + - dsr1-fp4-mi355x-sglang-disagg-mtp + description: + - "Bump SGL mori image to lmsysorg/sglang-rocm" + - "Add more high tput / low latency sweep configs" + - "Enable v2 mxfp4 DSR1 0528 model" + - "Enable fp4 disp / fp8 combine feature on mori" + - "Enable Mori SDMA + two batch overlapping feature" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1236 + +- config-keys: + - dsv4-fp4-mi355x-sglang + description: + - "Tune DSv4 FP4 MI355X SGLang runtime envs: enable fused compress and TileLang MHC post, and drop the Torch fallback env overrides for top-k transform and FP8 paged MQA logits" + - "Drive SGLang --max-running-requests and --cuda-graph-max-bs from the matrix CONC value instead of hard-coding 256, so each sweep point launches with matching serving capacity and graph capture size" + - "Adjust the sweep coverage: start 1k1k dp-attn=true at conc=16, add 1k1k dp-attn=false conc=1-16, and extend 8k1k dp-attn=false coverage to conc=16" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1272 + +- config-keys: + - dsv4-fp4-mi355x-sglang + description: + - "Bump image to rocm/sgl-dev:rocm720-mi35x-a8410de-20260502-DSv4 (one commit forward on amd/deepseek_v4: sglang PR #24249, fuse-compress-decode 0501)" + - "Drop the runtime sglang clone+pip overlay from benchmarks/single_node/dsv4_fp4_mi355x_sglang.sh — the new image bakes the same a8410de6 SHA the overlay was pinning, so the overlay is redundant. Future sglang bumps now go through an image tag bump" + - "Context: ep=8 dp-attn=true entries failed gsm8k eval after #1244 merged. PR sweep (run 25246535693) reported gsm8k strict-match=0.9318 because the launcher silently dropped --ep-size and sglang ran with ep_size=1 regardless of the matrix label; post-merge sweep (run 25262278289) ran with ep_size=8 and gsm8k strict-match dropped to 0.0000. The image bump is the candidate fix to verify on rerun" + - "ep=1 entries (dp-attn true and false) are unaffected by the EP=8 regression" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1266 + +- config-keys: + - dsv4-fp4-gb200-dynamo-vllm-mtp2 + description: + - "Fix result file parsing for non-disagg multinode benchmarks (decode num-worker=0)" + - "sa-bench result files without _ctx_/_gen_ suffix were silently dropped from the summary table" + - "Use vllm/vllm-openai:v0.20.1-ubuntu2404 directly for GB200 MTP2 instead of upgrading vLLM inside the v0.20.0 container" + - "Fix applies to all 7 multinode launch scripts, the benchmark-multinode-tmpl workflow, and process_result.py" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1256 + +- config-keys: + - qwen3.5-fp8-b200-sglang-mtp + description: + - "Update image to lmsysorg/sglang:nightly-dev-20260422-de962f32" + - "Add TP8 search-space point (conc 4) for 1k1k and 8k1k" + - "Dynamic scheduler-recv-interval: 30 for CONC>4, 10 otherwise" + - "Align B200 flags with B300: SGLANG_ENABLE_SPEC_V2=1, --enable-symm-mem, --expert-parallel-size" + - "Reduce prefill tokens from 32768 to 16384, drop flashinfer_allreduce_fusion" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1065 + +- config-keys: + - dsv4-fp8-h200-vllm-mtp + description: + - "Add DeepSeek-V4-Pro FP8 H200 vLLM MTP variant (mirrors dsv4-fp8-h200-vllm with --speculative-config {\"method\":\"mtp\",\"num_speculative_tokens\":1})" + - "Image: vllm/vllm-openai:v0.20.1" + - "Set VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 to skip the cudagraph-memory estimator (it overshoots the H200 + MTP memory budget at profile time even though actual cudagraph capture works fine)" + - "run_benchmark_serving uses --dsv4 (chat-formatted prompts) per the AGENTS.md MTP rule, since EAGLE-style speculative decoding regresses acceptance on raw random tokens" + - "Search space mirrors the non-MTP H200 entry: TP=8, EP=8, DP-attn=true, CONC 4-64 for both 1k1k and 8k1k, with spec-decoding: mtp" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1222 + +- config-keys: + - dsv4-fp8-h200-vllm-mtp + description: + - "Bump --speculative-config num_speculative_tokens from 1 to 2 (`{\"method\":\"mtp\",\"num_speculative_tokens\":2}`)" + - "Re-test whether H200 MTP kernels accept 2 draft tokens — Blackwell MTP runs at 2 (per @wzhao18's vLLM Blackwell MTP submission); checking if H200 has parity now" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1279 + +- config-keys: + - dsv4-fp4-b300-trt + description: + - "Update the TensorRT-LLM DeepSeek-V4-Pro image to ghcr.io/semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-9aa3715" + - "Enable TRTLLM fused MHC by default with the DeepSeek-V4 feature image" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1270 + +- config-keys: + - qwen3.5-fp4-b200-sglang-mtp + description: + - "Update image to lmsysorg/sglang:nightly-dev-20260422-de962f32" + - "Add tp:2 ep:1 conc 4-128 search-space for 1k1k and 8k1k" + - "Align server flags with FP4 B200 STP: --enable-symm-mem, --expert-parallel-size, dynamic scheduler-recv-interval" + - "Add MTP flags: SGLANG_ENABLE_SPEC_V2=1, EAGLE speculative decoding (steps=3, topk=1, draft=4)" + - "Reduce prefill/chunked from 32768 to 16384" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1257 + +- config-keys: + - dsv4-fp4-b200-trt + description: + - "Add B200 TensorRT-LLM DeepSeek-V4-Pro single-node coverage using the feat/deepseek_v4 image" + - "Mirror the B300 TRT launch path with OpenAI chat serving, FP8 KV cache, TRTLLM MoE, NCCL NVLS disabled by default, and fused MHC disabled for hidden size 7168 correctness" + - "Update the B200 DGXC Slurm partition from removed gpu to gpu-2 so single-node B200 jobs can allocate" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1277 + +- config-keys: + - dsv4-fp8-h200-vllm + - dsv4-fp8-h200-vllm-mtp + description: + - "Add pure TP (tp:8, ep:1) search-space row alongside the existing DP+EP row" + - "Raise conc-end from 64 to 256 on both 1k1k and 8k1k sweeps" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1287 + +- config-keys: + - dsv4-fp4-b300-trt-mtp + description: + - "Add DeepSeek-V4-Pro FP4 B300 TensorRT-LLM MTP coverage using ghcr.io/semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-9aa3715" + - "Mirror the B300 TRT STP search space with spec-decoding: mtp and TensorRT-LLM MTP num_nextn_predict_layers=2" + - "Benchmark serving uses the DeepSeek-V4 chat template for MTP acceptance-rate correctness" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1291 + +- config-keys: + - qwen3.5-fp8-b200-sglang-mtp + description: + - "Re-run qwen3.5-fp8-b200-sglang-mtp sweep after the B200 DGXC Slurm partition change (gpu → gpu-2)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1292 + +- config-keys: + - minimaxm2.5-fp8-mi355x-vllm + description: + - "Tune MiniMax-M2.5 FP8 MI355X vLLM scheduling thresholds for better throughput and stability across the 1k/1k and 8k/1k sweep points" + - "Default path: block-size=32, shuffled KV cache disabled (VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=0), async scheduling enabled" + - "1k/1k TP8/EP8: keep block-size=32 and shuffled KV cache disabled; disable async scheduling (--no-async-scheduling)" + - "1k/1k non-TP8/EP8: block-size=16 with shuffled KV cache enabled; disable async scheduling through c128" + - "8k/1k TP8/EP8: keep block-size=32 and shuffled KV cache disabled; disable AITER MoE (VLLM_ROCM_USE_AITER_MOE=0); disable async scheduling" + - "8k/1k non-TP8/EP8: disable async scheduling through c64; switch to block-size=16 with shuffled KV cache at c64 and above" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1276 + +- config-keys: + - dsv4-fp4-b200-trt-mtp + description: + - "Add DeepSeek-V4-Pro FP4 B200 TensorRT-LLM MTP coverage using ghcr.io/semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-9aa3715" + - "Mirror the B200 TRT STP search space with spec-decoding: mtp and TensorRT-LLM MTP num_nextn_predict_layers=2" + - "Benchmark serving uses the DeepSeek-V4 chat template for MTP acceptance-rate correctness" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1294 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang + description: + - "Overhaul the 8k/1k DSv4-Pro GB300 SGLang search space: switch decode to WideEP TP=16 across most concurrency points (TP=12 at the 21504 max-concurrency point), and scale concurrencies to 1024/1024/4096/8192/21504; keep a 1p1d TP=4 baseline at conc=1" + - "Rename and consolidate the per-concurrency recipe files to `disagg-gb300-{N}p1d-{topo}-{nodes}-c{conc}.yaml`" + - "Re-enable lm-eval scoring for dsv4-fp4-gb300-dynamo-sglang now that the srt-slurm pin includes the lm-eval orchestrator path" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1295 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang-mtp + description: + - "Add DeepSeek-V4-Pro FP4 GB300 disaggregated SGLang MTP coverage using lmsysorg/sglang-staging:deepseek-v4-grace-blackwell-dev" + - "Mirror the base 8k/1k disagg search space (1p1d-tp4, 1p6d-dep4-tp4, 1p1d-dep4-dep8, 1p1d-dep4-dep16, 2p1d-dep4-dep8, 4p1d-dep4-dep8) with spec-decoding: mtp and EAGLE on the decode side (num-steps=3, eagle-topk=1, num-draft-tokens=4)" + - "Recipes adapted from elvischenv/srt-slurm@dsv4-gb300-disagg-8k1k-mtp:recipes/dsv4-pro/sglang/gb300-fp4/8k1k/disagg/mtp/, repointed at the public sglang-staging container and the deepseek-v4-pro model alias" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1297 + +- config-keys: + - dsv4-fp4-mi355x-sglang + description: + - "Bump image to rocm/sgl-dev:rocm720-mi35x-bfd32b6-20260507-DSv4." + - "Tune DSv4 FP4 MI355X SGLang runtime envs: enable aiter MHC pre/post, and enable triton swa prepare kernel." + - "Add --context-length. Add --enable-prefill-delayer for dp config" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1300 + +- config-keys: + - dsv4-fp4-b200-dynamo-vllm + description: + - "Add DeepSeek-V4-Pro FP4 B200 disaggregated multi-node coverage using Dynamo vLLM" + - "9eff9734a30b6713a8566217d36f8277630fd2d31cec7f0a0292835901a23aa4" + - "Adapt the existing DSV4 GB200 vLLM disagg recipes to B200 by mapping each worker to one full 8-GPU B200 node" + - "Update the B200 DGXC Slurm launcher to support dsv4/fp4 with dynamo-vllm and local recipe overlays" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1303 + +- config-keys: + - dsv4-fp4-b300-dynamo-vllm + description: + - "Turn off explicit deep_gemm_mega_moe backend selection in B300 Dynamo vLLM throughput recipes" + - "Let vLLM choose the MoE backend automatically to avoid CUDA symmetric-memory rendezvous failures during startup" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1304 + +- config-keys: + - dsv4-fp4-b200-dynamo-vllm + description: + - "Fix B200 Dynamo vLLM recipe benchmark concurrencies to match the nvidia-master.yaml search space" + - "Propagate low-latency concurrencies 1/64/128, high-throughput concurrencies 1024/2048/4096/8192, and max-throughput concurrency 8192 into the srt-slurm recipe files" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1305 + +- config-keys: + - dsv4-fp4-mi355x-sglang + description: + - "Bump image to rocm/sgl-dev:rocm720-mi35x-0363e6c-20260509-DSv4." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1308 + +- config-keys: + - qwen3.5-fp8-mi355x-atom + - qwen3.5-fp8-mi355x-atom-mtp + description: + - "Add Qwen3.5-397B-A17B FP8 MI355X ATOM benchmark configs with and without MTP" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1310 + +- config-keys: + - minimaxm2.5-fp8-mi300x-vllm + description: + - "Update vLLM ROCm image from v0.16.0 to v0.21.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1405 + +- config-keys: + - kimik2.5-int4-b200-vllm + description: + - "Update vLLM image from v0.15.1 to v0.20.2" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1337 + +- config-keys: + - gptoss-fp4-h200-vllm + description: + - "Update vLLM image from v0.18.0 to v0.20.2" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1335 + +- config-keys: + - gptoss-fp4-h100-vllm + description: + - "Update vLLM image from v0.18.0 to v0.20.2" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1334 + +- config-keys: + - dsr1-fp4-b300-sglang + description: + - "Update SGLang image from v0.5.10.post1-cu130 to v0.5.11-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1321 + +- config-keys: + - dsr1-fp4-b300-sglang + description: + - "Update SGLang image from v0.5.10.post1-cu130 to v0.5.11-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1321 + +- config-keys: + - qwen3.5-fp8-h200-sglang-mtp + description: + - "Update SGLang image from v0.5.10.post1 to v0.5.11" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1347 + +- config-keys: + - qwen3.5-fp8-h200-sglang-mtp + description: + - "Rerun update SGLang image from v0.5.10.post1 to v0.5.11" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1347 + +- config-keys: + - dsr1-fp8-h200-sglang + description: + - "Update SGLang image from v0.5.9-cu130 to v0.5.11-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1326 + +- config-keys: + - glm5-fp8-b300-sglang-mtp + description: + - "Update SGLang image from v0.5.10.post1-cu130 to v0.5.11-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1332 + +- config-keys: + - glm5-fp4-b300-sglang-mtp + description: + - "Update SGLang image from v0.5.10.post1-cu130 to v0.5.11-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1330 + +- config-keys: + - glm5-fp8-b300-sglang + description: + - "Update SGLang image from v0.5.10.post1-cu130 to v0.5.11-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1331 + +- config-keys: + - dsr1-fp8-b300-sglang + description: + - "Update SGLang image from v0.5.10.post1-cu130 to v0.5.11-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1324 + +- config-keys: + - minimaxm2.5-fp8-h100-vllm + description: + - "Update vLLM image from v0.18.0 to v0.20.2" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1339 + +- config-keys: + - minimaxm2.5-fp8-h200-vllm + description: + - "Update vLLM image from v0.18.0 to v0.20.2" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1340 + +- config-keys: + - qwen3.5-bf16-b300-sglang + description: + - "Update SGLang image from v0.5.10.post1-cu130 to v0.5.11-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1341 + +- config-keys: + - qwen3.5-bf16-b300-sglang-mtp + description: + - "Update SGLang image from v0.5.10.post1-cu130 to v0.5.11-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1342 + +- config-keys: + - qwen3.5-fp4-b300-sglang + description: + - "Update SGLang image from v0.5.10.post1-cu130 to v0.5.11-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1343 + +- config-keys: + - qwen3.5-fp4-b300-sglang-mtp + description: + - "Update SGLang image from v0.5.10.post1-cu130 to v0.5.11-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1344 + +- config-keys: + - qwen3.5-fp8-b300-sglang-mtp + description: + - "Update SGLang image from v0.5.10.post1-cu130 to v0.5.11-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1346 + +- config-keys: + - dsr1-fp4-mi355x-sglang-disagg + - dsr1-fp4-mi355x-sglang-disagg-mtp + description: + - "Fix the eval result of dsr1 fp4 with fp8 blockwise combine" + - "Bump the image to May 19" + - "Add conc 512 new sweep point" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1566 + +- config-keys: + - kimik2.5-int4-h200-vllm + description: + - "Update vLLM image from v0.16.0 to v0.20.2" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1338 + +- config-keys: + - glm5-fp4-b300-sglang + description: + - "Update SGLang image from v0.5.10.post1-cu130 to v0.5.11-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1329 + +- config-keys: + - glm5-fp4-b200-sglang-mtp + description: + - "Update SGLang image from v0.5.10.post1-cu130 to v0.5.11-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1328 + +- config-keys: + - dsv4-fp4-mi355x-atom + description: + - "Add DeepSeek-V4-Pro FP4 MI355X ATOM benchmark config; bump image to rocm/atom-dev:nightly_202605101539, expand concurrency range (conc 4–1024), and simplify runtime script" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1311 + +- config-keys: + - glm5-fp8-mi355x-sglang + description: + - "Image: lmsysorg/sglang-rocm:v0.5.11-rocm720-mi35x-20260513" + - "Add --cuda-graph-max-bs to meet conc" + - "Turn to tp=4 for best perf" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1375 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang + description: + - "Enable W4A4 (MXFP4) megamoe by appending w4a4 related environ flags when megamoe is enabled" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1382 + +- config-keys: + - dsr1-fp8-b200-sglang-mtp + description: + - "Update SGLang image from v0.5.9-cu130 to v0.5.11-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1323 + +- config-keys: + - dsr1-fp8-b200-sglang + description: + - "Update SGLang image from v0.5.9-cu130 to v0.5.11-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1322 + +- config-keys: + - kimik2.5-fp4-mi355x-atom + description: + - "Bump ATOM image to rocm/atom:rocm7.2.3_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom20260511" + - "Improves tput/GPU by up to +31% at low concurrency (tp=4, isl=1024, c=4-16)" + - "Ref ATOM upstream benchmark run https://github.com/ROCm/ATOM/actions/runs/25686894636" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1387 + + +- config-keys: + - dsv4-fp4-b300-vllm-mtp + description: + - "Update image tag to vllm/vllm-openai:v0.20.2" + - "Add DEP configs for B300 vLLM MTP" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1271 + +- config-keys: + - minimaxm2.5-fp8-mi355x-vllm + description: + - "Update vLLM ROCm image from v0.19.0 to v0.21.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1410 + +- config-keys: + - gptoss-fp4-h200-vllm + description: + - "Update vLLM image from v0.20.2 to v0.21.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1400 + +- config-keys: + - dsv4-fp4-b300-vllm-mtp + description: + - "Update vLLM image from v0.20.2 to v0.21.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1397 + +- config-keys: + - kimik2.5-int4-b200-vllm + description: + - "Update vLLM image from v0.20.2 to v0.21.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1396 + +- config-keys: + - gptoss-fp4-mi355x-vllm + description: + - "Update vLLM ROCm image from v0.17.0 to v0.21.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1406 + +- config-keys: + - kimik2.5-fp4-mi355x-vllm + description: + - "Update vLLM ROCm image from v0.18.0 to v0.21.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1407 + +- config-keys: + - kimik2.5-int4-mi355x-vllm + description: + - "Update vLLM ROCm image from v0.18.0 to v0.21.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1408 + +- config-keys: + - minimaxm2.5-fp4-mi355x-vllm + description: + - "Update vLLM ROCm image from v0.19.1 to v0.21.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1409 + +- config-keys: + - minimaxm2.5-fp8-h200-vllm + description: + - "Update vLLM image from v0.20.2 to v0.21.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1402 + +- config-keys: + - dsr1-fp4-b300-sglang + description: + - "Update SGLang image from v0.5.11-cu130 to v0.5.12-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1418 + +- config-keys: + - kimik2.5-int4-h200-vllm + description: + - "Update vLLM image from v0.20.2 to v0.21.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1401 + +- config-keys: + - dsr1-fp8-b300-sglang + - dsr1-fp8-b300-sglang-mtp + description: + - "Update SGLang image from v0.5.11-cu130 to v0.5.12-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1419 + +- config-keys: + - glm5-fp4-b200-sglang + - glm5-fp4-b200-sglang-mtp + description: + - "Update SGLang image from v0.5.10.post1-cu130 to v0.5.12-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1417 + +- config-keys: + - qwen3.5-fp8-h200-sglang-mtp + description: + - "Update SGLang image from v0.5.11 to v0.5.12" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1424 + +- config-keys: + - gptoss-fp4-h100-vllm + description: + - "Update vLLM image from v0.20.2 to v0.21.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1398 + +- config-keys: + - qwen3.5-fp8-mi325x-sglang + description: + - "Update SGLang image from v0.5.10-rocm720-mi30x to v0.5.12-rocm720-mi30x" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1430 + +- config-keys: + - minimaxm2.5-fp8-h100-vllm + description: + - "Update vLLM image from v0.20.2 to v0.21.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1399 + +- config-keys: + - dsr1-fp8-h200-sglang + description: + - "Update SGLang image from v0.5.11-cu130 to v0.5.12-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1423 + +- config-keys: + - qwen3.5-bf16-mi325x-sglang + description: + - "Update SGLang image from v0.5.10-rocm720-mi30x to v0.5.12-rocm720-mi30x" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1429 + +- config-keys: + - dsr1-fp8-b200-sglang + - dsr1-fp8-b200-sglang-mtp + description: + - "Update SGLang image from v0.5.11-cu130 to v0.5.12-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1416 + +- config-keys: + - gptoss-fp4-b200-vllm + description: + - "Update vLLM image from v0.15.1 to v0.20.2" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1394 + +- config-keys: + - kimik2.5-int4-mi300x-vllm + description: + - "Update vLLM ROCm image from v0.18.0 to v0.21.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1404 + +- config-keys: + - qwen3.5-fp8-mi300x-sglang + description: + - "Update SGLang image from v0.5.10-rocm720-mi30x to v0.5.12-rocm720-mi30x" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1427 + +- config-keys: + - glm5-fp8-h200-sglang + description: + - "Update SGLang image from custom glm5-hopper tag (59d old) to v0.5.12-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1459 + +- config-keys: + - qwen3.5-fp8-h200-sglang + description: + - "Update SGLang image from v0.5.9-cu129-amd64 (74d old) to v0.5.12-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1458 + +- config-keys: + - dsv4-fp8-h200-vllm + - dsv4-fp8-h200-vllm-mtp + description: + - "Update vLLM image to v0.21.0 (from custom deepseekv4-cu129 / v0.20.1@sha256-pinned)" + - "Lower --gpu-memory-utilization from 0.95 to 0.90 in dsv4_fp8_h200.sh and dsv4_fp8_h200_mtp.sh — v0.21.0 uses more memory at load time, OOM'd on GPU 2 at 0.95" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1461 + +- config-keys: + - dsr1-fp8-mi325x-sglang + description: + - "Update SGLang image from v0.5.9-rocm700-mi30x to v0.5.12-rocm700-mi30x" + - "Workaround LlamaTokenizer.all_special_tokens_extended removal in newer transformers: prefer backend_request_func.get_tokenizer over vLLM's" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1428 + +- config-keys: + - dsr1-fp4-b200-sglang + description: + - "Update SGLang image from v0.5.11-cu130 to v0.5.12-cu130" + - "Temporarily disable agentic-coding scenario (blocked by e2e-tests.yml artifact-name mismatch)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1415 + +- config-keys: + - qwen3.5-bf16-b200-sglang + - qwen3.5-bf16-b200-sglang-mtp + description: + - "Update SGLang image from nightly-dev-20260216-d3bae71e (86d old) to v0.5.12-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1446 + +- config-keys: + - gptoss-fp4-mi325x-vllm + description: + - "Update vLLM ROCm image from v0.17.0 (70d old) to v0.21.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1467 + +- config-keys: + - kimik2.5-int4-mi325x-vllm + description: + - "Update vLLM ROCm image from v0.18.0 (52d old) to v0.21.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1468 + +- config-keys: + - minimaxm2.5-fp8-mi325x-vllm + description: + - "Update vLLM ROCm image from v0.18.0 (50d old) to v0.21.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1469 + +- config-keys: + - minimaxm2.5-fp8-b200-vllm + description: + - "Update vLLM image from v0.19.0-cu130 (25d old) to v0.21.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1449 + +- config-keys: + - minimaxm2.5-fp4-b200-vllm + description: + - "Update vLLM image from v0.19.0-cu130 (25d old) to v0.21.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1448 + +- config-keys: + - kimik2.5-fp4-b200-vllm + description: + - "Update vLLM image from v0.20.2 to v0.21.0" + - "Add VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 to disable aggressive CUDA-graph memory profiler that OOMs the KV cache" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1395 + +- config-keys: + - qwen3.5-bf16-mi300x-sglang + description: + - "Update SGLang image from v0.5.10-rocm720-mi30x to v0.5.12-rocm720-mi30x" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1426 + +- config-keys: + - glm5-fp8-h200-sglang-mtp + description: + - "Add MTP/EAGLE speculative-decoding sibling for glm5-fp8-h200-sglang on lmsysorg/sglang:v0.5.12-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1480 + +- config-keys: + - qwen3.5-bf16-mi325x-sglang-mtp + description: + - "Add MTP/EAGLE speculative-decoding sibling of qwen3.5-bf16-mi325x-sglang" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1483 + +- config-keys: + - kimik2.5-fp4-b300-vllm + description: + - "Update vLLM image from v0.19.0-cu130 (27d old) to v0.21.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1452 + +- config-keys: + - minimaxm2.5-fp8-b300-vllm + description: + - "Update vLLM image from v0.19.0-cu130 (26d old) to v0.21.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1454 + +- config-keys: + - dsr1-fp8-mi300x-sglang + description: + - "Update SGLang image from v0.5.9-rocm700-mi30x to v0.5.12-rocm700-mi30x" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1425 + +- config-keys: + - glm5-fp8-b300-sglang + - glm5-fp8-b300-sglang-mtp + description: + - "Update SGLang image from v0.5.11-cu130 to v0.5.12-cu130" + - "Disable JIT DeepGemm (SGL_ENABLE_JIT_DEEPGEMM=0) to bypass v0.5.12 DeepGemm TMA-descriptor regression on B300 — see sgl-project/sglang#25551" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1421 + +- config-keys: + - dsr1-fp8-b200-trt + - dsr1-fp8-b200-trt-mtp + description: + - "Update TensorRT-LLM image (off: v1.2.0rc6.post2 109d / mtp: v1.2.0rc6.post3 102d) to v1.3.0rc14 (latest pre-release)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1488 + +- config-keys: + - dsr1-fp8-h200-trt + - dsr1-fp8-h200-trt-mtp + description: + - "Update TensorRT-LLM image from v1.1.0rc2.post2 (154d/124d old) to v1.3.0rc14 (latest pre-release)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1487 + +- config-keys: + - dsv4-fp4-b200-vllm + - dsv4-fp4-b200-vllm-mtp + description: + - "Update vLLM image from v0.20.0-cu130 (20d/18d old) to v0.21.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1476 + +- config-keys: + - qwen3.5-fp4-b200-sglang + - qwen3.5-fp4-b200-sglang-mtp + description: + - "Update SGLang image from nightly-dev-20260422-de962f32 (17d/13d old) to v0.5.12-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1474 + +- config-keys: + - qwen3.5-fp8-b200-sglang + - qwen3.5-fp8-b200-sglang-mtp + description: + - "Update SGLang image from nightly-dev-20260422-de962f32 (18d/12d old) to v0.5.12-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1473 + +- config-keys: + - gptoss-fp4-b200-vllm + description: + - "Update vLLM image from v0.20.2 (1d old) to v0.21.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1466 + +- config-keys: + - kimik2.5-int4-b300-vllm + description: + - "Update vLLM image from v0.20.0-cu130 (14d old) to v0.21.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1457 + +- config-keys: + - dsv4-fp4-b300-vllm + description: + - "Update vLLM image from v0.20.0-cu130 (18d old) to v0.21.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1456 + +- config-keys: + - minimaxm2.5-fp4-b300-vllm + description: + - "Update vLLM image from v0.19.0-cu130 (26d old) to v0.21.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1453 + +- config-keys: + - gptoss-fp4-h200-trt + description: + - "Update TensorRT-LLM image from v1.3.0rc11 (34d old) to v1.3.0rc14 (latest pre-release)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1491 + +- config-keys: + - gptoss-fp4-b200-trt + description: + - "Update TensorRT-LLM image from v1.2.0rc2.post2 (102d old) to v1.3.0rc14 (latest pre-release)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1490 + +- config-keys: + - qwen3.5-bf16-b300-sglang + - qwen3.5-bf16-b300-sglang-mtp + description: + - "Update SGLang image from v0.5.11-cu130 to v0.5.12-cu130" + - "Add --mm-attention-backend triton_attn to bypass flash-attn cute sm_103 assertion (see sgl-project/sglang#25564)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1422 + +- config-keys: + - qwen3.5-fp8-h100-sglang + - qwen3.5-fp8-h100-sglang-mtp + description: + - "Add Qwen-3.5-397B-A17B FP8 sglang recipes (off + MTP/EAGLE) for H100 on lmsysorg/sglang:v0.5.12-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1509 + +- config-keys: + - dsr1-fp8-mi325x-sglang-mtp + description: + - "Add MTP/EAGLE speculative-decoding sibling of dsr1-fp8-mi325x-sglang on lmsysorg/sglang:v0.5.12-rocm700-mi30x" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1500 + +- config-keys: + - glm5-fp8-b200-sglang + - glm5-fp8-b200-sglang-mtp + description: + - "Update SGLang image from nightly-dev-cu13-20260317-1eea7448 (33d/29d old) to v0.5.12-cu130" + - "Add --fp8-gemm-runner-backend cutlass to bypass DeepGemm illegal-memory-access on CUDA-graph capture (KLAUD_DEBUG §4a, B200 sm_100 variant)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1447 + +- config-keys: + - qwen3.5-fp8-mi325x-sglang-mtp + description: + - "Add MTP/EAGLE speculative-decoding sibling of qwen3.5-fp8-mi325x-sglang" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1484 + +- config-keys: + - glm5-fp8-b200-dynamo-sglang + description: + - "Add GLM-5 FP8 B200 Dynamo SGLang disaggregated multi-node coverage using lmsysorg/sglang:v0.5.11-cu130" + - "1k1k and 8k1k STP low-latency and max-throughput srt-slurm recipes under benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/b200-fp8/" + - "Use the B200 local model alias glm5-fp8 mapped to /scratch/fsw/models/GLM-5-FP8" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1372 + +- config-keys: + - dsv4-fp4-mi355x-sglang + description: + - "Bump image to rocm/sgl-dev:rocm720-mi35x-b19052c-20260518-DSv4." + - "Enabled Triton attention backend, FlyDSL MoE, and fused hash topk." + - "Concurrency sweep up to 1024." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1355 + +- config-keys: + - dsr1-fp4-b200-trt + - dsr1-fp4-b200-trt-mtp + description: + - "Update TensorRT-LLM image (off: v1.2.0rc6.post2 104d / mtp: v1.2.0rc6.post3 101d) to v1.3.0rc14 (latest pre-release)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1489 + +- config-keys: + - dsr1-fp8-mi355x-sglang + description: + - "Update SGLang image to v0.5.12-rocm700-mi35x" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1432 + +- config-keys: + - kimik2.5-fp4-b200-vllm + description: + - "Update vLLM image from v0.20.2 to v0.21.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1504 + +- config-keys: + - dsv4-fp4-gb200-dynamo-vllm-mtp2 + description: + - "Upgrade GB200 DSV4 MTP2 1P/1D DEP8 MegaMOE mid-curve recipe to vLLM v0.21.0" + - "Extend mid-curve concurrency sweep to also benchmark conc=256, conc=512, and conc=1024" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1390 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang-mtp + description: + - "Add MI355X config: qwen3.5-fp4-sglang-mtp using lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260517" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1445 + +- config-keys: + - dsr1-fp4-mi355x-sglang + description: + - "Update SGLang image to v0.5.12-rocm700-mi35x" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1431 + +- config-keys: + - qwen3.5-bf16-mi355x-sglang + - qwen3.5-bf16-mi355x-sglang-mtp + description: + - "Update SGLang ROCm image from v0.5.10rc0-rocm720-mi35x-20260415 to v0.5.12-rocm720-mi35x-20260517" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1443 + +- config-keys: + - qwen3.5-fp8-mi355x-sglang + - qwen3.5-fp8-mi355x-sglang-mtp + description: + - "Update SGLang ROCm image from v0.5.10rc0-rocm720-mi35x-20260414 to v0.5.12-rocm720-mi35x-20260517" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1444 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang + description: + - "Update SGLang ROCm image from rocm/sgl-dev:v0.5.10rc0-rocm720-mi35x-20260413 (32d old, deprecated rocm/sgl-dev repo) to lmsysorg/sglang:v0.5.12-rocm720-mi35x (clean stable tag on the active lmsysorg/sglang repo)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1497 + +- config-keys: + - glm5-fp8-mi355x-sglang + - glm5-fp8-mi355x-sglang-mtp + description: + - "Update SGLang ROCm image from v0.5.11/v0.5.10rc0 to v0.5.12-rocm720-mi35x-20260517" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1440 + +- config-keys: + - kimik2.5-fp4-mi355x-vllm-disagg + description: + - "Add Kimi-K2.5-MXFP4 FP4 vLLM disagg PD recipe (1P2D, MoRI-EP + MoRI-IO) for MI355X on vllm/vllm-openai-rocm:nightly" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1569 + +- config-keys: + - minimaxm2.5-fp8-mi355x-vllm-disagg + description: + - "Add MiniMax-M2.5 FP8 vLLM disagg PD recipe (1P2D, MoRI-EP + MoRI-IO) for MI355X on vllm/vllm-openai-rocm:nightly" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1569 + +- config-keys: + - dsv4-fp4-mi355x-vllm + description: + - "Following recipe from https://github.com/vllm-project/recipes/pull/433" + - "Add DEP8 dp-attn=true validation probes at conc=64 for 1k1k and 8k1k" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1374 + +- config-keys: + - glm5-fp8-h200-sglang + description: + - "Update SGLang image from glm5-hopper to v0.5.10.post1-cu130" + - "Add --enable-flashinfer-allreduce-fusion to server launch" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1033 + +- config-keys: + - dsr1-fp8-mi355x-sglang-mtp + description: + - "Add MTP/EAGLE speculative-decoding sibling for dsr1-fp8-mi355x-sglang (model: deepseek-ai/DeepSeek-R1-0528) on lmsysorg/sglang:v0.5.12-rocm700-mi35x" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1521 + +- config-keys: + - dsr1-fp4-mi355x-atom + - dsr1-fp4-mi355x-atom-mtp + description: + - "Update Atom ROCm image (off: rocm7.1.1-...-atom0.1.1-MI350x 125d / mtp: rocm7.2.0-...-atom0.1.1 83d) to rocm7.2.3_..._atom20260511" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1518 + +- config-keys: + - glm5-fp8-mi325x-sglang + - glm5-fp8-mi325x-sglang-mtp + description: + - "Add GLM-5 FP8 SGLang ROCm recipes (off + MTP/EAGLE) for MI325X on lmsysorg/sglang:v0.5.12-rocm720-mi30x" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1485 + +- config-keys: + - dsr1-fp8-mi355x-atom + - dsr1-fp8-mi355x-atom-mtp + description: + - "Update Atom ROCm image (off: rocm7.1.1-...-atom0.1.1-MI350x 125d / mtp: rocm7.2.1-...-atom0.1.2) to rocm7.2.3_..._atom20260511" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1519 + +- config-keys: + - dsr1-fp4-mi355x-sglang-mtp + description: + - "Add MTP/EAGLE speculative-decoding sibling for dsr1-fp4-mi355x-sglang (model: amd/DeepSeek-R1-0528-MXFP4) on lmsysorg/sglang:v0.5.12-rocm700-mi35x" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1520 + +- config-keys: + - dsr1-fp8-h200-sglang-mtp + description: + - "Add MTP/EAGLE speculative-decoding sibling for dsr1-fp8-h200-sglang (model: deepseek-ai/DeepSeek-R1-0528) on lmsysorg/sglang:v0.5.12-cu130 — TP=8, EP=1, search-space conc 4..64 on 1k1k + 8k1k" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1523 + +- config-keys: + - dsr1-fp8-b200-sglang + - dsr1-fp8-b300-sglang + - dsr1-fp4-b200-sglang + - dsr1-fp4-b300-sglang + - dsr1-fp8-b200-sglang-mtp + - dsr1-fp8-b300-sglang-mtp + description: + - "Re-run DSR1 SGLang agg configs (B200/B300, FP8/FP4, no-MTP/MTP) — picks up tokenizer fix from #1381" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1502 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang + description: + - "Update SGLang container image from `lmsysorg/sglang-staging:deepseek-v4-grace-blackwell-dev` to `lmsysorg/sglang:nightly-dev-cu13-20260519-dbac4647` for all non-MTP disagg configs" + - "Switch moe-a2a-backend from `deepep` to `megamoe` for wideep configs" + - "Remove obsolete/redundant environment variables and replace deprecated ones" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1492 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang + description: + - "Update SGLang image from nightly-dev-cu13-20260518-c67b2870 to nightly-dev-cu13-20260519-dbac4647" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1492 + +- config-keys: + - dsr1-fp4-b200-sglang-mtp + description: + - "Add MTP/EAGLE speculative-decoding sibling for dsr1-fp4-b200-sglang (model: nvidia/DeepSeek-R1-0528-FP4-V2) on lmsysorg/sglang:v0.5.12-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1522 + +- config-keys: + - qwen3.5-fp8-mi355x-atom + description: + - "Bump ATOM image to rocm/atom:rocm7.2.3_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom20260511" + - "TP=4 shows +3.2% to +16.3% throughput improvement across 1k1k and 8k1k workloads (concurrency 4-256)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1411 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang + description: + - "Update SGLang image from nightly-dev-cu13-20260519-dbac4647 to nightly-dev-cu13-20260520-425dffbd for all non-MTP disagg configs" + - "Remove SGLANG_OPT_FP8_WO_A_GEMM=0 workaround (topk_v2 crash fixed upstream in sgl-project/sglang#25805)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1528 + +- config-keys: + - qwen3.5-fp4-b300-sglang + - qwen3.5-fp4-b300-sglang-mtp + description: + - "Update SGLang image from v0.5.11-cu130 (5d old) to v0.5.12-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1475 + +- config-keys: + - dsv4-fp4-mi355x-vllm + description: + - "Bump vLLM ROCm image from nightly-b50646e5effd7cb5884cd96fdff4c53c18521198 to nightly-4f940896a32c9e2a0eba7f50d521bf5f6b4de458" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1546 + +- config-keys: + - dsv4-fp4-mi355x-sglang + description: + - "Bump image to rocm/sgl-dev:rocm720-mi35x-8c3b5aa-20260521-DSv4" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1548 + +- config-keys: + - qwen3.5-fp8-b300-sglang + - qwen3.5-fp8-b300-sglang-mtp + description: + - "Update SGLang image from v0.5.10.post1-cu130 / v0.5.11-cu130 (30d old) to v0.5.12-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1451 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang-mtp + description: + - "Enable W4A4 (MXFP4) megamoe by setting SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS=1 and SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND=1 in the megamoe environment blocks (not the low-latency 1p1d-tp4-tp4 or the 4p1d recipe)" + - "Update SGLang image from nightly-dev-cu13-20260510-2473659e to nightly-dev-20260527-14f81a67" + - "Switch moe-a2a-backend from deepep to megamoe and drop the deepep-config override" + - "Add two high concurrency configs" + - "Clean up obsolete environs in the 8k1k disagg recipes: drop SGLANG_OPT_USE_JIT_NORM / SGLANG_OPT_USE_JIT_INDEXER_METADATA / SGLANG_OPT_USE_TOPK_V2 (now default-on); drop the auto-set MegaMoE companions (SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE, SGLANG_OPT_FIX_HASH_MEGA_MOE, SGLANG_OPT_FIX_MEGA_MOE_MEMORY, SGLANG_OPT_FIX_NEXTN_MEGA_MOE, SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK); drop SGLANG_OPT_USE_FAST_MASK_EP which no longer exists in sglang environ.py (SGLANG_RADIX_DISABLE_REUSE is kept)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1378 + +- config-keys: + - dsr1-fp8-b200-sglang + - dsr1-fp8-b300-sglang + - dsr1-fp4-b200-sglang + - dsr1-fp4-b300-sglang + - dsr1-fp8-b200-sglang-mtp + - dsr1-fp8-b300-sglang-mtp + description: + - "Truncate sweep to conc=1 and conc=2 only: set conc-start=1, conc-end=2 in every search-space across all six DSR1 SGLang agg configs" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1534 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang-mtp + description: + - "Add --use-chat-template to run_benchmark_serving so prompts are formatted with the Qwen chat template (matching the other Qwen MTP recipes)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1554 + +- config-keys: + - glm5-fp4-gb300-dynamo-sglang + description: + - "Add GLM-5 FP4 GB300 Dynamo SGLang disaggregated multi-node coverage using lmsysorg/sglang:v0.5.11-cu130" + - "1k1k and 8k1k STP low-latency and max-throughput srt-slurm recipes under benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp4/ (ported from upstream srt-slurm PR #152)" + - "Wire glm5/fp4 model + dynamo-sglang framework branches into runners/launch_gb300-nv.sh" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1514 + +- config-keys: + - minimaxm2.5-fp8-h100-vllm + description: + - "Update minimaxm2.5-fp8-h100-vllm recipe (v0.19.1)" + - "Image: vllm/vllm-openai:v0.21.0 -> v0.19.1-cu130" + - "Replace recipe flags: drop PIECEWISE/0.90 mem util/256 max-num-seqs/no-prefix-caching/explicit max-model-len; add --enable-auto-tool-choice, --tool-call-parser minimax_m2, --reasoning-parser minimax_m2_append_think, --compilation-config mode:3+fuse_minimax_qk_norm" + - "Search-space: tp:8 ep:8 (TEP=8), conc-end 128 chosen at saturation per local sweep" + - "Local bench: TEP=8 peaks at C=128 with 26923 tot tps (+178% vs TEP=4 peak at C=32 in May 6 j11600242 sweep)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1516 + +- config-keys: + - qwen3.5-fp8-mi355x-atom-mtp + description: + - "Add --use-chat-template to run_benchmark_serving so prompts are formatted with the Qwen chat template (matching the other Qwen MTP recipes)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1555 + +- config-keys: + - minimaxm2.5-fp8-h200-vllm + description: + - "Update MiniMax-M2.5 FP8 H200 vLLM to vllm/vllm-openai:v0.20.1-ubuntu2404" + - "Set vLLM serving knobs in benchmarks/single_node/minimaxm2.5_fp8_h200.sh: generated benchmark max-model-len, previous eval max-model-len handling, fp8 KV cache, FlashInfer attention/autotune, Triton MoE, and MiniMax QK norm fusion" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1354 + +- config-keys: + - dsv4-fp4-mi355x-sglang + description: + - "Bump image to rocm/sgl-dev:rocm720-mi35x-f96ac98-20260526-DSv4" + - "Add args to avoid kvcache pool full issue on high conc" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1568 + +- config-keys: + - qwen3.5-fp8-h200-sglang + - dsr1-fp8-mi355x-sglang + description: + - "Validates measured-power aggregation pipeline (PR #1558) on both NVIDIA (H200) and AMD (MI355X) hardware — different SMI tools (nvidia-smi vs amd-smi), different CSV schemas (power.draw [W] vs socket_power), same aggregator. No config change. Entry intentionally kept past merge so run-sweep produces canonical agg JSONs with avg_power_w + joules_per_output_token on main for both vendors, seeding the dashboard's day-zero data." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1558 + +- config-keys: + - qwen3.5-fp8-mi355x-sglang-disagg + description: + - "Add Qwen3.5-397B-A17B-FP8 MI355X SGLang disaggregated prefill-decode benchmark" + - "Image: lmsysorg/sglang-rocm:v0.5.11-rocm700-mi35x-20260511" + - "1P+1D TP8/EP1 smoke sweep for 1k1k and 8k1k (conc 8-512); MoRI transfer backend" + - "Add models.yaml server flags and multinode launch script qwen3.5_fp8_mi355x_sglang-disagg.sh" + - "8k1k row uses dp-attn=false (matches 1k1k): with --enable-dp-attention + --moe-a2a-backend mori, sglang auto-promotes moe_ep_size=tp_size=8, but is_deepep_class_backend() excludes MoRI, so num_shared_slots stays at the global value (1) and the (num_experts - num_shared_slots) % moe_ep_size assertion in fused_moe_triton/layer.py fires for Qwen3.5 (512 routed + 1 shared). Track upstream sglang; flip back to dp-attn=true once MoRI is added to is_deepep_class_backend() or shared-slot accounting is reconciled." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1570 + +- config-keys: + - glm5-fp8-mi355x-sglang-disagg + description: + - "Add GLM-5-FP8 MI355X SGLang disaggregated prefill-decode benchmark" + - "Image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523 (bumped from .v0.5.12-...-20260517 to unlock the PD-disagg MoRI overlay; matches chun-chang/sglang-disagg-qwen3.5)" + - "Adds patches/mori_conn.py overlay (bind-mounted via job.slurm) to fix sglang v0.5.12.post1 MoRI/PD startup crashes for hybrid-attention models (GLM-5 NSA, etc.): sender flatten, state_types plural fallback, consumer normalize, SWA/DSA rank/length normalize. Validated: GSM8K=0.971 strict/0.970 flexible on chun-chang. Auto-applied for v0.5.12.post1 images; opt-out via MORI_CONN_PATCH=skip." + - "1P+1D TP8/EP1 CI smoke sweep for 1k1k and 8k1k (conc 8-512)" + - "Add GLM-5-FP8 models.yaml flags, setup_deps.sh (aiter gluon + transformers glm_moe_dsa), GLM-5 env tuning in env.sh" + - "Add multinode launch script glm5_fp8_mi355x_sglang-disagg.sh; server.sh sources setup_deps.sh" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1572 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang-disagg + description: + - "Add Qwen3.5-397B-A17B-MXFP4 MI355X SGLang PD-disaggregation" + - "Bump image to lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523, 1P1D TP8/EP1, dp-attn false, conc [8..512]" + - "MoRI conn.py overlay (48e459bd) via job.slurm; launcher qwen3.5_fp4_mi355x_sglang-disagg.sh" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1579 + +- config-keys: + - dsv4-fp4-b300-sglang + description: + - "Update sglang image to nightly-dev-cu13-20260529-a8cfae0b" + - "Refactor benchmark script to dispatch by CONC instead of nested DP_ATTENTION/CONC/EP_SIZE" + - "Switch CONC 2048/4096/8192 from --moe-a2a-backend deepep to megamoe" + - "Remove env vars deleted from sglang main (SGLANG_OPT_USE_JIT_NORM, SGLANG_OPT_USE_FAST_MASK_EP, SGLANG_OPT_FIX_NEXTN_MEGA_MOE, SGLANG_OPT_FIX_HASH_MEGA_MOE)" + - "Remove env vars redundant with sglang defaults (SGLANG_OPT_USE_JIT_INDEXER_METADATA, SGLANG_OPT_USE_TOPK_V2, SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2)" + - "Remove env vars auto-set by megamoe backend (SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE, SGLANG_OPT_FIX_MEGA_MOE_MEMORY)" + - "Remove --deepep-config and SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK (unused by megamoe/StandardDispatcher)" + - "Fix CONC=512 yaml ep from 4 to 1 (flashinfer_mxfp4 does not set ep=tp)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1506 + + +- config-keys: + - dsr1-fp4-mi355x-sglang-disagg-8k1k-mtp + description: + - "Bump the image to May 26" + - "Add conc 128/256 new sweep point" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1584 + +- config-keys: + - glm5-fp8-gb300-dynamo-sglang + description: + - "Add GLM-5 FP8 GB300 Dynamo SGLang disaggregated multi-node coverage using lmsysorg/sglang:v0.5.11-cu130" + - "1k1k and 8k1k STP hightpt and lowlat srt-slurm recipes under benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/ (resolved from upstream srt-slurm PR #160 via srtctl resolve-override)" + - "Wire glm5/fp8 model + dynamo-sglang framework branches into runners/launch_gb300-nv.sh with SA upstream defaults (SLURM_PARTITION=batch_1, SLURM_ACCOUNT=benchmark, SQUASH_FILE under /home/sa-shared/gharunners/squash/)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1557 + +- config-keys: + - dsv4-fp4-b200-vllm + description: + - "Update vLLM image tag to v0.22.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1384 + +- config-keys: + - glm5-fp4-gb300-dynamo-sglang + description: + - "Update GB300 FP4 GLM-5 8k1k low-latency sweep to mirror NVIDIA/srt-slurm#175: add a 5th 1p17d topology (decode_nodes/workers=17), and lower decode max-running-requests / cuda-graph-max-bs / benchmark concurrency per-zip-index from a flat 4096/1024 to 128/64/32/16/1 (mrr & cuda-graph) and 128/64/32/16/12 (concurrency)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1583 + +- config-keys: + - glm5.1-fp4-mi355x-sglang + description: + - "Bump SGLang ROCm image from v0.5.10rc0-rocm720-mi35x-20260415 to v0.5.12.post1-rocm720-mi35x-20260529" + - "Picks up the fix for the GSM8K accuracy regression reported in sgl-project/sglang#25742 (v0.5.12-20260517 collapsed to ~0.32 at TP=2)" + - "Local eval-only runs on MI355X recover to gsm8k strict-match 0.975 at TP=2/conc=64 and 0.974 at TP=4/conc=16, well above the 0.92 upstream gate added in sgl-project/sglang#26396" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1593 + +- config-keys: + - minimaxm2.5-fp8-mi325x-vllm + description: + - "Pin AITER FA attention backend in benchmarks/single_node/minimaxm2.5_fp8_mi325x.sh to recover the ~38% MI325X throughput regression introduced when vLLM PR #36702 (between v0.18.0 and v0.21.0) flipped the dense full-attention default on ROCm from ROCM_AITER_FA to ROCM_ATTN (vllm-project/vllm#43029)" + - "Export VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=1 to enable the AITER asm/hip paged-attention auto-dispatch" + - "Pass --attention-backend ROCM_AITER_FA to vllm serve, aligning with the merged upstream MiniMax ROCm recipe (vllm-project/recipes#481)" + - "Pin image vllm/vllm-openai-rocm:v0.20.2 — the version the upstream recipe explicitly validates (`min_vllm_version: 0.20.2`). v0.21.0 separately crashes during AITER MoE CUDA-graph capture on MiniMax-M2.5 (silent worker death, `Engine core initialization failed`) reproducible via the recipe's exact flags; v0.20.2 + recipe completes a 100-prompt vllm bench serve cleanly at 2030 tok/s total throughput on MI325X (TP=4)" + - "Add --compilation-config '{\"mode\":3,\"cudagraph_mode\":\"PIECEWISE\"}' to vllm serve, mirroring `model.base_args` from the upstream recipe. `pass_config.fuse_minimax_qk_norm` from the recipe is intentionally omitted — it triggers an upstream NameError on ROCm because vllm/compilation/passes/pass_manager.py imports MiniMaxQKNormPass under `is_cuda()` (NVIDIA-only) while using it unconditionally" + - "Conditionally enable VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=1 per (TP, EP, CONC) — on for shapes where the AITER ASM paged-attention kernel exists in the gfx942 heuristic table (TP=2 EP=1 CONC<=16, TP=8 EP=8 CONC<=64), off otherwise. Above the thresholds vllm/v1/attention/backends/rocm_aiter_fa.py routes decode through aiter pa_fwd_asm and crashes with `RuntimeError: get_heuristic_kernel: cannot get heuristic kernel!` for MiniMax-M2.5's attention shape (gqa=6 block_size=32 qTile=0); below them the ASM auto-dispatch is the perf win the recipe wants. Thresholds confirmed across 17 bench cells + 3 eval cells in PR #1594 sweep run 26692603804. Mirrors the per-shape toggle pattern in benchmarks/single_node/minimaxm2.5_fp8_mi355x.sh; can collapse to unconditional SHUFFLE=1 once AITER registers the missing kernel on gfx942" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1594 + +- config-keys: + - gptoss-fp4-mi355x-vllm + description: + - "Update vLLM ROCm image from v0.21.0 to v0.22.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1623 + +- config-keys: + - minimaxm2.5-fp8-h200-vllm + description: + - "Update vLLM image from v0.21.0 to v0.22.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1610 + +- config-keys: + - gptoss-fp4-h200-vllm + description: + - "Update vLLM image from v0.21.0 to v0.22.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1606 + +- config-keys: + - kimik2.5-int4-h200-vllm + description: + - "Update vLLM image from v0.21.0 to v0.22.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1600 + +- config-keys: + - minimaxm2.5-fp8-h100-vllm + description: + - "Update vLLM image from v0.19.1-cu130 to v0.22.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1609 + +- config-keys: + - minimaxm2.5-fp8-mi325x-vllm + description: + - "Update vLLM ROCm image from v0.20.2 to v0.22.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1619 + +- config-keys: + - gptoss-fp4-mi325x-vllm + description: + - "Update vLLM ROCm image from v0.21.0 to v0.22.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1622 + +- config-keys: + - minimaxm2.5-fp4-mi355x-vllm + description: + - "Update vLLM ROCm image from v0.21.0 to v0.22.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1620 + +- config-keys: + - minimaxm2.5-fp8-mi355x-vllm + description: + - "Update vLLM ROCm image from v0.21.0 to v0.22.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1617 + +- config-keys: + - minimaxm2.5-fp4-b200-vllm + description: + - "Update vLLM image from v0.21.0 to v0.22.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1611 + +- config-keys: + - gptoss-fp4-b200-vllm + description: + - "Update vLLM image from v0.21.0 to v0.22.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1604 + +- config-keys: + - kimik2.5-int4-b200-vllm + description: + - "Update vLLM image from v0.21.0 to v0.22.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1598 + +- config-keys: + - minimaxm2.5-fp8-b200-vllm + description: + - "Update vLLM image from v0.21.0 to v0.22.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1607 + +- config-keys: + - kimik2.5-fp4-b200-vllm + description: + - "Update vLLM image from v0.21.0 to v0.22.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1602 + +- config-keys: + - dsv4-fp4-mi355x-vllm + description: + - "Update vLLM ROCm image from nightly-4f940896a32c9e2a0eba7f50d521bf5f6b4de458 to v0.22.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1624 + +- config-keys: + - dsr1-fp8-mi355x-atom-mtp + description: + - "Update ATOM image to rocm/atom:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.3" + - "isl=1024/osl=1024: +47% to +116% improvement across conc 4-256 vs prior InferenceX numbers" + - "isl=8192/osl=1024: +47% to +131% improvement across conc 4-256" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1628 + +- config-keys: + - kimik2.5-fp4-mi355x-vllm + description: + - "Update vLLM ROCm image from v0.21.0 to v0.22.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1616 + +- config-keys: + - dsv4-fp4-mi355x-atom + description: + - "Add DeepSeek-V4-Pro FP4 MI355X ATOM DP-attention benchmark; image rocm/atom:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.3" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1626 + +- config-keys: + - dsv4-fp4-mi355x-vllm-mtp + description: + - "Add MTP speculative-decoding sibling for dsv4-fp4-mi355x-vllm (model: deepseek-ai/DeepSeek-V4-Pro) on vllm/vllm-openai-rocm:v0.22.0, per vllm-project/vllm#43385" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1630 + +- config-keys: + - dsv4-fp4-mi355x-sglang-mtp + description: + - "Add MTP/EAGLE speculative-decoding sibling for dsv4-fp4-mi355x-sglang (model: deepseek-ai/DeepSeek-V4-Pro) on v0.5.12.post1-rocm720-mi35x-20260601 (mainline ROCm nightly carrying sgl-project/sglang#26383); routes around the absent deep_gemm via SGLANG_OPT_FP8_WO_A_GEMM=0 + SGLANG_TOPK_TRANSFORM_512_TORCH=1." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1631 + +- config-keys: + - dsv4-fp4-b300-vllm + description: + - "Update DSv4 FP4 B300 vLLM image tag to v0.22.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1588 + +- config-keys: + - dsv4-fp4-mi355x-atom-mtp + description: + - "Add DeepSeek-V4-Pro FP4 MI355X ATOM MTP3 benchmark; image rocm/atom:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.3" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1627 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang-mtp + description: + - "Update SGLang image from nightly-dev-cu13-20260509-9ee83034 to nightly-dev-20260527-14f81a67" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1637 + +- config-keys: + - minimaxm2.5-fp4-gb200-dynamo-vllm + description: + - "Add MiniMax-M2.5 NVFP4 GB200 disaggregated multinode vLLM benchmarks via Dynamo" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1642 + +- config-keys: + - minimaxm2.5-fp8-b200-dynamo-vllm + description: + - "Add MiniMax-M2.5 FP8 B200 disaggregated multinode vLLM benchmarks via Dynamo" + - "Add 1k1k/8k1k FP8 recipe set under benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m2.5-b200-fp8/" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1649 + +- config-keys: + - minimaxm2.5-fp4-b200-dynamo-vllm + description: + - "Add MiniMax-M2.5 NVFP4 B200 disaggregated multinode vLLM benchmarks via Dynamo" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1643 + +- config-keys: + - minimaxm2.5-fp4-b300-dynamo-vllm + description: + - "Add MiniMax-M2.5 NVFP4 B300 disaggregated multinode vLLM benchmarks via Dynamo" + - "Image: vllm/vllm-openai:v0.20.1" + - "Same 1k/1k and 8k/1k search space as gb300, plus a new tp8-1p1d at low concurrencies for both ISLs" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1652 + +- config-keys: + - minimaxm2.5-fp4-gb300-dynamo-vllm + description: + - "Add MiniMax-M2.5 NVFP4 GB300 disaggregated multinode vLLM benchmarks via Dynamo" + - "Add 1k1k/8k1k minimax recipe set under benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m2.5/" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1641 + +- config-keys: + - dsv4-fp4-b200-vllm + description: + - "Enable EPLB for DEP configs" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1655 + +- config-keys: + - minimaxm2.5-fp8-gb300-dynamo-vllm + description: + - "Add MiniMax-M2.5 FP8 GB300 disaggregated multinode vLLM benchmarks via Dynamo" + - "Add 1k1k/8k1k FP8 recipe set under benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m2.5-fp8/" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1647 + +- config-keys: + - minimaxm2.5-fp8-gb200-dynamo-vllm + description: + - "Add MiniMax-M2.5 FP8 GB200 disaggregated multinode vLLM benchmarks via Dynamo" + - "Add 1k1k/8k1k FP8 recipe set under benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m2.5-gb200-fp8/" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1648 + +- config-keys: + - minimaxm2.5-fp8-b300-dynamo-vllm + description: + - "Add MiniMax-M2.5 FP8 B300 disaggregated multinode vLLM benchmarks via Dynamo" + - "Add 1k1k/8k1k FP8 recipe set under benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m2.5-b300-fp8/" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1663 + +- config-keys: + - qwen3.5-fp8-h100-sglang + description: + - "Tune Qwen3.5-397B-A17B-FP8 H100 SGLang aggregate recipe for 1k/1k and 8k/1k sweeps" + - "Use TP8/EP1 for conc 1-8 and TP8/EP8 for conc 16-256" + - "Use scheduler-recv-interval values 2/60/30/1200/600/1920 for conc 1-4/8/16/32/64/128-256" + - "Set max-running-requests=256, chunked-prefill-size=16384, mem-fraction-static=0.8, cuda-graph-max-bs=CONC, and enable symm-mem" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1544 + +- config-keys: + - qwen3.5-fp8-mi355x-sglang + - qwen3.5-fp8-mi355x-sglang-mtp + description: + - "Bump image to lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260528." + - "Update script for aiter attention backend from triton." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1669 + +- config-keys: + - minimaxm2.5-fp8-h200-vllm + description: + - "Switch attention backend from FLASHINFER to FLASH_ATTN for the 8k/1k cell of MiniMax-M2.5 FP8 H200 vLLM." + - "1k/1k cell not changed in this PR: at 1k/1k all three measured configs." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1668 + +- config-keys: + - gptoss-fp4-mi355x-vllm + description: + - "Update GPT-OSS model for MI355X vLLM from amd/gpt-oss-120b-w-mxfp4-a-fp8 to openai/gpt-oss-120b" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1670 + +- config-keys: + - kimik2.5-fp4-b300-vllm + description: + - "Update vLLM image from v0.21.0 to v0.22.0" + - "Expand concurrency sweep for the 1k/1k and 8k/1k cells: TP4/EP1 conc 4-64 -> 1-128, TP8/EP1 conc-start 4 -> 1 (conc-end 4 unchanged)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1674 + +- config-keys: + - kimik2.5-fp4-b200-vllm + description: + - "Expand concurrency sweep for the 1k/1k and 8k/1k cells: TP4/EP1 conc 4-64 -> 1-128, TP8/EP1 conc-start 4 -> 1 (conc-end 4 unchanged)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1677 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang + description: + - "Add wide-EP sweep configs (EP=12/16/24/32/40) matching srt-slurm PR#173 topology (18 nodes total)" + - "EP=12 15P+3D conc=12000, EP=16 14P+4D conc=8192, EP=24 12P+6D conc=3000, EP=32 10P+8D conc=2500, EP=40 8P+10D conc=2048" + - "Aligned decode params with Weiliang config: swa-full-tokens-ratio=0.20, max-running-requests=18432, moe-dense-tp-size=1; added prefill enable-dp-lm-head and cuda-graph-max-bs=512" + - "Remove 4 dominated old configs (4p-dep16-8n, 8p-dep16-12n, 10p-dep16-14n, 12p-dep12-15n) superseded by wide-EP frontier" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1586 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang + - qwen3.5-fp4-mi355x-sglang-mtp + description: + - "Bump image to lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260604." + - "Update script for aiter attention backend from triton." + - "Enable aiter unfied attention." + - "Enable aiter allreduce fusion." + - "Remove sweep config mtp+cc256+tp2, which may OOM." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1680 + +- config-keys: + - minimaxm2.5-fp4-mi355x-vllm + description: + - "Enable vLLM Rust request frontend by exporting VLLM_USE_RUST_FRONTEND=1 in benchmarks/single_node/minimaxm2.5_fp4_mi355x.sh (v0.22.0 ROCm image ships the vllm-rs binary, so the flag engages it). Environment-only change; serve flags, TP/EP, attention/kernel settings unchanged" + - "The Rust frontend replaces only the Python serving/API layer (HTTP, tokenization, scheduling glue, detokenization) and spawns the same Python EngineCore, so GPU kernels/attention/MoE GEMM/KV cache are untouched" + - "A/B sweep (28 single-node points, 1k1k + 8k1k, TP 1/2/4) vs the Python-frontend baseline (run 26696260751): throughput Pareto-neutral (peak tok/s/GPU within <1.5%, frontiers coincident) and TPOT flat (+-0.5%); TTFT improves ~8% at 1k1k and ~22% at 8k1k (every point), the expected signature of lower frontend CPU latency before first token, scaling with input length" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1634 + +- config-keys: + - dsv4-fp4-b200-trt + - dsv4-fp4-b200-trt-mtp + description: + - "Update the B200 TensorRT-LLM DeepSeek-V4-Pro image to ghcr.io/semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066" + - "Sync the dsv4-fp4-b200-trt and dsv4-fp4-b200-trt-mtp recipes with the B200 aggregated frontier config (worker GC off, NCCL graph mixing off, mimalloc/PyTorch alloc tweaks, higher KV cache fractions by DP path, stream_interval 100, use_low_precision_moe_combine, DP batching_wait_iters 30, max_num_tokens drops the OSL term)" + - "MTP recipe uses max_draft_len with a variable default draft length, enable_lm_head_tp_in_adp on the DP-attn path, and removes timeout_iters from the DP config" + - "Raise dsv4-fp4-b200-trt-mtp DP-attn conc-end (1k ISL: 512->1024; 8k ISL: 128->256) to cover the new high-concurrency regime" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1699 + +- config-keys: + - dsv4-fp4-mi355x-atom-disagg + description: + - "Add multi-node ATOM disaggregated prefill/decode support for DeepSeek-V4-Pro on MI355X" + - "Add server_atom.sh: multi-node launcher using atom.entrypoints.openai_server with mooncake RDMA KV transfer and atomesh router" + - "Add env_atom.sh: ATOM/mooncake-specific environment (mooncake LD_LIBRARY_PATH, ATOM_MOE_GU_ITLV=1, AITER_BF16_FP8_MOE_BOUND=0)" + - "Add models_atom.yaml: per-model configs for ATOM engine" + - "Add atom-disagg sweep: 2P1D DPA+TP8 (conc 256-2048) and 1P1D TP8 (conc 4-256) at isl=8192/osl=1024" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1683 + + +- config-keys: + - dsv4-fp4-b300-trt + - dsv4-fp4-b300-trt-mtp + description: + - "Update the B300 TensorRT-LLM DeepSeek-V4-Pro image to ghcr.io/semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066" + - "B300 analog of PR #1699 (B200): sync the dsv4-fp4-b300-trt and dsv4-fp4-b300-trt-mtp recipes with the agg frontier config (worker GC off, NCCL graph mixing off, mimalloc/PyTorch alloc tweaks, higher KV cache fractions by DP path, stream_interval 100, use_low_precision_moe_combine, DP batching_wait_iters 30, max_num_tokens drops the OSL term)" + - "MTP recipe uses max_draft_len with a variable default draft length, enable_lm_head_tp_in_adp on the DP-attn path, and removes timeout_iters from the DP config" + - "Cap cuda_graph_config.max_batch_size at 1024 on both recipes: TRTLLM_MLA_EXTRA_OVERLAP hands MLA prologue tensors across streams without record_stream(), so CUDA-graph warmup at decode batch >1024 (repros at 1088, e.g. tp8/ep8 dp-attn conc-2048 on B300) use-after-frees into CUDA_ERROR_ILLEGAL_ADDRESS; workaround until NVIDIA/TensorRT-LLM#15265 ships in the image. Runtime --max_batch_size stays = CONC, so batches >1024 run eager" + - "B300-specific bits preserved (MODEL_PATH download block, TRTLLM_MHC_ENABLE_FUSED_HC=1, trtllm-serve MODEL_PATH); drop the 1k1k conc-2048 point on the tp8/ep8 DP-attn row (both recipes), the batch regime that triggers the MLA-overlap crash above; rest of the search space unchanged" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1703 + +- config-keys: + - dsr1-fp4-b200-dynamo-sglang-mtp + description: + - "Move both the 1k1k and 8k1k scenarios of the DeepSeek-R1 FP4 B200 dynamo-sglang MTP disagg sweep to local split recipes (one flat recipe YAML per topology under benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/{1k1k,8k1k}/disagg/mtp/), rather than referencing recipes from the srt-slurm repo" + - "1k1k: 4 MTP variants, behavior unchanged from the previous srt-slurm 1k1k recipe — 2 low-latency (dep4-1p prefill / tep8 decode at 5 and 6 decode nodes, conc up to 512) + 2 max-throughput (dep4-1p prefill / dep8 decode at 1 and 2 decode nodes, conc up to 1024)" + - "8k1k: 6-variant sweep — 3 low-latency (1p5d / 1p3d / 1p1d, TP4 prefill / TP8 decode, conc up to 512) + 3 MTP2 high-throughput (2p1d / 3p1d / 5p1d, DEP4 prefill / DEP8 decode, single concurrency 768 / 1024 / 2048); MTP2 recipes use scheduler-recv-interval=1, enable-dp-lm-head, spec 2 steps / 3 draft tokens, and UCX_TLS in the prefill/decode environments" + - "Bump container image to lmsysorg/sglang:v0.5.12.post1 (from v0.5.8.post1-cu130); the 1k1k recipes keep the dynamo-sglang container alias and follow the config image" + - "Clone srt-slurm at NVIDIA/srt-slurm@main for the srtctl/dynamo tooling and copy the local b200-fp4 recipes into the checkout" + - "Pin the 1k1k MTP recipes to the same dynamo source hash as the 8k1k recipes (dynamo.hash=5b4bc1dd70965017a737c71b19db5a0aeaa88727, install: true), so both scenarios build dynamo from an identical revision instead of relying on whatever is baked into the dynamo-sglang container alias" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1688 + +- config-keys: + - dsv4-fp4-mi355x-sglang + description: + - "Switch back to the main branch on InferenceX benchmarking. Since the AMD v4 model code has been merged into main, we now use the main branch instead of the previously used amd/deepseek_v4 branch. The image is bumped to the main branch image rocm/sgl-dev:v0.5.12.post1-rocm720-mi35x-20260610." + - "Refresh the script's env vars. The main branch introduced several env var renames and new default settings, so we refreshed the env vars in the script accordingly." + - "Enable the unified KV attention kernel. Set export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton for better performance." + - "Specify the prefill delay explicitly. Set --prefill-delayer-max-delay-ms 5000 to define a concrete prefill delay, allowing DP to batch more requests together per execution." + pr-link: + https://github.com/SemiAnalysisAI/InferenceX/pull/1701 + +- config-keys: + - dsv4-fp4-b300-sglang-mtp + description: + - "Align MTP env vars to GB300: replace PRECOMPILE=0 with FAST_WARMUP=1, add RADIX_FORCE_MISS, DEFAULT_THINKING, DSV4_REASONING_EFFORT=max" + - "Replace DP-attn env vars with shared GB300 block: MEGA_MOE_USE_FP4_ACTS, USE_MXF4_KIND, NUM_MAX_TOKENS_PER_RANK=8192" + - "Unify EAGLE spec-decoding to (3,1,4) for both DP-attn and TP-only paths, add --enable-deepseek-v4-fp4-indexer" + - "Bump image to nightly-dev-cu13-20260610-f332e526" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1700 + +- config-keys: + - dsr1-fp4-mi355x-sglang + description: + - "MI355x DSR1-FP4: Include TP4 configurations for 8k1k" + - "Expand the TP sweep (included TP=4) for 8k/1k configuration for conc=4 to 64" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1692 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang + - qwen3.5-fp4-mi355x-sglang-mtp + description: + - "Bump image from lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260604 to lmsysorg/sglang-rocm:v0.5.13-rocm720-mi35x-20260612" + - "Enable AITER_FLYDSL_FORCE=1 in both non-MTP and MTP benchmark scripts" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1716 + +- config-keys: + - dsv4-fp4-mi355x-sglang + description: + - "Bump image to lmsysorg/sglang-rocm:v0.5.13-rocm720-mi35x-20260612." + - "Fix the intermediate_pad setting in the MoE computation in sglang PR#27858. This avoids the unnecessary overhead of computing useless padding." + - "Correct the chunk prefill setting size under tp8/dp8 config." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1715 + + +- config-keys: + - dsv4-fp4-gb200-dynamo-sglang + description: + - "Initial submission: DSv4-Pro FP4 disagg on GB200 with SGLang (8k/1k)." + - "Image: lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" + - "8 topologies sweeping low-latency (1p1d-tp8-tp8) through max throughput (6p1d-dep8-dep12)." + - "Updated 1p1d and 2p1d configs to match https://github.com/shyeh25/srt-slurm/commit/ede724d7cc9a780be5b84659f599733bf9fd0097" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1675 + +- config-keys: + - glm5-fp4-b200-sglang + - glm5-fp4-b200-sglang-mtp + - glm5-fp8-b200-sglang + - glm5-fp8-b200-sglang-mtp + description: + - "Update SGLang image from v0.5.12-cu130 to nightly-dev-cu13-20260605-7dc73766" + - "glm5-fp4-b200-sglang-mtp: override --mem-fraction-static from 0.85 to 0.8 when ISL=8192, OSL=1024, and CONC>128 (default 0.85 elsewhere)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1567 + +- config-keys: + - minimaxm2.5-fp4-b300-trt + description: + - "Add MiniMax-M2.5 FP4 (NVFP4) B300 TensorRT-LLM benchmark (model: nvidia/MiniMax-M2.5-NVFP4)" + - "Image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1712 + +- config-keys: + - minimaxm3-fp8-b300-vllm + description: + - "Initial submission: MiniMax-M3 MXFP8 day-zero single-node vLLM benchmark on B300 (model: MiniMaxAI/MiniMax-M3-MXFP8, 427B total / 26B active MoE with MSA sparse attention)" + - "Image: vllm/vllm-openai:minimax-m3 (already the cu130 build; M3 support is unmerged upstream — vllm-project/vllm#45381)" + - "--block-size 128 is mandatory (MSA sparse/index cache alignment); --language-model-only skips the vision encoder for text-only throughput; conc-scaled --max-cudagraph-capture-size" + - "Layouts: TP8 and TP4 (latency), TP4+EP4 / TP8+EP8 (TEP throughput), tp2-ep2, TP8+EP8 dp-attn (DEP) across 1k1k and 8k1k" + - "Serves from the launch_b300-nv.sh MODEL/MODEL_PATH split (model not in the SRE-staged /scratch/models list -> writable /data/models)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1724 + +- config-keys: + - minimaxm3-fp8-b200-vllm + description: + - "Initial submission: MiniMax-M3 MXFP8 day-zero single-node vLLM benchmark on B200 (model: MiniMaxAI/MiniMax-M3-MXFP8, 427B total / 26B active MoE with MSA sparse attention)" + - "Image: vllm/vllm-openai:minimax-m3 (M3 support is unmerged upstream; the dedicated image is built from the m3_release branch, vllm-project/vllm#45381)" + - "--block-size 128 is mandatory (MSA sparse/index cache alignment); --language-model-only skips the vision encoder for text-only throughput; conc-scaled --max-cudagraph-capture-size" + - "Layouts: TP8 and TP4 (latency), TP4+EP4 / TP8+EP8 (TEP throughput), TP8+EP8 dp-attn (DEP) across 1k1k and 8k1k" + - "Weights are not SRE-staged: launch_b200-dgxc.sh adds a MODEL_PATH case for /lustre/fsw/gharunners/models/MiniMax-M3-MXFP8; runners.yaml adds a b200-dgxc runner-type group" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1723 + +- config-keys: + - minimaxm3-fp8-b300-vllm-mtp + description: + - "Initial submission: MiniMax-M3 MXFP8 B300 vLLM benchmark with EAGLE3 speculative decoding (target: MiniMaxAI/MiniMax-M3-MXFP8, draft: Inferact/MiniMax-M3-EAGLE3, 3 speculative tokens)" + - "Image: vllm/vllm-openai:minimax-m3 (same m3_release-branch build as the non-MTP entry)" + - "Serve shape follows minimaxm3-fp8-b300-vllm (--block-size 128, --language-model-only); cudagraph capture scaled to CONC * (1 + spec tokens); prompts routed through the chat template for realistic acceptance" + - "Drafter pinned to FLASH_ATTN via speculative-config attention_backend: the EAGLE3 head is MHA and FlashInfer only supports the mandatory page size 128 through its GQA-only trtllm-gen kernel" + - "Layouts: TP8 / TP4 (latency), TP8+EP8 / TP4+EP4 (TEP), TP8+EP8 dp-attn (DEP) across 1k1k and 8k1k — non-MTP search space trimmed at the extreme-concurrency end, tp2-ep2 dropped (draft weights + draft KV headroom)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1733 + +- config-keys: + - minimaxm3-fp8-h200-vllm + - minimaxm3-fp8-h100-vllm + description: + - "Day-zero MiniMax-M3 MXFP8 single-node recipes for H200 and H100 (vLLM)." + - "Image: vllm/vllm-openai:minimax-m3 (dedicated day-zero image; M3 not in a stable release yet)." + - "Sweeps TP4/TP8, TP+EP (TEP), and DP-attention+EP (DEP) per https://recipes.vllm.ai/MiniMaxAI/MiniMax-M3; H100 is TP8-only (MXFP8 weights ~427 GB)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1731 + +- config-keys: + - minimaxm2.5-fp4-b200-trt + description: + - "Add MiniMax-M2.5 NVFP4 B200 TensorRT-LLM single-node benchmark (1k1k and 8k1k)" + - "Image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc18" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1722 + +- config-keys: + - minimaxm3-fp8-h200-vllm-mtp + - minimaxm3-fp8-h100-vllm-mtp + description: + - "Initial submission: MiniMax-M3 MXFP8 H200 + H100 vLLM benchmarks with EAGLE3 speculative decoding (target: MiniMaxAI/MiniMax-M3-MXFP8, draft: Inferact/MiniMax-M3-EAGLE3, 3 speculative tokens) — spec-decoding=mtp variants of the day-zero H200/H100 recipes (PR #1731)" + - "Image: vllm/vllm-openai:minimax-m3 (same m3_release-branch build as the non-MTP entries)" + - "Serve shape follows the non-MTP H200/H100 scripts (--block-size 128, --language-model-only, HF_HUB_OFFLINE serve to dodge the shared-FS download-lock race); cudagraph capture scaled to CONC * (1 + spec tokens); prompts routed through the chat template for realistic acceptance" + - "Drafter pinned to FLASH_ATTN via speculative-config attention_backend: the EAGLE3 head is MHA and FlashInfer only supports the mandatory page size 128 through its GQA-only trtllm-gen kernel" + - "H200 layouts: TP4 / TP8 (latency), TP4+EP4 / TP8+EP8 (TEP), TP8+EP8 dp-attn (DEP) across 1k1k and 8k1k — non-MTP search space trimmed at the extreme-concurrency end. H100 is TP8-only (no room below TP8 at ~56 GB weights/GPU); DEP omitted (KV-cache init fails at high conc, draft only tightens it)" + - "Adds SPEC_SUFFIX to the three H100 launchers (cw, cr, dgxc-slurm) so spec-decoding=mtp routes to the _mtp script — they hardcoded _h100.sh and never gained the _mtp routing the H200 launchers have had since #392" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1739 + +- config-keys: + - minimaxm3-fp8-b200-vllm-mtp + description: + - "Initial submission: MiniMax-M3 MXFP8 B200 vLLM benchmark with EAGLE3 speculative decoding (target: MiniMaxAI/MiniMax-M3-MXFP8, draft: Inferact/MiniMax-M3-EAGLE3, 3 speculative tokens) — spec-decoding=mtp variant of the day-zero B200 recipe" + - "Image: vllm/vllm-openai:minimax-m3 (same m3_release-branch build as the non-MTP entry)" + - "Serve shape follows minimaxm3-fp8-b200-vllm (--block-size 128, --language-model-only); cudagraph capture scaled to CONC * (1 + spec tokens); prompts routed through the chat template for realistic acceptance" + - "Drafter pinned to FLASH_ATTN via speculative-config attention_backend: the EAGLE3 head is MHA and FlashInfer only supports the mandatory page size 128 through its GQA-only trtllm-gen kernel" + - "Layouts: TP8 / TP4 (latency), TP8+EP8 / TP4+EP4 (TEP), TP8+EP8 dp-attn (DEP) across 1k1k and 8k1k — non-MTP search space trimmed at the extreme-concurrency end, mirroring the minimaxm3-fp8-b300-vllm-mtp search space" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1741 + +- config-keys: + - minimaxm3-fp8-mi355x-vllm + description: + - "Initial submission: MiniMax-M3 MXFP8 day-zero single-node vLLM benchmark on MI355X / gfx950 (model: MiniMaxAI/MiniMax-M3-MXFP8, 427B total / 26B active MoE with MSA sparse attention)" + - "Image: vllm/vllm-openai-rocm:minimax-m3" + - "Matches vllm-project/recipes@2a3728ed MI355X guidance with --block-size 128, --attention-backend TRITON_ATTN, --kv-cache-dtype fp8, MiniMax-M3 tool/reasoning parsers, and automatic tool choice" + - "MI355X deviations required for fixed-sequence text benchmarking: --language-model-only, scenario-specific --max-model-len, and --enforce-eager to bypass the M3 decode CUDA-graph assertion" + - "B300-parity layouts and concurrency ranges: TP8, TP8+EP8, TP4, TP4+EP4, TP2+EP2, and TP8+EP8 dp-attn (DEP) across 1k1k and 8k1k" + - "launch_mi355x-amds.sh routes M3 weights to NFS /it-share/hf-hub-cache instead of node-local /var/lib NVMe" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1725 + +- config-keys: + - minimaxm3-fp8-h200-vllm-mtp + - minimaxm3-fp8-h100-vllm-mtp + description: + - "Start the TP-only latency rows of the MiniMax-M3 EAGLE3 MTP sweeps (H200, H100) at concurrency 1 instead of 4, matching the conc-1 start used on the non-MTP day-zero recipes — captures the single-request latency point. TEP/DEP rows keep their higher concurrency starts." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1743 + +- config-keys: + - minimaxm3-fp8-mi355x-vllm-mtp + description: + - "Initial submission: MiniMax-M3 MXFP8 MI355X (gfx950) vLLM benchmark with EAGLE3 speculative decoding (target: MiniMaxAI/MiniMax-M3-MXFP8, draft: Inferact/MiniMax-M3-EAGLE3, 3 speculative tokens) — spec-decoding=mtp variant of the MI355X day-zero recipe" + - "Image: vllm/vllm-openai-rocm:minimax-m3 (same day-zero ROCm build as the non-MTP entry)" + - "Serve shape follows minimaxm3-fp8-mi355x-vllm (--block-size 128, --language-model-only, --kv-cache-dtype fp8, --attention-backend TRITON_ATTN, --enforce-eager, minimax_m3 parsers); prompts routed through the chat template for realistic acceptance" + - "No attention_backend override on the drafter: the server runs on TRITON_ATTN, so the FlashInfer page-128/MHA limitation that forced FLASH_ATTN on the CUDA recipes does not apply on ROCm" + - "Layouts: TP8 / TP4 (latency), TP8+EP8 / TP4+EP4 (TEP), TP8+EP8 dp-attn (DEP) across 1k1k and 8k1k — non-MTP search space trimmed at the extreme-concurrency end, tp2-ep2 dropped, mirroring the minimaxm3-fp8-b300-vllm-mtp search space" + - "[AI generated draft test] The shipped ROCm image's AMD MiniMax-M3 model lacks SupportsEagle3, so the recipe patches it in-place at runtime (functionstackx/vllm#1, ported from nvidia/model.py) before serving — validates EAGLE3 on MI355X ahead of an image rebuild" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1745 + +- config-keys: + - minimaxm3-fp8-mi300x-vllm + description: + - "Initial submission: MiniMax-M3 MXFP8 day-zero single-node vLLM benchmark on MI300X / gfx942" + - "Image and serving shape reuse the MI355X recipe: vllm/vllm-openai-rocm:minimax-m3, block size 128, prefix caching disabled, TRITON_ATTN, language-model-only, and eager execution" + - "Use the default BF16 KV cache on MI300X because MiniMax-M3-MXFP8 lacks calibrated q/prob scales for ROCm FP8 attention; vLLM warns that its fallback scale of 1.0 may cause accuracy issues" + - "H100-aligned layouts and concurrency ranges: TP8 and TP8+EP8 across 1k1k and 8k1k" + - "Fix launch_mi300x-amds.sh node exclusion to use the current short Slurm node name" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1746 + +- config-keys: + - minimaxm3-fp8-mi300x-vllm-mtp + description: + - "Initial submission: MiniMax-M3 MXFP8 MI300X (gfx942) vLLM benchmark with EAGLE3 speculative decoding (target: MiniMaxAI/MiniMax-M3-MXFP8, draft: Inferact/MiniMax-M3-EAGLE3, 3 speculative tokens) — spec-decoding=mtp variant of the MI300X day-zero recipe" + - "Image: vllm/vllm-openai-rocm:minimax-m3 (same day-zero ROCm build as the non-MTP entry)" + - "Serve shape follows minimaxm3-fp8-mi300x-vllm: --block-size 128, --no-enable-prefix-caching, --language-model-only, --attention-backend TRITON_ATTN, --enforce-eager, minimax_m3 parsers, and the default BF16 KV cache (gfx942 lacks calibrated ROCm FP8 attention scales); prompts routed through the chat template for realistic acceptance" + - "TP8-only search space (gfx942 192 GB is memory-tight, like H100): TP8 latency rows started at conc 1, TP8+EP8 (TEP) at high concurrency, across 1k1k and 8k1k" + - "[AI generated draft test] The shipped ROCm image's AMD MiniMax-M3 model lacks SupportsEagle3, so the recipe patches it in-place at runtime (functionstackx/vllm#1, upstream vllm-project/vllm#45546; validated green on MI355X) before serving — validates EAGLE3 on MI300X ahead of an image rebuild" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1749 + +- config-keys: + - minimaxm3-fp8-mi300x-vllm + description: + - "Enable CUDA graphs for MiniMax-M3 MXFP8 on MI300X and set VLLM_USE_BREAKABLE_CUDAGRAPH=0 per AMD guidance" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1750 + +- config-keys: + - minimaxm3-fp8-mi300x-vllm-mtp + description: + - "Run the MiniMax-M3 MXFP8 MI300X EAGLE3 MTP recipe with CUDA graphs instead of --enforce-eager" + - "Drop --enforce-eager and set VLLM_USE_BREAKABLE_CUDAGRAPH=0 (matching the non-MTP MI300X recipe, #1750), which avoids the M3-decode breakable-cudagraph path that previously forced eager execution" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1756 + +- config-keys: + - minimaxm3-fp8-mi355x-vllm + description: + - "Enable CUDA graphs for MiniMax-M3 MXFP8 on MI355X, set VLLM_USE_BREAKABLE_CUDAGRAPH=0, and disable prefix caching" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1754 + +- config-keys: + - minimaxm3-fp8-mi325x-vllm + description: + - "Initial submission: MiniMax-M3 MXFP8 day-zero single-node vLLM benchmark on MI325X / gfx942" + - "Follows the official MI325X MXFP8 recipe with vllm/vllm-openai-rocm:minimax-m3, block size 128, TRITON_ATTN, and MiniMax-M3 tool/reasoning parsers; fixed-sequence text benchmarking adds language-model-only, the default BF16 KV cache, disabled prefix caching, and a scenario-specific max model length" + - "H200-aligned layouts and concurrency ranges: TP4 and TP8 latency, TP4/TP8 expert parallelism, and TP8 data-parallel attention across 1k1k and 8k1k" + - "Route the MI325X Hugging Face cache and runtime compiler caches to node-local storage, and mount ROCm GPU devices explicitly" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1748 + +- config-keys: + - minimaxm3-fp8-mi355x-vllm-mtp + description: + - "Run the MiniMax-M3 MXFP8 MI355X EAGLE3 MTP recipe with CUDA graphs instead of --enforce-eager" + - "Drop --enforce-eager and set VLLM_USE_BREAKABLE_CUDAGRAPH=0, which avoids the M3-decode breakable-cudagraph path that previously forced eager execution (the non-MTP MI355X recipe already got this in #1754)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1755 + +- config-keys: + - minimaxm3-fp8-mi325x-vllm-mtp + description: + - "Initial submission: MiniMax-M3 MXFP8 MI325X (gfx942) vLLM benchmark with EAGLE3 speculative decoding (target: MiniMaxAI/MiniMax-M3-MXFP8, draft: Inferact/MiniMax-M3-EAGLE3, 3 speculative tokens) — spec-decoding=mtp variant of the MI325X day-zero recipe (#1748)" + - "Image: vllm/vllm-openai-rocm:minimax-m3 (same day-zero ROCm build as the non-MTP entry)" + - "Serve shape follows minimaxm3-fp8-mi325x-vllm: --block-size 128, --no-enable-prefix-caching, --language-model-only, --attention-backend TRITON_ATTN, minimax_m3 parsers, default BF16 KV cache (gfx942 lacks calibrated ROCm FP8 attention scales). Runs with CUDA graphs (no --enforce-eager, VLLM_USE_BREAKABLE_CUDAGRAPH=0); prompts via chat template for realistic acceptance" + - "H200-style search space (TP4/TP8 latency, TP4+EP4/TP8+EP8 TEP, TP8+EP8 dp-attn DEP) trimmed at the extreme-concurrency end with TP-only latency rows started at conc 1" + - "[AI generated draft test] The shipped ROCm image's AMD MiniMax-M3 model lacks SupportsEagle3, so the recipe patches it in-place at runtime (functionstackx/vllm#1, upstream vllm-project/vllm#45546; validated green on MI355X/MI300X) before serving; also adds SPEC_SUFFIX to launch_mi325x-amds.sh so spec-decoding=mtp routes to the _mtp script" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1759 + +- config-keys: + - minimaxm3-fp8-mi300x-vllm + - minimaxm3-fp8-mi325x-vllm + description: + - "Extend the MiniMax-M3 MXFP8 MI300X and MI325X non-MTP sweeps down to concurrency 1 on the TP-only latency rows (was conc 4), to capture the single-request latency point; TEP/DEP rows keep their higher concurrency starts" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1760 + +- config-keys: + - minimaxm3-fp8-h100-vllm + - minimaxm3-fp8-h200-vllm + description: + - "Extend MiniMax-M3 MXFP8 H100/H200 non-MTP sweeps to concurrency 1 on the latency rows (H100: TP8; H200: TP4 and TP8) and add full TEP coverage from conc 1 to 256 (H100: TP8+EP8; H200: TP4+EP4 and TP8+EP8, incl. a new TP4+EP4 row for 8k1k). H200 TP8+EP8 upper bound moves 512->256 (high concurrency stays covered by the TP8+EP8 dp-attn DEP rows). DEP rows unchanged" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1761 + +- config-keys: + - dsv4-fp4-mi355x-sglang + description: + - "Switch fixed-seq-len search space from TP8 to TP4 for both isl=1024 and isl=8192 scenarios" + - "Expand isl=8192 coverage: add TP4 dp-attn sweep (conc 32–2048) and TP4 TP-only sweep (conc 1–32)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1762 + + +- config-keys: + - dsv4-fp4-gb300-dynamo-trt + - dsv4-fp4-gb300-dynamo-trt-mtp + description: + - "Add DeepSeek-V4-Pro MXFP4 GB300 disaggregated TRT-LLM benchmarks via Dynamo (27 STP + 27 MTP configs)" + - "New configs: dsv4-fp4-gb300-dynamo-trt (STP) and dsv4-fp4-gb300-dynamo-trt-mtp (MTP)" + - "Covers ISL 1024/OSL 1024 (14 STP + 14 MTP) and ISL 8192/OSL 1024 (13 STP + 13 MTP)" + - "Container: nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-deepseek-v4-dev.1" + - "Recipes sourced from NVIDIA/srt-slurm branch sa-submission-q2-2026" + - "Runner script updated to support dsv4 model prefix with dynamo-trt framework on GB300" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1689 + +- config-keys: + - minimaxm3-fp8-b200-vllm + description: + - "Align MiniMax-M3 B200 vLLM fixed-sequence serving with MiniMax-M2.5 FP8 B200 settings by setting VLLM_FLOAT32_MATMUL_PRECISION=high and restoring max cudagraph capture size 2048." + - "Add TP4+EP4 coverage for MiniMax-M3 B200: DP-attention rows for 1k1k/8k1k and the missing non-DP-attention row for 8k1k." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1779 + +- config-keys: + - minimaxm3-fp8-b200-vllm + description: + - "Align MiniMax-M3 B200 vLLM fixed-sequence serving with MiniMax-M2.5 FP8 B200 settings by setting VLLM_FLOAT32_MATMUL_PRECISION=high and restoring max cudagraph capture size 2048." + - "Add TP4+EP4 coverage for MiniMax-M3 B200: DP-attention rows for 1k1k/8k1k and the missing non-DP-attention row for 8k1k." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1779 + +- config-keys: + - dsv4-fp4-mi355x-sglang + description: + - "Switch fixed-seq-len search space from TP8 to TP4 for both isl=1024 and isl=8192 scenarios" + - "Expand isl=8192 coverage: add TP4 dp-attn sweep (conc 32–2048) and TP4 TP-only sweep (conc 1–32)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1762 + +- config-keys: + - kimik2.5-int4-mi355x-vllm + description: + - "Replace triton w4a16 MoE with FlyDSL w4a16 MoE" + - "Image: vllm/vllm-openai-rocm:nightly" + - "Add more sweep points" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1785 + +- config-keys: + - minimaxm3-fp8-b200-vllm-mtp + - minimaxm3-fp8-b300-vllm-mtp + description: + - "Align MiniMax-M3 B200/B300 EAGLE3 MTP serving with the MiniMax-M2.5 FP8 serving settings by setting VLLM_FLOAT32_MATMUL_PRECISION=high and using max cudagraph capture size 2048." + - "Add TP4+EP4 MTP coverage: DP-attention rows for 1k1k/8k1k and the missing non-DP-attention row for 8k1k." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1784 + +- config-keys: + - minimaxm3-fp8-b300-vllm + description: + - "Align MiniMax-M3 B300 vLLM fixed-sequence serving with MiniMax-M2.5 FP8 settings by setting VLLM_FLOAT32_MATMUL_PRECISION=high and restoring max cudagraph capture size 2048." + - "Add TP4+EP4 coverage for MiniMax-M3 B300: DP-attention rows for 1k1k/8k1k and the missing non-DP-attention row for 8k1k." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1781 + +- config-keys: + - qwen3.5-fp4-b200-trt + description: + - "Add Qwen3.5-397B-A17B-NVFP4 B200 single-node TensorRT-LLM benchmark (1k/1k and 8k/1k) with a TP/TEP/DEP parallelism sweep" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1711 + +- config-keys: + - minimaxm3-fp8-b200-vllm + - minimaxm3-fp8-b300-vllm + - minimaxm3-fp8-b200-vllm-mtp + - minimaxm3-fp8-b300-vllm-mtp + description: + - "Use the Marlin MoE backend for MiniMax-M3 B200/B300 TP-only vLLM configurations by adding --moe-backend marlin when expert parallelism is disabled." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1809 + +- config-keys: + - dsr1-fp8-gb300-dynamo-trt + description: + - "Fix gsm8k accuracy at 88% instead of 95% for a single point." + - "In previous submission, there was an numeric issue causing accuracy degradation and performance anomaly in some MTP points at certain concurrency." + - "This issue is now fixed in the latest TRTLLM release." + - "Also update all configs for DSR1 TRTLLM FP8 to reflect latest released image usage" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1767 + +- config-keys: + - minimaxm3-fp4-mi355x-atom + description: + - "Add day-zero MiniMax-M3 MXFP4 (amd/MiniMax-M3-MXFP4) single-node atom benchmark on MI355X, following the ROCm/ATOM MiniMax-M3 recipe (TP4, block size 128 for MSA, default KV cache dtype)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1813 + + +- config-keys: + - glm5-fp4-gb300-dynamo-trt + description: + - "Add GLM-5 NVFP4 GB300 disaggregated TRT-LLM (STP, non-MTP) benchmarks via Dynamo (22 STP configs: 13 for 1K/1K, 9 for 8K/1K)" + - "Container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13" + - "Recipes sourced from NVIDIA/srt-slurm branch sa-submission-q2-2026 (gb300_nvfp4 STP recipes)" + - "Runner script launch_gb300-nv.sh: added dynamo-trt-specific glm5-fp4 case with SERVED_MODEL_NAME and SRT_SLURM_MODEL_PREFIX=nvidia/GLM-5-NVFP4" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1798 + +- config-keys: + - dsv4-fp4-mi355x-atom + description: + - "Update image to rocm/atom:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.4_20260612" + - "Update ISL=8192 search-space: TP8-only from conc=4-64, DPA from conc=128-1024 (previously conc=1-64 and DPA conc=64-512)" + - "Update Applied TBO on high concurrencies" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1717 + +- config-keys: + - dsv4-fp4-mi355x-atom + description: + - "Update image to rocm/atom:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.4_20260612" + - "Update ISL=8192 search-space: TP8-only from conc=4-64, DPA from conc=128-1024 (previously conc=1-64 and DPA conc=64-512)" + - "Update Applied TBO on high concurrencies" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1717 + +- config-keys: + - minimaxm3-fp8-b200-vllm-mtp + description: + - "Update the MiniMax-M3 B200 single-node image to vllm/vllm-openai:minimax-m3-0618-x86_64-cu130." + - "Enable FlashInfer TRT-LLM attention with FP8 indexer KV and KV cache; use TRITON_ATTN for the EAGLE3 drafter." + - "Switch TP-only configurations from explicit Marlin MoE to the new image's default FlashInfer TRT-LLM MoE backend." + - "Patch the image's MiniMax M3 MSA prefill path to materialize sliced top-k indices before CSR construction." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1832 + +- config-keys: + - minimaxm3-fp8-b200-vllm + description: + - "Update the MiniMax-M3 B200 single-node image to vllm/vllm-openai:minimax-m3-0618-x86_64-cu130." + - "Enable FlashInfer TRT-LLM attention with FP8 indexer KV and KV cache." + - "Switch TP-only configurations from explicit Marlin MoE to the new image's default FlashInfer TRT-LLM MoE backend." + - "Patch the image's MiniMax M3 MSA prefill path to materialize sliced top-k indices before CSR construction." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1833 + +- config-keys: + - minimaxm3-fp8-b300-vllm + description: + - "Update the MiniMax-M3 B300 single-node image to vllm/vllm-openai:minimax-m3-0618-x86_64-cu130." + - "Enable FlashInfer TRT-LLM attention with FP8 indexer KV and KV cache." + - "Switch TP-only configurations from explicit Marlin MoE to the new image's default FlashInfer TRT-LLM MoE backend." + - "Patch the image's MiniMax M3 MSA prefill path to materialize sliced top-k indices before CSR construction." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1834 + +- config-keys: + - minimaxm3-fp4-mi355x-atom + description: + - "Expand search space for minimaxm3-fp4-mi355x-atom: add TP2 and TP8 configurations, extend concurrency range to 256 for ISL1024 and ISL8192, and add TP8 conc=1-2 for ISL8192." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1825 + +- config-keys: + - minimaxm3-fp4-mi355x-atom + - minimaxm3-fp4-mi355x-atom-mtp + description: + - "Add minimaxm3-fp4-mi355x-atom-mtp: MiniMax-M3 MXFP4 on MI355X with EAGLE3 speculative decoding (3 draft tokens)" + - "Bump image to rocm/atom-dev:MiniMax-M3-20260623 for both fp4 atom entries" + - "Search space: TP2/TP4, ISL=1024,8192, OSL=1024, conc 1–256" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1917 + +- config-keys: + - minimaxm3-fp8-b300-vllm-mtp + description: + - "Update the MiniMax-M3 B300 single-node image to vllm/vllm-openai:minimax-m3-0618-x86_64-cu130." + - "Enable FlashInfer TRT-LLM attention with FP8 indexer KV and KV cache; use TRITON_ATTN for the EAGLE3 drafter." + - "Switch TP-only configurations from explicit Marlin MoE to the new image's default FlashInfer TRT-LLM MoE backend." + - "Patch the image's MiniMax M3 MSA prefill path to materialize sliced top-k indices before CSR construction." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1835 + +- config-keys: + - minimaxm3-fp8-mi300x-vllm-mtp + description: + - "Update the MI300X MiniMax-M3 EAGLE3 vLLM image to vllm/vllm-openai-rocm:nightly-b53b1c7ffe7aebdafd0876350f30e51d1226c92a" + - "Use FP8 KV cache" + - "Remove the runtime SupportsEagle3 source patch now included in the pinned nightly" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1843 + +- config-keys: + - minimaxm3-fp8-mi355x-atom + - minimaxm3-fp8-mi355x-atom-mtp + description: + - "Add minimaxm3-fp8-mi355x-atom: MiniMax-M3 MXFP8 single-node benchmark on MI355X using ATOM framework" + - "Uses rocm/atom-dev:MiniMax-M3-20260623; TP4, block size 128, ISL=1024,8192 OSL=1024" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1916 + +- config-keys: + - minimaxm3-fp8-mi355x-atom-mtp + description: + - "Add minimaxm3-fp8-mi355x-atom-mtp: same with EAGLE3 speculative decoding (3 draft tokens)" + - "Both use rocm/atom-dev:MiniMax-M3-20260619; search space mirrors FP4 atom variants (ISL=1024,8192 OSL=1024 TP2/TP4/TP8)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1867 + +- config-keys: + - minimaxm3-fp8-gb300-dynamo-vllm + description: + - "Switch to vllm/vllm-openai:nightly-aarch64 — contains upstream head_ratio fix (vllm-project/vllm#45879), avoids gemm1_alpha crash in minimax-m3-0618" + - "Add --moe-backend marlin for TP-only prefill/decode workers (no EP, no DP-attention) per PR #1809 pattern" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1735 + +- config-keys: + - kimik2.5-fp4-gb200-dynamo-trt + description: + - "Update Kimi K2.5 NVFP4 GB200 TRT-LLM Dynamo disaggregated configurations" + - "Bump image from nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.1.0-dev.2 to 1.3.0-dev.1-cuda13" + - "Replace allconc-style recipes with 25 per-concurrency STP recipes (13 ISL1K/OSL1K + 12 ISL8K/OSL1K)" + - "Recipes from srt-slurm sa-submission-q2-2026 branch (srt-slurm PR#89)" + - "ISL1K/OSL1K: conc 8–8192 across ctx1dep4 and ctx2dep4 topologies" + - "ISL8K/OSL1K: conc 5–4301 across ctx1dep4 to ctx9dep4 topologies" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1797 + +- config-keys: + - kimik2.5-fp4-gb300-dynamo-vllm + description: + - "Add Kimi-K2.5 NVFP4 GB300 disaggregated multinode vLLM benchmarks via Dynamo" + - "1k/1k topologies (5 shapes): 1p1d-dep4-dep16 (conc 2048,4096), 1p1d-dep4-dep24 (conc 1024), 1p2d-dep4-dep4 (conc 6144), 1p7d-tep4-tp4 (conc 8-128), 2p3d-dep4-dep8 (conc 8192)" + - "8k/1k topologies (6 shapes): 1p4d-dep4-tp8 (conc 4), 1p8d-dep4-tp4 (conc 32,128), 2p1d-dep4-dep24 (conc 2048), 3p1d-dep4-dep16 (conc 2048), 4p1d-dep4-dep8 (conc 3072, multi-frontend), 8p1d-dep4-dep24 (conc 15360, multi-frontend)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1861 + +- config-keys: + - kimik2.5-fp4-gb200-dynamo-trt + description: + - "Recover the skipped official ingest for PR #1797 from validated sweep run 27591355916 (attempt 6)" + - "No benchmark configuration change; reuse the exact 25-point fixed-sequence matrix and 2 eval jobs" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1869 + +- config-keys: + - glm5.1-fp4-mi355x-sglang + description: + - "Bump SGLang ROCm image from v0.5.12.post1-rocm720-mi35x-20260529 to v0.5.13.post1-rocm720-mi35x-20260622" + - "Enable aiter allreduce fusion via --enable-aiter-allreduce-fusion in benchmarks/single_node/fixed_seq_len/glm5.1_fp4_mi355x.sh" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1905 + +- config-keys: + - minimaxm3-fp8-b300-dynamo-vllm + description: + - "Add MiniMax-M3 MXFP8 B300 disaggregated vLLM benchmarks via Dynamo for 1k1k and 8k1k STP." + - "Add local srt-slurm recipes under benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8 and wire the B300 launcher to overlay them into srt-slurm." + - "Add right-sized TP4 decode variants with expert parallelism disabled and the Marlin MoE backend for selected low-concurrency 1k1k and 8k1k shapes." + - "Colocate the six-GPU TP4 prefill/decode pairs on one B300 node and enable CUDA IPC for NIXL KV transfer." + - "Patch the 0618 image's MiniMax M3 MSA prefill top-k slice to be contiguous before CSR construction." + - "Align 8k1k expert-parallel settings with the 1k1k recipes and correct the decode CUDA graph capture limit." + - "Backport NVIDIA/srt-slurm#38 to sanitize Slurm node-IP discovery output on the pinned submission branch." + - "Backport vllm-project/vllm#45879 so NIXL validates heterogeneous-TP KV block lengths using the GQA KV-head ratio." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1863 + +- config-keys: + - kimik2.5-fp4-gb300-dynamo-trt + description: + - "Add Kimi K2.5 NVFP4 GB300 TRT-LLM Dynamo disaggregated multinode configurations" + - "Image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13" + - "Recipes from srt-slurm sa-submission-q2-2026 branch (srt-slurm PR#214)" + - "STP-only: 14 configs for ISL1K/OSL1K (conc 8–8192), 11 configs for ISL8K/OSL1K (conc 4–2253)" + - "Topology range: 1–8 prefill workers (TP4/EP4), decode 1d-dep8 to 1d-dep32 and 4d/5d-tep8/tep4" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1796 + +- config-keys: + - kimik2.5-fp4-gb200-dynamo-vllm + description: + - "Refresh GB200 dynamo-vllm disagg sweep: bump image to vllm/vllm-openai:v0.21.0, dynamo 1.2.1, switch to checked-in kimi-k2.5-fp4 recipes" + - "1k/1k: 1p1d-dep4-dep8 (conc 4096,12288), 1p4d-dep4-tp8 (conc 4-128), 1p1d-dep4-dep16 (conc 4096,6144)" + - "8k/1k: 1p4d-dep4-tep4 (conc 128), 1p4d-dep4-tp8 (conc 4-256), 3p1d-dep4-dep16 (conc 1024), 6p1d-dep4-dep16 (conc 3072), 8p1d-dep4-dep16 (conc 6144)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1862 + +- config-keys: + - minimaxm3-fp4-mi355x-atom-disagg + description: + - "Add minimaxm3-fp4-mi355x-atom-disagg CI script: multi-node disaggregated PD on MI355X via ATOM for MiniMax-M3-MXFP4" + - "No MTP, KV_CACHE_DTYPE=auto (MXFP4 native, no fp8 override), MAX_MODEL_LEN=32768, MAX_NUM_BATCHED_TOKENS=32768" + - "server_atom.sh: conditional --kv_cache_dtype, MAX_MODEL_LEN/MAX_NUM_BATCHED_TOKENS/CUDAGRAPH_OPT support, syntax fixes" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1856 + +- config-keys: + - dsv4-fp4-mi355x-sglang + description: + - "Bump image to lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260618." + - "Enable SGLANG_DP_USE_GATHERV=1. Change to use all_gatherv + reduce_scatterv for tp + dp config." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1824 + +- config-keys: + - kimik2.5-fp4-gb300-dynamo-trt + description: + - "Recover the failed official ingest for PR #1796 from validated sweep run 27663808752 (attempt 2)" + - "Artifact-only recovery: reuse 23 fixed-sequence rows and 2 eval rows without rerunning benchmarks" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1884 + +- config-keys: + - qwen3.5-fp8-gb200-dynamo-sglang + description: + - "Add Qwen3.5-397B-A17B-FP8 GB200 disaggregated multinode SGLang benchmarks via Dynamo" + - "Image: lmsysorg/sglang:nightly-dev-cu13-20260608-303757cc" + - "6 topologies across 1k/1k and 8k/1k: 1P1D TP4 STP + wide-EP (DEP4 prefill / DEP16 decode) from 1P1D up to 8P1D, recipes under benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1810 + +- config-keys: + - minimaxm3-fp8-b300-dynamo-vllm + description: + - "Add MiniMax-M3 MXFP8 B300 disaggregated vLLM benchmarks via Dynamo for 1k1k and 8k1k STP." + - "Add local srt-slurm recipes under benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8 and wire the B300 launcher to overlay them into srt-slurm." + - "Add right-sized TP4 decode variants with expert parallelism disabled and the Marlin MoE backend for selected low-concurrency 1k1k and 8k1k shapes." + - "Colocate the six-GPU TP4 prefill/decode pairs on one B300 node and enable CUDA IPC for NIXL KV transfer." + - "Patch the 0618 image's MiniMax M3 MSA prefill top-k slice to be contiguous before CSR construction." + - "Align 8k1k expert-parallel settings with the 1k1k recipes and correct the decode CUDA graph capture limit." + - "Backport NVIDIA/srt-slurm#38 to sanitize Slurm node-IP discovery output on the pinned submission branch." + - "Backport vllm-project/vllm#45879 so NIXL validates heterogeneous-TP KV block lengths using the GQA KV-head ratio." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1893 + +- config-keys: + - minimaxm3-fp8-gb200-dynamo-vllm + description: + - "Initial addition of MiniMax-M3 FP8 GB200 disaggregated Dynamo-vLLM benchmark using vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" + - "Allocate FlashInfer MNNVL workspace for one-shot all-reduce and materialize the MSA prefill top-k slice before CSR construction" + - "Preserve current Qwen3.5 and Kimi-K2.5 GB200 launcher paths while adding MiniMax-M3 shared-FS staging and atomic image import" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1938 + +- config-keys: + - minimaxm3-fp8-gb300-dynamo-vllm + description: + - "Update the GB300 MiniMax-M3 Dynamo-vLLM image to vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" + - "Use the dedicated ARM64 MiniMax-M3 performance image; benchmark settings unchanged" + - "Allocate FlashInfer MNNVL workspace for one-shot TP8 all-reduce during CUDA graph capture" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1888 + +- config-keys: + - minimaxm3-fp8-gb300-dynamo-vllm + description: + - "Update the GB300 MiniMax-M3 Dynamo-vLLM image to vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" + - "Use the dedicated ARM64 MiniMax-M3 performance image; benchmark settings unchanged" + - "Allocate FlashInfer MNNVL workspace for one-shot TP8 all-reduce during CUDA graph capture" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1897 + +- config-keys: + - qwen3.5-fp4-b200-trt-mtp + description: + - "Add Qwen3.5-397B-A17B-NVFP4 B200 single-node TensorRT-LLM benchmark with MTP speculative decode (1k/1k and 8k/1k) across a TP/TEP/DEP parallelism sweep" + - "Image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1894 + +- config-keys: + - dsr1-fp4-b200-sglang + description: + - "Update B200 FP4 SGLang image from lmsysorg/sglang:v0.5.12-cu130 to lmsysorg/sglang:v0.5.12.post1." + - "Update search space: 1k/1k TP4/EP4 conc 1-256 and TP8/EP8 conc 1; 8k/1k TP4/EP4 conc 1-128, add TP4/EP4 DP-attention conc 64-256, and keep TP8/EP8 conc 1." + - "For DP-attention runs, use TP-sized data parallelism with DP attention local-control broadcast, DP LM head, prefill delayer, scheduler recv interval 1, chunked prefill size 32768, and schedule conservativeness 3.33." + - "Set SGLANG_RADIX_FORCE_MISS=1, remove --disable-radix-cache, and explicitly pass --disable-piecewise-cuda-graph." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1792 + +- config-keys: + - dsv4-fp4-gb200-dynamo-sglang-mtp + description: + - "Initial submission: DSv4-Pro FP4 disagg on GB200 with SGLang + MTP speculative decoding (8k/1k)." + - "Image: lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" + - "8 topologies: low-latency 1p1d-tp8-tp8 + 1p6d-dep8-tp8; mid-curve 1p1d through 6p1d-dep8-dep16." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1676 + +- config-keys: + - minimaxm3-fp8-b300-dynamo-vllm + description: + - "Run the PR #1891 MiniMax-M3 MXFP8 B300 Dynamo-vLLM recipe set on top of current main." + - "Uses the vllm/vllm-openai:minimax-m3-0618-x86_64-cu130 image and the TEP4/TEP8 8k1k topologies not covered by PR #1890." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1891 + +- config-keys: + - dsv4-fp4-b300-sglang + description: + - "Update B300 FP4 SGLang (non-MTP) image to latest nightly: lmsysorg/sglang:nightly-dev-cu13-20260624-b2c8f7a2 (was nightly-dev-cu13-20260529-a8cfae0b)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1913 + +- config-keys: + - minimaxm3-fp8-mi355x-atom-disagg + description: + - "Add minimaxm3-fp8-mi355x-atom-disagg CI recipe: multi-node disaggregated PD on MI355X via ATOM for MiniMax-M3-MXFP8" + - "Settings aligned with slurm reference: MEM_FRAC_STATIC=0.8, MAX_NUM_SEQS=128, BLOCK_SIZE=128, MAX_MODEL_LEN=32768, KV_CACHE_DTYPE=auto" + - "server_atom.sh: fix _MAX_CONC assignment before cudagraph size check; gate ATOM_MOE_GU_ITLV/AITER_BF16_FP8_MOE_BOUND on DeepSeek-V4-Pro only" + - "Search space: ISL=8192 and ISL=1024, 1P1D TP4, conc 1-512" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1865 + +- config-keys: + - minimaxm3-fp8-mi355x-vllm-disagg + description: + - "Initial submission: MiniMax-M3 MXFP8 MI355X vLLM disaggregated (prefill/decode) smoke test on the day-zero ROCm image (vllm/vllm-openai-rocm:minimax-m3) — 1 prefill (TP8) + 1 decode (TP8) across conc 1,2,4,8,16, validating the MoRI-IO KV-transfer disagg pipeline end-to-end for M3" + - "Layered on the MoRI-IO patch-removal infra (#1585): uses benchmarks/multi_node/amd_utils with the runtime MoRI patches removed" + - "Per-worker serve flags (models_vllm.yaml MiniMax-M3-MXFP8): --block-size 128 (MSA), --language-model-only, --kv-cache-dtype fp8, --attention-backend TRITON_ATTN, minimax_m3 parsers; no EP (TP8, MoE experts TP-sharded)" + - "M3 disagg script points MODEL_PATH at the cluster's shared HF cache (/it-share/hf-hub-cache) where the ~414 GB MiniMax-M3-MXFP8 checkpoint is pre-staged, instead of the launcher default /it-share/data; scoped to M3 only (other disagg models keep /it-share/data)" + - "Sweeps conc 1,2,4,8,16,32,64,128,256,512,1024 at both 1k1k and 8k1k (1P TP8 + 1D TP8). The 8k1k point makes the multi-node eval policy (8k1k + conc >= 16) mark one lm-eval on the highest-max-conc layout (eval-conc=median), validating the disagg pipeline's correctness; run with non-canary-full-sweep-enabled so the eval entry actually runs" + - "Adds two asymmetric prefill/decode layouts at both 1k1k and 8k1k alongside the TP8+TP8 sweep: 1P TP4 + 1D TP8 (smaller prefill, full-node decode) at conc 1,2,4,8,16,32,64,128,256; and balanced 1P TP4 + 1D TP4 at conc 64,128,256,512,1024. Per-worker TP comes from the master-config prefill/decode tp (server_vllm.sh rewrites the models_vllm.yaml --tensor-parallel-size placeholder); no EP, dp-attn off, PREFILL_NODES=1/DECODE_NODES=1 (TP4 uses half an 8-GPU node)" + - "Adds a 2P TP4 + 1D TP8 layout at both 1k1k and 8k1k for high conc 256,512,768,1024: two TP4 prefill workers (num-worker 2, PREFILL_NODES=2, each TP4 on half an 8-GPU node) feeding one TP8 decode (DECODE_NODES=1); 3 nodes total" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1762 + +- config-keys: + - minimaxm3-fp8-mi355x-atom-disagg + description: + - "Add minimaxm3-fp8-mi355x-atom-disagg CI recipe: multi-node disaggregated PD on MI355X via ATOM for MiniMax-M3-MXFP8" + - "Settings aligned with slurm reference: MEM_FRAC_STATIC=0.8, MAX_NUM_SEQS=128, BLOCK_SIZE=128, MAX_MODEL_LEN=32768, KV_CACHE_DTYPE=auto" + - "server_atom.sh: fix _MAX_CONC assignment before cudagraph size check; gate ATOM_MOE_GU_ITLV/AITER_BF16_FP8_MOE_BOUND on DeepSeek-V4-Pro only" + - "Search space: ISL=8192 and ISL=1024, 1P1D TP4, conc 1-512" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1927 + +- config-keys: + - dsv4-fp4-b200-dynamo-vllm + description: + - "Update the DeepSeek-V4-Pro B200 disaggregated Dynamo-vLLM benchmark to the vllm/vllm-openai:v0.23.0 image" + - "Lower max-num-batched-tokens to 16384 and gpu-memory-utilization to 0.9 on the high-throughput and max-throughput recipes to avoid OOM" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1899 + +- config-keys: + - minimaxm3-fp4-mi355x-vllm-disagg + description: + - "Initial submission: MiniMax-M3 MXFP4 disagg (prefill/decode) on MI355X with vLLM over the MoRI-IO KV connector (8k/1k)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1914 + +- config-keys: + - minimaxm3-fp4-b200-vllm + description: + - "Add MiniMax-M3 NVFP4 (nvidia/MiniMax-M3-NVFP4) B200 single-node aggregated vLLM benchmark (no spec decode), runner b200-dgxc" + - "Image vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41 (bakes in MiniMax-M3 modelopt NVFP4 support, vllm-project/vllm PR #46380; no runtime patch needed)" + - "Weights pre-staged at /scratch/fsw/models/MiniMax-M3-NVFP4 (added minimaxm3-fp4 MODEL_PATH branch to launch_b200-dgxc.sh); --block-size 128 (MSA), --language-model-only" + - "Sweeps tp 4/8 with and without EP and dp-attn at 1k1k and 8k1k, conc 1-1024" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1932 + +- config-keys: + - minimaxm3-fp8-gb300-dynamo-vllm + description: + - "Refresh GB300 MiniMax-M3 disagg recipe set: replace disagg-gb300-* files with new naming convention; drop TP4+Marlin variants." + - "1k/1k topologies (6 shapes): 1p1d-dep2-dep4 (conc 8192), 1p2d-dep2-tep8 (conc 4,16,64,128,256), 2p2d-dep2-tep8 (conc 32), 2p3d-dep2-dep4 (conc 8192), 2p4d-dep2-dep4 (conc 8192), 4p2d-dep2-dep8 (conc 1024,4096)." + - "8k/1k topologies (9 shapes): 1p1d-dep2-dep8 (conc 256), 1p1d-dep2-tep8 (conc 128), 1p2d-dep2-tep8 (conc 32,64,128), 2p1d-dep2-dep8 (conc 512), 2p2d-dep2-tep8 (conc 16), 2p4d-dep2-tep4 (conc 4), 3p1d-dep2-dep8 (conc 1024), 3p2d-dep2-dep8 (conc 512), 6p1d-dep2-dep8 (conc 2048)." + - "Image unchanged: vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1925 + +- config-keys: + - minimaxm3-fp4-b300-vllm + description: + - "Add MiniMax-M3 NVFP4 (nvidia/MiniMax-M3-NVFP4) B300 single-node aggregated vLLM benchmark (no spec decode)" + - "Image vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41 (bakes in MiniMax-M3 modelopt NVFP4 support, vllm-project/vllm PR #46380; no runtime patch needed)" + - "Weights pre-staged read-only at /scratch/models/MiniMax-M3-NVFP4 (added MiniMax-M3-NVFP4 to launch_b300-nv.sh STAGED_MODELS); --block-size 128 (MSA), --language-model-only" + - "Sweeps tp 2/4/8 with and without EP and dp-attn at 1k1k and 8k1k, conc 1-1024" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1928 + +- config-keys: + - minimaxm3-fp4-b300-vllm-mtp + description: + - "Add MiniMax-M3 NVFP4 (nvidia/MiniMax-M3-NVFP4) B300 single-node aggregated vLLM benchmark with EAGLE3 speculative decoding (spec-decoding: mtp, 3 draft tokens via Inferact/MiniMax-M3-EAGLE3)" + - "Image vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41 (bakes in MiniMax-M3 modelopt NVFP4 support, vllm-project/vllm PR #46380; no runtime patch needed); prompts routed through the chat template" + - "Target weights pre-staged read-only at /scratch/models/MiniMax-M3-NVFP4 (added MiniMax-M3-NVFP4 to launch_b300-nv.sh STAGED_MODELS); EAGLE3 draft downloaded to the writable /data/models; --block-size 128 (MSA), --language-model-only" + - "Sweeps tp 4/8 with and without EP and dp-attn at 1k1k and 8k1k, conc 1-512" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1929 + +- config-keys: + - minimaxm3-fp4-mi355x-vllm + description: + - "Add a MiniMax-M3 MXFP4 single-node vLLM benchmark on MI355X using amd/MiniMax-M3-MXFP4." + - "Pin vllm/vllm-openai-rocm:nightly-3f5a1e1733200760169ff31ebe60a271072b199e, which includes upstream Quark MXFP4 support from vllm-project/vllm#45794." + - "Serve through the text-only language-model path with block size 128, TRITON_ATTN, MiniMax-M3 tool/reasoning parsers, automatic tool choice, and VLLM_USE_BREAKABLE_CUDAGRAPH=0; let vLLM select the MoE backend and retain the default KV-cache dtype." + - "Mirror the MiniMax-M3 MXFP8 MI355X TP/EP/DP-attention search space at 1k1k and 8k1k for a direct precision comparison." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1935 + +- config-keys: + - qwen3.5-fp4-gb300-dynamo-sglang + description: + - "Add Qwen3.5-397B-A17B-NVFP4 GB300 disaggregated multinode SGLang benchmarks via Dynamo." + - "Image: lmsysorg/sglang:nightly-dev-cu13-20260608-303757cc." + - "8k/1k STP recipes: 1P1D TP4 (conc 1-256), 5P1D DEP4+1D DEP16 (conc 2048, NIXL), 6P1D and 7P1D DEP4+1D DEP16 (conc 5120, Mooncake)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1921 + +- config-keys: + - minimaxm3-fp4-mi355x-vllm-mtp + description: + - "Add a MiniMax-M3 MXFP4 MI355X vLLM benchmark with EAGLE3 speculative decoding using amd/MiniMax-M3-MXFP4 and Inferact/MiniMax-M3-EAGLE3 with three speculative tokens." + - "Reuse the pinned vllm/vllm-openai-rocm:nightly-3f5a1e1733200760169ff31ebe60a271072b199e image, text-only target path, TRITON_ATTN, automatic tool choice, MiniMax-M3 parsers, VLLM_USE_BREAKABLE_CUDAGRAPH=0, default KV-cache dtype, and automatic MoE backend selection." + - "Pass --use-chat-template for MTP acceptance and mirror the existing MiniMax-M3 MXFP8 MI355X MTP TP/EP/DP-attention search space at 1k1k and 8k1k." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1939 + +- config-keys: + - minimaxm3-fp8-mi355x-vllm-mtp + description: + - "Update the MiniMax-M3 MXFP8 MI355X vLLM EAGLE3 benchmark image from vllm/vllm-openai-rocm:minimax-m3 to vllm/vllm-openai-rocm:nightly-3f5a1e1733200760169ff31ebe60a271072b199e." + - "Benchmark configuration, EAGLE3 draft model, serving flags, and search space are unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1941 + +- config-keys: + - minimaxm3-fp8-mi355x-vllm + description: + - "Update the MiniMax-M3 MXFP8 MI355X vLLM image from vllm/vllm-openai-rocm:minimax-m3 to vllm/vllm-openai-rocm:nightly-3f5a1e1733200760169ff31ebe60a271072b199e." + - "Benchmark serving flags and TP/EP/DP-attention search space are unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1942 + +- config-keys: + - dsv4-fp4-b300-dynamo-vllm + description: + - "Update the DeepSeek-V4-Pro B300 disaggregated Dynamo-vLLM benchmark to the vllm/vllm-openai:v0.23.0 image" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1952 + +- config-keys: + - minimaxm3-fp4-mi355x-vllm + description: + - "Enable AITER MoE on MiniMax-M3 MXFP4 MI355X single-node vLLM STP: export VLLM_ROCM_USE_AITER=1, VLLM_ROCM_USE_AITER_MOE=1, and VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1; pass --moe-backend aiter." + - "Pin vllm/vllm-openai-rocm:nightly-4559c43a9526597c00cbcc4f59979496500268d1 (from nightly-3f5a1e1733200760169ff31ebe60a271072b199e) for AITER MoE and shared-expert fusion support (vllm-project/vllm#46419, vllm-project/vllm#46545)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1954 + +- config-keys: + - minimaxm3-fp8-mi300x-vllm + description: + - "Update the MiniMax-M3 MXFP8 MI300X vLLM image from vllm/vllm-openai-rocm:minimax-m3 to vllm/vllm-openai-rocm:nightly-4559c43a9526597c00cbcc4f59979496500268d1." + - "Enable AITER kernels for MiniMax-M3 MXFP8 on MI300X/gfx942 via the single master toggle VLLM_ROCM_USE_AITER=1: the stock image left it unset, so the hot decode GEMMs and fused MoE ran on the generic kernels. The per-component AITER flags (MoE, linear, RMSNorm, FP8 batched-GEMM) default to True and are gated behind the master flag, so they are left at their defaults. Keep attention on TRITON_ATTN (VLLM_ROCM_USE_AITER_MHA=0, which defaults to True) because the MXFP8 checkpoint lacks calibrated q/prob scales for ROCm FP8 attention." + - "Add AMD-recommended, numerically-inert MI300X runtime knobs: TORCH_BLAS_PREFER_HIPBLASLT=1, NCCL_MIN_NCHANNELS=112 (raises RCCL channels above the ~32-64 default for TP8), GPU_MAX_HW_QUEUES=2 (caps HIP streams below the default of 4)." + - "Stack two accuracy-safe scheduling levers: --async-scheduling (overlaps CPU input-prep with GPU decode) and --max-num-batched-tokens 16384 (amortizes the per-step BF16-emulated MoE weight read of ~95 GB/rank over more prompt tokens, halving prefill weight-reads vs the 8192 default). Both are token-for-token identical (scheduling only); GSM8K exact-match holds at 0.959." + - "Switch the 1k1k conc256 search-space row from TP8/EP8 to TP8/EP1: the EP8 topology regressed high-concurrency throughput (434 vs 905 tok/s/gpu @ conc256, EP8 vs EP1) and EP1 matches the topology the prior AITER uplift was measured against." + - "Measured uplift on 8xMI300X, 1k1k random sweep (total tok/s/gpu): conc256 782.7->905 (EP8->EP1 + scheduling levers), conc128 598.9->628 (+4.9%), conc64 365.1->429 (+17.5%), conc32 295.6->327.4 (+10.8%), conc16 203.1->216.5 (+6.6%), conc8 127.6->136.6 (+7.1%), conc4 80.1->84.6 (+5.6%); conc1-2 unchanged (latency-bound). GSM8K exact-match holds at 0.959." + - "Drop the remaining TP8/EP8 row from the 8k1k search space (conc128-256), same fix as the 1k1k row above: the CI eval-only job at 8k1k TP8/EP8 conc256 caught GSM8K collapsing to ~0 (0.0000 strict / ~0.01 flexible). Reproduced locally on 8xMI300X against this image: --enable-expert-parallel (EP8) returns garbled/incoherent tokens on both /v1/completions and /v1/chat/completions for trivial prompts (e.g. 'The capital of France is' -> gibberish), while the same request against plain TP8 (EP1) answers correctly and scores 0.92 exact-match on a 50-question GSM8K sample. Root cause is EP8 itself (garbage generation), not --async-scheduling, which was tried and ruled out first. 8k1k is now TP8/EP1 across the full conc1-256 range." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1951 + +- config-keys: + - minimaxm3-fp8-mi355x-vllm + - minimaxm3-fp8-mi355x-vllm-mtp + description: + - "Update the MiniMax-M3 MXFP8 MI355X vLLM benchmark image from vllm/vllm-openai-rocm:minimax-m3 to vllm/vllm-openai-rocm:nightly-4559c43a9526597c00cbcc4f59979496500268d1, which includes the gfx950 mxfp8 MoE/linear tuning (vllm-project/vllm#45725), fused shared-experts MoE for the mxfp8 model (#46545), and the AITER flydsl MoE backend (#46184)." + - "Align the standard and EAGLE3 (MTP) bench scripts with vllm-project/recipes#581: gate VLLM_ROCM_USE_AITER on expert parallelism (on for EP/DP-attention runs, where the AITER fused MoE is the auto-selected backend; off for TP-only runs, which fall back to the native MXFP8 path since the master switch otherwise yields degenerate MiniMax-M3 output), export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 unconditionally (the router-append shared-experts fusion is independent of the master switch and self-disables under EP), and export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT6 (INT6 quick all-reduce)." + - "Retune the TP/EP search space to the best layout per concurrency band and drop redundant points (full TP8/EP8, TP2/EP2, DP-attention)." + - "minimaxm3-fp8-mi355x-vllm: 1k1k sweeps TP8 (conc 1-32), TP4 (conc 4-32), TP4/EP4 (conc 64-512); 8k1k sweeps TP8 (conc 1-2), TP4 (conc 2-128)." + - "minimaxm3-fp8-mi355x-vllm-mtp: 1k1k sweeps TP8 (conc 4-32), TP8/EP8 (conc 1-256), TP4 (conc 1-2 and 32-64), TP4/EP4 (conc 128-256); 8k1k sweeps TP8 (conc 1 and 4-16), TP4 (conc 16-128)." + - "Serving flags are otherwise unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1946 + +- config-keys: + - minimaxm3-fp4-b300-dynamo-vllm + description: + - "Add MiniMax-M3 NVFP4 B300 disaggregated vLLM benchmarks via Dynamo for 1k1k and 8k1k STP (no MTP)" + - "Use nvidia/MiniMax-M3-NVFP4 from /scratch/models/MiniMax-M3-NVFP4 with vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41, which includes vllm-project/vllm PR #46380; no runtime patch needed" + - "Reuse the existing MXFP8 B300 topology and concurrency matrix across 15 srt-slurm recipes, while dropping the FP8-only Marlin override from TP4 decode" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1931 + +- config-keys: + - minimaxm3-fp4-mi355x-vllm-mtp + description: + - "Enable AITER MoE on MiniMax-M3 MXFP4 MI355X single-node vLLM MTP (EAGLE3), mirroring the STP recipe: export VLLM_ROCM_USE_AITER=1, VLLM_ROCM_USE_AITER_MOE=1, VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1; pass --moe-backend aiter unconditionally (including expert parallelism)." + - "Fixes the ~8h engine-core startup hang on EP configs: with moe_backend=auto, EP fell back to Mxfp4MoeBackend.EMULATION, which deadlocked all expert-parallel workers building the Quark hw-emulation C++ kernel into a shared torch_extensions dir. Forcing --moe-backend aiter selects AITER_MXFP4_MXFP4 (no emulation build)." + - "Pin vllm/vllm-openai-rocm:nightly-4559c43a9526597c00cbcc4f59979496500268d1 (from nightly-3f5a1e1733200760169ff31ebe60a271072b199e), matching the STP recipe." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1964 + +- config-keys: + - minimaxm3-fp4-b300-dynamo-vllm + description: + - "Add MiniMax-M3 NVFP4 B300 disaggregated vLLM benchmarks via Dynamo for 1k1k and 8k1k STP (no MTP)" + - "Use nvidia/MiniMax-M3-NVFP4 from /scratch/models/MiniMax-M3-NVFP4 with vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41, which includes vllm-project/vllm PR #46380; no runtime patch needed" + - "Reuse the existing MXFP8 B300 topology and concurrency matrix across 15 srt-slurm recipes, while dropping the FP8-only Marlin override from TP4 decode" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1966 + +- config-keys: + - dsv4-fp4-b200-dynamo-vllm + description: + - "Add point-specific DeepSeek-V4-Pro FP4 B200 Dynamo-vLLM disaggregated recipes for the 8k/1k concurrency points c=1, 32, 64, 128, 256, 512." + - "Wire those rows in nvidia-master.yaml; c=16, 1024, 8192, and 12345 are not part of this update." + - "Recipes use vllm/vllm-openai:v0.23.0, DSV4 revision 0366e4e064385807ea86b088a5c6c878ff23343b, max-model-len 9280, with per-point P1/D1 (1 prefill node DEP=8 / 1 decode node) serving knobs." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1963 + +- config-keys: + - minimaxm3-fp4-b200-vllm-mtp + description: + - "Add MiniMax-M3 NVFP4 (nvidia/MiniMax-M3-NVFP4) B200 single-node aggregated vLLM benchmark with EAGLE3 speculative decoding (spec-decoding: mtp, 3 draft tokens via Inferact/MiniMax-M3-EAGLE3), runner b200-dgxc" + - "Image vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41 (bakes in MiniMax-M3 modelopt NVFP4 support, vllm-project/vllm PR #46380; no runtime patch needed); prompts routed through the chat template" + - "Target weights pre-staged at /scratch/fsw/models/MiniMax-M3-NVFP4 (added minimaxm3-fp4 MODEL_PATH branch to launch_b200-dgxc.sh); EAGLE3 draft fetched next to the target weights; --block-size 128 (MSA), --language-model-only" + - "Sweeps tp 4/8 with and without EP at 1k1k and 8k1k, conc 1-256" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1933 + +- config-keys: + - dsv4-fp4-b200-sglang + description: + - "Bump SGLang image from lmsysorg/sglang:deepseek-v4-blackwell (digest sha256:df18bfc4...) to mainline nightly lmsysorg/sglang:nightly-dev-cu13-20260628-da802ddc." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1923 + +- config-keys: + - dsr1-fp4-b200-sglang-mtp + description: + - "Update the B200 FP4 SGLang MTP image to lmsysorg/sglang:v0.5.12.post1." + - "Update the MTP search space to include the low-latency TP4/EP1 lane with MTP speculative decoding enabled." + - "Use 1k/1k TP4/EP1 c1-c32 and TP4/EP4 c64-c256; use 8k/1k TP4/EP1 c1-c32 and TP4/EP4 DP-attention c64-c256." + - "Enable the MTP DP-attention path with local-control broadcast, DP LM head, prefill delayer, chunked prefill size 32768, scheduler recv interval 1, and schedule conservativeness 3.33." + - "Use SGLANG_RADIX_FORCE_MISS=1, remove --disable-radix-cache and --enable-symm-mem, and explicitly pass --disable-piecewise-cuda-graph." + - "Align MTP runtime settings with the non-MTP aggregate: cuda graph max batch size 256, max running requests 256, mem fraction static 0.85, stream interval 10, and no explicit max-prefill-tokens." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1962 + +- config-keys: + - dsr1-fp4-b200-sglang + description: + - "Update the B200 FP4 SGLang aggregate search space to low-latency lanes: TP4/EP1 for the low-concurrency range and TP4/EP4 for the higher-concurrency range." + - "Use 1k/1k TP4/EP1 c1-c32 and TP4/EP4 c64-c256; use 8k/1k TP4/EP1 c1-c32 and TP4/EP4 DP-attention c64-c256." + - "Drop the TP8/EP8 single-concurrency points." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1975 + +- config-keys: + - minimaxm3-fp4-b300-vllm + description: + - "Update Minimax M3 b300 vllm image tag" + - "Update search space to cover more configs" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1990 + +- config-keys: + - minimaxm3-fp4-b200-vllm + description: + - "Update Minimax M3 b200 vllm image tag" + - "Update search space to cover more configs" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1978 + +- config-keys: + - minimaxm3-fp4-mi355x-atom + - minimaxm3-fp4-mi355x-atom-mtp + - minimaxm3-fp8-mi355x-atom + - minimaxm3-fp8-mi355x-atom-mtp + description: + - "Bump ATOM image from rocm/atom-dev:MiniMax-M3-20260623 to rocm/atom-dev:nightly_202607011530 for all single-node MiniMax-M3 ATOM recipes." + - "Add --online_quant_config with ptpc_fp8 and MoE layer exclusions (*block_sparse_moe) to all scripts." + - "Replace deprecated AITER_QUICK_REDUCE_CAST_BF16_TO_FP16=0 and ATOM_M3_SPARSE_USE_ASM_PA=1 with ATOM_FORCE_ATTN_TRITON=1." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2001 + +- config-keys: + - minimaxm3-fp8-mi355x-vllm + description: + - "Bump the MiniMax-M3 MXFP8 MI355X vLLM image to nightly-09663abde0f50944a8d5ea30120666024b503faa" + - "Use --linear-backend emulation for the MXFP8 dense-linear path (beats the stock nightly native MXFP8 linear: ~+26% tput / -21% TPOT at 8k1k conc1, ~+2-3% at high concurrency)" + - "Add --max-num-batched-tokens 32768 (env MAX_NUM_BATCHED_TOKENS) to enlarge the per-step prefill budget and improve TP4 throughput at high concurrency" + - "Enable the AITER master switch for TP-only (no-EP) runs via --moe-backend aiter: the earlier degenerate-output issue that forced it off for TP-only is fixed by vllm-project/vllm#47158, so TP4 uses the AITER_MXFP8 MoE path (verified GSM8K 0.9613 flex / 0.9621 strict on this nightly)" + - "Simplify both search spaces to a single TP4 conc 1-512 sweep for 1k1k and 8k1k (drop TP8 and TP4/EP4: TP8 has poor throughput/GPU and plain TP4 matches or beats TP4/EP4 at high concurrency)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2003 + +- config-keys: + - minimaxm3-fp8-mi355x-vllm-mtp + description: + - "Bump the MiniMax-M3 MXFP8 MI355X vLLM MTP (EAGLE3) image to nightly-09663abde0f50944a8d5ea30120666024b503faa, which natively supports SupportsEagle3 (the in-place EAGLE3 patch is now a no-op) and carries vllm-project/vllm#47158" + - "Port the non-MTP serve-command tuning to the MTP recipe: --moe-backend aiter, --linear-backend emulation, --max-num-batched-tokens 32768, and the AITER master switch on for TP-only runs (kept --speculative-config eagle3 with 3 draft tokens)" + - "Simplify both search spaces to a single TP4 conc 1-512 sweep for 1k1k and 8k1k (drop TP8 and TP4/EP4, matching the non-MTP entry; verified locally on this nightly at TP4 conc512, 5120/5120 completed)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2003 + +- config-keys: + - dsv4-fp4-mi355x-atom + description: + - "Remove --hf-overrides use_index_cache/index_topk_freq indexer-skipping override from the ATOM serve command (not allowed: reduces model architecture FLOPs per PR_REVIEW_CHECKLIST.md)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2037 + +- config-keys: + - dsv4-fp4-mi355x-vllm + description: + - "Bump DeepSeek-V4-Pro FP4 MI355X single-node vLLM STP image from vllm/vllm-openai-rocm:v0.22.0 to the latest nightly vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa." + - "The nightly enables two-stage attention kernels (split-KV decode), which reduce decode attention latency across all concurrency levels." + - "Employ the AITER MLA attention backend for the DeepSeek-V4 MLA path." + - "Switch the MoE backend from triton_unfused to AITER MoE (VLLM_ROCM_USE_AITER_MOE=1 + --moe-backend aiter) for the FP4 experts." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1980 + +- config-keys: + - qwen3.5-fp8-h200-sglang + - qwen3.5-fp8-h200-sglang-mtp + description: + - "Update SGLang image from qwen3.5-fp8-h200-sglang: lmsysorg/sglang:v0.5.12-cu130 / qwen3.5-fp8-h200-sglang-mtp: lmsysorg/sglang:v0.5.12 to lmsysorg/sglang:v0.5.14-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2061 + +- config-keys: + - kimik2.5-int4-b300-vllm + description: + - "Update vLLM image from vllm/vllm-openai:v0.21.0 to vllm/vllm-openai:v0.24.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2070 + +- config-keys: + - qwen3.5-fp4-b300-sglang + - qwen3.5-fp4-b300-sglang-mtp + description: + - "Update SGLang image from lmsysorg/sglang:v0.5.12-cu130 to lmsysorg/sglang:v0.5.14-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2057 + +- config-keys: + - qwen3.5-bf16-b200-sglang + - qwen3.5-bf16-b200-sglang-mtp + description: + - "Update SGLang image from lmsysorg/sglang:v0.5.12-cu130 to lmsysorg/sglang:v0.5.14-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2054 + +- config-keys: + - dsv4-fp4-mi355x-vllm-mtp + description: + - "Bump DeepSeek-V4-Pro FP4 MI355X single-node vLLM MTP image from vllm/vllm-openai-rocm:v0.22.0 to the latest nightly vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa." + - "The nightly enables two-stage attention kernels (split-KV decode), which reduce decode attention latency across all concurrency levels." + - "Employ the AITER MLA attention backend for the DeepSeek-V4 MLA path." + - "Switch the MoE backend from triton_unfused to AITER MoE (VLLM_ROCM_USE_AITER_MOE=1 + --moe-backend aiter) for the FP4 experts." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1981 + +- config-keys: + - qwen3.5-fp8-h100-sglang + - qwen3.5-fp8-h100-sglang-mtp + description: + - "Update SGLang image from lmsysorg/sglang:v0.5.12-cu130 to lmsysorg/sglang:v0.5.14-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2060 + +- config-keys: + - qwen3.5-fp8-b200-sglang + - qwen3.5-fp8-b200-sglang-mtp + description: + - "Update SGLang image from lmsysorg/sglang:v0.5.12-cu130 to lmsysorg/sglang:v0.5.14-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2058 + +- config-keys: + - kimik2.5-int4-b200-vllm + description: + - "Update vLLM image from vllm/vllm-openai:v0.22.0 to vllm/vllm-openai:v0.24.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2073 + +- config-keys: + - qwen3.5-fp4-b200-sglang + - qwen3.5-fp4-b200-sglang-mtp + description: + - "Update SGLang image from lmsysorg/sglang:v0.5.12-cu130 to lmsysorg/sglang:v0.5.14-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2056 + +- config-keys: + - qwen3.5-fp8-mi355x-sglang + - qwen3.5-fp8-mi355x-sglang-mtp + description: + - "Update SGLang ROCm image from lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260528 to lmsysorg/sglang:v0.5.14-rocm720-mi35x" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2067 + +- config-keys: + - kimik2.5-fp4-mi355x-vllm + description: + - "Update vLLM ROCm image from vllm/vllm-openai-rocm:v0.22.0 to vllm/vllm-openai-rocm:v0.24.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2074 + +- config-keys: + - kimik2.5-int4-mi355x-vllm + description: + - "Update vLLM ROCm image from vllm/vllm-openai-rocm:nightly-b8336c3c7c298e0878f22a7bf70f4e295b2f4e01 to vllm/vllm-openai-rocm:v0.24.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2077 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm + description: + - "Refresh DSV4 8k/1k vLLM GB300 recipes with new w4a4 container and updated configs" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2010 + +- config-keys: + - minimaxm3-fp4-b200-vllm-mtp + description: + - "Update image tag to nightly" + - "Update B200 MTP search space" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2007 + +- config-keys: + - minimaxm3-fp4-b300-vllm-mtp + description: + - "Update image tag to nightly" + - "Update B300 MTP search space" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2006 + +- config-keys: + - minimaxm3-fp4-b300-vllm + description: + - "Add high concurrency configs" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1994 + +- config-keys: + - minimaxm3-fp8-mi355x-vllm-mtp + description: + - "Pin the EAGLE3 drafter to TRITON_ATTN via the speculative-config attention_backend override; without it the drafter falls back to a slow default backend." + - "Speeds up the draft forward measured on MI355X TP4 MXFP8, 8k/1k throughput gains growing with concurrency vs no override." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2107 + +- config-keys: + - glm5-fp8-gb200-dynamo-sglang + description: + - "Add GLM-5.1-FP8 GB200 disaggregated multinode SGLang benchmarks via Dynamo" + - "Image: lmsysorg/sglang:v0.5.12" + - "14 topologies across 1k/1k and 8k/1k: prefill TP8 STP + decode wide-EP (DEP16/DEP32 high-throughput) and per-node TP8 low-latency, recipes under benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1895 + +- config-keys: + - glm5-fp4-gb200-dynamo-sglang + description: + - "Initial submission: GLM-5 FP4 disagg on GB200 with Dynamo SGLang (8k1k + 1k1k, STP / no-MTP)." + - "Model: nvidia/GLM-5.1-NVFP4 (GB200 Lustre path: /mnt/lustre01/models/GLM-5.1-NVFP4)." + - "Image: lmsysorg/sglang:v0.5.11-cu130" + - "Recipes ported from srt-slurm PR #211 (recipes/gb200-fp4/glm5.yaml), split per topology under recipes/sglang/glm5/gb200-fp4/." + - "8k1k: 4 wide-EP (TP=32) max-throughput + 6 per-node (TP=4) low-latency topologies." + - "1k1k: 1 wide-EP max-throughput + 6 low-latency (DEP16/DEP32 + TP=4) topologies." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1780 + +- config-keys: + - dsr1-fp4-b200-dynamo-sglang-mtp + description: + - "Prune the 8k/1k MTP disagg low-latency points: 1p5d-tp8 to conc 4-32, 1p3d-tp8 to conc 32-64, and 1p1d-tp8 to conc 32." + - "Add 8k/1k MTP disagg high-throughput points: 1p1d and 4p1d (DEP4 prefill / DEP8 decode) at conc 512." + - "Add an 8k/1k MTP disagg 4p1d (DEP4 prefill / DEP8 decode) high-throughput point at conc 2048 with a dedicated engine tune (max-running-requests 2048, decode scheduler-recv-interval 1, prefill/decode stream-interval 100/34, SGLANG_HACK_SEQ_BOOTSTRAP_ROOM)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2113 + +- config-keys: + - glm5-fp4-gb200-dynamo-sglang-mtp + description: + - "Add GLM-5.1 NVFP4 GB200 disaggregated dynamo-sglang MTP config covering 8k1k and 1k1k, high-throughput and low-latency topologies" + - "Image: lmsysorg/sglang:v0.5.13.post1-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2115 + +- config-keys: + - glm5-fp4-gb300-dynamo-sglang-mtp + description: + - "Add GLM-5.1 NVFP4 GB300 disaggregated dynamo-sglang MTP config covering 8k1k and 1k1k, high-throughput and low-latency topologies" + - "Image: lmsysorg/sglang:v0.5.13.post1-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2114 + +- config-keys: + - kimik2.7-fp4-mi355x-vllm-agentic + description: + - "Add Kimi-K2.7-Code-MXFP4 agentic-coding config on MI355X (cluster:gbt350docker->mi355x-amds) via vLLM v0.24.0, TP4" + - "Sweep KV-offloading none vs dram/LMCache at conc [1,4,8,16,32,48]; LMCache MP server + LMCacheMPConnector, dram-utilization 0.80" + - "Recipe benchmarks/single_node/agentic/kimik2.7_fp4_mi355x.sh clones LMCache to container-local /opt/lmcache-src (pinned) so CI checkout never trips over root-owned build artifacts" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/PLACEHOLDER From a7afb20cef0b385cbe86ee54d55da02781ea2d89 Mon Sep 17 00:00:00 2001 From: Hyukjoon Lee Date: Thu, 9 Jul 2026 15:49:27 +0900 Subject: [PATCH 6/9] agentx: adjust kimik2.7 agentic sweep top conc 48 -> 42 --- configs/amd-master.yaml | 4 ++-- perf-changelog.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index addf1ea374..064f236297 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -820,8 +820,8 @@ kimik2.7-fp4-mi355x-vllm-agentic: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, kv-offloading: none, conc-list: [1, 4, 8, 16, 32, 48] } - - { tp: 4, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [1, 4, 8, 16, 32, 48] } + - { tp: 4, kv-offloading: none, conc-list: [1, 4, 8, 16, 32, 42] } + - { tp: 4, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [1, 4, 8, 16, 32, 42] } kimik2.5-fp4-mi355x-atom: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 7c684c4c88..7ef522fe2e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4639,6 +4639,6 @@ - kimik2.7-fp4-mi355x-vllm-agentic description: - "Add Kimi-K2.7-Code-MXFP4 agentic-coding config on MI355X (cluster:gbt350docker->mi355x-amds) via vLLM v0.24.0, TP4" - - "Sweep KV-offloading none vs dram/LMCache at conc [1,4,8,16,32,48]; LMCache MP server + LMCacheMPConnector, dram-utilization 0.80" + - "Sweep KV-offloading none vs dram/LMCache at conc [1,4,8,16,32,42]; LMCache MP server + LMCacheMPConnector, dram-utilization 0.80" - "Recipe benchmarks/single_node/agentic/kimik2.7_fp4_mi355x.sh clones LMCache to container-local /opt/lmcache-src (pinned) so CI checkout never trips over root-owned build artifacts" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/PLACEHOLDER From e90cd18b09ee89d231904f0c9e79187e22458678 Mon Sep 17 00:00:00 2001 From: Hyukjoon Lee Date: Thu, 9 Jul 2026 15:58:04 +0900 Subject: [PATCH 7/9] agentx: set kimik2.7 perf-changelog pr-link to #2126 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 7ef522fe2e..19abec7da0 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4641,4 +4641,4 @@ - "Add Kimi-K2.7-Code-MXFP4 agentic-coding config on MI355X (cluster:gbt350docker->mi355x-amds) via vLLM v0.24.0, TP4" - "Sweep KV-offloading none vs dram/LMCache at conc [1,4,8,16,32,42]; LMCache MP server + LMCacheMPConnector, dram-utilization 0.80" - "Recipe benchmarks/single_node/agentic/kimik2.7_fp4_mi355x.sh clones LMCache to container-local /opt/lmcache-src (pinned) so CI checkout never trips over root-owned build artifacts" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/PLACEHOLDER + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2126 From 753ffaa77fae583aaf7267c6ad399e1c3db17f1b Mon Sep 17 00:00:00 2001 From: Hyukjoon Lee Date: Thu, 9 Jul 2026 16:23:02 +0900 Subject: [PATCH 8/9] agentx: reshape kimik2.7 sweep to Pareto frontier + address review - Frontier-only sweep: none [1,4,8] (low conc, fits GPU HBM) + lmcache [16,32] (above the KV cliff). Drops dominated/cliff points per review. - Fix recipe header: 'Variant of kimik2.5_fp4_mi355x.sh' (was self-referential) - perf-changelog: correct runner (cluster:mi355x-amds) and conc lists --- benchmarks/single_node/agentic/kimik2.7_fp4_mi355x.sh | 2 +- configs/amd-master.yaml | 4 ++-- perf-changelog.yaml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x.sh b/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x.sh index b9a197a9b7..10184a18ba 100755 --- a/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x.sh +++ b/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x.sh @@ -4,7 +4,7 @@ set -x # Agentic trace replay benchmark for Kimi-K2.7 FP4 on MI355X using vLLM. # -# Variant of kimik2.7_fp4_mi355x.sh that supports TWO KV configs: +# Variant of kimik2.5_fp4_mi355x.sh that supports TWO KV configs: # KV_OFFLOADING=none -> GPU KV only # KV_OFFLOADING=dram KV_OFFLOAD_BACKEND=lmcache -> LMCache MP server + connector # diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 064f236297..5d9b51bdc5 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -820,8 +820,8 @@ kimik2.7-fp4-mi355x-vllm-agentic: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, kv-offloading: none, conc-list: [1, 4, 8, 16, 32, 42] } - - { tp: 4, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [1, 4, 8, 16, 32, 42] } + - { tp: 4, kv-offloading: none, conc-list: [1, 4, 8] } + - { tp: 4, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [16, 32] } kimik2.5-fp4-mi355x-atom: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 19abec7da0..8abd495343 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4638,7 +4638,7 @@ - config-keys: - kimik2.7-fp4-mi355x-vllm-agentic description: - - "Add Kimi-K2.7-Code-MXFP4 agentic-coding config on MI355X (cluster:gbt350docker->mi355x-amds) via vLLM v0.24.0, TP4" - - "Sweep KV-offloading none vs dram/LMCache at conc [1,4,8,16,32,42]; LMCache MP server + LMCacheMPConnector, dram-utilization 0.80" + - "Add Kimi-K2.7-Code-MXFP4 agentic-coding config on MI355X (runner: cluster:mi355x-amds) via vLLM v0.24.0, TP4" + - "Sweep KV-offloading none (conc [1,4,8]) vs dram/LMCache (conc [16,32]); LMCache MP server + LMCacheMPConnector, dram-utilization 0.80" - "Recipe benchmarks/single_node/agentic/kimik2.7_fp4_mi355x.sh clones LMCache to container-local /opt/lmcache-src (pinned) so CI checkout never trips over root-owned build artifacts" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2126 From 48d59702a32499a8a28a35352abfe26f21f8401d Mon Sep 17 00:00:00 2001 From: Hyukjoon Lee Date: Thu, 9 Jul 2026 16:52:48 +0900 Subject: [PATCH 9/9] agentx: trim redundant comments in kimik2.7 recipe --- .../agentic/kimik2.7_fp4_mi355x.sh | 30 +++++-------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x.sh b/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x.sh index 10184a18ba..7f084cd69f 100755 --- a/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x.sh +++ b/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x.sh @@ -3,8 +3,6 @@ set -euo pipefail set -x # Agentic trace replay benchmark for Kimi-K2.7 FP4 on MI355X using vLLM. -# -# Variant of kimik2.5_fp4_mi355x.sh that supports TWO KV configs: # KV_OFFLOADING=none -> GPU KV only # KV_OFFLOADING=dram KV_OFFLOAD_BACKEND=lmcache -> LMCache MP server + connector # @@ -25,9 +23,6 @@ if [ -n "${ROCR_VISIBLE_DEVICES:-}" ]; then export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" fi -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. if [[ -n "${MODEL_PATH:-}" ]]; then if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then hf download "$MODEL" --local-dir "$MODEL_PATH" @@ -120,8 +115,6 @@ wait_for_lmcache_ready() { exit 1 } -# Resolve the effective offload backend. When KV_OFFLOADING=none there is no -# backend; when dram, KV_OFFLOAD_BACKEND selects native vs lmcache. if [[ "$KV_OFFLOADING" == "none" ]]; then OFFLOAD_MODE="none" else @@ -130,21 +123,16 @@ fi case "$OFFLOAD_MODE" in none) - # GPU-only KV, no DRAM offload. Leave prefix caching at vLLM's - # default (ON) so this is an honest no-offload baseline that still - # reuses shared prefixes on-GPU — apples-to-apples vs the lmcache - # cell, which extends that same reuse into DRAM. (Matches the - # kimik2.5 / dsv4 agentic recipes.) + # GPU-only KV baseline: keep on-GPU prefix caching ON (no DRAM offload) + # so it's apples-to-apples vs the lmcache cell. PREFIX_CACHE_ARGS=(--enable-prefix-caching) ;; lmcache) unset VLLM_USE_SIMPLE_KV_OFFLOAD - # Build LMCache against ROCm if the connector isn't already importable - # (prebuilt kimi-lmcache images already ship it). Clone to a - # container-local dir (NOT the bind-mounted /workspace) so the CI - # checkout's `clean: true` never trips over root-owned build artifacts - # on the next job. Pin a ref for reproducibility. + # Build LMCache against ROCm if the connector isn't importable. Clone to + # a container-local dir (NOT bind-mounted /workspace) so the next job's + # checkout `clean: true` won't trip over root-owned build artifacts. if ! python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null 2>&1; then LMCACHE_SRC_DIR="${LMCACHE_SRC_DIR:-/opt/lmcache-src}" LMCACHE_GIT_REF="${LMCACHE_GIT_REF:-aaf7c0d3}" @@ -161,11 +149,9 @@ case "$OFFLOAD_MODE" in LMCACHE_PORT="${LMCACHE_PORT:-5555}" LMCACHE_HTTP_PORT="${LMCACHE_HTTP_PORT:-8080}" LMCACHE_CONNECT_HOST="${LMCACHE_CONNECT_HOST:-tcp://$LMCACHE_HOST}" - # Let the external MP server own the whole CPU KV pool. The requested - # budget is TOTAL_CPU_DRAM_GB, but LMCache's L1 is SHM-backed: if L1 > - # /dev/shm free it silently disables SHM and falls back to the slow - # pickle path (crashes at load — see kimik27 CI shm-overflow note). - # Cap L1 to 90% of current /dev/shm free space so SHM stays enabled. + # LMCache L1 is SHM-backed: if L1 > /dev/shm free it silently disables + # SHM and falls back to the pickle path (crashes at load). Cap L1 to 90% + # of /dev/shm free so SHM stays enabled. SHM_FREE_GB=$(df -BG --output=avail /dev/shm 2>/dev/null | tail -1 | tr -dc '0-9') SHM_CAP_GB=$(( SHM_FREE_GB * 90 / 100 )) LMCACHE_L1_SIZE_GB="${LMCACHE_L1_SIZE_GB:-$TOTAL_CPU_DRAM_GB}"