From 59794db0bccc2c5e8bf14d5bc07c7095a62cfaf2 Mon Sep 17 00:00:00 2001 From: Jason Li Date: Fri, 10 Jul 2026 06:38:24 -0700 Subject: [PATCH 1/4] feat: update MiniMax-M3 B300 FlashInfer nightly --- .../fixed_seq_len/minimaxm3_fp4_b300.sh | 23 ++++++++++++++++++- configs/nvidia-master.yaml | 5 +--- perf-changelog.yaml | 8 +++++++ 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300.sh index f91419edb7..e3572abf2d 100755 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300.sh +++ b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300.sh @@ -3,7 +3,9 @@ # MiniMax-M3 NVFP4 B300 single-node vLLM recipe. # Same shape as minimaxm3_fp8_b300.sh but uses the nvidia/MiniMax-M3-NVFP4 # checkpoint. MiniMax-M3 modelopt NVFP4 support (vllm-project/vllm PR #46380) is -# baked into the perf container image, so no runtime patch is needed. +# baked into the perf container image. The FlashInfer runtime is pinned to the +# first nightly containing the upstream SM100 low-M MXFP8 split-K kernel +# (flashinfer-ai/flashinfer#3847). source "$(dirname "$0")/../../benchmark_lib.sh" @@ -19,6 +21,23 @@ check_env_vars \ RANDOM_RANGE_RATIO \ RESULT_FILENAME +FLASHINFER_VERSION=0.6.15.dev20260710 +FLASHINFER_RELEASE_URL="https://github.com/flashinfer-ai/flashinfer/releases/download/nightly-v0.6.15-20260710" + +python3 -m pip uninstall -y flashinfer-python flashinfer-cubin flashinfer-jit-cache + +python3 -m pip install --no-deps \ + "${FLASHINFER_RELEASE_URL}/flashinfer_python-${FLASHINFER_VERSION}-py3-none-any.whl" \ + || { echo "FlashInfer ${FLASHINFER_VERSION} install failed" >&2; exit 1; } + +python3 -m pip install --no-deps \ + "${FLASHINFER_RELEASE_URL}/flashinfer_cubin-${FLASHINFER_VERSION}-py3-none-any.whl" \ + || { echo "FlashInfer cubin ${FLASHINFER_VERSION} install failed" >&2; exit 1; } + +python3 -m pip install --no-deps \ + "${FLASHINFER_RELEASE_URL}/flashinfer_jit_cache-${FLASHINFER_VERSION}+cu130-cp39-abi3-manylinux_2_28_x86_64.whl" \ + || { echo "FlashInfer JIT cache ${FLASHINFER_VERSION}+cu130 install failed" >&2; exit 1; } + if [[ -n "${MODEL_PATH:-}" ]]; then if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then hf download "$MODEL" --local-dir "$MODEL_PATH" @@ -39,6 +58,7 @@ SERVER_LOG=/workspace/server.log export VLLM_ENGINE_READY_TIMEOUT_S=3600 export VLLM_FLOAT32_MATMUL_PRECISION=high export VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm +export VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800 if [ "${DP_ATTENTION}" = "true" ]; then PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel" @@ -57,6 +77,7 @@ start_gpu_monitor set -x vllm serve "$MODEL_PATH" --served-model-name "$MODEL" --host 0.0.0.0 --port $PORT \ $PARALLEL_ARGS \ +--attention_config.indexer_kv_dtype fp8 \ --gpu-memory-utilization 0.95 \ --max-model-len $MAX_MODEL_LEN \ --kv-cache-dtype fp8 \ diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 6be8e006f9..bd8e0709e9 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -13559,7 +13559,7 @@ minimaxm3-fp8-b300-vllm: # weights are pre-staged read-only at /scratch/models/MiniMax-M3-NVFP4 (added to # the STAGED_MODELS allow-list in launch_b300-nv.sh). minimaxm3-fp4-b300-vllm: - image: vllm/vllm-openai:nightly-93d8f834dd8acf33eb0e2a75b2711b628cb6e226 + image: vllm/vllm-openai:nightly-2afa3f7e950264bb179d030c23a1ed1f46558fd9 model: nvidia/MiniMax-M3-NVFP4 model-prefix: minimaxm3 runner: b300 @@ -13575,15 +13575,12 @@ minimaxm3-fp4-b300-vllm: - { tp: 4, conc-start: 1, conc-end: 64 } - { tp: 2, conc-start: 4, conc-end: 256 } - { tp: 4, ep: 4, conc-start: 64, conc-end: 512 } - - { tp: 2, ep: 2, dp-attn: true, conc-start: 512, conc-end: 512 } - - { tp: 2, ep: 2, dp-attn: true, conc-start: 4096, conc-end: 4096 } - isl: 8192 osl: 1024 search-space: - { tp: 8, conc-start: 1, conc-end: 2 } - { tp: 4, conc-start: 1, conc-end: 2 } - { tp: 2, conc-start: 4, conc-end: 256 } - - { tp: 2, ep: 2, dp-attn: true, conc-start: 512, conc-end: 512 } # EAGLE3 speculative-decoding (spec-decoding: mtp) variant of MiniMax-M3 NVFP4 # (nvidia/MiniMax-M3-NVFP4) B300 single-node vLLM, pairing the target with the diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c1a3c3aee8..13f335263a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4699,3 +4699,11 @@ - "Clean the export envs" - "Enable two batch overlap" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2093 + +- config-keys: + - minimaxm3-fp4-b300-vllm + description: + - "Bump the vLLM image to nightly-2afa3f7e950264bb179d030c23a1ed1f46558fd9" + - "Install FlashInfer 0.6.15.dev20260710 with upstream SM100 low-M MXFP8 split-K support" + - "Use FP8 indexer KV cache, extend the model execution timeout, and remove DP-attention sweep points" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2148 From edc6ed7be5d35637ca794b64165dfd6741a8342b Mon Sep 17 00:00:00 2001 From: Jason Li Date: Fri, 10 Jul 2026 07:58:19 -0700 Subject: [PATCH 2/4] test bf16 gate --- .../fixed_seq_len/minimaxm3_fp4_b300.sh | 14 ++++++++++++++ .../patches/vllm-minimaxm3-gate-bf16.patch | 11 +++++++++++ perf-changelog.yaml | 3 ++- 3 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 benchmarks/single_node/fixed_seq_len/patches/vllm-minimaxm3-gate-bf16.patch diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300.sh index e3572abf2d..adee8fb560 100755 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300.sh +++ b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300.sh @@ -38,6 +38,20 @@ python3 -m pip install --no-deps \ "${FLASHINFER_RELEASE_URL}/flashinfer_jit_cache-${FLASHINFER_VERSION}+cu130-cp39-abi3-manylinux_2_28_x86_64.whl" \ || { echo "FlashInfer JIT cache ${FLASHINFER_VERSION}+cu130 install failed" >&2; exit 1; } +# Test the BF16 MiniMax-M3 routing gate with the upstream FlashInfer nightly. +VLLM_GATE_PATCH="$(dirname "$0")/patches/vllm-minimaxm3-gate-bf16.patch" +if ! command -v patch >/dev/null 2>&1; then + apt-get update -y && apt-get install -y --no-install-recommends patch \ + || { echo "Failed to install patch(1)" >&2; exit 1; } +fi +VLLM_PACKAGE_DIR=$(python3 -c "import importlib.util; print(importlib.util.find_spec('vllm').submodule_search_locations[0])") \ + || { echo "Could not locate the installed vllm package" >&2; exit 1; } +VLLM_SITE_PACKAGES=$(dirname "$VLLM_PACKAGE_DIR") +patch --dry-run -p1 -d "$VLLM_SITE_PACKAGES" < "$VLLM_GATE_PATCH" >/dev/null \ + || { echo "vLLM MiniMax-M3 GateLinear BF16 patch does not apply" >&2; exit 1; } +patch -p1 -d "$VLLM_SITE_PACKAGES" < "$VLLM_GATE_PATCH" \ + || { echo "vLLM MiniMax-M3 GateLinear BF16 patch failed" >&2; exit 1; } + if [[ -n "${MODEL_PATH:-}" ]]; then if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then hf download "$MODEL" --local-dir "$MODEL_PATH" diff --git a/benchmarks/single_node/fixed_seq_len/patches/vllm-minimaxm3-gate-bf16.patch b/benchmarks/single_node/fixed_seq_len/patches/vllm-minimaxm3-gate-bf16.patch new file mode 100644 index 0000000000..831480e370 --- /dev/null +++ b/benchmarks/single_node/fixed_seq_len/patches/vllm-minimaxm3-gate-bf16.patch @@ -0,0 +1,11 @@ +--- a/vllm/models/minimax_m3/nvidia/model.py ++++ b/vllm/models/minimax_m3/nvidia/model.py +@@ -228,7 +228,7 @@ + config.hidden_size, + config.num_local_experts, + bias=False, +- params_dtype=torch.float32, ++ params_dtype=torch.bfloat16, + out_dtype=torch.float32, + prefix=f"{prefix}.gate", + ) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 13f335263a..8f595ba68c 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4706,4 +4706,5 @@ - "Bump the vLLM image to nightly-2afa3f7e950264bb179d030c23a1ed1f46558fd9" - "Install FlashInfer 0.6.15.dev20260710 with upstream SM100 low-M MXFP8 split-K support" - "Use FP8 indexer KV cache, extend the model execution timeout, and remove DP-attention sweep points" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2148 + - "Test the BF16 MiniMax-M3 routing gate as a branch of PR #2148" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX From 48b834c67c8d7e65cd5a79bd374c3c9c99767148 Mon Sep 17 00:00:00 2001 From: Jason Li Date: Fri, 10 Jul 2026 08:00:19 -0700 Subject: [PATCH 3/4] link test pr --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 8f595ba68c..9573fc7730 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4707,4 +4707,4 @@ - "Install FlashInfer 0.6.15.dev20260710 with upstream SM100 low-M MXFP8 split-K support" - "Use FP8 indexer KV cache, extend the model execution timeout, and remove DP-attention sweep points" - "Test the BF16 MiniMax-M3 routing gate as a branch of PR #2148" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2149 From fd7ca2a2d475bc6ce6cc4244ab3aa52b74de1560 Mon Sep 17 00:00:00 2001 From: Jason Li Date: Fri, 10 Jul 2026 08:06:58 -0700 Subject: [PATCH 4/4] compare bf16 gate against main --- .../fixed_seq_len/minimaxm3_fp4_b300.sh | 25 ++----------------- configs/nvidia-master.yaml | 2 +- perf-changelog.yaml | 6 ++--- 3 files changed, 5 insertions(+), 28 deletions(-) diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300.sh index adee8fb560..adc37fe420 100755 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300.sh +++ b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300.sh @@ -3,9 +3,7 @@ # MiniMax-M3 NVFP4 B300 single-node vLLM recipe. # Same shape as minimaxm3_fp8_b300.sh but uses the nvidia/MiniMax-M3-NVFP4 # checkpoint. MiniMax-M3 modelopt NVFP4 support (vllm-project/vllm PR #46380) is -# baked into the perf container image. The FlashInfer runtime is pinned to the -# first nightly containing the upstream SM100 low-M MXFP8 split-K kernel -# (flashinfer-ai/flashinfer#3847). +# baked into the perf container image, so no runtime patch is needed. source "$(dirname "$0")/../../benchmark_lib.sh" @@ -21,24 +19,7 @@ check_env_vars \ RANDOM_RANGE_RATIO \ RESULT_FILENAME -FLASHINFER_VERSION=0.6.15.dev20260710 -FLASHINFER_RELEASE_URL="https://github.com/flashinfer-ai/flashinfer/releases/download/nightly-v0.6.15-20260710" - -python3 -m pip uninstall -y flashinfer-python flashinfer-cubin flashinfer-jit-cache - -python3 -m pip install --no-deps \ - "${FLASHINFER_RELEASE_URL}/flashinfer_python-${FLASHINFER_VERSION}-py3-none-any.whl" \ - || { echo "FlashInfer ${FLASHINFER_VERSION} install failed" >&2; exit 1; } - -python3 -m pip install --no-deps \ - "${FLASHINFER_RELEASE_URL}/flashinfer_cubin-${FLASHINFER_VERSION}-py3-none-any.whl" \ - || { echo "FlashInfer cubin ${FLASHINFER_VERSION} install failed" >&2; exit 1; } - -python3 -m pip install --no-deps \ - "${FLASHINFER_RELEASE_URL}/flashinfer_jit_cache-${FLASHINFER_VERSION}+cu130-cp39-abi3-manylinux_2_28_x86_64.whl" \ - || { echo "FlashInfer JIT cache ${FLASHINFER_VERSION}+cu130 install failed" >&2; exit 1; } - -# Test the BF16 MiniMax-M3 routing gate with the upstream FlashInfer nightly. +# Test the BF16 MiniMax-M3 routing gate against the main branch runtime. VLLM_GATE_PATCH="$(dirname "$0")/patches/vllm-minimaxm3-gate-bf16.patch" if ! command -v patch >/dev/null 2>&1; then apt-get update -y && apt-get install -y --no-install-recommends patch \ @@ -72,7 +53,6 @@ SERVER_LOG=/workspace/server.log export VLLM_ENGINE_READY_TIMEOUT_S=3600 export VLLM_FLOAT32_MATMUL_PRECISION=high export VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm -export VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800 if [ "${DP_ATTENTION}" = "true" ]; then PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel" @@ -91,7 +71,6 @@ start_gpu_monitor set -x vllm serve "$MODEL_PATH" --served-model-name "$MODEL" --host 0.0.0.0 --port $PORT \ $PARALLEL_ARGS \ ---attention_config.indexer_kv_dtype fp8 \ --gpu-memory-utilization 0.95 \ --max-model-len $MAX_MODEL_LEN \ --kv-cache-dtype fp8 \ diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index bd8e0709e9..bb4e712c54 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -13559,7 +13559,7 @@ minimaxm3-fp8-b300-vllm: # weights are pre-staged read-only at /scratch/models/MiniMax-M3-NVFP4 (added to # the STAGED_MODELS allow-list in launch_b300-nv.sh). minimaxm3-fp4-b300-vllm: - image: vllm/vllm-openai:nightly-2afa3f7e950264bb179d030c23a1ed1f46558fd9 + image: vllm/vllm-openai:nightly-93d8f834dd8acf33eb0e2a75b2711b628cb6e226 model: nvidia/MiniMax-M3-NVFP4 model-prefix: minimaxm3 runner: b300 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index a1e8d5fe3f..1b55573632 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4711,8 +4711,6 @@ - config-keys: - minimaxm3-fp4-b300-vllm description: - - "Bump the vLLM image to nightly-2afa3f7e950264bb179d030c23a1ed1f46558fd9" - - "Install FlashInfer 0.6.15.dev20260710 with upstream SM100 low-M MXFP8 split-K support" - - "Use FP8 indexer KV cache, extend the model execution timeout, and remove DP-attention sweep points" - - "Test the BF16 MiniMax-M3 routing gate as a branch of PR #2148" + - "Test the BF16 MiniMax-M3 routing gate against the main branch image and bundled FlashInfer runtime" + - "Keep the existing non-DP serving configuration and omit DP-attention sweep points" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2149