diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300.sh index f91419edb..adc37fe42 100755 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300.sh +++ b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300.sh @@ -19,6 +19,20 @@ check_env_vars \ RANDOM_RANGE_RATIO \ RESULT_FILENAME +# Test the BF16 MiniMax-M3 routing gate against the main branch runtime. +VLLM_GATE_PATCH="$(dirname "$0")/patches/vllm-minimaxm3-gate-bf16.patch" +if ! command -v patch >/dev/null 2>&1; then + apt-get update -y && apt-get install -y --no-install-recommends patch \ + || { echo "Failed to install patch(1)" >&2; exit 1; } +fi +VLLM_PACKAGE_DIR=$(python3 -c "import importlib.util; print(importlib.util.find_spec('vllm').submodule_search_locations[0])") \ + || { echo "Could not locate the installed vllm package" >&2; exit 1; } +VLLM_SITE_PACKAGES=$(dirname "$VLLM_PACKAGE_DIR") +patch --dry-run -p1 -d "$VLLM_SITE_PACKAGES" < "$VLLM_GATE_PATCH" >/dev/null \ + || { echo "vLLM MiniMax-M3 GateLinear BF16 patch does not apply" >&2; exit 1; } +patch -p1 -d "$VLLM_SITE_PACKAGES" < "$VLLM_GATE_PATCH" \ + || { echo "vLLM MiniMax-M3 GateLinear BF16 patch failed" >&2; exit 1; } + if [[ -n "${MODEL_PATH:-}" ]]; then if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then hf download "$MODEL" --local-dir "$MODEL_PATH" diff --git a/benchmarks/single_node/fixed_seq_len/patches/vllm-minimaxm3-gate-bf16.patch b/benchmarks/single_node/fixed_seq_len/patches/vllm-minimaxm3-gate-bf16.patch new file mode 100644 index 000000000..831480e37 --- /dev/null +++ b/benchmarks/single_node/fixed_seq_len/patches/vllm-minimaxm3-gate-bf16.patch @@ -0,0 +1,11 @@ +--- a/vllm/models/minimax_m3/nvidia/model.py ++++ b/vllm/models/minimax_m3/nvidia/model.py +@@ -228,7 +228,7 @@ + config.hidden_size, + config.num_local_experts, + bias=False, +- params_dtype=torch.float32, ++ params_dtype=torch.bfloat16, + out_dtype=torch.float32, + prefix=f"{prefix}.gate", + ) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 6be8e006f..bb4e712c5 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -13575,15 +13575,12 @@ minimaxm3-fp4-b300-vllm: - { tp: 4, conc-start: 1, conc-end: 64 } - { tp: 2, conc-start: 4, conc-end: 256 } - { tp: 4, ep: 4, conc-start: 64, conc-end: 512 } - - { tp: 2, ep: 2, dp-attn: true, conc-start: 512, conc-end: 512 } - - { tp: 2, ep: 2, dp-attn: true, conc-start: 4096, conc-end: 4096 } - isl: 8192 osl: 1024 search-space: - { tp: 8, conc-start: 1, conc-end: 2 } - { tp: 4, conc-start: 1, conc-end: 2 } - { tp: 2, conc-start: 4, conc-end: 256 } - - { tp: 2, ep: 2, dp-attn: true, conc-start: 512, conc-end: 512 } # EAGLE3 speculative-decoding (spec-decoding: mtp) variant of MiniMax-M3 NVFP4 # (nvidia/MiniMax-M3-NVFP4) B300 single-node vLLM, pairing the target with the diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f4a77dd6a..1b5557363 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4707,3 +4707,10 @@ - "Clean the export envs" - "Enable two batch overlap" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2093 + +- config-keys: + - minimaxm3-fp4-b300-vllm + description: + - "Test the BF16 MiniMax-M3 routing gate against the main branch image and bundled FlashInfer runtime" + - "Keep the existing non-DP serving configuration and omit DP-attention sweep points" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2149