[WIP] Comparison branch for #2148: BF16 gate on main

benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300.sh

@@ -19,6 +19,20 @@ check_env_vars \
     RANDOM_RANGE_RATIO \
     RESULT_FILENAME
 
+# Test the BF16 MiniMax-M3 routing gate against the main branch runtime.
+VLLM_GATE_PATCH="$(dirname "$0")/patches/vllm-minimaxm3-gate-bf16.patch"
+if ! command -v patch >/dev/null 2>&1; then
+    apt-get update -y && apt-get install -y --no-install-recommends patch \
+        || { echo "Failed to install patch(1)" >&2; exit 1; }
+fi
+VLLM_PACKAGE_DIR=$(python3 -c "import importlib.util; print(importlib.util.find_spec('vllm').submodule_search_locations[0])") \
+    || { echo "Could not locate the installed vllm package" >&2; exit 1; }
+VLLM_SITE_PACKAGES=$(dirname "$VLLM_PACKAGE_DIR")
+patch --dry-run -p1 -d "$VLLM_SITE_PACKAGES" < "$VLLM_GATE_PATCH" >/dev/null \
+    || { echo "vLLM MiniMax-M3 GateLinear BF16 patch does not apply" >&2; exit 1; }
+patch -p1 -d "$VLLM_SITE_PACKAGES" < "$VLLM_GATE_PATCH" \
+    || { echo "vLLM MiniMax-M3 GateLinear BF16 patch failed" >&2; exit 1; }
+
 if [[ -n "${MODEL_PATH:-}" ]]; then
     if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
         hf download "$MODEL" --local-dir "$MODEL_PATH"

benchmarks/single_node/fixed_seq_len/patches/vllm-minimaxm3-gate-bf16.patch

@@ -0,0 +1,11 @@
+--- a/vllm/models/minimax_m3/nvidia/model.py
++++ b/vllm/models/minimax_m3/nvidia/model.py
+@@ -228,7 +228,7 @@
+             config.hidden_size,
+             config.num_local_experts,
+             bias=False,
+-            params_dtype=torch.float32,
++            params_dtype=torch.bfloat16,
+             out_dtype=torch.float32,
+             prefix=f"{prefix}.gate",
+         )

configs/nvidia-master.yaml

@@ -13575,15 +13575,12 @@ minimaxm3-fp4-b300-vllm:
       - { tp: 4, conc-start: 1, conc-end: 64 }
       - { tp: 2, conc-start: 4, conc-end: 256 }
       - { tp: 4, ep: 4, conc-start: 64, conc-end: 512 }
-      - { tp: 2, ep: 2, dp-attn: true, conc-start: 512, conc-end: 512 }
-      - { tp: 2, ep: 2, dp-attn: true, conc-start: 4096, conc-end: 4096 }
     - isl: 8192
       osl: 1024
       search-space:
       - { tp: 8, conc-start: 1, conc-end: 2 }
       - { tp: 4, conc-start: 1, conc-end: 2 }
       - { tp: 2, conc-start: 4, conc-end: 256 }
-      - { tp: 2, ep: 2, dp-attn: true, conc-start: 512, conc-end: 512 }
 
 # EAGLE3 speculative-decoding (spec-decoding: mtp) variant of MiniMax-M3 NVFP4
 # (nvidia/MiniMax-M3-NVFP4) B300 single-node vLLM, pairing the target with the

perf-changelog.yaml

@@ -4707,3 +4707,10 @@
     - "Clean the export envs"
     - "Enable two batch overlap"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2093
+
+- config-keys:
+    - minimaxm3-fp4-b300-vllm
+  description:
+    - "Test the BF16 MiniMax-M3 routing gate against the main branch image and bundled FlashInfer runtime"
+    - "Keep the existing non-DP serving configuration and omit DP-attention sweep points"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2149