diff --git a/benchmarks/multi_node/amd_utils/models_vllm.yaml b/benchmarks/multi_node/amd_utils/models_vllm.yaml index 13456c2dc..9c046b4cf 100644 --- a/benchmarks/multi_node/amd_utils/models_vllm.yaml +++ b/benchmarks/multi_node/amd_utils/models_vllm.yaml @@ -52,10 +52,17 @@ gpt-oss-120b: MiniMax-M3-MXFP8: # MiniMax-M3 MXFP8 disagg, no EP. The --tensor-parallel-size 8 below is just a # placeholder: server_vllm.sh sed-rewrites it to PREFILL_TP_SIZE/DECODE_TP_SIZE - # from the master-config prefill/decode tp (the sweep mixes TP8 and TP4 layouts). + # from the master-config prefill/decode tp (the sweep uses TP4 workers only, + # varying the prefill:decode worker ratio rather than TP). # --block-size 128 is mandatory (MSA sparse/index cache); text-only benchmark # so --language-model-only frees the vision encoder. gfx950 uses FP8 KV cache. - prefill_flags: "--tensor-parallel-size 8 --block-size 128 --language-model-only --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --no-enable-prefix-caching --gpu-memory-utilization 0.90 --tool-call-parser minimax_m3 --reasoning-parser minimax_m3 --enable-auto-tool-choice" - decode_flags: "--tensor-parallel-size 8 --block-size 128 --language-model-only --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --no-enable-prefix-caching --gpu-memory-utilization 0.90 --tool-call-parser minimax_m3 --reasoning-parser minimax_m3 --enable-auto-tool-choice" - env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_USE_BREAKABLE_CUDAGRAPH=0 VLLM_ENGINE_READY_TIMEOUT_S=3600" + # Serve flags + env kept in sync with the single-node recipe + # benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi355x.sh: + # --moe-backend aiter (AITER fused MoE), --linear-backend emulation, and the + # larger --max-num-batched-tokens 32768 prefill budget; plus the AITER + # router-append shared-experts fusion (self-disables under EP; the sweep is + # EP1 so it stays active) and INT6 quick all-reduce quantization. + prefill_flags: "--tensor-parallel-size 8 --block-size 128 --language-model-only --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --moe-backend aiter --linear-backend emulation --max-num-batched-tokens 32768 --no-enable-prefix-caching --gpu-memory-utilization 0.90 --tool-call-parser minimax_m3 --reasoning-parser minimax_m3 --enable-auto-tool-choice" + decode_flags: "--tensor-parallel-size 8 --block-size 128 --language-model-only --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --moe-backend aiter --linear-backend emulation --max-num-batched-tokens 32768 --no-enable-prefix-caching --gpu-memory-utilization 0.90 --tool-call-parser minimax_m3 --reasoning-parser minimax_m3 --enable-auto-tool-choice" + env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT6 VLLM_USE_BREAKABLE_CUDAGRAPH=0 VLLM_ENGINE_READY_TIMEOUT_S=3600" hf_dir: "models--MiniMaxAI--MiniMax-M3-MXFP8" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 03c7a296c..f665df32b 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -2910,14 +2910,21 @@ minimaxm3-fp8-mi325x-vllm-mtp: - { tp: 8, ep: 8, conc-start: 256, conc-end: 256, spec-decoding: mtp } - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 256, spec-decoding: mtp } -# MiniMax-M3 MXFP8 MI355X vLLM disaggregated (prefill/decode) smoke test on the -# day-zero ROCm image. Minimal 1 prefill (TP8) + 1 decode (TP8) at conc 1 to -# validate the MoRI-IO KV-transfer disagg pipeline end-to-end for M3. Layered on -# the MoRI-patch-removal infra (#1585). No EP (TP8 only); MoE experts are -# TP-sharded as in the single-node M3 TP8 recipe. Per-worker serve flags live in +# MiniMax-M3 MXFP8 MI355X vLLM disaggregated (prefill/decode) sweep on the +# day-zero ROCm image, over the MoRI-IO KV-transfer pipeline (MoRI-patch-removal +# infra #1585). All workers are TP4, no EP: the single-node M3 MXFP8 recipe +# (minimaxm3-fp8-mi355x-vllm, PR #2003) found plain TP4 beats both TP8 and +# TP4/EP4 on tok/s/GPU for this model on gfx950, so prefill and decode both use +# TP4 and we tune the prefill:decode worker ratio (xP:yD) instead of TP. The +# mi355x-disagg pool has 3 nodes and the launcher places one worker per node +# (NUM_NODES = xP + yD), so every layout keeps xP + yD <= 3: +# - 1P-TP4 / 1D-TP4 (2 nodes): balanced, full concurrency curve. +# - 1P-TP4 / 2D-TP4 (3 nodes): decode-heavy, for the decode-bound 1k1k tail. +# - 2P-TP4 / 1D-TP4 (3 nodes): prefill-heavy, for the prefill-bound 8k1k tail. +# Per-worker serve flags live in # benchmarks/multi_node/amd_utils/models_vllm.yaml (MiniMax-M3-MXFP8). minimaxm3-fp8-mi355x-vllm-disagg: - image: vllm/vllm-openai-rocm:nightly-556bc4e3a089378e9df2482659898192da18db15 + image: vllm/vllm-openai-rocm:nightly-2afa3f7e950264bb179d030c23a1ed1f46558fd9 model: MiniMaxAI/MiniMax-M3-MXFP8 model-prefix: minimaxm3 runner: mi355x-disagg @@ -2927,29 +2934,15 @@ minimaxm3-fp8-mi355x-vllm-disagg: disagg: true scenarios: fixed-seq-len: + # 1k1k is decode-bound (1024-token prefill, then 1024 decode steps): pair the + # balanced layout with a decode-heavy 1P/2D layout to lift high-concurrency + # throughput. Evals do not run at 1k1k, so concurrency is free to run high. - isl: 1024 osl: 1024 search-space: + # Balanced 1P TP4 + 1D TP4 (2 nodes) across the full curve. - spec-decoding: "none" conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - # Asymmetric 1P TP4 + 1D TP8 (smaller prefill, full-node decode) across - # conc 1,2,4,8,16,32,64,128,256. - - spec-decoding: "none" - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256 ] prefill: num-worker: 1 tp: 4 @@ -2959,14 +2952,15 @@ minimaxm3-fp8-mi355x-vllm-disagg: - "PREFILL_NODES=1" decode: num-worker: 1 - tp: 8 + tp: 4 ep: 1 dp-attn: false additional-settings: - "DECODE_NODES=1" - # Balanced half-node 1P TP4 + 1D TP4 at high conc 64,128,256,512,1024. + # Decode-heavy 1P TP4 + 2D TP4 (3 nodes): double the decode engines to + # absorb the decode-bound 1k1k tail at high concurrency. - spec-decoding: "none" - conc-list: [ 64, 128, 256, 512, 1024 ] + conc-list: [ 256, 512, 1024, 2048 ] prefill: num-worker: 1 tp: 4 @@ -2975,75 +2969,23 @@ minimaxm3-fp8-mi355x-vllm-disagg: additional-settings: - "PREFILL_NODES=1" decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - # 2P TP4 + 1D TP8: two half-node TP4 prefill workers (PREFILL_NODES=2) - # feeding one full-node TP8 decode, at high conc 256,512,768,1024. - - spec-decoding: "none" - conc-list: [ 256, 512, 768, 1024 ] - prefill: num-worker: 2 tp: 4 ep: 1 dp-attn: false additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - # 8k1k disagg sweep across four P/D layouts (1P TP8 + 1D TP8 conc 1..1024; - # 1P TP4 + 1D TP8 conc 1..256; 1P TP4 + 1D TP4 conc 64..1024; 2P TP4 + 1D TP8 - # conc 256..1024). The multi-node eval policy (8k1k + conc >= 16) marks one - # lm-eval on the highest-max-conc layout (TP8+TP8, eval-conc=median=128) — - # validating the M3 MoRI-IO disagg pipeline's correctness end-to-end. + - "DECODE_NODES=2" + # 8k1k is prefill-bound (8192-token prompts vs 1024 decode steps): pair the + # balanced layout with a prefill-heavy 2P/1D layout. Concurrency is capped at + # 512 so the multi-node eval policy (8k1k + conc >= 16, highest eligible conc) + # marks lm-eval at conc 512 — matching the range NVIDIA's aggregated 8k1k + # sweep tops out at and keeping the lm-eval async client stable. - isl: 8192 osl: 1024 search-space: + # Balanced 1P TP4 + 1D TP4 (2 nodes) across the full curve. - spec-decoding: "none" - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - # Asymmetric 1P TP4 + 1D TP8 (smaller prefill, full-node decode) across - # conc 1,2,4,8,16,32,64,128,256. - - spec-decoding: "none" - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - # Balanced half-node 1P TP4 + 1D TP4 at high conc 64,128,256,512,1024. - - spec-decoding: "none" - conc-list: [ 64, 128, 256, 512, 1024 ] + conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256, 512 ] prefill: num-worker: 1 tp: 4 @@ -3058,10 +3000,10 @@ minimaxm3-fp8-mi355x-vllm-disagg: dp-attn: false additional-settings: - "DECODE_NODES=1" - # 2P TP4 + 1D TP8: two half-node TP4 prefill workers (PREFILL_NODES=2) - # feeding one full-node TP8 decode, at high conc 256,512,768,1024. + # Prefill-heavy 2P TP4 + 1D TP4 (3 nodes): two half-node TP4 prefill workers + # keep the single TP4 decode engine fed for the prefill-bound 8k1k tail. - spec-decoding: "none" - conc-list: [ 256, 512, 768, 1024 ] + conc-list: [ 128, 256, 512 ] prefill: num-worker: 2 tp: 4 @@ -3071,7 +3013,7 @@ minimaxm3-fp8-mi355x-vllm-disagg: - "PREFILL_NODES=2" decode: num-worker: 1 - tp: 8 + tp: 4 ep: 1 dp-attn: false additional-settings: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index be28208ab..48d2c0a5a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4716,3 +4716,12 @@ - "Clean the export envs" - "Enable two batch overlap" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2093 + +- config-keys: + - minimaxm3-fp8-mi355x-vllm-disagg + description: + - "Bump image to vllm/vllm-openai-rocm:nightly-2afa3f7e950264bb179d030c23a1ed1f46558fd9" + - "Sync per-worker vLLM serve flags with the single-node minimaxm3-fp8-mi355x-vllm recipe (PR #2003): add --moe-backend aiter, --linear-backend emulation, --max-num-batched-tokens 32768; export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 and VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT6" + - "Retune the P/D search space to TP4 across all workers (drop the TP8 and TP4/EP layouts, which regress tok/s/GPU on gfx950) and tune the prefill:decode worker ratio instead. 1k1k: balanced 1P-TP4/1D-TP4 (conc 1-1024) + decode-heavy 1P-TP4/2D-TP4 (conc 256-2048). 8k1k: balanced 1P-TP4/1D-TP4 (conc 1-512) + prefill-heavy 2P-TP4/1D-TP4 (conc 128-512). All layouts keep prefill+decode workers <= 3 for the 3-node mi355x-disagg pool" + - "On-box MI355X per-GPU throughput vs the original 6/24 disagg config: 8k1k peak ~2085 -> ~6003 tok/s/gpu (~2.9x), 1k1k peak ~1736 -> ~2586 tok/s/gpu (~1.5x)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2144