diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml new file mode 100644 index 0000000000..38265e7809 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml @@ -0,0 +1,225 @@ +base: + name: gb200-fp8-glm5-mtp + model: + path: glm-5.1-fp8 + container: "lmsysorg/sglang:v0.5.13.post1-cu130" + precision: fp8 + identity: + model: + repo: zai-org/GLM-5-FP8 + revision: 4f96cc5eec29dcee5d6ded54f7ffe889438f9516 + frameworks: + dynamo: 1.2.1 + sglang: 0.5.13.post1 + resources: + gpu_type: gb200 + gpus_per_node: 4 + frontend: + type: dynamo + dynamo: + version: 1.2.1 + backend: + type: sglang + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + sglang_config: + prefill: + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + chunked-prefill-size: 131072 + max-prefill-tokens: 16384 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + enable-flashinfer-allreduce-fusion: false + max-running-requests: 136 + cuda-graph-max-bs: 136 + mem-fraction-static: 0.7 + context-length: 9280 + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + decode: + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + enable-flashinfer-allreduce-fusion: false + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + max-running-requests: 28 + cuda-graph-max-bs: 28 + chunked-prefill-size: 64 + mem-fraction-static: 0.94 + context-length: 9280 + skip-tokenizer-init: true + stream-interval: 30 + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + health_check: + max_attempts: 360 + interval_seconds: 10 + benchmark: + type: sa-bench + req_rate: inf + # MTP: EAGLE-style spec decoding is trained against chat-formatted + # inputs — keep the chat template on explicitly rather than relying + # on the srtctl schema default. + use_chat_template: true +# ################# 8k1k ################# +zip_override_mtp_8k1k_hightpt: + name: [8k1k-1p1d_dep16, 8k1k-2p1d_dep16, 8k1k-4p1d_dep16] + backend: + decode_environment: + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + sglang_config: + decode: + cuda-graph-max-bs: [111, 234, 535] + data-parallel-size: 16 + deepep-config: /configs/deepep_config.json + deepep-mode: low_latency + enable-dp-attention: true + enable-dp-lm-head: true + ep-dispatch-algorithm: static + ep-num-redundant-experts: 0 + expert-parallel-size: 16 + max-running-requests: [111, 234, 535] + mem-fraction-static: 0.85 + moe-a2a-backend: deepep + moe-dense-tp-size: 1 + moe-runner-backend: deep_gemm + tensor-parallel-size: 16 + benchmark: + concurrencies: [[137], [620], [1305]] + isl: 8192 + osl: 1024 + resources: + decode_nodes: 4 + decode_workers: 1 + prefill_nodes: [2, 4, 8] + prefill_workers: [1, 2, 4] +zip_override_mtp_8k1k_lowlat: + name: [8k1k-1p4d_tp8, 8k1k-1p4d_tp8-conc16, 8k1k-1p8d_tp8, 8k1k-1p8d_tp8-conc1, 8k1k-1p8d_tp8-conc4] + backend: + sglang_config: + decode: + cuda-graph-max-bs: [28, 8, 12, 1, 2] + max-running-requests: [28, 8, 12, 1, 2] + benchmark: + concurrencies: [[136], [56], [120], [12], [31]] + isl: 8192 + osl: 1024 + resources: + decode_nodes: [8, 8, 16, 16, 16] + decode_workers: [4, 4, 8, 8, 8] + prefill_nodes: 2 + prefill_workers: 1 +# ################# 1k1k ################# +zip_override_mtp_1k1k_hightpt: + name: [1k1k-1p1d_dep16, 1k1k-1p1d_dep16-conc1024, 1k1k-1p1d_dep16-conc256, 1k1k-1p1d_dep16-conc512, 1k1k-1p1d_dep32] + backend: + decode_environment: + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: ['512', '512', '512', '512', '768'] + sglang_config: + decode: + context-length: 2112 + cuda-graph-max-bs: [1430, 969, 242, 482, 1251] + data-parallel-size: [16, 16, 16, 16, 32] + deepep-config: /configs/deepep_config.json + deepep-mode: low_latency + enable-dp-attention: true + enable-dp-lm-head: true + ep-dispatch-algorithm: static + ep-num-redundant-experts: 0 + expert-parallel-size: [16, 16, 16, 16, 32] + max-running-requests: [1430, 969, 242, 482, 1251] + mem-fraction-static: [0.85, 0.85, 0.85, 0.85, 0.7] + moe-a2a-backend: deepep + moe-dense-tp-size: 1 + moe-runner-backend: deep_gemm + tensor-parallel-size: [16, 16, 16, 16, 32] + prefill: + context-length: 2112 + cuda-graph-max-bs: 256 + max-running-requests: 256 + benchmark: + concurrencies: [[1614], [1213], [320], [625], [1446]] + isl: 1024 + osl: 1024 + resources: + decode_nodes: [4, 4, 4, 4, 8] + decode_workers: 1 + prefill_nodes: 2 + prefill_workers: 1 +zip_override_mtp_1k1k_lowlat: + name: [1k1k-1p8d_tp8-conc1, 1k1k-1p8d_tp8-conc16, 1k1k-1p8d_tp8-conc4] + backend: + sglang_config: + decode: + context-length: 2112 + cuda-graph-max-bs: [1, 8, 2] + max-running-requests: [1, 8, 2] + mem-fraction-static: 0.8 + prefill: + context-length: 2112 + cuda-graph-max-bs: 256 + max-running-requests: 256 + benchmark: + concurrencies: [[13], [113], [32]] + isl: 1024 + osl: 1024 + resources: + decode_nodes: 16 + decode_workers: 8 + prefill_nodes: 2 + prefill_workers: 1 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 68a125b5e0..d7179c761e 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -11690,6 +11690,274 @@ glm5-fp8-gb200-dynamo-sglang: ep: 1 dp-attn: false +glm5-fp8-gb200-dynamo-sglang-mtp: + image: lmsysorg/sglang:v0.5.13.post1-cu130 + model: zai-org/GLM-5.1-FP8 + model-prefix: glm5.1 + runner: gb200 + precision: fp8 + framework: dynamo-sglang + multinode: true + disagg: true + scenarios: + fixed-seq-len: + # ---------- 8k1k high-throughput (wide-EP TP=16 decode, MTP) ---------- + - isl: 8192 + osl: 1024 + search-space: + # 1p1d dep16. 6 nodes (2P + 4D). + - spec-decoding: "mtp" + conc-list: [137] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[0]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 2p1d dep16. 8 nodes (4P + 4D). + - spec-decoding: "mtp" + conc-list: [620] + prefill: + num-worker: 2 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[1]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 4p1d dep16. 12 nodes (8P + 4D). + - spec-decoding: "mtp" + conc-list: [1305] + prefill: + num-worker: 4 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[2]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # ---------- 8k1k low-latency (TP=8 decode workers, MTP) ---------- + - isl: 8192 + osl: 1024 + search-space: + # 1p4d tp8. 10 nodes. + - spec-decoding: "mtp" + conc-list: [136] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[0]" + decode: + num-worker: 4 + tp: 8 + ep: 1 + dp-attn: false + # 1p4d tp8, conc16. 10 nodes. + - spec-decoding: "mtp" + conc-list: [56] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[1]" + decode: + num-worker: 4 + tp: 8 + ep: 1 + dp-attn: false + # 1p8d tp8. 18 nodes. + - spec-decoding: "mtp" + conc-list: [120] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[2]" + decode: + num-worker: 8 + tp: 8 + ep: 1 + dp-attn: false + # 1p8d tp8, conc1. 18 nodes. + - spec-decoding: "mtp" + conc-list: [12] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[3]" + decode: + num-worker: 8 + tp: 8 + ep: 1 + dp-attn: false + # 1p8d tp8, conc4. 18 nodes. + - spec-decoding: "mtp" + conc-list: [31] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[4]" + decode: + num-worker: 8 + tp: 8 + ep: 1 + dp-attn: false + # ---------- 1k1k high-throughput (wide-EP decode, MTP) ---------- + - isl: 1024 + osl: 1024 + search-space: + # 1p1d dep16. 6 nodes. + - spec-decoding: "mtp" + conc-list: [1614] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_1k1k_hightpt[0]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 1p1d dep16, conc1024. 6 nodes. + - spec-decoding: "mtp" + conc-list: [1213] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_1k1k_hightpt[1]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 1p1d dep16, conc256. 6 nodes. + - spec-decoding: "mtp" + conc-list: [320] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_1k1k_hightpt[2]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 1p1d dep16, conc512. 6 nodes. + - spec-decoding: "mtp" + conc-list: [625] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_1k1k_hightpt[3]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 1p1d dep32. 10 nodes. + - spec-decoding: "mtp" + conc-list: [1446] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_1k1k_hightpt[4]" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + # ---------- 1k1k low-latency (TP=8 decode workers, MTP) ---------- + - isl: 1024 + osl: 1024 + search-space: + # 1p8d tp8, conc1. 18 nodes. + - spec-decoding: "mtp" + conc-list: [13] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_1k1k_lowlat[0]" + decode: + num-worker: 8 + tp: 8 + ep: 1 + dp-attn: false + # 1p8d tp8, conc16. 18 nodes. + - spec-decoding: "mtp" + conc-list: [113] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_1k1k_lowlat[1]" + decode: + num-worker: 8 + tp: 8 + ep: 1 + dp-attn: false + # 1p8d tp8, conc4. 18 nodes. + - spec-decoding: "mtp" + conc-list: [32] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_1k1k_lowlat[2]" + decode: + num-worker: 8 + tp: 8 + ep: 1 + dp-attn: false + glm5-fp4-gb300-dynamo-sglang-mtp: image: lmsysorg/sglang:v0.5.13.post1-cu130 model: nvidia/GLM-5.1-NVFP4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index fc79650307..dc8c10aa09 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4634,3 +4634,10 @@ - "Add GLM-5.1 NVFP4 GB300 disaggregated dynamo-sglang MTP config covering 8k1k and 1k1k, high-throughput and low-latency topologies" - "Image: lmsysorg/sglang:v0.5.13.post1-cu130" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2114 + +- config-keys: + - glm5-fp8-gb200-dynamo-sglang-mtp + description: + - "Add GLM-5.1 FP8 GB200 disaggregated dynamo-sglang MTP config covering 8k1k and 1k1k, high-throughput and low-latency topologies" + - "Image: lmsysorg/sglang:v0.5.13.post1-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2123