Add GLM-5.1 FP8 GB200 disaggregated dynamo-sglang MTP config / 添加 GLM-5.1 FP8 GB200 分离式 dynamo-sglang MTP 配置

benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml

@@ -0,0 +1,225 @@
+base:
+  name: gb200-fp8-glm5-mtp
+  model:
+    path: glm-5.1-fp8
+    container: "lmsysorg/sglang:v0.5.13.post1-cu130"
+    precision: fp8
+  identity:
+    model:
+      repo: zai-org/GLM-5-FP8
+      revision: 4f96cc5eec29dcee5d6ded54f7ffe889438f9516
+    frameworks:
+      dynamo: 1.2.1
+      sglang: 0.5.13.post1
+  resources:
+    gpu_type: gb200
+    gpus_per_node: 4
+  frontend:
+    type: dynamo
+  dynamo:
+    version: 1.2.1
+  backend:
+    type: sglang
+    prefill_environment:
+      TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800'
+      PYTHONUNBUFFERED: '1'
+      DYN_SKIP_SGLANG_LOG_FORMATTING: '1'
+      SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000'
+      SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000'
+      SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000'
+      MC_TE_METRIC: 'true'
+      MC_FORCE_MNNVL: '1'
+      NCCL_MNNVL_ENABLE: '1'
+      NCCL_CUMEM_ENABLE: '1'
+      SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
+      SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0'
+      SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
+      DYN_REQUEST_PLANE: nats
+    decode_environment:
+      TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800'
+      PYTHONUNBUFFERED: '1'
+      DYN_SKIP_SGLANG_LOG_FORMATTING: '1'
+      SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000'
+      SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000'
+      SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000'
+      MC_TE_METRIC: 'true'
+      MC_FORCE_MNNVL: '1'
+      NCCL_MNNVL_ENABLE: '1'
+      NCCL_CUMEM_ENABLE: '1'
+      SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
+      SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0'
+      SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
+      DYN_REQUEST_PLANE: nats
+    sglang_config:
+      prefill:
+        served-model-name: GLM-5-FP8
+        trust-remote-code: true
+        quantization: fp8
+        kv-cache-dtype: fp8_e4m3
+        disaggregation-mode: prefill
+        disaggregation-transfer-backend: nixl
+        tensor-parallel-size: 8
+        data-parallel-size: 8
+        expert-parallel-size: 1
+        enable-dp-attention: true
+        enable-dp-lm-head: true
+        chunked-prefill-size: 131072
+        max-prefill-tokens: 16384
+        nsa-prefill-backend: trtllm
+        nsa-decode-backend: trtllm
+        moe-runner-backend: flashinfer_trtllm
+        enable-flashinfer-allreduce-fusion: false
+        max-running-requests: 136
+        cuda-graph-max-bs: 136
+        mem-fraction-static: 0.7
+        context-length: 9280
+        disable-radix-cache: true
+        weight-loader-prefetch-checkpoints: true
+        model-loader-extra-config: '{"enable_multithread_load": true}'
+      decode:
+        served-model-name: GLM-5-FP8
+        trust-remote-code: true
+        quantization: fp8
+        kv-cache-dtype: fp8_e4m3
+        disaggregation-mode: decode
+        disaggregation-transfer-backend: nixl
+        tensor-parallel-size: 8
+        data-parallel-size: 1
+        expert-parallel-size: 1
+        enable-dp-attention: false
+        enable-dp-lm-head: false
+        nsa-prefill-backend: trtllm
+        nsa-decode-backend: trtllm
+        moe-runner-backend: flashinfer_trtllm
+        enable-flashinfer-allreduce-fusion: false
+        speculative-algorithm: EAGLE
+        speculative-num-steps: 2
+        speculative-eagle-topk: 1
+        speculative-num-draft-tokens: 3
+        max-running-requests: 28
+        cuda-graph-max-bs: 28
+        chunked-prefill-size: 64
+        mem-fraction-static: 0.94
+        context-length: 9280
+        skip-tokenizer-init: true
+        stream-interval: 30
+        disable-radix-cache: true
+        weight-loader-prefetch-checkpoints: true
+        model-loader-extra-config: '{"enable_multithread_load": true}'
+  health_check:
+    max_attempts: 360
+    interval_seconds: 10
+  benchmark:
+    type: sa-bench
+    req_rate: inf
+    # MTP: EAGLE-style spec decoding is trained against chat-formatted
+    # inputs — keep the chat template on explicitly rather than relying
+    # on the srtctl schema default.
+    use_chat_template: true
+# ################# 8k1k #################
+zip_override_mtp_8k1k_hightpt:
+  name: [8k1k-1p1d_dep16, 8k1k-2p1d_dep16, 8k1k-4p1d_dep16]
+  backend:
+    decode_environment:
+      SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512'
+    sglang_config:
+      decode:
+        cuda-graph-max-bs: [111, 234, 535]
+        data-parallel-size: 16
+        deepep-config: /configs/deepep_config.json
+        deepep-mode: low_latency
+        enable-dp-attention: true
+        enable-dp-lm-head: true
+        ep-dispatch-algorithm: static
+        ep-num-redundant-experts: 0
+        expert-parallel-size: 16
+        max-running-requests: [111, 234, 535]
+        mem-fraction-static: 0.85
+        moe-a2a-backend: deepep
+        moe-dense-tp-size: 1
+        moe-runner-backend: deep_gemm
+        tensor-parallel-size: 16
+  benchmark:
+    concurrencies: [[137], [620], [1305]]
+    isl: 8192
+    osl: 1024
+  resources:
+    decode_nodes: 4
+    decode_workers: 1
+    prefill_nodes: [2, 4, 8]
+    prefill_workers: [1, 2, 4]
+zip_override_mtp_8k1k_lowlat:
+  name: [8k1k-1p4d_tp8, 8k1k-1p4d_tp8-conc16, 8k1k-1p8d_tp8, 8k1k-1p8d_tp8-conc1, 8k1k-1p8d_tp8-conc4]
+  backend:
+    sglang_config:
+      decode:
+        cuda-graph-max-bs: [28, 8, 12, 1, 2]
+        max-running-requests: [28, 8, 12, 1, 2]
+  benchmark:
+    concurrencies: [[136], [56], [120], [12], [31]]
+    isl: 8192
+    osl: 1024
+  resources:
+    decode_nodes: [8, 8, 16, 16, 16]
+    decode_workers: [4, 4, 8, 8, 8]
+    prefill_nodes: 2
+    prefill_workers: 1
+# ################# 1k1k #################
+zip_override_mtp_1k1k_hightpt:
+  name: [1k1k-1p1d_dep16, 1k1k-1p1d_dep16-conc1024, 1k1k-1p1d_dep16-conc256, 1k1k-1p1d_dep16-conc512, 1k1k-1p1d_dep32]
+  backend:
+    decode_environment:
+      SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: ['512', '512', '512', '512', '768']
+    sglang_config:
+      decode:
+        context-length: 2112
+        cuda-graph-max-bs: [1430, 969, 242, 482, 1251]
+        data-parallel-size: [16, 16, 16, 16, 32]
+        deepep-config: /configs/deepep_config.json
+        deepep-mode: low_latency
+        enable-dp-attention: true
+        enable-dp-lm-head: true
+        ep-dispatch-algorithm: static
+        ep-num-redundant-experts: 0
+        expert-parallel-size: [16, 16, 16, 16, 32]
+        max-running-requests: [1430, 969, 242, 482, 1251]
+        mem-fraction-static: [0.85, 0.85, 0.85, 0.85, 0.7]
+        moe-a2a-backend: deepep
+        moe-dense-tp-size: 1
+        moe-runner-backend: deep_gemm
+        tensor-parallel-size: [16, 16, 16, 16, 32]
+      prefill:
+        context-length: 2112
+        cuda-graph-max-bs: 256
+        max-running-requests: 256
+  benchmark:
+    concurrencies: [[1614], [1213], [320], [625], [1446]]
+    isl: 1024
+    osl: 1024
+  resources:
+    decode_nodes: [4, 4, 4, 4, 8]
+    decode_workers: 1
+    prefill_nodes: 2
+    prefill_workers: 1
+zip_override_mtp_1k1k_lowlat:
+  name: [1k1k-1p8d_tp8-conc1, 1k1k-1p8d_tp8-conc16, 1k1k-1p8d_tp8-conc4]
+  backend:
+    sglang_config:
+      decode:
+        context-length: 2112
+        cuda-graph-max-bs: [1, 8, 2]
+        max-running-requests: [1, 8, 2]
+        mem-fraction-static: 0.8
+      prefill:
+        context-length: 2112
+        cuda-graph-max-bs: 256
+        max-running-requests: 256
+  benchmark:
+    concurrencies: [[13], [113], [32]]
+    isl: 1024
+    osl: 1024
+  resources:
+    decode_nodes: 16
+    decode_workers: 8
+    prefill_nodes: 2
+    prefill_workers: 1