diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 4277e40c13..5dca7ef286 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -195,7 +195,7 @@ jobs: ${{ inputs.spec-decoding != 'none' && inputs.spec-decoding || '' }} ${{ inputs.kv-offloading != '' && inputs.kv-offloading != 'none' && format('{0} KV offload', inputs.kv-offloading) || '' }} ${{ inputs.kv-offload-backend != '' && inputs.kv-offload-backend != 'none' && inputs.kv-offload-backend != 'default' && inputs.kv-offload-backend || '' }} - c${{ inputs.conc != '' && inputs.conc || join(fromJson(inputs.conc-list), 'x') }}${{ inputs.eval-only && ' | eval-only' || (inputs.run-eval && ' | eval' || '') }} + c${{ join(fromJson(inputs.conc-list), 'x') }}${{ inputs.eval-only && ' | eval-only' || (inputs.run-eval && ' | eval' || '') }} steps: - name: Slurm cleanup (pre-run) diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index 05cdf107a9..88ea3dba57 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -551,6 +551,7 @@ jobs: decode-ep: ${{ matrix.config.decode.ep }} decode-dp-attn: ${{ matrix.config.decode.dp-attn }} decode-additional-settings: ${{ toJson(matrix.config.decode.additional-settings) }} + conc: ${{ matrix.config.conc[0] }} kv-offloading: ${{ matrix.config.kv-offloading }} kv-offload-backend: ${{ matrix.config.kv-offload-backend }} duration: ${{ matrix.config.duration }} diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/downgrade-sgl-deep-gemm.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/downgrade-sgl-deep-gemm.sh new file mode 100755 index 0000000000..89c0f38585 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/downgrade-sgl-deep-gemm.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# WAR for sglang issue #30399: +# "[Bug] PD disaggregation: GB200 Deepseek v4 Pro DeepGEMM grid sync timeout" +# Bisected in that issue to an sgl-deep-gemm version upgrade (sglang #29554). +# Only trips on cross-node DEP8 prefill (single-node DEP4 is fine). Downgrade +# sgl-deep-gemm to the pre-regression version in every worker container. +# +# Runs via `setup_script: pin-sgl-deep-gemm.sh` (before dynamo install + worker +# startup, inside each worker's container). Prints before/after so the run log +# shows what the container shipped vs what we pinned. +set -euo pipefail +export PIP_BREAK_SYSTEM_PACKAGES=1 + +TARGET="${SGL_DEEP_GEMM_VERSION:-0.1.3}" + +echo "[pin-sgl-deep-gemm] before:" +pip show sgl-deep-gemm 2>/dev/null | grep -iE '^(Name|Version):' || echo " (not installed)" + +echo "[pin-sgl-deep-gemm] installing sgl-deep-gemm==${TARGET} ..." +pip install --force-reinstall --no-deps \ + --extra-index-url https://www.piwheels.org/simple \ + "sgl-deep-gemm==${TARGET}" + +echo "[pin-sgl-deep-gemm] after:" +pip show sgl-deep-gemm 2>/dev/null | grep -iE '^(Name|Version):' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml new file mode 100644 index 0000000000..3c33c33209 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml @@ -0,0 +1,155 @@ +name: "agg-gb300-tp4-mtp-kvoffload" + +# Agentic-coding SGLang aggregated recipe for DeepSeek-V4-Pro on GB300 +# (single aggregated worker, TP4, MTP + hierarchical-cache KV offload). +# +# Uses the flat single-variant schema the agentic CI flow expects. Concurrency is NOT +# a recipe field here: the GHA matrix fans out one job per concurrency from +# the master-config conc-list, exporting CONC into agentic_srt.sh. Modeled on +# the flat vllm/deepseek-v4/agentic recipes (agentic infra + benchmark wiring) +# and the flat sglang/deepseek-v4/8k1k recipes (sglang backend schema). + +model: + path: "deepseek-v4-pro" + container: "dynamo-sglang" + precision: "fp4" + +dynamo: + # 5/14 tot + hash: "2d5d7eb603af21926887580fe3a212c7926fbaad" + install: true + +setup_script: downgrade-sgl-deep-gemm.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 4 + +infra: + # Dedicated etcd/nats node — matches the vllm agentic recipes; under the + # dynamo router + multiple frontends the infra services need their own node. + etcd_nats_dedicated_node: true + # cc-traces prompts (~125K tokens) serialize to ~2.8MB; the 1 MiB NATS + # default drops them. 32 MiB matches the agentic vllm recipes (~10x headroom + # over the largest observed payload); schema recommends 24+ for long ISL. + nats_max_payload_mb: 32 + +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Correlation-ID + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + DYN_ROUTER_TEMPERATURE: "10000000" + # The sglang image is PEP 668 externally-managed; the runtime dynamo + # install (dynamo_wheels.py / source build) runs pip and fails with + # "externally-managed-environment" without this. Lets pip install into + # the system env. Applies to both dynamo.hash (source) and dynamo.wheel. + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: "kv" + router-reset-states: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + + aggregated_environment: + SGLANG_SIMULATE_ACC_LEN: '2.49' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + # Lets the runtime dynamo install's pip bypass PEP 668 on the sglang image. + PIP_BREAK_SYSTEM_PACKAGES: "1" + # from submission for B300 + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "0" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_TOPK_V2: "True" + # for kvcache offload + SGLANG_ENABLE_UNIFIED_RADIX_TREE: "1" + + sglang_config: + aggregated: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + stream-interval: 50 + watchdog-timeout: 1000000 + mem-fraction-static: 0.90 + chunked-prefill-size: 8192 # can only support 11000 for TP + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + swa-full-tokens-ratio: 0.1 + max-running-requests: 128 + cuda-graph-max-bs: 128 + + scheduler-recv-interval: 30 + dp-size: 1 + tp-size: 4 + ep-size: 1 + # kv-cache offload + enable-hierarchical-cache: true + hicache-ratio: 1 + hicache-write-policy: write_through + hicache-io-backend: kernel + # mtp section + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + # gb300-nv: pyxis maps the calling user into the container as a non-root + # uid; agentic_srt.sh's apt-get install git step needs EUID 0. Remap to + # uid 0 inside the container. srt-slurm renders empty-string values as + # flag-only srun args. + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + # Aggregated: one worker serves both prefill and decode, so GPU accounting + # is the single-worker form num_gpus = TP (not the disagg prefill+decode + # sum). The multinode post-processing path assumes disagg and would divide + # throughput by prefill_gpus + decode_gpus; force the single-node code path + # in process_agentic_result.py instead. TP must match sglang_config tp-size. + IS_MULTINODE: "false" + TP: "4" + # Enable dynamo conv-aware routing (per-session affinity so multi-turn + # KV cache is reused across turns). Previously opted out because the + # frontend 400'd on aiperf's nvext.session_control actions; re-enabled + # to test with the current build. + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + # Container-side path of the aiperf mmap dataset cache; host-side mount + # wired via launch_gb300-*.sh srtslurm.yaml default_mounts. Without it, + # aiperf re-tokenizes + re-writes the dataset mmap on every run. + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + # Persistent HF hub cache (also via default_mounts) so the trace dataset + # isn't re-downloaded each run; overrides the workflow-level HF_HUB_CACHE. + HF_HUB_CACHE: "/hf_hub_cache" + diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1024-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1024-mtp-kvoffload.yaml new file mode 100644 index 0000000000..f866d6910b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1024-mtp-kvoffload.yaml @@ -0,0 +1,214 @@ +name: "disagg-gb300-12p4d-dep8-dep16-c1024-mtp-kvoffload" + +# Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 +# (12P x DEP8 / 4D x DEP16, MTP + hierarchical-cache KV offload), tuned for concurrency 1024. +# +# Uses the flat single-variant srtctl schema the agentic CI flow expects; +# resources + backend (prefill/decode env + sglang_config) are normalized +# from the Pareto run. +# Concurrency is exported into agentic_srt.sh from the master-config conc-list. + +model: + path: "deepseek-v4-pro" + container: "dynamo-sglang" + precision: "fp4" + +dynamo: + hash: "2d5d7eb603af21926887580fe3a212c7926fbaad" + install: true + +setup_script: downgrade-sgl-deep-gemm.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 12 + decode_nodes: 4 + prefill_workers: 6 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 16 + +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 32 + +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Correlation-ID + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + DYN_ROUTER_TEMPERATURE: "10000000" + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: "kv" + router-reset-states: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + + prefill_environment: + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_DSV4_MHC_PREWARM: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_REASONING_EFFORT: max + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' + SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + + decode_environment: + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_SIMULATE_ACC_LEN: '2.49' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_REASONING_EFFORT: max + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + + sglang_config: + prefill: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 8 + dp-size: 8 + ep-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + disaggregation-transfer-backend: mooncake + disaggregation-mode: prefill + load-balance-method: total_tokens + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.02 + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + chunked-prefill-size: 32768 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-ratio: 1 + hicache-io-backend: direct + + decode: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 16 + dp-size: 16 + ep-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + disaggregation-transfer-backend: mooncake + disaggregation-mode: decode + load-balance-method: total_tokens + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.02 + max-running-requests: 3072 + cuda-graph-max-bs: 1024 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c8-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c8-mtp-kvoffload.yaml new file mode 100644 index 0000000000..c113512088 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c8-mtp-kvoffload.yaml @@ -0,0 +1,213 @@ +name: "disagg-gb300-2p1d-dep8-tp4-c8-mtp-kvoffload" + +# Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 +# (2P x DEP8 / 1D x TP4, MTP + hierarchical-cache KV offload), tuned for concurrency 8. +# +# Uses the flat single-variant srtctl schema the agentic CI flow expects; +# resources + backend (prefill/decode env + sglang_config) are normalized +# from the Pareto run. +# Concurrency is exported into agentic_srt.sh from the master-config conc-list. + +model: + path: "deepseek-v4-pro" + container: "dynamo-sglang" + precision: "fp4" + +dynamo: + hash: "2d5d7eb603af21926887580fe3a212c7926fbaad" + install: true + +setup_script: downgrade-sgl-deep-gemm.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 2 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 4 + +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 32 + +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Correlation-ID + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + DYN_ROUTER_TEMPERATURE: "10000000" + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: "kv" + router-reset-states: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + + prefill_environment: + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_DSV4_MHC_PREWARM: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_REASONING_EFFORT: max + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' + SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + + decode_environment: + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_SIMULATE_ACC_LEN: '2.49' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_REASONING_EFFORT: max + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '2048' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + + sglang_config: + prefill: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 8 + dp-size: 8 + ep-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + disaggregation-transfer-backend: mooncake + disaggregation-mode: prefill + load-balance-method: total_tokens + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.02 + max-running-requests: 128 + cuda-graph-max-bs: 128 + chunked-prefill-size: 32768 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-ratio: 1 + hicache-io-backend: direct + + decode: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 4 + dp-size: 1 + ep-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + moe-runner-backend: flashinfer_mxfp4 + disaggregation-transfer-backend: mooncake + disaggregation-mode: decode + load-balance-method: total_tokens + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.02 + max-running-requests: 128 + cuda-graph-max-bs: 128 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml new file mode 100644 index 0000000000..3da24365db --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml @@ -0,0 +1,214 @@ +name: "disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload" + +# Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 +# (2P x DEP8 / 4D x DEP16, MTP + hierarchical-cache KV offload), tuned for concurrency 256. +# +# Uses the flat single-variant srtctl schema the agentic CI flow expects; +# resources + backend (prefill/decode env + sglang_config) are normalized +# from the Pareto run. +# Concurrency is exported into agentic_srt.sh from the master-config conc-list. + +model: + path: "deepseek-v4-pro" + container: "dynamo-sglang" + precision: "fp4" + +dynamo: + hash: "2d5d7eb603af21926887580fe3a212c7926fbaad" + install: true + +setup_script: downgrade-sgl-deep-gemm.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 2 + decode_nodes: 4 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 16 + +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 32 + +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Correlation-ID + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + DYN_ROUTER_TEMPERATURE: "10000000" + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: "kv" + router-reset-states: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + + prefill_environment: + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_DSV4_MHC_PREWARM: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_REASONING_EFFORT: max + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' + SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + + decode_environment: + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_SIMULATE_ACC_LEN: '2.49' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_REASONING_EFFORT: max + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + + sglang_config: + prefill: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 8 + dp-size: 8 + ep-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + disaggregation-transfer-backend: mooncake + disaggregation-mode: prefill + load-balance-method: total_tokens + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.02 + max-running-requests: 256 + cuda-graph-max-bs: 256 + chunked-prefill-size: 32768 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-ratio: 1 + hicache-io-backend: direct + + decode: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 16 + dp-size: 16 + ep-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + disaggregation-transfer-backend: mooncake + disaggregation-mode: decode + load-balance-method: total_tokens + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.02 + max-running-requests: 3072 + cuda-graph-max-bs: 256 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c256-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c256-mtp-kvoffload.yaml new file mode 100644 index 0000000000..2a82eca46b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c256-mtp-kvoffload.yaml @@ -0,0 +1,214 @@ +name: "disagg-gb300-4p4d-dep8-dep16-c256-mtp-kvoffload" + +# Agentic-coding SGLang disaggregated recipe for DeepSeek-V4-Pro on GB300 +# (4P x DEP8 / 4D x DEP16, MTP + hierarchical-cache KV offload), tuned for concurrency 256. +# +# DEP8-prefill variant aligned with the measured Pareto point: prefill uses +# tp/dp/ep 8, four nodes, and SGLANG_DSV4_MHC_PREWARM=1. +# Concurrency is exported into agentic_srt.sh +# from the master-config conc-list. + +model: + path: "deepseek-v4-pro" + container: "dynamo-sglang" + precision: "fp4" + +dynamo: + hash: "2d5d7eb603af21926887580fe3a212c7926fbaad" + install: true + +setup_script: downgrade-sgl-deep-gemm.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 4 + decode_nodes: 4 + prefill_workers: 2 + decode_workers: 1 + + +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 32 + +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Correlation-ID + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + DYN_ROUTER_TEMPERATURE: "10000000" + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: "kv" + router-reset-states: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + + prefill_environment: + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_REASONING_EFFORT: max + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' + SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + SGLANG_DSV4_MHC_PREWARM: '1' + + decode_environment: + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_SIMULATE_ACC_LEN: '2.49' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_REASONING_EFFORT: max + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + SGLANG_DSV4_MHC_PREWARM: '1' + + sglang_config: + prefill: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 8 + dp-size: 8 + ep-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + disaggregation-transfer-backend: mooncake + disaggregation-mode: prefill + load-balance-method: total_tokens + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.02 + max-running-requests: 256 + cuda-graph-max-bs: 256 + chunked-prefill-size: 32768 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-ratio: 1 + hicache-io-backend: direct + + decode: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 16 + dp-size: 16 + ep-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + disaggregation-transfer-backend: mooncake + disaggregation-mode: decode + load-balance-method: total_tokens + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.02 + max-running-requests: 3072 + cuda-graph-max-bs: 256 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c512-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c512-mtp-kvoffload.yaml new file mode 100644 index 0000000000..19aa7ad4ef --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c512-mtp-kvoffload.yaml @@ -0,0 +1,214 @@ +name: "disagg-gb300-6p4d-dep8-dep16-c512-mtp-kvoffload" + +# Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 +# (6P x DEP8 / 4D x DEP16, MTP + hierarchical-cache KV offload), tuned for concurrency 512. +# +# Uses the flat single-variant srtctl schema the agentic CI flow expects; +# resources + backend (prefill/decode env + sglang_config) are normalized +# from the Pareto run. +# Concurrency is exported into agentic_srt.sh from the master-config conc-list. + +model: + path: "deepseek-v4-pro" + container: "dynamo-sglang" + precision: "fp4" + +dynamo: + hash: "2d5d7eb603af21926887580fe3a212c7926fbaad" + install: true + +setup_script: downgrade-sgl-deep-gemm.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 6 + decode_nodes: 4 + prefill_workers: 3 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 16 + +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 32 + +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Correlation-ID + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + DYN_ROUTER_TEMPERATURE: "10000000" + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: "kv" + router-reset-states: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + + prefill_environment: + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_DSV4_MHC_PREWARM: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_REASONING_EFFORT: max + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' + SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + + decode_environment: + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_SIMULATE_ACC_LEN: '2.49' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_REASONING_EFFORT: max + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + + sglang_config: + prefill: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 8 + dp-size: 8 + ep-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + disaggregation-transfer-backend: mooncake + disaggregation-mode: prefill + load-balance-method: total_tokens + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.02 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 32768 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-ratio: 1 + hicache-io-backend: direct + + decode: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 16 + dp-size: 16 + ep-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + disaggregation-transfer-backend: mooncake + disaggregation-mode: decode + load-balance-method: total_tokens + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.02 + max-running-requests: 3072 + cuda-graph-max-bs: 512 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 6be8e006f9..4dcde9c9dd 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -14563,3 +14563,112 @@ glm5-fp4-gb300-dynamo-trt-mtp: ep: 16 dp-attn: true +dsv4-fp4-gb300-dynamo-sglang-agentic-agg: + image: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-sglang + multinode: true + disagg: false + scenarios: + agentic-coding: + - search-space: + - spec-decoding: mtp + conc-list: [1] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + +dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: + image: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-sglang + multinode: true + disagg: true + scenarios: + agentic-coding: + - search-space: + - spec-decoding: mtp + conc-list: [8] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c8-mtp-kvoffload.yaml" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: mtp + conc-list: [256] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: mtp + conc-list: [256] + prefill: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c256-mtp-kvoffload.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: mtp + conc-list: [512] + prefill: + num-worker: 3 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c512-mtp-kvoffload.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: mtp + conc-list: [1024] + prefill: + num-worker: 6 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1024-mtp-kvoffload.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 028631183e..24124e6de1 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4723,3 +4723,11 @@ - "Bump the SGLang ROCm image from 20260706 to 20260709 and retire the DeepSeek-V4 compress-state, DSA paged-MQA, AITER instruction-shape, and HiCache host-pool shims that are already included upstream." - "Remove the SWA re-prefill and unified-KV HiCache overlays after their upstream PRs merged; retain only patch code without a merged upstream equivalent." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2127 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang-agentic-agg + - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg + description: + - "Add GB300 DeepSeek-V4 dynamo-sglang agentic recipes (agg-tp4 + dep8 disagg pareto)" + - "Prune agentic SGLang configs to measured Pareto points and enable prefill-side MTP to match decode" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 5328ded02d..e408c06ce2 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -20,6 +20,17 @@ export AIPERF_MMAP_CACHE_HOST_PATH="/data/home/sa-shared/gharunners/ai-perf-cach export HF_HUB_CACHE_HOST_PATH="/data/home/sa-shared/gharunners/hf-hub-cache" mkdir -p "$HF_HUB_CACHE_HOST_PATH" +# Persistent dynamo source-build cache. srtctl's hash-pinned dynamo install +# (_hash_cached_source_install) caches the built wheel + src tarball at +# /configs/dynamo-wheels/ with a .complete sentinel; on a warm cache the +# install is just `pip install` from the cache (no apt, no root). In CI /configs +# is the per-job srt-slurm checkout (cold every job → cold build needs apt + +# root, which the non-root server containers can't do), so persist and share +# the cache across jobs by bind-mounting this host dir at /configs/dynamo-wheels. +# Seed it once with a --container-remap-root build. 777 for multi-user runners. +export DYNAMO_WHEELS_CACHE_HOST_PATH="/data/home/sa-shared/gharunners/dynamo-wheels" +mkdir -p "$DYNAMO_WHEELS_CACHE_HOST_PATH" + export MODEL_PATH=$MODEL if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp4" ]]; then @@ -122,7 +133,22 @@ SRT_REPO_DIR="${GITHUB_WORKSPACE}/srt-slurm-${GITHUB_RUN_ID:-manual}-${GITHUB_RU SRTCTL_SETUP_SCRIPT="" rm -rf "$SRT_REPO_DIR" -if [[ "$IS_AGENTIC" == "1" ]]; then +if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsv4" ]]; then + # DSv4 GB300 sglang agentic: NVIDIA/srt-slurm@03c0e8c has the nginx + # client_max_body_size fix (>1 MiB agentic warmup bodies), the + # session-affinity frontend, and the BenchmarkType.CUSTOM / extra_mount + # schema these recipes need. + git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + git checkout 03c0e8cbdd522fae378b7e216962afef6abffaeb + mkdir -p recipes/sglang/deepseek-v4/agentic + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic" \ + recipes/sglang/deepseek-v4/agentic + SRTCTL_SETUP_SCRIPT="downgrade-sgl-deep-gemm.sh" + cp \ + "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/configs/$SRTCTL_SETUP_SCRIPT" \ + "configs/$SRTCTL_SETUP_SCRIPT" +elif [[ "$IS_AGENTIC" == "1" ]]; then # Agentic multi-node uses cquil11/srt-slurm-nv@cam/no-preflight-flag, # a thin branch off NVIDIA/srt-slurm@127597c that adds one CLI flag # (`srtctl apply --no-preflight`) — needed because: @@ -266,6 +292,9 @@ srtctl_root: "${SRTCTL_ROOT}" default_mounts: "${AIPERF_MMAP_CACHE_HOST_PATH}": "/aiperf_mmap_cache" "${HF_HUB_CACHE_HOST_PATH}": "/hf_hub_cache" + # Warm dynamo source-build cache (nested over the auto /configs mount) so the + # hash-pinned install is a cache hit (pip-only, no apt/root) on every job. + "${DYNAMO_WHEELS_CACHE_HOST_PATH}": "/configs/dynamo-wheels" # Model path aliases model_paths: