From 8da351729db81bb2b43d28d1a24f3eb243e72b48 Mon Sep 17 00:00:00 2001 From: Sahithi Chigurupati Date: Wed, 8 Jul 2026 13:36:00 -0700 Subject: [PATCH 1/6] Add GB300 AgentX Qwen3.5 recipes - 9 Qwen3.5-397B-A17B-NVFP4 GB300 sglang AgentX recipes (agg + disagg pareto) - configs/nvidia-master.yaml: qwen3.5-fp4-gb300-dynamo-sglang{,-agentic-agg,-agentic-disagg} - perf-changelog.yaml: agentic pareto entry - runners/launch_gb300-nv.sh: qwen3.5 launcher branch + dynamo-wheels cache --- ...g-gb300-tp4-c1-mtp-hicache-jid2191933.yaml | 111 +++++++++ ...-gb300-tp4-c96-mtp-hicache-jid2195211.yaml | 111 +++++++++ ...c128-mtp-hicache-convaware-jid2220685.yaml | 206 ++++++++++++++++ ...-c32-mtp-hicache-convaware-jid2212783.yaml | 204 ++++++++++++++++ ...-tp4-tp4-c64-mtp-convaware-jid2214439.yaml | 228 ++++++++++++++++++ ...-tp4-tp4-c96-mtp-convaware-jid2214440.yaml | 204 ++++++++++++++++ ...c192-mtp-hicache-convaware-jid2230562.yaml | 206 ++++++++++++++++ ...c256-mtp-hicache-convaware-jid2228613.yaml | 206 ++++++++++++++++ ...c384-mtp-hicache-convaware-jid2239740.yaml | 206 ++++++++++++++++ configs/nvidia-master.yaml | 159 ++++++++++++ perf-changelog.yaml | 10 + runners/launch_gb300-nv.sh | 97 +++++--- 12 files changed, 1916 insertions(+), 32 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c1-mtp-hicache-jid2191933.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c96-mtp-hicache-jid2195211.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-dep4-dep4-c128-mtp-hicache-convaware-jid2220685.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-convaware-jid2212783.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-convaware-jid2214439.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-convaware-jid2214440.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-2p1d-dep4-dep4-c192-mtp-hicache-convaware-jid2230562.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p1d-dep4-dep8-c256-mtp-hicache-convaware-jid2228613.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-4p1d-dep4-dep8-c384-mtp-hicache-convaware-jid2239740.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c1-mtp-hicache-jid2191933.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c1-mtp-hicache-jid2191933.yaml new file mode 100644 index 0000000000..5a24f6573b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c1-mtp-hicache-jid2191933.yaml @@ -0,0 +1,111 @@ +name: "agg-gb300-tp4-c1-mtp-hicache-jid2191933" + +# Agentic-coding SGLang aggregated recipe for Qwen3.5-397B-A17B-NVFP4 on GB300 +# (single aggregated worker, TP4, NEXTN MTP + hierarchical cache). +# +# Ported from the manually-run srtctl base:/zip_override_conc: starter to the +# flat single-variant agentic schema. Pure sglang (no dynamo): frontend.type +# sglang serves directly. Concurrency comes from the GHA matrix (exported as +# CONC into agentic_srt.sh), not the recipe. Serving tuning preserved verbatim +# from the source run; only CI scaffolding is added. + +model: + path: "qwen3.5-fp4" + container: "dynamo-sglang" + precision: "fp4" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 4 + +infra: + nats_max_payload_mb: 8 + +frontend: + type: sglang + +backend: + type: sglang + aggregated_environment: + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + sglang_config: + aggregated: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-path: /model/ + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-symm-mem: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 8192 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs: 256 + max-running-requests: 1024 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.8 + max-mamba-cache-size: 1024 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 1.01 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-write-policy: write_through + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + # gb300-nv: pyxis maps the calling user into the container as a non-root uid; + # agentic_srt.sh's apt-get install git step needs EUID 0. Remap to uid 0 inside + # the container. srt-slurm renders empty-string values as flag-only srun args. + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + # Aggregated: one worker serves both prefill and decode, so GPU accounting is + # the single-worker form num_gpus = TP (not the disagg prefill+decode sum). + # Force the single-node post-proc path; TP must match sglang_config tp-size. + IS_MULTINODE: "false" + TP: "4" + # Match the source run's dataset (256k cc-traces-with-subagents variant). + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + # Container-side path of the aiperf mmap dataset cache; host-side mount wired + # via launch_gb300-nv.sh srtslurm.yaml default_mounts. + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + # Persistent HF hub cache (also via default_mounts). + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c96-mtp-hicache-jid2195211.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c96-mtp-hicache-jid2195211.yaml new file mode 100644 index 0000000000..667b6088d1 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c96-mtp-hicache-jid2195211.yaml @@ -0,0 +1,111 @@ +name: "agg-gb300-tp4-c96-mtp-hicache-jid2195211" + +# Agentic-coding SGLang aggregated recipe for Qwen3.5-397B-A17B-NVFP4 on GB300 +# (single aggregated worker, TP4, NEXTN MTP + hierarchical cache). +# +# Ported from the manually-run srtctl base:/zip_override_conc: starter to the +# flat single-variant agentic schema. Pure sglang (no dynamo): frontend.type +# sglang serves directly. Concurrency comes from the GHA matrix (exported as +# CONC into agentic_srt.sh), not the recipe. Serving tuning preserved verbatim +# from the source run; only CI scaffolding is added. + +model: + path: "qwen3.5-fp4" + container: "dynamo-sglang" + precision: "fp4" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 4 + +infra: + nats_max_payload_mb: 8 + +frontend: + type: sglang + +backend: + type: sglang + aggregated_environment: + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + sglang_config: + aggregated: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-path: /model/ + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-symm-mem: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 8192 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs: 256 + max-running-requests: 1024 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.8 + max-mamba-cache-size: 1024 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 1.05 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-write-policy: write_through + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + # gb300-nv: pyxis maps the calling user into the container as a non-root uid; + # agentic_srt.sh's apt-get install git step needs EUID 0. Remap to uid 0 inside + # the container. srt-slurm renders empty-string values as flag-only srun args. + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + # Aggregated: one worker serves both prefill and decode, so GPU accounting is + # the single-worker form num_gpus = TP (not the disagg prefill+decode sum). + # Force the single-node post-proc path; TP must match sglang_config tp-size. + IS_MULTINODE: "false" + TP: "4" + # Match the source run's dataset (256k cc-traces-with-subagents variant). + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + # Container-side path of the aiperf mmap dataset cache; host-side mount wired + # via launch_gb300-nv.sh srtslurm.yaml default_mounts. + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + # Persistent HF hub cache (also via default_mounts). + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-dep4-dep4-c128-mtp-hicache-convaware-jid2220685.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-dep4-dep4-c128-mtp-hicache-convaware-jid2220685.yaml new file mode 100644 index 0000000000..8ba346e939 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-dep4-dep4-c128-mtp-hicache-convaware-jid2220685.yaml @@ -0,0 +1,206 @@ +name: "disagg-gb300-1p1d-dep4-dep4-c128-mtp-hicache-convaware-jid2220685" + +# Agentic-coding SGLang disaggregated 1P1D DEP recipe for Qwen3.5-397B-A17B-NVFP4 +# on GB300, ported to the flat single-variant agentic schema. Concurrency is +# NOT a recipe field: the GHA matrix fans out one job per concurrency and +# exports CONC into agentic_srt.sh. Serving tuning (sglang_config, env, +# frontend, dynamo hash, topology) is preserved verbatim from the source run. + +model: + path: "qwen3.5-fp4" + container: "dynamo-sglang" + precision: "fp4" + +dynamo: + hash: "3169427d2ed7ea6a0451e47ce443822d26893bae" + install: true + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 4 + +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 8 + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: kv + router-reset-states: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + + prefill_environment: + SGLANG_DISAGG_STAGING_BUFFER: '1' + NO_COLOR: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + PYTHONUNBUFFERED: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + MC_FORCE_MNNVL: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + PIP_BREAK_SYSTEM_PACKAGES: "1" + decode_environment: + SGLANG_DISAGG_STAGING_BUFFER: '1' + NO_COLOR: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + PYTHONUNBUFFERED: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + PIP_BREAK_SYSTEM_PACKAGES: "1" + + sglang_config: + prefill: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-path: /model/ + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + watchdog-timeout: 1000000 + log-level: info + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 1.01 + decode: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + moe-dense-tp-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + prefill-round-robin-balance: true + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: decode + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_cutedsl + linear-attn-decode-backend: flashinfer + disable-shared-experts-fusion: true + moe-a2a-backend: deepep + deepep-mode: low_latency + ep-dispatch-algorithm: static + eplb-algorithm: deepseek + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + speculative-moe-runner-backend: deep_gemm + speculative-moe-a2a-backend: deepep + chunked-prefill-size: 4096 + mem-fraction-static: 0.75 + max-mamba-cache-size: 1024 + max-running-requests: 512 + cuda-graph-max-bs: 64 + watchdog-timeout: 1000000 + decode-log-interval: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-convaware-jid2212783.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-convaware-jid2212783.yaml new file mode 100644 index 0000000000..8b70edc935 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-convaware-jid2212783.yaml @@ -0,0 +1,204 @@ +name: "disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-convaware-jid2212783" + +# Agentic-coding SGLang disaggregated 1P1D recipe for Qwen3.5-397B-A17B-NVFP4 +# on GB300, ported to the flat single-variant agentic schema. Concurrency is +# NOT a recipe field: the GHA matrix fans out one job per concurrency and +# exports CONC into agentic_srt.sh. Serving tuning (sglang_config, env, +# frontend, dynamo hash, topology) is preserved verbatim from the source run. + +model: + path: "qwen3.5-fp4" + container: "dynamo-sglang" + precision: "fp4" + +dynamo: + hash: "3169427d2ed7ea6a0451e47ce443822d26893bae" + install: true + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 4 + +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 8 + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: kv + router-reset-states: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + + prefill_environment: + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + NO_COLOR: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + PYTHONUNBUFFERED: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + MC_FORCE_MNNVL: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + PIP_BREAK_SYSTEM_PACKAGES: "1" + decode_environment: + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + NO_COLOR: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + PYTHONUNBUFFERED: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + PIP_BREAK_SYSTEM_PACKAGES: "1" + + sglang_config: + prefill: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-path: /model/ + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + mamba-ssm-dtype: bfloat16 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + watchdog-timeout: 1000000 + log-level: info + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 1.01 + decode: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + prefill-round-robin-balance: true + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: decode + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + disable-shared-experts-fusion: true + ep-dispatch-algorithm: static + eplb-algorithm: deepseek + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + chunked-prefill-size: 4096 + mem-fraction-static: 0.75 + max-mamba-cache-size: 1024 + max-running-requests: 512 + cuda-graph-max-bs: 64 + watchdog-timeout: 1000000 + decode-log-interval: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-convaware-jid2214439.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-convaware-jid2214439.yaml new file mode 100644 index 0000000000..5a80bf23d6 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-convaware-jid2214439.yaml @@ -0,0 +1,228 @@ +name: "disagg-gb300-1p1d-tp4-tp4-c64-mtp-convaware-jid2214439" + +# Agentic-coding SGLang disaggregated 1P1D recipe for Qwen3.5-397B-A17B-NVFP4 +# on GB300 (TP4 prefill / TP4 decode, NEXTN MTP + hierarchical cache). +# +# Ported from the manually-run srtctl base:/zip_override_conc: starter to the +# flat single-variant schema the agentic CI flow expects (modeled on the +# proven sglang/deepseek-v4/agentic recipes). Concurrency is NOT a recipe +# field here: the GHA matrix fans out one job per concurrency from the +# master-config conc-list, exporting CONC into agentic_srt.sh. Serving tuning +# (sglang_config, prefill/decode env, frontend, dynamo hash, topology) is +# preserved verbatim from the source run; only the CI scaffolding is added. + +model: + path: "qwen3.5-fp4" + container: "dynamo-sglang" + precision: "fp4" + +dynamo: + hash: "3169427d2ed7ea6a0451e47ce443822d26893bae" + install: true + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 4 + +infra: + # Dedicated etcd/nats node — under the dynamo router + multiple frontends + # the infra services need their own node (matches the proven agentic recipes). + etcd_nats_dedicated_node: true + nats_max_payload_mb: 8 + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + # The sglang image is PEP 668 externally-managed; the runtime dynamo + # install (dynamo.hash source build) runs pip and fails with + # "externally-managed-environment" without this. + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: kv + router-reset-states: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + + prefill_environment: + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + NO_COLOR: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + PYTHONUNBUFFERED: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + MC_FORCE_MNNVL: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + # Lets the runtime dynamo install's pip bypass PEP 668 on the sglang image. + PIP_BREAK_SYSTEM_PACKAGES: "1" + decode_environment: + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + NO_COLOR: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + PYTHONUNBUFFERED: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + # Lets the runtime dynamo install's pip bypass PEP 668 on the sglang image. + PIP_BREAK_SYSTEM_PACKAGES: "1" + + sglang_config: + prefill: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-path: /model/ + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + mamba-ssm-dtype: bfloat16 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + watchdog-timeout: 1000000 + log-level: info + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: false + hicache-write-policy: write_back + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 1.01 + decode: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + prefill-round-robin-balance: true + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: decode + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + disable-shared-experts-fusion: true + ep-dispatch-algorithm: static + eplb-algorithm: deepseek + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + chunked-prefill-size: 4096 + mem-fraction-static: 0.75 + max-mamba-cache-size: 1024 + max-running-requests: 512 + cuda-graph-max-bs: 64 + watchdog-timeout: 1000000 + decode-log-interval: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + # gb300-nv: pyxis maps the calling user into the container as a non-root + # uid; agentic_srt.sh's apt-get install git step needs EUID 0. Remap to + # uid 0 inside the container. srt-slurm renders empty-string values as + # flag-only srun args. + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + # Match the source run's dataset (256k cc-traces-with-subagents variant). + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + # Per-session affinity so multi-turn KV cache is reused across turns + # (recipe name: "convaware"); build_replay_cmd adds + # --use-dynamo-conv-aware-routing for dynamo-* frameworks. + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + # Container-side path of the aiperf mmap dataset cache; host-side mount + # wired via launch_gb300-nv.sh srtslurm.yaml default_mounts. + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + # Persistent HF hub cache (also via default_mounts) so the trace dataset + # isn't re-downloaded each run. + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-convaware-jid2214440.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-convaware-jid2214440.yaml new file mode 100644 index 0000000000..4292eb18e1 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-convaware-jid2214440.yaml @@ -0,0 +1,204 @@ +name: "disagg-gb300-1p1d-tp4-tp4-c96-mtp-convaware-jid2214440" + +# Agentic-coding SGLang disaggregated 1P1D recipe for Qwen3.5-397B-A17B-NVFP4 +# on GB300, ported to the flat single-variant agentic schema. Concurrency is +# NOT a recipe field: the GHA matrix fans out one job per concurrency and +# exports CONC into agentic_srt.sh. Serving tuning (sglang_config, env, +# frontend, dynamo hash, topology) is preserved verbatim from the source run. + +model: + path: "qwen3.5-fp4" + container: "dynamo-sglang" + precision: "fp4" + +dynamo: + hash: "3169427d2ed7ea6a0451e47ce443822d26893bae" + install: true + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 4 + +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 8 + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: kv + router-reset-states: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + + prefill_environment: + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + NO_COLOR: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + PYTHONUNBUFFERED: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + MC_FORCE_MNNVL: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + PIP_BREAK_SYSTEM_PACKAGES: "1" + decode_environment: + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + NO_COLOR: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + PYTHONUNBUFFERED: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + PIP_BREAK_SYSTEM_PACKAGES: "1" + + sglang_config: + prefill: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-path: /model/ + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + mamba-ssm-dtype: bfloat16 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + watchdog-timeout: 1000000 + log-level: info + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: false + hicache-write-policy: write_back + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 1.01 + decode: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + prefill-round-robin-balance: true + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: decode + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + disable-shared-experts-fusion: true + ep-dispatch-algorithm: static + eplb-algorithm: deepseek + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + chunked-prefill-size: 4096 + mem-fraction-static: 0.75 + max-mamba-cache-size: 1024 + max-running-requests: 512 + cuda-graph-max-bs: 64 + watchdog-timeout: 1000000 + decode-log-interval: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-2p1d-dep4-dep4-c192-mtp-hicache-convaware-jid2230562.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-2p1d-dep4-dep4-c192-mtp-hicache-convaware-jid2230562.yaml new file mode 100644 index 0000000000..ae2219fa78 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-2p1d-dep4-dep4-c192-mtp-hicache-convaware-jid2230562.yaml @@ -0,0 +1,206 @@ +name: "disagg-gb300-2p1d-dep4-dep4-c192-mtp-hicache-convaware-jid2230562" + +# Agentic-coding SGLang disaggregated 2P1D DEP recipe for Qwen3.5-397B-A17B-NVFP4 +# on GB300, ported to the flat single-variant agentic schema. Concurrency is +# NOT a recipe field: the GHA matrix fans out one job per concurrency and +# exports CONC into agentic_srt.sh. Serving tuning (sglang_config, env, +# frontend, dynamo hash, topology) is preserved verbatim from the source run. + +model: + path: "qwen3.5-fp4" + container: "dynamo-sglang" + precision: "fp4" + +dynamo: + hash: "3169427d2ed7ea6a0451e47ce443822d26893bae" + install: true + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 2 + decode_nodes: 1 + prefill_workers: 2 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 4 + +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 8 + +frontend: + type: dynamo + enable_multiple_frontends: false + num_additional_frontends: 0 + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: kv + router-reset-states: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + + prefill_environment: + SGLANG_DISAGG_STAGING_BUFFER: '1' + NO_COLOR: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + PYTHONUNBUFFERED: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + MC_FORCE_MNNVL: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + PIP_BREAK_SYSTEM_PACKAGES: "1" + decode_environment: + SGLANG_DISAGG_STAGING_BUFFER: '1' + NO_COLOR: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + PYTHONUNBUFFERED: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + PIP_BREAK_SYSTEM_PACKAGES: "1" + + sglang_config: + prefill: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-path: /model/ + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + watchdog-timeout: 1000000 + log-level: info + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 1.01 + decode: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + moe-dense-tp-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + prefill-round-robin-balance: true + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: decode + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_cutedsl + linear-attn-decode-backend: flashinfer + disable-shared-experts-fusion: true + moe-a2a-backend: deepep + deepep-mode: low_latency + ep-dispatch-algorithm: static + eplb-algorithm: deepseek + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + speculative-moe-runner-backend: deep_gemm + speculative-moe-a2a-backend: deepep + chunked-prefill-size: 4096 + mem-fraction-static: 0.75 + max-mamba-cache-size: 1024 + max-running-requests: 512 + cuda-graph-max-bs: 64 + watchdog-timeout: 1000000 + decode-log-interval: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p1d-dep4-dep8-c256-mtp-hicache-convaware-jid2228613.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p1d-dep4-dep8-c256-mtp-hicache-convaware-jid2228613.yaml new file mode 100644 index 0000000000..eee58316a1 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p1d-dep4-dep8-c256-mtp-hicache-convaware-jid2228613.yaml @@ -0,0 +1,206 @@ +name: "disagg-gb300-3p1d-dep4-dep8-c256-mtp-hicache-convaware-jid2228613" + +# Agentic-coding SGLang disaggregated 3P1D DEP recipe for Qwen3.5-397B-A17B-NVFP4 +# on GB300, ported to the flat single-variant agentic schema. Concurrency is +# NOT a recipe field: the GHA matrix fans out one job per concurrency and +# exports CONC into agentic_srt.sh. Serving tuning (sglang_config, env, +# frontend, dynamo hash, topology) is preserved verbatim from the source run. + +model: + path: "qwen3.5-fp4" + container: "dynamo-sglang" + precision: "fp4" + +dynamo: + hash: "3169427d2ed7ea6a0451e47ce443822d26893bae" + install: true + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 3 + decode_nodes: 2 + prefill_workers: 3 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 8 + +frontend: + type: dynamo + enable_multiple_frontends: false + num_additional_frontends: 0 + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: kv + router-reset-states: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + + prefill_environment: + SGLANG_DISAGG_STAGING_BUFFER: '1' + NO_COLOR: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + PYTHONUNBUFFERED: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + MC_FORCE_MNNVL: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + PIP_BREAK_SYSTEM_PACKAGES: "1" + decode_environment: + SGLANG_DISAGG_STAGING_BUFFER: '1' + NO_COLOR: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + PYTHONUNBUFFERED: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + PIP_BREAK_SYSTEM_PACKAGES: "1" + + sglang_config: + prefill: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-path: /model/ + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + watchdog-timeout: 1000000 + log-level: info + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 1.01 + decode: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + moe-dense-tp-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + prefill-round-robin-balance: true + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: decode + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_cutedsl + linear-attn-decode-backend: flashinfer + disable-shared-experts-fusion: true + moe-a2a-backend: deepep + deepep-mode: low_latency + ep-dispatch-algorithm: static + eplb-algorithm: deepseek + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + speculative-moe-runner-backend: deep_gemm + speculative-moe-a2a-backend: deepep + chunked-prefill-size: 4096 + mem-fraction-static: 0.75 + max-mamba-cache-size: 1024 + max-running-requests: 512 + cuda-graph-max-bs: 64 + watchdog-timeout: 1000000 + decode-log-interval: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-4p1d-dep4-dep8-c384-mtp-hicache-convaware-jid2239740.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-4p1d-dep4-dep8-c384-mtp-hicache-convaware-jid2239740.yaml new file mode 100644 index 0000000000..0a72c6cde9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-4p1d-dep4-dep8-c384-mtp-hicache-convaware-jid2239740.yaml @@ -0,0 +1,206 @@ +name: "disagg-gb300-4p1d-dep4-dep8-c384-mtp-hicache-convaware-jid2239740" + +# Agentic-coding SGLang disaggregated 4P1D DEP recipe for Qwen3.5-397B-A17B-NVFP4 +# on GB300, ported to the flat single-variant agentic schema. Concurrency is +# NOT a recipe field: the GHA matrix fans out one job per concurrency and +# exports CONC into agentic_srt.sh. Serving tuning (sglang_config, env, +# frontend, dynamo hash, topology) is preserved verbatim from the source run. + +model: + path: "qwen3.5-fp4" + container: "dynamo-sglang" + precision: "fp4" + +dynamo: + hash: "3169427d2ed7ea6a0451e47ce443822d26893bae" + install: true + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 4 + decode_nodes: 2 + prefill_workers: 4 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 8 + +frontend: + type: dynamo + enable_multiple_frontends: false + num_additional_frontends: 0 + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: kv + router-reset-states: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + + prefill_environment: + SGLANG_DISAGG_STAGING_BUFFER: '1' + NO_COLOR: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + PYTHONUNBUFFERED: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + MC_FORCE_MNNVL: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + PIP_BREAK_SYSTEM_PACKAGES: "1" + decode_environment: + SGLANG_DISAGG_STAGING_BUFFER: '1' + NO_COLOR: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + PYTHONUNBUFFERED: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + PIP_BREAK_SYSTEM_PACKAGES: "1" + + sglang_config: + prefill: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-path: /model/ + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + watchdog-timeout: 1000000 + log-level: info + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 1.01 + decode: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + moe-dense-tp-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + prefill-round-robin-balance: true + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: decode + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_cutedsl + linear-attn-decode-backend: flashinfer + disable-shared-experts-fusion: true + moe-a2a-backend: deepep + deepep-mode: low_latency + ep-dispatch-algorithm: static + eplb-algorithm: deepseek + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + speculative-moe-runner-backend: deep_gemm + speculative-moe-a2a-backend: deepep + chunked-prefill-size: 4096 + mem-fraction-static: 0.75 + max-mamba-cache-size: 1024 + max-running-requests: 512 + cuda-graph-max-bs: 64 + watchdog-timeout: 1000000 + decode-log-interval: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 6be8e006f9..9a6c152087 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -11380,6 +11380,165 @@ qwen3.5-fp4-gb300-dynamo-sglang: ep: 16 dp-attn: true + +qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: + image: lmsysorg/sglang:nightly-dev-cu13-20260624-b2c8f7a2 + model: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-prefix: qwen3.5 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-sglang + multinode: true + disagg: false + scenarios: + agentic-coding: + - search-space: + - spec-decoding: none + conc-list: [1] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c1-mtp-hicache-jid2191933.yaml" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: none + conc-list: [96] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c96-mtp-hicache-jid2195211.yaml" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + + +qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: + image: lmsysorg/sglang:nightly-dev-cu13-20260624-b2c8f7a2 + model: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-prefix: qwen3.5 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-sglang + multinode: true + disagg: true + scenarios: + agentic-coding: + - search-space: + # 1P1D STP (TP4 prefill / TP4 decode). 2 GB300 nodes. + - spec-decoding: none + conc-list: [32] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-convaware-jid2212783.yaml" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: none + conc-list: [64] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-convaware-jid2214439.yaml" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: none + conc-list: [96] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-convaware-jid2214440.yaml" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + # 1P1D wide-EP (DEP4 prefill / DEP4 decode). 2 GB300 nodes. + - spec-decoding: none + conc-list: [128] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-dep4-dep4-c128-mtp-hicache-convaware-jid2220685.yaml" + decode: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + # 2P1D wide-EP (2x DEP4 prefill / DEP4 decode). 3 GB300 nodes. + - spec-decoding: none + conc-list: [192] + prefill: + num-worker: 2 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-2p1d-dep4-dep4-c192-mtp-hicache-convaware-jid2230562.yaml" + decode: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + # 3P1D wide-EP (3x DEP4 prefill / DEP8 decode). 5 GB300 nodes. + - spec-decoding: none + conc-list: [256] + prefill: + num-worker: 3 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p1d-dep4-dep8-c256-mtp-hicache-convaware-jid2228613.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + # 4P1D wide-EP (4x DEP4 prefill / DEP8 decode). 6 GB300 nodes. + - spec-decoding: none + conc-list: [384] + prefill: + num-worker: 4 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-4p1d-dep4-dep8-c384-mtp-hicache-convaware-jid2239740.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + glm5-fp4-gb300-dynamo-sglang: image: lmsysorg/sglang:v0.5.11-cu130 model: nvidia/GLM-5.1-NVFP4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c1a3c3aee8..fb97a145bc 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4363,6 +4363,16 @@ - "Reuse the existing MXFP8 B300 topology and concurrency matrix across 15 srt-slurm recipes, while dropping the FP8-only Marlin override from TP4 decode" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1931 +- config-keys: + - qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg + - qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg + description: + - "Add Qwen3.5-397B-A17B-NVFP4 FP4 GB300 SGLang AgentX Pareto-frontier benchmarks via the srtctl/dynamo stack" + - "agg: single aggregated node (TP4, MTP/NEXTN, hierarchical KV cache), concurrencies [1, 96]" + - "disagg: 1P1D-4P1D (TP4/DEP4 prefill, TP4/DEP4/DEP8 decode, MTP/NEXTN, hicache, conv-aware routing) over 7 Pareto points at concurrency 32-384" + - "Image: lmsysorg/sglang:nightly-dev-cu13-20260624-b2c8f7a2; runner: gb300-nv" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2121 + - config-keys: - minimaxm3-fp4-mi355x-vllm-mtp description: diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 5328ded02d..eef609ec70 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -17,6 +17,17 @@ export ENROOT_ROOTFS_WRITABLE=1 # write to it. export AIPERF_MMAP_CACHE_HOST_PATH="/data/home/sa-shared/gharunners/ai-perf-cache" +# Persistent dynamo source-build cache. srtctl's hash-pinned dynamo install +# (_hash_cached_source_install) caches the built wheel + src tarball at +# /configs/dynamo-wheels/ with a .complete sentinel; on a warm cache the +# install is just `pip install` from the cache (no apt, no root). In CI /configs +# is the per-job srt-slurm checkout (cold every job → cold build needs apt + +# root, which the non-root server containers can't do), so persist and share +# the cache across jobs by bind-mounting this host dir at /configs/dynamo-wheels. +# Seed it once with a --container-remap-root build. 777 for multi-user runners. +export DYNAMO_WHEELS_CACHE_HOST_PATH="/data/home/sa-shared/gharunners/dynamo-wheels" +mkdir -p "$DYNAMO_WHEELS_CACHE_HOST_PATH" + export HF_HUB_CACHE_HOST_PATH="/data/home/sa-shared/gharunners/hf-hub-cache" mkdir -p "$HF_HUB_CACHE_HOST_PATH" @@ -123,38 +134,57 @@ SRTCTL_SETUP_SCRIPT="" rm -rf "$SRT_REPO_DIR" if [[ "$IS_AGENTIC" == "1" ]]; then - # Agentic multi-node uses cquil11/srt-slurm-nv@cam/no-preflight-flag, - # a thin branch off NVIDIA/srt-slurm@127597c that adds one CLI flag - # (`srtctl apply --no-preflight`) — needed because: - # - # - We want MODEL_PATH=/scratch/models/DeepSeek-V4-Pro (node-local - # NVMe, fast) instead of the NFS path under /data/home/sa-shared. - # - /scratch only exists on GB300 compute nodes; it is NOT mounted - # on the GHA runner pod that invokes srtctl. - # - srtctl's pre-submit model check (_preflight_model in - # src/srtctl/core/validation.py) does a Path.is_dir() in-process - # on the invoking node — so it fails before sbatch is ever - # called with "Model alias 'X' resolved to '/scratch/...', - # but that path is unavailable". - # - --no-preflight skips just the optional Python-level FS check. - # vLLM still fails loudly at runtime if the path is genuinely - # missing on the compute node. - # - # All other upstream schema features we need are inherited from - # NVIDIA HEAD: - # - BenchmarkType.CUSTOM + benchmark.command + benchmark.env - # (hook that hands off to benchmarks/multi_node/agentic_srt.sh) - # - DynamoConfig.wheel (so vllm recipes can pin the ai-dynamo wheel) - # - sbatch_directives / srun_options (top-level recipe fields) - git clone https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - # 854b3fd = --no-preflight flag - # 6e34b8b = benchmark_stage propagates srun_options (needed for - # container-remap-root to reach the agentic_srt.sh srun) - git checkout 6e34b8b83229634d732e41a4e2d6595f46ef60b5 - mkdir -p recipes/vllm/deepseek-v4/agentic - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic" \ - recipes/vllm/deepseek-v4/agentic + if [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "qwen3.5" ]]; then + # Qwen3.5 agentic uses upstream NVIDIA/srt-slurm@main: the two features + # the cquil11 fork was pinned for are now merged to main — + # - `srtctl apply --no-preflight` (skip the in-process model FS check): + # model.path resolves to /scratch/models/Qwen3.5-397B-A17B-NVFP4 + # (compute-node-only NVMe), which the GHA runner pod can't stat, so + # the Path.is_dir() preflight would fail before sbatch is ever + # called. The engine still fails loudly at runtime if the path is + # genuinely missing on the compute node. + # - benchmark_stage propagates srun_options (container-remap-root must + # reach the agentic_srt.sh srun). + git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + git checkout main + mkdir -p recipes/sglang/qwen3.5 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5" \ + recipes/sglang/qwen3.5 + else + # Agentic multi-node uses cquil11/srt-slurm-nv@cam/no-preflight-flag, + # a thin branch off NVIDIA/srt-slurm@127597c that adds one CLI flag + # (`srtctl apply --no-preflight`) — needed because: + # + # - We want MODEL_PATH=/scratch/models/DeepSeek-V4-Pro (node-local + # NVMe, fast) instead of the NFS path under /data/home/sa-shared. + # - /scratch only exists on GB300 compute nodes; it is NOT mounted + # on the GHA runner pod that invokes srtctl. + # - srtctl's pre-submit model check (_preflight_model in + # src/srtctl/core/validation.py) does a Path.is_dir() in-process + # on the invoking node — so it fails before sbatch is ever + # called with "Model alias 'X' resolved to '/scratch/...', + # but that path is unavailable". + # - --no-preflight skips just the optional Python-level FS check. + # vLLM still fails loudly at runtime if the path is genuinely + # missing on the compute node. + # + # All other upstream schema features we need are inherited from + # NVIDIA HEAD: + # - BenchmarkType.CUSTOM + benchmark.command + benchmark.env + # (hook that hands off to benchmarks/multi_node/agentic_srt.sh) + # - DynamoConfig.wheel (so vllm recipes can pin the ai-dynamo wheel) + # - sbatch_directives / srun_options (top-level recipe fields) + git clone https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + # 854b3fd = --no-preflight flag + # 6e34b8b = benchmark_stage propagates srun_options (needed for + # container-remap-root to reach the agentic_srt.sh srun) + git checkout 6e34b8b83229634d732e41a4e2d6595f46ef60b5 + mkdir -p recipes/vllm/deepseek-v4/agentic + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic" \ + recipes/vllm/deepseek-v4/agentic + fi elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" @@ -265,6 +295,9 @@ srtctl_root: "${SRTCTL_ROOT}" # re-tokenized + re-written every job. default_mounts: "${AIPERF_MMAP_CACHE_HOST_PATH}": "/aiperf_mmap_cache" + # Warm dynamo source-build cache (nested over the auto /configs mount) so the + # hash-pinned install is a cache hit (pip-only, no apt/root) on every job. + "${DYNAMO_WHEELS_CACHE_HOST_PATH}": "/configs/dynamo-wheels" "${HF_HUB_CACHE_HOST_PATH}": "/hf_hub_cache" # Model path aliases From 798435b5c96b4fd1009db91b9d639eba4a17e467 Mon Sep 17 00:00:00 2001 From: Sahithi Chigurupati Date: Wed, 8 Jul 2026 14:11:39 -0700 Subject: [PATCH 2/6] fix(sweep): run + collect the multi-node agentic sweep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - run-sweep.yml: pass the list-valued conc correctly to the multi-node agentic template — conc-list via toJson, conc as conc[0]. Passing the raw list to the scalar 'conc' input made GitHub reject the job ("A sequence was not expected"), so no agentic matrix jobs instantiated. - run-sweep.yml: add sweep-agentic + sweep-multi-node-agentic to the collect-results gate so agentic-only sweeps collect their results. - benchmark-multinode-tmpl.yml: job-name label shows the full concurrency batch. --- .github/workflows/benchmark-multinode-tmpl.yml | 2 +- .github/workflows/run-sweep.yml | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 4277e40c13..5dca7ef286 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -195,7 +195,7 @@ jobs: ${{ inputs.spec-decoding != 'none' && inputs.spec-decoding || '' }} ${{ inputs.kv-offloading != '' && inputs.kv-offloading != 'none' && format('{0} KV offload', inputs.kv-offloading) || '' }} ${{ inputs.kv-offload-backend != '' && inputs.kv-offload-backend != 'none' && inputs.kv-offload-backend != 'default' && inputs.kv-offload-backend || '' }} - c${{ inputs.conc != '' && inputs.conc || join(fromJson(inputs.conc-list), 'x') }}${{ inputs.eval-only && ' | eval-only' || (inputs.run-eval && ' | eval' || '') }} + c${{ join(fromJson(inputs.conc-list), 'x') }}${{ inputs.eval-only && ' | eval-only' || (inputs.run-eval && ' | eval' || '') }} steps: - name: Slurm cleanup (pre-run) diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index 1f5e1e7bc0..34fdcf9145 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -536,7 +536,7 @@ jobs: model-prefix: ${{ matrix.config.model-prefix }} framework: ${{ matrix.config.framework }} precision: ${{ matrix.config.precision }} - conc-list: '[${{ matrix.config.conc }}]' + conc-list: ${{ toJson(matrix.config.conc) }} spec-decoding: ${{ matrix.config.spec-decoding }} disagg: ${{ matrix.config.disagg }} prefill-hardware: ${{ matrix.config.prefill.hardware }} @@ -551,7 +551,7 @@ jobs: decode-ep: ${{ matrix.config.decode.ep }} decode-dp-attn: ${{ matrix.config.decode.dp-attn }} decode-additional-settings: ${{ toJson(matrix.config.decode.additional-settings) }} - conc: ${{ matrix.config.conc }} + conc: ${{ matrix.config.conc[0] }} kv-offloading: ${{ matrix.config.kv-offloading }} kv-offload-backend: ${{ matrix.config.kv-offload-backend }} duration: ${{ matrix.config.duration }} @@ -667,7 +667,9 @@ jobs: needs.sweep-single-node-1k1k.result != 'skipped' || needs.sweep-single-node-8k1k.result != 'skipped' || needs.sweep-multi-node-1k1k.result != 'skipped' || - needs.sweep-multi-node-8k1k.result != 'skipped' + needs.sweep-multi-node-8k1k.result != 'skipped' || + needs.sweep-agentic.result != 'skipped' || + needs.sweep-multi-node-agentic.result != 'skipped' ) }} uses: ./.github/workflows/collect-results.yml From e132f97a7bc62aff02de32d98f919434a79b9883 Mon Sep 17 00:00:00 2001 From: Sahithi Chigurupati Date: Wed, 8 Jul 2026 14:22:11 -0700 Subject: [PATCH 3/6] chore(perf-changelog): move Qwen3.5 agentic entry to end of file --- perf-changelog.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index fb97a145bc..55ecbd81aa 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4363,16 +4363,6 @@ - "Reuse the existing MXFP8 B300 topology and concurrency matrix across 15 srt-slurm recipes, while dropping the FP8-only Marlin override from TP4 decode" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1931 -- config-keys: - - qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg - - qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg - description: - - "Add Qwen3.5-397B-A17B-NVFP4 FP4 GB300 SGLang AgentX Pareto-frontier benchmarks via the srtctl/dynamo stack" - - "agg: single aggregated node (TP4, MTP/NEXTN, hierarchical KV cache), concurrencies [1, 96]" - - "disagg: 1P1D-4P1D (TP4/DEP4 prefill, TP4/DEP4/DEP8 decode, MTP/NEXTN, hicache, conv-aware routing) over 7 Pareto points at concurrency 32-384" - - "Image: lmsysorg/sglang:nightly-dev-cu13-20260624-b2c8f7a2; runner: gb300-nv" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2121 - - config-keys: - minimaxm3-fp4-mi355x-vllm-mtp description: @@ -4709,3 +4699,13 @@ - "Clean the export envs" - "Enable two batch overlap" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2093 + +- config-keys: + - qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg + - qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg + description: + - "Add Qwen3.5-397B-A17B-NVFP4 FP4 GB300 SGLang AgentX Pareto-frontier benchmarks via the srtctl/dynamo stack" + - "agg: single aggregated node (TP4, MTP/NEXTN, hierarchical KV cache), concurrencies [1, 96]" + - "disagg: 1P1D-4P1D (TP4/DEP4 prefill, TP4/DEP4/DEP8 decode, MTP/NEXTN, hicache, conv-aware routing) over 7 Pareto points at concurrency 32-384" + - "Image: lmsysorg/sglang:nightly-dev-cu13-20260624-b2c8f7a2; runner: gb300-nv" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2121 From e571715ed36024908db0613b3529c5ae45e3a8c5 Mon Sep 17 00:00:00 2001 From: Sahithi Chigurupati Date: Wed, 8 Jul 2026 14:39:01 -0700 Subject: [PATCH 4/6] fix(qwen3.5): mark GB300 agentic entries as spec-decoding mtp The 9 qwen3.5 agentic-agg/disagg entries declared spec-decoding: none, but their recipes enable NEXTN MTP (speculative-algorithm NEXTN). SPEC_DECODING flows into RESULT_FILENAME and the aggregated result JSON's spec_decoding field, so dashboards/Pareto would misclassify these MTP runs as non-MTP. Set spec-decoding: "mtp" to match the engine + the file's MTP convention. --- configs/nvidia-master.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 9a6c152087..10b67e60fc 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -11393,7 +11393,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: scenarios: agentic-coding: - search-space: - - spec-decoding: none + - spec-decoding: "mtp" conc-list: [1] prefill: num-worker: 1 @@ -11407,7 +11407,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: tp: 4 ep: 1 dp-attn: false - - spec-decoding: none + - spec-decoding: "mtp" conc-list: [96] prefill: num-worker: 1 @@ -11436,7 +11436,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: agentic-coding: - search-space: # 1P1D STP (TP4 prefill / TP4 decode). 2 GB300 nodes. - - spec-decoding: none + - spec-decoding: "mtp" conc-list: [32] prefill: num-worker: 1 @@ -11450,7 +11450,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: tp: 4 ep: 1 dp-attn: false - - spec-decoding: none + - spec-decoding: "mtp" conc-list: [64] prefill: num-worker: 1 @@ -11464,7 +11464,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: tp: 4 ep: 1 dp-attn: false - - spec-decoding: none + - spec-decoding: "mtp" conc-list: [96] prefill: num-worker: 1 @@ -11479,7 +11479,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false # 1P1D wide-EP (DEP4 prefill / DEP4 decode). 2 GB300 nodes. - - spec-decoding: none + - spec-decoding: "mtp" conc-list: [128] prefill: num-worker: 1 @@ -11494,7 +11494,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 4 dp-attn: true # 2P1D wide-EP (2x DEP4 prefill / DEP4 decode). 3 GB300 nodes. - - spec-decoding: none + - spec-decoding: "mtp" conc-list: [192] prefill: num-worker: 2 @@ -11509,7 +11509,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 4 dp-attn: true # 3P1D wide-EP (3x DEP4 prefill / DEP8 decode). 5 GB300 nodes. - - spec-decoding: none + - spec-decoding: "mtp" conc-list: [256] prefill: num-worker: 3 @@ -11524,7 +11524,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true # 4P1D wide-EP (4x DEP4 prefill / DEP8 decode). 6 GB300 nodes. - - spec-decoding: none + - spec-decoding: "mtp" conc-list: [384] prefill: num-worker: 4 From 9f716dad10daca2fe4b4a8476878fcfefc03378a Mon Sep 17 00:00:00 2001 From: Sahithi Chigurupati Date: Wed, 8 Jul 2026 17:16:51 -0700 Subject: [PATCH 5/6] feat(qwen3.5): force golden acceptance-length for GB300 agentic MTP Pin synthetic acceptance to the committed golden AL (golden_al_distribution/ qwen3.5_mtp.yaml, thinking_on, draft-len 3 = 3.39) on the decode worker (disagg) / aggregated worker (agg), per the AgentX fairness guideline, instead of the draft head's variable acceptance: SGLANG_SIMULATE_ACC_LEN: '3.39' SGLANG_SIMULATE_ACC_METHOD: match-expected SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" real-draft-token keeps real per-token timing (non-zero ITL); the real NEXTN speculative-* settings are unchanged. --- .../agentic/agg-gb300-tp4-c1-mtp-hicache-jid2191933.yaml | 3 +++ .../agentic/agg-gb300-tp4-c96-mtp-hicache-jid2195211.yaml | 3 +++ ...0-1p1d-dep4-dep4-c128-mtp-hicache-convaware-jid2220685.yaml | 3 +++ ...b300-1p1d-tp4-tp4-c32-mtp-hicache-convaware-jid2212783.yaml | 3 +++ ...disagg-gb300-1p1d-tp4-tp4-c64-mtp-convaware-jid2214439.yaml | 3 +++ ...disagg-gb300-1p1d-tp4-tp4-c96-mtp-convaware-jid2214440.yaml | 3 +++ ...0-2p1d-dep4-dep4-c192-mtp-hicache-convaware-jid2230562.yaml | 3 +++ ...0-3p1d-dep4-dep8-c256-mtp-hicache-convaware-jid2228613.yaml | 3 +++ ...0-4p1d-dep4-dep8-c384-mtp-hicache-convaware-jid2239740.yaml | 3 +++ 9 files changed, 27 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c1-mtp-hicache-jid2191933.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c1-mtp-hicache-jid2191933.yaml index 5a24f6573b..02c88cc299 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c1-mtp-hicache-jid2191933.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c1-mtp-hicache-jid2191933.yaml @@ -37,6 +37,9 @@ frontend: backend: type: sglang aggregated_environment: + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" TORCH_CUDA_ARCH_LIST: '10.0' PYTHONNOUSERSITE: '1' NCCL_NVLS_ENABLE: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c96-mtp-hicache-jid2195211.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c96-mtp-hicache-jid2195211.yaml index 667b6088d1..5891df5b32 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c96-mtp-hicache-jid2195211.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c96-mtp-hicache-jid2195211.yaml @@ -37,6 +37,9 @@ frontend: backend: type: sglang aggregated_environment: + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" TORCH_CUDA_ARCH_LIST: '10.0' PYTHONNOUSERSITE: '1' NCCL_NVLS_ENABLE: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-dep4-dep4-c128-mtp-hicache-convaware-jid2220685.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-dep4-dep4-c128-mtp-hicache-convaware-jid2220685.yaml index 8ba346e939..5440eebe22 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-dep4-dep4-c128-mtp-hicache-convaware-jid2220685.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-dep4-dep4-c128-mtp-hicache-convaware-jid2220685.yaml @@ -75,6 +75,9 @@ backend: SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' PIP_BREAK_SYSTEM_PACKAGES: "1" decode_environment: + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGG_STAGING_BUFFER: '1' NO_COLOR: '1' TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-convaware-jid2212783.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-convaware-jid2212783.yaml index 8b70edc935..809ca3c622 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-convaware-jid2212783.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-convaware-jid2212783.yaml @@ -77,6 +77,9 @@ backend: SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' PIP_BREAK_SYSTEM_PACKAGES: "1" decode_environment: + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGG_STAGING_BUFFER: '1' SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-convaware-jid2214439.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-convaware-jid2214439.yaml index 5a80bf23d6..c2b92efff7 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-convaware-jid2214439.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-convaware-jid2214439.yaml @@ -88,6 +88,9 @@ backend: # Lets the runtime dynamo install's pip bypass PEP 668 on the sglang image. PIP_BREAK_SYSTEM_PACKAGES: "1" decode_environment: + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGG_STAGING_BUFFER: '1' SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-convaware-jid2214440.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-convaware-jid2214440.yaml index 4292eb18e1..b12ff8085b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-convaware-jid2214440.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-convaware-jid2214440.yaml @@ -77,6 +77,9 @@ backend: SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' PIP_BREAK_SYSTEM_PACKAGES: "1" decode_environment: + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGG_STAGING_BUFFER: '1' SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-2p1d-dep4-dep4-c192-mtp-hicache-convaware-jid2230562.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-2p1d-dep4-dep4-c192-mtp-hicache-convaware-jid2230562.yaml index ae2219fa78..e6f0a353f7 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-2p1d-dep4-dep4-c192-mtp-hicache-convaware-jid2230562.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-2p1d-dep4-dep4-c192-mtp-hicache-convaware-jid2230562.yaml @@ -75,6 +75,9 @@ backend: SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' PIP_BREAK_SYSTEM_PACKAGES: "1" decode_environment: + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGG_STAGING_BUFFER: '1' NO_COLOR: '1' TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p1d-dep4-dep8-c256-mtp-hicache-convaware-jid2228613.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p1d-dep4-dep8-c256-mtp-hicache-convaware-jid2228613.yaml index eee58316a1..e748ac9408 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p1d-dep4-dep8-c256-mtp-hicache-convaware-jid2228613.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p1d-dep4-dep8-c256-mtp-hicache-convaware-jid2228613.yaml @@ -75,6 +75,9 @@ backend: SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' PIP_BREAK_SYSTEM_PACKAGES: "1" decode_environment: + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGG_STAGING_BUFFER: '1' NO_COLOR: '1' TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-4p1d-dep4-dep8-c384-mtp-hicache-convaware-jid2239740.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-4p1d-dep4-dep8-c384-mtp-hicache-convaware-jid2239740.yaml index 0a72c6cde9..753f506c10 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-4p1d-dep4-dep8-c384-mtp-hicache-convaware-jid2239740.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-4p1d-dep4-dep8-c384-mtp-hicache-convaware-jid2239740.yaml @@ -75,6 +75,9 @@ backend: SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' PIP_BREAK_SYSTEM_PACKAGES: "1" decode_environment: + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGG_STAGING_BUFFER: '1' NO_COLOR: '1' TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' From f644fd790bbf2dc4c3de8883f96ed57e15a34dc0 Mon Sep 17 00:00:00 2001 From: Sahithi Chigurupati Date: Thu, 9 Jul 2026 16:01:36 -0700 Subject: [PATCH 6/6] chore(qwen3.5): bump GB300 agentic image to nightly-dev-cu13-20260709-074bb928 Align the agentic-agg and agentic-disagg keys: 20260624-b2c8f7a2 -> 20260709-074bb928. --- configs/nvidia-master.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 10b67e60fc..3b28cd667f 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -11382,7 +11382,7 @@ qwen3.5-fp4-gb300-dynamo-sglang: qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: - image: lmsysorg/sglang:nightly-dev-cu13-20260624-b2c8f7a2 + image: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928 model: nvidia/Qwen3.5-397B-A17B-NVFP4 model-prefix: qwen3.5 runner: cluster:gb300-nv @@ -11424,7 +11424,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: - image: lmsysorg/sglang:nightly-dev-cu13-20260624-b2c8f7a2 + image: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928 model: nvidia/Qwen3.5-397B-A17B-NVFP4 model-prefix: qwen3.5 runner: cluster:gb300-nv