From e196e0add06331eeb0dc4cac497770c4d3240617 Mon Sep 17 00:00:00 2001 From: Oasis-Git Date: Thu, 9 Jul 2026 18:23:27 -0700 Subject: [PATCH 1/8] dsv4 sglang agentic: MegaMoE DEP + TP-only low-latency recipes (B300) Rework the B300 agentic sglang recipe into three regimes: - DP-attention (megamoe): MegaMoE DeepGEMM MoE (--moe-a2a-backend megamoe + mega_moe env + fused shared experts + autotune), mem-fraction 0.835, swa 0.075, prefill-delayer. Rank-adjusted sizing: DEP8 chunk 65536 / cuda-graph-max-bs-decode 544; DEP4 chunk 32768 / decode 128 (effective chunk 8192 for both). Measured DEP8 conc128: 24,466 -> 33,220 tok/s/gpu (vLLM v0.23.0 reference 28,962). - TP-only low-latency (TP8 or TP4, non-DP, conc <= 16): SGLang cookbook low-latency single-node recipe with speculative decoding removed: flashinfer_mxfp4 + --enable-deepseek-v4-fp4-indexer + fused shared experts, chunked-prefill 8192, mem-fraction 0.90. - TP8 mid concurrency (32-52): unchanged flashinfer_mxfp4 baseline. Builds on #2112 (image bump to nightly-20260707). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../agentic/dsv4_fp4_b300_sglang.sh | 63 ++++++++++++++++--- 1 file changed, 53 insertions(+), 10 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang.sh index 5599ca4717..692cc10a42 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang.sh @@ -105,7 +105,28 @@ PARALLEL_ARGS=(--tp "$TP") METRICS_ARGS=(--enable-metrics) MEM_FRACTION_STATIC=0.88 CHUNKED_PREFILL_SIZE=8192 +SWA_FULL_TOKENS_RATIO=0.1 +# Default (non-DP) attention path: compressed attention, shared-expert fusion off. +MODEL_ARGS=( + --attention-backend compressed + --page-size 256 + --disable-shared-experts-fusion +) +CUDA_GRAPH_ARGS=() +EXTRA_ARGS=() if [ "$DP_ATTENTION" = "true" ]; then + # DP-attention high-throughput agentic recipe (SGLang cookbook: DSV4-Pro FP4 + # B300, high-throughput, single-node): MegaMoE DeepGEMM MoE (DeepEP dispatch + + # fused shared experts + kernel autotune), tuned for conc128. + # Measured DEP8 conc128 vs the previous flashinfer_mxfp4 path: + # 24,466 -> 33,220 tok/s/gpu (surpasses the vLLM reference of 28,962). + export SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE=1 + export SGLANG_OPT_FIX_HASH_MEGA_MOE=1 + export SGLANG_OPT_USE_FAST_MASK_EP=1 + export SGLANG_OPT_FIX_MEGA_MOE_MEMORY=1 + export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8320 + export SGLANG_OPT_FIX_NEXTN_MEGA_MOE=1 + export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=0 PARALLEL_ARGS+=( --dp "$TP" --tokenizer-worker-num "$TP" @@ -115,11 +136,36 @@ if [ "$DP_ATTENTION" = "true" ]; then --stream-interval 20 --dist-init-addr "127.0.0.1:$((PORT + 2000))" --ep-size "$EP_SIZE" + --moe-a2a-backend megamoe + ) + MEM_FRACTION_STATIC=0.835 + SWA_FULL_TOKENS_RATIO=0.075 + # MegaMoE requires the default (non-compressed) attention path and fused shared experts. + MODEL_ARGS=() + EXTRA_ARGS=(--enable-prefill-delayer) + # chunked-prefill is divided by dp_size (=TP) under DP-attention; keep the + # effective chunk at 8192. Size the decode graph to served concurrency: + # DEP8 (dp 8) serves conc up to 512, DEP4 (dp 4) tops out near 128. + if [ "$TP" = "8" ]; then + CHUNKED_PREFILL_SIZE=65536 + CUDA_GRAPH_ARGS=(--cuda-graph-max-bs-decode 544) + else + CHUNKED_PREFILL_SIZE=32768 + CUDA_GRAPH_ARGS=(--cuda-graph-max-bs-decode 128) + fi +elif { [ "$TP" = "8" ] || [ "$TP" = "4" ]; } && [ "${CONC:-999}" -le 16 ]; then + # TP-only low-latency (TP4 or TP8, non-DP, conc <= 16): SGLang cookbook + # DSV4-Pro FP4 B300 low-latency single-node recipe (mirrors the fixed_seq_len + # MTP TP-only path) with speculative decoding (EAGLE) removed. Default + # attention + fused shared experts + DSV4 FP4 sparse-attention indexer; + # mem-fraction 0.90. + PARALLEL_ARGS+=( --moe-runner-backend flashinfer_mxfp4 --disable-flashinfer-autotune + --enable-deepseek-v4-fp4-indexer ) - MEM_FRACTION_STATIC=0.95 - CHUNKED_PREFILL_SIZE=16384 + MEM_FRACTION_STATIC=0.90 + MODEL_ARGS=(--page-size 256) else PARALLEL_ARGS+=( --moe-runner-backend flashinfer_mxfp4 @@ -127,17 +173,13 @@ else ) fi -MODEL_ARGS=( - --attention-backend compressed - --page-size 256 - --disable-shared-experts-fusion -) - # AgentX concurrency counts live session trees, not individual requests. # Allow subagent fan-out to exceed CONC without clipping request bursts. MAX_RUNNING_REQUESTS=$((2 * CONC)) CUDA_GRAPH_MAX_BS=$CONC [ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 +# Non-DP path keeps the concurrency-scaled decode graph; DP path set 544 above. +[ ${#CUDA_GRAPH_ARGS[@]} -eq 0 ] && CUDA_GRAPH_ARGS=(--cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS") export PYTHONNOUSERSITE=1 export TORCH_CUDA_ARCH_LIST=10.0 @@ -172,9 +214,9 @@ SGLANG_CMD=( --trust-remote-code "${PARALLEL_ARGS[@]}" --mem-fraction-static "$MEM_FRACTION_STATIC" - --swa-full-tokens-ratio 0.1 + --swa-full-tokens-ratio "$SWA_FULL_TOKENS_RATIO" --max-running-requests "$MAX_RUNNING_REQUESTS" - --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" + "${CUDA_GRAPH_ARGS[@]}" --allow-auto-truncate --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" --tool-call-parser deepseekv4 @@ -183,6 +225,7 @@ SGLANG_CMD=( --watchdog-timeout 1800 "${MODEL_ARGS[@]}" "${METRICS_ARGS[@]}" + "${EXTRA_ARGS[@]}" "${CACHE_ARGS[@]}" ) From c4239958bbabc62ad819370b182f643d4231f774 Mon Sep 17 00:00:00 2001 From: Oasis-Git Date: Thu, 9 Jul 2026 18:54:55 -0700 Subject: [PATCH 2/8] dsv4 sglang agentic B200: low-latency mem-fraction + DEP conc-split recipe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit B200 sglang agentic (dsv4_fp4_b200_sglang.sh): - low-latency TP-only path (DP_ATTENTION=false): mem-fraction 0.90 (was 0.88), matching the SGLang cookbook low-latency recipe (flashinfer_mxfp4, chunked 8192, swa 0.1, no spec, GPU-only). - DEP path now splits by concurrency via DEP_HIGH_CONC: conc < 54 -> conservative recipe (chunked 32768, mem 0.88, swa 0.1, NUM_MAX_TOKENS_PER_RANK 4096, --cuda-graph-max-bs) conc >= 54 -> cookbook high-throughput recipe (chunked 65536, mem 0.835, swa 0.075, NUM_MAX_TOKENS_PER_RANK 8192, --cuda-graph-max-bs-decode 544, --enable-prefill-delayer). The 8192 tokens/rank cap keeps chunked 65536 on the DeepGEMM MoE path instead of the fp4-incompatible Triton fallback. configs/nvidia-master.yaml: - dsv4-fp4-b200-sglang-agentic-hicache: refresh TP/DEP conc-lists. - dsv4-fp4-b300-sglang-agentic-hicache: refresh conc-lists. 中文：B200 sglang 智能体脚本按并发拆分 DEP 配方（conc>=54 使用 cookbook 高吞吐配方， 含 chunked 65536 / prefill delayer / mem 0.835 / swa 0.075，并将 tokens/rank 上限提到 8192，避免 fp4 MoE 回退到 Triton 导致崩溃）；低延迟 TP-only 路径 mem 提到 0.90； 更新 master.yaml 中 B200/B300 sglang 智能体的并发列表。 Co-Authored-By: Claude Opus 4.8 --- .../agentic/dsv4_fp4_b200_sglang.sh | 48 ++++++++++++++++--- configs/nvidia-master.yaml | 18 +++---- 2 files changed, 50 insertions(+), 16 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_sglang.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_sglang.sh index 4321644d8c..3c211543af 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b200_sglang.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b200_sglang.sh @@ -99,13 +99,20 @@ fi PARALLEL_ARGS=(--tp "$TP") METRICS_ARGS=(--enable-metrics) CHUNKED_PREFILL_SIZE=8192 +# DEP recipe splits by concurrency. The high-throughput tail (conc >= 54) uses +# the SGLang cookbook recipe (chunked 65536 + prefill delayer + tighter mem/SWA +# + cuda-graph-max-bs-decode 544, plus the 8192 tokens/rank cap that chunked +# 65536 needs to stay on the DeepGEMM MoE path). conc < 54 keeps the +# conservative recipe (chunked 32768) that avoids the pre-54 throughput cliff. +DEP_HIGH_CONC=false +[ "$DP_ATTENTION" = "true" ] && [ "$CONC" -ge 54 ] && DEP_HIGH_CONC=true +DEP_EXTRA_ARGS=() if [ "$DP_ATTENTION" = "true" ]; then DEEPEP_CONFIG='{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' export SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE=1 export SGLANG_OPT_FIX_HASH_MEGA_MOE=1 export SGLANG_OPT_USE_FAST_MASK_EP=1 export SGLANG_OPT_FIX_MEGA_MOE_MEMORY=1 - export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=4096 export SGLANG_OPT_FIX_NEXTN_MEGA_MOE=1 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=0 PARALLEL_ARGS+=( @@ -120,7 +127,14 @@ if [ "$DP_ATTENTION" = "true" ]; then --moe-a2a-backend deepep --deepep-config "$DEEPEP_CONFIG" ) - CHUNKED_PREFILL_SIZE=32768 + if [ "$DEP_HIGH_CONC" = "true" ]; then + export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8192 + CHUNKED_PREFILL_SIZE=65536 + DEP_EXTRA_ARGS=(--enable-prefill-delayer) + else + export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=4096 + CHUNKED_PREFILL_SIZE=32768 + fi else PARALLEL_ARGS+=( --moe-runner-backend flashinfer_mxfp4 @@ -134,13 +148,32 @@ MODEL_ARGS=() # DeepGEMM's DSv4 indexer needs a multi-GiB temporary allocation at long # contexts. Leave the same HBM headroom used by the B300 recipe so a nearly # full GPU KV cache does not OOM while HiCache is spilling to host memory. -MEM_FRACTION_STATIC=0.88 +# The low-latency TP-only path (conc <= 16) runs GPU-only with no HiCache +# spill, so it can take a larger static fraction for more KV headroom. +if [ "$DP_ATTENTION" = "true" ]; then + if [ "$DEP_HIGH_CONC" = "true" ]; then + MEM_FRACTION_STATIC=0.835 + else + MEM_FRACTION_STATIC=0.88 + fi +else + MEM_FRACTION_STATIC=0.90 +fi # AgentX concurrency counts live session trees, not individual requests. # Allow subagent fan-out to exceed CONC without clipping request bursts. MAX_RUNNING_REQUESTS=$((2 * CONC)) -CUDA_GRAPH_MAX_BS=$CONC -[ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 +# The cookbook DEP tail captures a large decode graph (batch 544); other paths +# scale the unified cuda-graph batch with CONC (capped at 64). +if [ "$DEP_HIGH_CONC" = "true" ]; then + CUDA_GRAPH_ARGS=(--cuda-graph-max-bs-decode 544) +else + CUDA_GRAPH_MAX_BS=$CONC + [ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 + CUDA_GRAPH_ARGS=(--cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS") +fi +SWA_FULL_TOKENS_RATIO=0.1 +[ "$DEP_HIGH_CONC" = "true" ] && SWA_FULL_TOKENS_RATIO=0.075 export PYTHONNOUSERSITE=1 export TORCH_CUDA_ARCH_LIST=10.0 @@ -175,10 +208,11 @@ SGLANG_CMD=( --trust-remote-code "${PARALLEL_ARGS[@]}" --mem-fraction-static "$MEM_FRACTION_STATIC" - --swa-full-tokens-ratio 0.1 + --swa-full-tokens-ratio "$SWA_FULL_TOKENS_RATIO" --max-running-requests "$MAX_RUNNING_REQUESTS" - --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" + "${CUDA_GRAPH_ARGS[@]}" --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" + "${DEP_EXTRA_ARGS[@]}" --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --chat-template "$SCRIPT_DIR/../chat_templates/deepseek_v4_thinking.jinja" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 6be8e006f9..22add6901e 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -14084,12 +14084,12 @@ dsv4-fp4-b200-sglang-agentic-hicache: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 3, 4, 5] } - - { tp: 8, kv-offloading: dram, kv-offload-backend: hicache, conc-list: [8, 10, 16, 32, 40, 44] } + - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 16] } + - { tp: 8, kv-offloading: dram, kv-offload-backend: hicache, conc-list: [8, 16, 32] } # DEP without HiCache is already degraded at conc 52 (2,098 tok/s/GPU), # so resolve its pre-52 cliff instead of repeating the collapsed tail. - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [16, 24, 32, 38, 44, 48, 50, 52] } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: hicache, conc-list: [16, 32, 38, 44, 50, 56, 64, 66, 68] } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [16, 32, 38, 44, 46, 48, 50, 52, 54, 56, 58, 60] } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: hicache, conc-list: [16, 32, 38, 40, 44, 48, 50, 54, 56, 60, 64, 66, 68] } # GB200 DeepSeek-V4 disaggregated AgentX frontier. The 3P/2D TEP8/TP8 curve # covers the middle/high-interactivity range omitted by the one-decode DEP @@ -14106,11 +14106,11 @@ dsv4-fp4-b300-sglang-agentic-hicache: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, kv-offloading: none, conc-list: [1, 4, 8, 16, 20, 24, 32] } - - { tp: 8, kv-offloading: none, conc-list: [1, 4, 8, 16, 32, 40, 48, 52, 56, 60, 64, 72] } - - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, conc-list: [8, 16, 24, 32, 40, 64] } - - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: hicache, conc-list: [32, 40, 48, 56, 64, 72, 80, 88, 96, 128] } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [52, 72, 100, 128, 144, 196, 512] } + - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 8, 16] } + - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32, 40, 48] } + - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, conc-list: [8, 16, 24, 32, 40] } + - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: hicache, conc-list: [32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 128] } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [52, 72, 100, 128, 136, 144, 152, 196] } # DEP8 prefill uses an 8K batch because 16K OOMs in the FP4 MoE intermediate; # decode uses FULL_DECODE_ONLY after the controlled graph test restored decode From 783261d8cf23dd02d81856b4440899f6f2e0d3cd Mon Sep 17 00:00:00 2001 From: Oasis-Git Date: Thu, 9 Jul 2026 22:09:30 -0700 Subject: [PATCH 3/8] dsv4 sglang agentic: trim conc points (B200 DEP8 no-offload; B300 tp8 none) --- configs/nvidia-master.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 22add6901e..945b65a2fd 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -14088,7 +14088,7 @@ dsv4-fp4-b200-sglang-agentic-hicache: - { tp: 8, kv-offloading: dram, kv-offload-backend: hicache, conc-list: [8, 16, 32] } # DEP without HiCache is already degraded at conc 52 (2,098 tok/s/GPU), # so resolve its pre-52 cliff instead of repeating the collapsed tail. - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [16, 32, 38, 44, 46, 48, 50, 52, 54, 56, 58, 60] } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [16, 32, 38, 44, 48, 52, 54, 56, 58, 60] } - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: hicache, conc-list: [16, 32, 38, 40, 44, 48, 50, 54, 56, 60, 64, 66, 68] } # GB200 DeepSeek-V4 disaggregated AgentX frontier. The 3P/2D TEP8/TP8 curve @@ -14107,7 +14107,7 @@ dsv4-fp4-b300-sglang-agentic-hicache: - dram-utilization: 0.80 search-space: - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 8, 16] } - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32, 40, 48] } + - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32, 40] } - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, conc-list: [8, 16, 24, 32, 40] } - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: hicache, conc-list: [32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 128] } - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [52, 72, 100, 128, 136, 144, 152, 196] } From 716d955d365f9c01312f772cd0a5f3caae85a332 Mon Sep 17 00:00:00 2001 From: Oasis-Git Date: Thu, 9 Jul 2026 22:17:43 -0700 Subject: [PATCH 4/8] perf-changelog: trigger dsv4 sglang agentic B200/B300 MegaMoE + low-latency sweep --- perf-changelog.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 3425fd7c7c..bc47846aaa 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4669,3 +4669,15 @@ - "Image: lmsysorg/sglang:v0.5.11-cu130" - "11 topologies across 1k/1k and 8k/1k (prefill TP4 + decode wide-EP: 8k/1k DEP16/24/32/40, 1k/1k DEP48/56; per-node TP4 low-latency); MTP flags: speculative-algorithm EAGLE, num-steps 2, eagle-topk 1, num-draft-tokens 3" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1907 + +- config-keys: + - dsv4-fp4-b200-sglang-agentic-hicache + - dsv4-fp4-b300-sglang-agentic-hicache + description: + - "B300: rework dsv4_fp4_b300_sglang.sh into three regimes selected at launch — DP-attention -> MegaMoE DeepGEMM; pure-TP (TP8/TP4) at conc<=16 -> low-latency; otherwise flashinfer_mxfp4 baseline" + - "B300 MegaMoE (DP-attention path): switch --moe-a2a-backend to megamoe with the DeepGEMM MegaMoE env stack (SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE=1, FIX_HASH_MEGA_MOE=1, USE_FAST_MASK_EP=1, FIX_MEGA_MOE_MEMORY=1, DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8320, FIX_NEXTN_MEGA_MOE=1, SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=0); add --enable-prefill-delayer, mem-fraction-static 0.835, swa-full-tokens-ratio 0.075" + - "B300 MegaMoE rank-aware sizing: DEP8 uses chunked-prefill-size 65536 + cuda-graph-max-bs-decode 544; DEP4 uses chunked-prefill-size 32768 + cuda-graph-max-bs-decode 128 (effective per-DP chunk held at 8192)" + - "B300 low-latency (pure TP, conc<=16): --moe-runner-backend flashinfer_mxfp4 --disable-flashinfer-autotune --enable-deepseek-v4-fp4-indexer, mem-fraction-static 0.90, page-size 256 (SGLang DSV4 cookbook low-latency recipe; EAGLE speculative decoding intentionally omitted since it is unavailable here)" + - "B200: split mem-fraction-static (0.88 under DP-attention, 0.90 otherwise)" + - "nvidia-master.yaml: refine B200/B300 agentic-hicache conc-list search spaces" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2145 From 2cbb3d0d23cf0dfe85c79e9e1a5437d0f9f82710 Mon Sep 17 00:00:00 2001 From: Oasis-Git Date: Fri, 10 Jul 2026 00:43:53 -0700 Subject: [PATCH 5/8] dsv4 sglang agentic B300: set DEP4 mem-fraction to 0.93 (0.835 leaves no KV room) --- benchmarks/single_node/agentic/dsv4_fp4_b300_sglang.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang.sh index 692cc10a42..f258f8a148 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang.sh @@ -150,6 +150,13 @@ if [ "$DP_ATTENTION" = "true" ]; then CHUNKED_PREFILL_SIZE=65536 CUDA_GRAPH_ARGS=(--cuda-graph-max-bs-decode 544) else + # DEP4 (TP4/EP4) shards the model across only 4 GPUs, so per-GPU weights + # are ~2x DEP8 (233 GB loaded, ~32 GB free). At the DEP8-tuned 0.835 the + # KV pool cannot allocate (profiler floor ~0.879). Set 0.93: ~12 GB KV + # for the no-offload conc points, while keeping ~18 GB of activation/ + # CUDA-graph headroom for the MegaMoE workspace (DEP4's footprint is + # lighter than DEP8's: decode graph 128 vs 544, same 8192 eff chunk). + MEM_FRACTION_STATIC=0.93 CHUNKED_PREFILL_SIZE=32768 CUDA_GRAPH_ARGS=(--cuda-graph-max-bs-decode 128) fi From dda80a080b3cfdcc8830fdffc7b1b7d7fbec3657 Mon Sep 17 00:00:00 2001 From: Oasis-Git Date: Fri, 10 Jul 2026 01:04:24 -0700 Subject: [PATCH 6/8] dsv4 sglang agentic B200: trim DEP8/hicache conc points (nvidia-master) --- configs/nvidia-master.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 945b65a2fd..3cb8d26775 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -14085,11 +14085,11 @@ dsv4-fp4-b200-sglang-agentic-hicache: - dram-utilization: 0.80 search-space: - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 16] } - - { tp: 8, kv-offloading: dram, kv-offload-backend: hicache, conc-list: [8, 16, 32] } + - { tp: 8, kv-offloading: dram, kv-offload-backend: hicache, conc-list: [8, 16, 24] } # DEP without HiCache is already degraded at conc 52 (2,098 tok/s/GPU), # so resolve its pre-52 cliff instead of repeating the collapsed tail. - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [16, 32, 38, 44, 48, 52, 54, 56, 58, 60] } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: hicache, conc-list: [16, 32, 38, 40, 44, 48, 50, 54, 56, 60, 64, 66, 68] } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [16, 32, 38, 44, 48, 52, 56, 58, 60] } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: hicache, conc-list: [16, 32, 38, 40, 44, 48, 50, 54, 56, 60, 64, 68] } # GB200 DeepSeek-V4 disaggregated AgentX frontier. The 3P/2D TEP8/TP8 curve # covers the middle/high-interactivity range omitted by the one-decode DEP From 32dc320de416e9092b4136bbfb29918400c58099 Mon Sep 17 00:00:00 2001 From: Oasis-Git Date: Fri, 10 Jul 2026 01:46:09 -0700 Subject: [PATCH 7/8] dsv4 sglang agentic B300: bump DEP8 mem-fraction 0.835 -> 0.85 --- benchmarks/single_node/agentic/dsv4_fp4_b300_sglang.sh | 2 +- perf-changelog.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang.sh index f258f8a148..8b451c792d 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang.sh @@ -138,7 +138,7 @@ if [ "$DP_ATTENTION" = "true" ]; then --ep-size "$EP_SIZE" --moe-a2a-backend megamoe ) - MEM_FRACTION_STATIC=0.835 + MEM_FRACTION_STATIC=0.85 SWA_FULL_TOKENS_RATIO=0.075 # MegaMoE requires the default (non-compressed) attention path and fused shared experts. MODEL_ARGS=() diff --git a/perf-changelog.yaml b/perf-changelog.yaml index ac70e5ea20..41d05ff2ce 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4705,7 +4705,7 @@ - dsv4-fp4-b300-sglang-agentic-hicache description: - "B300: rework dsv4_fp4_b300_sglang.sh into three regimes selected at launch — DP-attention -> MegaMoE DeepGEMM; pure-TP (TP8/TP4) at conc<=16 -> low-latency; otherwise flashinfer_mxfp4 baseline" - - "B300 MegaMoE (DP-attention path): switch --moe-a2a-backend to megamoe with the DeepGEMM MegaMoE env stack (SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE=1, FIX_HASH_MEGA_MOE=1, USE_FAST_MASK_EP=1, FIX_MEGA_MOE_MEMORY=1, DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8320, FIX_NEXTN_MEGA_MOE=1, SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=0); add --enable-prefill-delayer, mem-fraction-static 0.835, swa-full-tokens-ratio 0.075" + - "B300 MegaMoE (DP-attention path): switch --moe-a2a-backend to megamoe with the DeepGEMM MegaMoE env stack (SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE=1, FIX_HASH_MEGA_MOE=1, USE_FAST_MASK_EP=1, FIX_MEGA_MOE_MEMORY=1, DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8320, FIX_NEXTN_MEGA_MOE=1, SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=0); add --enable-prefill-delayer, mem-fraction-static 0.85 (DEP8) / 0.93 (DEP4, whose heavier per-GPU weights need a higher floor to fit the KV pool), swa-full-tokens-ratio 0.075" - "B300 MegaMoE rank-aware sizing: DEP8 uses chunked-prefill-size 65536 + cuda-graph-max-bs-decode 544; DEP4 uses chunked-prefill-size 32768 + cuda-graph-max-bs-decode 128 (effective per-DP chunk held at 8192)" - "B300 low-latency (pure TP, conc<=16): --moe-runner-backend flashinfer_mxfp4 --disable-flashinfer-autotune --enable-deepseek-v4-fp4-indexer, mem-fraction-static 0.90, page-size 256 (SGLang DSV4 cookbook low-latency recipe; EAGLE speculative decoding intentionally omitted since it is unavailable here)" - "B200: split mem-fraction-static (0.88 under DP-attention, 0.90 otherwise)" From 2449b424cc5325284a4b177ee8fe048cf89f837a Mon Sep 17 00:00:00 2001 From: Oasis-Git Date: Fri, 10 Jul 2026 01:50:59 -0700 Subject: [PATCH 8/8] dsv4 sglang agentic B200: DEP high-conc mem-fraction 0.835->0.85 + conc-list tweaks --- benchmarks/single_node/agentic/dsv4_fp4_b200_sglang.sh | 2 +- configs/nvidia-master.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_sglang.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_sglang.sh index 3c211543af..2174f69fd4 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b200_sglang.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b200_sglang.sh @@ -152,7 +152,7 @@ MODEL_ARGS=() # spill, so it can take a larger static fraction for more KV headroom. if [ "$DP_ATTENTION" = "true" ]; then if [ "$DEP_HIGH_CONC" = "true" ]; then - MEM_FRACTION_STATIC=0.835 + MEM_FRACTION_STATIC=0.85 else MEM_FRACTION_STATIC=0.88 fi diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 3cb8d26775..70f2cc5857 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -14088,8 +14088,8 @@ dsv4-fp4-b200-sglang-agentic-hicache: - { tp: 8, kv-offloading: dram, kv-offload-backend: hicache, conc-list: [8, 16, 24] } # DEP without HiCache is already degraded at conc 52 (2,098 tok/s/GPU), # so resolve its pre-52 cliff instead of repeating the collapsed tail. - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [16, 32, 38, 44, 48, 52, 56, 58, 60] } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: hicache, conc-list: [16, 32, 38, 40, 44, 48, 50, 54, 56, 60, 64, 68] } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [16, 32, 38, 44, 48, 52, 54, 56, 58] } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: hicache, conc-list: [16, 32, 38, 40, 44, 48, 52, 54, 56, 60, 64, 68] } # GB200 DeepSeek-V4 disaggregated AgentX frontier. The 3P/2D TEP8/TP8 curve # covers the middle/high-interactivity range omitted by the one-decode DEP