From b6a9877d592fc3b23a57a0acc381f36f062341bb Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 9 Jul 2026 16:41:24 -0500 Subject: [PATCH 01/15] [AMD][AgentX] Add DeepSeek V4 MI355X agentic disagg --- .../agentic/dsv4_fp4_mi355x_sglang-disagg.sh | 172 ++++++++++ benchmarks/multi_node/amd_utils/env.sh | 134 +++++++- benchmarks/multi_node/amd_utils/job.slurm | 93 +++++- benchmarks/multi_node/amd_utils/models.yaml | 43 +++ .../multi_node/amd_utils/patches/README.md | 3 + .../patches/decode_tp_queue_agree.patch | 107 +++++++ .../multi_node/amd_utils/server_sglang.sh | 293 ++++++++++++++++-- benchmarks/multi_node/amd_utils/setup_deps.sh | 249 +++++++++++---- benchmarks/multi_node/amd_utils/submit.sh | 10 + .../multi_node/amd_utils/trace_replay.sh | 160 ++++++++++ configs/amd-master.yaml | 33 ++ perf-changelog.yaml | 7 + runners/launch_mi355x-amds.sh | 64 +++- utils/agentic/aggregation/backends/sglang.py | 2 + .../test_process_agentic_result.py | 1 + 15 files changed, 1272 insertions(+), 99 deletions(-) create mode 100755 benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh create mode 100644 benchmarks/multi_node/amd_utils/patches/decode_tp_queue_agree.patch create mode 100644 benchmarks/multi_node/amd_utils/trace_replay.sh diff --git a/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh b/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh new file mode 100755 index 0000000000..e5dde8d120 --- /dev/null +++ b/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash + +# Agentic trace-replay recipe for a disaggregated SGLang server on MI355X +# (DeepSeek-V4-Pro FP4, 1P1D TP8). +# +# CI-style sibling of dsr1_fp4_mi355x_sglang-disagg.sh: driven entirely by +# environment variables and submits a SLURM job via submit.sh. The agentic / +# HiCache-offload configuration mirrors the DSR1 recipe but uses DSV4-Pro +# specific flags (dsv4 attention backend, page-size 256, SWA settings). + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/../../benchmark_lib.sh" + +check_env_vars \ + CONC_LIST \ + ISL \ + OSL \ + IMAGE \ + SPEC_DECODING \ + MODEL_PATH \ + PREFILL_NUM_WORKERS \ + PREFILL_TP \ + PREFILL_EP \ + PREFILL_DP_ATTN \ + DECODE_NUM_WORKERS \ + DECODE_TP \ + DECODE_EP \ + DECODE_DP_ATTN \ + PREFILL_NODES \ + DECODE_NODES \ + RANDOM_RANGE_RATIO \ + DURATION \ + KV_OFFLOADING \ + IS_AGENTIC \ + FRAMEWORK + +if [[ -n "$SLURM_JOB_ID" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +fi + +set -x + +# Use upstreamed multi_node scripts (no external clone needed) +cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 + +# Set up SGL launch script-specific environment variables +export TIME_LIMIT="${TIME_LIMIT:-08:00:00}" +export MODEL_PATH=$MODEL_PATH +export MODEL_NAME=$MODEL_NAME +export CONTAINER_IMAGE=$IMAGE + +# ── Identity / result naming ── +export MODEL_PREFIX="${MODEL_PREFIX:-dsv4}" +export PRECISION="${PRECISION:-fp4}" +export RESULT_FILENAME="${RESULT_FILENAME:-${RUNNER_NAME:-dsv4-fp4-agentic}}" + +# ── Agentic benchmark params ── +export DURATION="${DURATION:-1800}" +# DSV4-Pro max model len for agentic traces (matches single-node recipe). +export MAX_MODEL_LEN="${MAX_MODEL_LEN:-1000000}" + +# ── In-tree sglang patches ── +# mori_conn.py targets hybrid-state bugs (GLM-5, Qwen3.5). DSV4-Pro uses a +# pure MoE/DSA architecture without hybrid state; skip to avoid interference. +export MORI_CONN_PATCH="${MORI_CONN_PATCH:-skip}" + +# ── Aiter fault mitigation ── +# --disable-custom-all-reduce avoids a known aiter fault on MI355X. +export DISABLE_CUSTOM_ALL_REDUCE="${DISABLE_CUSTOM_ALL_REDUCE:-0}" + +# ── KV cache offloading (HiCache) ── +# KV_OFFLOADING=none | dram (passed from YAML; default none for disagg). +# KV_OFFLOAD_BACKEND selects the backend when offloading is on; this recipe +# only implements HiCache, so "hicache" is the only supported value. +# HICACHE_TIER: L2 -> GPU + CPU-DRAM host pool. L3 -> + Mooncake store. +export KV_OFFLOADING="${KV_OFFLOADING:-none}" +if [[ "$KV_OFFLOADING" != "none" ]]; then + export KV_OFFLOAD_BACKEND="${KV_OFFLOAD_BACKEND:-hicache}" +fi +# HiCache/Mooncake tunables only matter when KV offloading is enabled. +if [[ "$KV_OFFLOADING" != "none" && "${KV_OFFLOAD_BACKEND:-}" == "hicache" ]]; then + export HICACHE_TIER="${HICACHE_TIER:-L2}" + export HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-1}" + # DSV4 uses page-size 256 (set in models.yaml); HiCache must match. + export HICACHE_PAGE_SIZE="${HICACHE_PAGE_SIZE:-256}" + # HiCache ratio (host pool = ratio * GPU KV pool). Default derived in server_sglang.sh. + export HICACHE_RATIO="${HICACHE_RATIO:-}" + + # ── HiCache layout/backend by tier ── + # L3 (Mooncake): page_first + direct + write_through + storage=mooncake + # L2 (CPU DRAM): layer_first + direct + write_through_selective + storage=none + # NOTE: write_through_selective evicts only under GPU memory pressure, avoiding + # the mori RDMA race that causes GPU memory access faults with write_through. + if [[ "${HICACHE_TIER^^}" == "L3" ]]; then + export HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first}" + export HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" + export HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" + export HICACHE_STORAGE_BACKEND="${HICACHE_STORAGE_BACKEND:-mooncake}" + else + export HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first}" + export HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" + export HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" + export HICACHE_STORAGE_BACKEND="${HICACHE_STORAGE_BACKEND:-}" + fi + export HICACHE_PREFETCH_POLICY="${HICACHE_PREFETCH_POLICY:-best_effort}" + # Shared nodes: use non-default Mooncake ports to avoid collisions. + export MC_MASTER_PORT="${MC_MASTER_PORT:-58137}" + export MC_METADATA_PORT="${MC_METADATA_PORT:-8080}" + export MC_METRICS_PORT="${MC_METRICS_PORT:-19003}" + export MC_MASTER_THREADS="${MC_MASTER_THREADS:-64}" + export MC_EVICTION_HIGH_WATERMARK="${MC_EVICTION_HIGH_WATERMARK:-0.95}" + export MC_PATCH_HOSTPOOL="${MC_PATCH_HOSTPOOL:-1}" + export MC_PROTOCOL="${MC_PROTOCOL:-tcp}" + export MC_GLOBAL_SEG="${MC_GLOBAL_SEG:-64gb}" + export MC_DEVICE="${MC_DEVICE:-}" + export MC_MASTER_ADDR="${MC_MASTER_ADDR:-}" + export MC_METADATA_SERVER="${MC_METADATA_SERVER:-}" +fi + +# ── MoRIIO RDMA Send Queue tuning ── +export MORI_IO_SQ_BACKOFF_TIMEOUT_US="${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-500000}" +export MORI_IO_QP_MAX_SEND_WR="${MORI_IO_QP_MAX_SEND_WR:-32768}" + +# ── SGLang PD router policy + server metrics ── +export PREFILL_ROUTER_POLICY="${PREFILL_ROUTER_POLICY:-cache_aware}" +export ENABLE_METRICS="${ENABLE_METRICS:-1}" + +# ── MTP ── +export DECODE_MTP_SIZE="${DECODE_MTP_SIZE:-0}" + +# Derive EP/DP enable flags from the topology inputs. +if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then +export PREFILL_ENABLE_EP=false +else +export PREFILL_ENABLE_EP=true +fi + +if [[ "$PREFILL_DP_ATTN" == "true" ]]; then +export PREFILL_ENABLE_DP=true +else +export PREFILL_ENABLE_DP=false +fi + +if [[ "${DECODE_EP:-1}" -eq 1 ]]; then +export DECODE_ENABLE_EP=false +else +export DECODE_ENABLE_EP=true +fi + +if [[ "$DECODE_DP_ATTN" == "true" ]]; then +export DECODE_ENABLE_DP=true +else +export DECODE_ENABLE_DP=false +fi + +# Launch the job. CONC_LIST is space-delimited in YAML; submit.sh wants 'x'. +JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ + $PREFILL_NUM_WORKERS \ + $DECODE_NODES \ + $DECODE_NUM_WORKERS \ + $ISL $OSL "${CONC_LIST// /x}" inf \ + ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ + ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ + ${PREFILL_TP} ${DECODE_TP} \ + ${RANDOM_RANGE_RATIO}) + +if [[ $? -ne 0 ]]; then + echo "Failed to submit job" >&2 + exit 1 +fi + +echo "$JOB_ID" diff --git a/benchmarks/multi_node/amd_utils/env.sh b/benchmarks/multi_node/amd_utils/env.sh index a05182bf16..2a3005c644 100755 --- a/benchmarks/multi_node/amd_utils/env.sh +++ b/benchmarks/multi_node/amd_utils/env.sh @@ -12,6 +12,23 @@ set -x ENGINE="${ENGINE:-sglang-disagg}" export PYTHONDONTWRITEBYTECODE=1 +# ============================================================================= +# HiCache / Mooncake settings from job.slurm +# ============================================================================= +# job.slurm writes the recipe-provided HiCache/Mooncake tunables to +# hicache_mc_.env and mounts it read-only at /config/hicache_mc.env. Source +# it here (auto-export) so values like HICACHE_PAGE_SIZE=256 reach the container +# before server_sglang.sh applies its "${VAR:-default}" fallbacks. Without this +# the vars arrive unset and server_sglang.sh defaults HICACHE_PAGE_SIZE to 1, +# overriding the recipe's --page-size. Empty values in the file are harmless: +# the "${VAR:-default}" fallbacks still treat "" as unset. +if [[ -f /config/hicache_mc.env ]]; then + set -a + source /config/hicache_mc.env + set +a + echo "[env.sh] sourced HiCache config from /config/hicache_mc.env (HICACHE_PAGE_SIZE=${HICACHE_PAGE_SIZE:-unset})" +fi + # ============================================================================= # Shared: IBDEVICES detection # ============================================================================= @@ -50,11 +67,11 @@ export NCCL_IB_HCA=${NCCL_IB_HCA:-$IBDEVICES} # ============================================================================= # Shared by the vLLM MoRIIOConnector and the SGLang/MoRI KV-transfer path. -export MORI_IO_SQ_BACKOFF_TIMEOUT_US=50000 -export MORI_IO_QP_MAX_SEND_WR=16384 -export MORI_IO_QP_MAX_CQE=32768 -export MORI_IO_QP_MAX_SGE=2 -export MORI_IO_TC_DISABLE=0 +export MORI_IO_SQ_BACKOFF_TIMEOUT_US="${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-50000}" +export MORI_IO_QP_MAX_SEND_WR="${MORI_IO_QP_MAX_SEND_WR:-16384}" +export MORI_IO_QP_MAX_CQE="${MORI_IO_QP_MAX_CQE:-32768}" +export MORI_IO_QP_MAX_SGE="${MORI_IO_QP_MAX_SGE:-2}" +export MORI_IO_TC_DISABLE="${MORI_IO_TC_DISABLE:-0}" # QoS/DSCP configuration # Priority order: 1) Set by runner, 2) Detect via nicctl, 3) Detect from hostname @@ -66,8 +83,12 @@ elif command -v nicctl &> /dev/null; then $1 == "DSCP" && $2 == ":" && $NF == p { print $3; exit }') + # nicctl may emit trailing commas (e.g. "24,"); keep the leading integer so the + # arithmetic can't choke and unparseable output falls back to hostname detection. + ND_PRIO="${ND_PRIO%%,*}"; ND_PRIO="${ND_PRIO//[!0-9]/}" + ND_DSCP="${ND_DSCP%%,*}"; ND_DSCP="${ND_DSCP//[!0-9]/}" - if [[ -n "$ND_DSCP" ]] && [[ -n "$ND_PRIO" ]]; then + if [[ "$ND_DSCP" =~ ^[0-9]+$ ]] && [[ "$ND_PRIO" =~ ^[0-9]+$ ]]; then TC=$(( 4 * ND_DSCP )) export MORI_RDMA_SL=$ND_PRIO export MORI_IO_SL=$ND_PRIO @@ -149,7 +170,11 @@ if [[ "$ENGINE" == "vllm-disagg" ]]; then $1 == "DSCP" && $2 == ":" && $NF == p { print $3; exit }') - if [[ -n "$ND_DSCP" ]] && [[ -n "$ND_PRIO" ]]; then + # nicctl may emit trailing commas (e.g. "24,"); keep the leading integer so the + # arithmetic can't choke and unparseable output falls back to hostname detection. + ND_PRIO="${ND_PRIO%%,*}"; ND_PRIO="${ND_PRIO//[!0-9]/}" + ND_DSCP="${ND_DSCP%%,*}"; ND_DSCP="${ND_DSCP//[!0-9]/}" + if [[ "$ND_DSCP" =~ ^[0-9]+$ ]] && [[ "$ND_PRIO" =~ ^[0-9]+$ ]]; then export UCX_IB_TRAFFIC_CLASS=$(( 4 * ND_DSCP )) export UCX_IB_SL=$ND_PRIO echo "[INFO] Detected QoS from nicctl: UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS, UCX_IB_SL=$UCX_IB_SL" @@ -189,14 +214,27 @@ else export AITER_LOG_LEVEL=ERROR export SGLANG_MORI_DISPATCH_DTYPE=auto - export MORI_COMBINE_DTYPE_PREFILL=fp8_direct_cast - export MORI_COMBINE_DTYPE_DECODE=fp8 + # export MORI_COMBINE_DTYPE_PREFILL=fp8_direct_cast + # export MORI_COMBINE_DTYPE_DECODE=fp8 + export MORI_COMBINE_DTYPE_PREFILL="" + export MORI_COMBINE_DTYPE_DECODE="" export SGLANG_MORI_QP_PER_TRANSFER=4 export SGLANG_MORI_NUM_WORKERS=4 + # Keep these as overridable defaults (not hard assignments), otherwise + # later tuning blocks cannot raise them for high-concurrency runs. + # export MORI_IO_SQ_BACKOFF_TIMEOUT_US="${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-500000}" + + # export MORI_IO_QP_MAX_SEND_WR="${MORI_IO_QP_MAX_SEND_WR:-16384}" + # export MORI_IO_QP_MAX_CQE=32768 + # export MORI_IO_QP_MAX_SGE=1 + + # export MORI_IO_TC_DISABLE=0 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600 export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600 + export SGLANG_HEALTH_CHECK_TIMEOUT=600 + # GLM-5: uses NSA (not MLA), needs fused-decode-MLA disabled + fast loading if [[ "$MODEL_NAME" == "GLM-5-FP8" ]]; then export SGLANG_ROCM_FUSED_DECODE_MLA=0 @@ -237,4 +275,80 @@ else # FIXME: WA for latest upstream 0305 image export PYTHONPATH=/sgl-workspace/aiter:${PYTHONPATH} -fi + # ========================================================================= + # DeepSeek-V4-Pro PD recipe overrides + # Placed at the end of the SGLang env block so it wins over the global + # MoRI/SGLang defaults set above. Mirrors the validated DSv4 manual PD + # commands (ported from InferenceX amd/dsv4_sgl_di). These SGLANG_OPT_* / + # AITER_* kernel-routing knobs steer DSv4 away from the default aiter CK + # fused-MoE path, which raises "Unsupported kernel config for moe heuristic + # dispatch" at decode time on this fp4 model (job 19034 crash). Only the + # SGLang/MoRI env knobs are pinned here; CLI flags live in models.yaml and + # the cluster NIC/socket vars stay runner-derived. + # ========================================================================= + if [[ "$MODEL_NAME" == "DeepSeek-V4-Pro" ]]; then + # MoRI RDMA send-queue depth for DSv4 (overrides the global default above). + export MORI_IO_QP_MAX_SEND_WR=32767 + # Unified radix tree: cache impl with per-component (full-attn / SWA) + # management for hybrid-attention models. Set unconditionally (not gated on + # hicache) so all SGLang runs use it. + export SGLANG_ENABLE_UNIFIED_RADIX_TREE=1 + # Proactively free out-of-window SWA KV slots during chunked prefill. + # Without it, in-flight requests pin SWA KV for their whole context, keeping + # the SWA pool under constant eviction pressure; under LRU the trailing + # window of cached sessions gets flushed, making prefix-cache hits bimodal + # and collapsing the effective hit rate on multi-turn agentic workloads. + export SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS=1 + + # MoRI dispatch/combine dtypes: auto for both roles (not the fp8 split default) + export SGLANG_MORI_DISPATCH_DTYPE=auto + export MORI_COMBINE_DTYPE_PREFILL=auto + export MORI_COMBINE_DTYPE_DECODE=auto + + # Per-role MoRI dispatch sizing (used by the harness chunked/MoE math) + export MORI_MAX_DISPATCH_TOKENS_PREFILL=8192 + export MORI_MAX_DISPATCH_TOKENS_DECODE=64 + unset MORI_MOE_MAX_INPUT_TOKENS_PREFILL + unset MORI_MOE_MAX_INPUT_TOKENS_DECODE + + # PER_RANK dispatch tokens pinned independently (16384 prefill / 128 + # decode); server_sglang.sh prefers these over the MORI_MAX_DISPATCH_* + # coupling when set. + export MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_PREFILL=16384 + export MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_DECODE=128 + + # Fixed inter-kernel switch threshold (not derived). + export SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD=4096 + + # Overlap plan stream on for DSv4 (global default is 0) + # export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 + + # DSv4 model kernel routing (mirrors the single-node / manual PD recipe) + export SGLANG_DEFAULT_THINKING=1 + export SGLANG_DSV4_REASONING_EFFORT=max + export SGLANG_OPT_DEEPGEMM_HC_PRENORM=false + export SGLANG_USE_AITER=1 + export SGLANG_USE_ROCM700A=0 + export SGLANG_OPT_USE_FUSED_COMPRESS=true + export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton + export SGLANG_OPT_FP8_WO_A_GEMM=false + export SGLANG_OPT_USE_JIT_INDEXER_METADATA=false + export SGLANG_OPT_USE_TOPK_V2=false + export SGLANG_OPT_USE_AITER_INDEXER=${SGLANG_OPT_USE_AITER_INDEXER:-true} + export SGLANG_OPT_USE_TILELANG_INDEXER=false + export SGLANG_OPT_USE_TILELANG_MHC_PRE=false + export SGLANG_OPT_USE_TILELANG_MHC_POST=false + export SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1 + export SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true + export SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false + export SGLANG_ROCM_USE_MULTI_STREAM=false + export AITER_BF16_FP8_MOE_BOUND=0 + export SGLANG_EAGER_INPUT_NO_COPY=true + export SGLANG_SHARED_EXPERT_TP1=1 + export SGLANG_DP_SHARED_EXPERT_LOCAL=1 + export SGLANG_DP_USE_GATHERV=1 + export SGLANG_DP_USE_REDUCE_SCATTER=1 + export GPU_MAX_HW_QUEUES=5 + fi + +fi \ No newline at end of file diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm index 4f50a4962c..72f4d9dc9d 100755 --- a/benchmarks/multi_node/amd_utils/job.slurm +++ b/benchmarks/multi_node/amd_utils/job.slurm @@ -202,11 +202,41 @@ else fi } - if check_model_path "$MODEL_DIR/$MODEL_NAME" "$MODEL_DIR"; then + # Extract hf_dir from models.yaml (same as vllm-disagg path above) + SGL_DISK_DIR_NAME=$(awk '/^'"$MODEL_NAME"':/{found=1; next} + found && /^[^ ]/{exit} + found && /hf_dir:/{gsub(/[" ]/, "", $2); print $2; exit}' "$MODELS_YAML") + SGL_DISK_DIR_NAME="${SGL_DISK_DIR_NAME:-$MODEL_NAME}" + + # Prefer the caller-supplied MODEL_PATH (recipe scripts set this explicitly); + # fall back to MODEL_DIR/hf_dir then MODEL_DIR/MODEL_NAME. + if [[ -n "${MODEL_PATH:-}" && "$MODEL_PATH" != "$MODEL_DIR" ]]; then + # Caller already resolved the path (e.g. MODEL_PATH=/it-share/hf_cache/models--...) + # Use it directly if it exists on all nodes, otherwise try subdirectory combos. + if check_model_path "$MODEL_PATH" "MODEL_PATH (caller-supplied)"; then + echo "Selected MODEL_PATH: $MODEL_PATH (caller-supplied, available on all nodes)" + elif check_model_path "$MODEL_PATH/$SGL_DISK_DIR_NAME" "$MODEL_PATH/$SGL_DISK_DIR_NAME"; then + MODEL_PATH="$MODEL_PATH/$SGL_DISK_DIR_NAME" + echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" + elif check_model_path "$MODEL_PATH/$MODEL_NAME" "$MODEL_PATH/$MODEL_NAME"; then + MODEL_PATH="$MODEL_PATH/$MODEL_NAME" + echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" + else + echo "FATAL ERROR: Model '$MODEL_NAME' not found on ALL allocated nodes in:" + echo " - $MODEL_PATH" + echo " - $MODEL_PATH/$SGL_DISK_DIR_NAME" + echo " - $MODEL_PATH/$MODEL_NAME" + exit 1 + fi + elif check_model_path "$MODEL_DIR/$SGL_DISK_DIR_NAME" "$MODEL_DIR/$SGL_DISK_DIR_NAME"; then + MODEL_PATH="$MODEL_DIR/$SGL_DISK_DIR_NAME" + echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" + elif check_model_path "$MODEL_DIR/$MODEL_NAME" "$MODEL_DIR"; then MODEL_PATH="$MODEL_DIR/$MODEL_NAME" echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" else echo "FATAL ERROR: Model '$MODEL_NAME' not found on ALL allocated nodes in:" + echo " - $MODEL_DIR/$SGL_DISK_DIR_NAME" echo " - $MODEL_DIR/$MODEL_NAME" exit 1 fi @@ -366,7 +396,6 @@ DOCKER_ENV_COMMON=( -e BENCH_MAX_CONCURRENCY=\$BENCH_MAX_CONCURRENCY -e BENCH_REQUEST_RATE=\$BENCH_REQUEST_RATE -e TQDM_MININTERVAL=\$TQDM_MININTERVAL - -e DRY_RUN=\$DRY_RUN -e BENCHMARK_LOGS_DIR=/benchmark_logs -e ENGINE=\$ENGINE -e WS_PATH=${WS_PATH} @@ -379,14 +408,44 @@ DOCKER_ENV_COMMON=( -e RUNNER_TYPE=\$RUNNER_TYPE -e RESULT_FILENAME=\$RESULT_FILENAME -e SPEC_DECODING=\$SPEC_DECODING + # DISAGG was never forwarded into the container at all (not even under a + # different name), so process_agentic_result.py's env_bool("DISAGG") always + # defaulted to false in the result JSON regardless of the actual topology. + -e DISAGG=\${DISAGG:-false} -e PREFILL_TP_SIZE=\$PREFILL_TP_SIZE + # PREFILL_TP/DECODE_TP/*_NUM_WORKERS (below, undecorated -- distinct from the + # *_SIZE vars server_sglang.sh uses for launch args) are what + # process_agentic_result.py's _gpu_shape() reads for multinode runs. Without + # these the container never sees them (only *_SIZE was passed), so agentic + # result JSONs silently recorded tp=0 / prefill_tp=0 / prefill_num_workers=0 + # for every multinode run. + -e PREFILL_TP=\$PREFILL_TP + -e PREFILL_NUM_WORKERS=\$PREFILL_NUM_WORKERS -e PREFILL_ENABLE_EP=\$PREFILL_ENABLE_EP -e PREFILL_ENABLE_DP=\$PREFILL_ENABLE_DP + -e PREFILL_CONTEXT_LENGTH=\${PREFILL_CONTEXT_LENGTH:-} + -e PREFILL_CHUNKED_PREFILL_SIZE=\${PREFILL_CHUNKED_PREFILL_SIZE:-} + -e SGLANG_AITER_MLA_PERSIST=\${SGLANG_AITER_MLA_PERSIST:-0} + -e DISABLE_CUSTOM_ALL_REDUCE=\${DISABLE_CUSTOM_ALL_REDUCE:-0} + -e MAX_MODEL_LEN=\${MAX_MODEL_LEN:-} + -e DURATION=\${DURATION:-1800} + -e IS_AGENTIC=\${IS_AGENTIC:-0} + -e KV_OFFLOADING=\${KV_OFFLOADING:-none} + -e KV_OFFLOAD_BACKEND=\${KV_OFFLOAD_BACKEND:-} + -e TOTAL_CPU_DRAM_GB=\${TOTAL_CPU_DRAM_GB:-} + -e ENABLE_METRICS=\${ENABLE_METRICS:-0} + -e PREFILL_ROUTER_POLICY=\${PREFILL_ROUTER_POLICY:-random} + -e DECODE_ROUTER_POLICY=\${DECODE_ROUTER_POLICY:-random} + -e MORI_IO_SQ_BACKOFF_TIMEOUT_US=\${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-} + -e MORI_IO_QP_MAX_SEND_WR=\${MORI_IO_QP_MAX_SEND_WR:-} -e DECODE_TP_SIZE=\$DECODE_TP_SIZE + -e DECODE_TP=\$DECODE_TP + -e DECODE_NUM_WORKERS=\$DECODE_NUM_WORKERS -e DECODE_ENABLE_EP=\$DECODE_ENABLE_EP -e DECODE_ENABLE_DP=\$DECODE_ENABLE_DP -e DECODE_MTP_SIZE=\$DECODE_MTP_SIZE -e IS_MULTINODE=\$IS_MULTINODE + -e DRY_RUN=\${DRY_RUN:-0} ) # Engine-specific env vars @@ -427,6 +486,35 @@ else ) fi +# HiCache / Mooncake settings are delivered via a bind-mounted config file rather +# than a long list of docker -e flags. Write it once to the shared benchmark-logs +# dir (already a host path, visible on every node) and mount it read-only at +# /config/hicache_mc.env, where env.sh sources it before applying its defaults. +# Empty values are preserved so env.sh's "${VAR:-default}" fallbacks still apply. +HICACHE_MC_CONFIG="${BENCHMARK_LOGS_DIR}/hicache_mc_${SLURM_JOB_ID}.env" +cat > "$HICACHE_MC_CONFIG" < $HICACHE_MC_CONFIG" + # Engine-specific container filter for pre-clean CONT_FILTER="name=^container_${ENGINE}_" @@ -588,6 +676,7 @@ fi -v /tmp:/run_logs \ -v ${BENCHMARK_LOGS_DIR}:/benchmark_logs \ -v ${DI_REPO_DIR}:${DOCKER_MOUNT_PATH} \ + -v ${HICACHE_MC_CONFIG}:/config/hicache_mc.env:ro \ ${EXTRA_DOCKER_MOUNTS:-} \ \${RDMA_MOUNTS[@]+"\${RDMA_MOUNTS[@]}"} \ ${DOCKER_ENV_COMMON[*]} \ diff --git a/benchmarks/multi_node/amd_utils/models.yaml b/benchmarks/multi_node/amd_utils/models.yaml index 605a377be9..698fcbc5eb 100644 --- a/benchmarks/multi_node/amd_utils/models.yaml +++ b/benchmarks/multi_node/amd_utils/models.yaml @@ -349,3 +349,46 @@ DeepSeek-R1-0528-MXFP4-v2: max_running_requests: 128 chunked_prefill_size: 262144 cuda_graph_bs_range: "1-128" + +# DeepSeek-V4-Pro PD-disaggregation recipe (MI355X, SGLang + MoRI). +# KV transfer = mori for both topologies (pure-TP and DEP); the DP path additionally +# routes the MoE all-to-all through mori (--moe-a2a-backend mori) with dp-attention. +# DSv4-specific kernel routing (unified_kv_triton, AITER indexer, fp8 wo_a fallback, +# thinking/reasoning-effort, dispatch dtypes, per-role PER_RANK dispatch tokens) is set +# in env.sh's DeepSeek-V4-Pro block. The bench client uses --dsv4 framing (bench.sh). +# prefill.disable_cuda_graph routes prefill to --disable-cuda-graph; decode keeps +# --cuda-graph-bs. See dsv4_mi355x_sglang_disagg_plan.md. hf_dir: "models--deepseek-ai--DeepSeek-V4-Pro" +DeepSeek-V4-Pro: + base_flags: "--watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend dsv4 --page-size 256 --swa-full-tokens-ratio 0.1 --disable-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --disaggregation-transfer-backend mori --log-level error --log-level-http error" + dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" + ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" + prefill: + mem_fraction_static: 0.8 + disable_radix_cache: true + disable_cuda_graph: true + dp: + max_running_requests: 1024 + chunked_prefill_size: 65280 # dsv4 compressor kernel uint16 token cap (255*256) + context_length: 1048576 + max_total_tokens: 1048576 + no_dp: + max_running_requests: 128 + # Small prefill chunks interleave long-context agentic prefills across + # requests instead of letting one ~100K-token prefill monopolize the + # engine (the conc>=16 queue-saturation / decode-stall failure mode). + # Mirrors the single-node DSv4 agentic recipe (dsv4_fp4_mi355x.sh=8192). + # Was 65280 (255*256, the dsv4 compressor kernel uint16 token cap); 8192 + # (32*256) stays a page-size multiple well under that cap. + chunked_prefill_size: 8192 + context_length: 1048576 + max_total_tokens: 1048576 + decode: + mem_fraction_static: 0.85 + prefill_round_robin_balance: true + disagg_decode_enable_radix_cache: false + dp: + max_running_requests: 1024 + cuda_graph_bs_range: "1-128" + no_dp: + max_running_requests: 128 + cuda_graph_bs_range: "1-128" diff --git a/benchmarks/multi_node/amd_utils/patches/README.md b/benchmarks/multi_node/amd_utils/patches/README.md index 765d571b27..55abad7a5f 100644 --- a/benchmarks/multi_node/amd_utils/patches/README.md +++ b/benchmarks/multi_node/amd_utils/patches/README.md @@ -8,6 +8,9 @@ block our benchmark + accuracy configs — so we can keep reusing the - `mori_conn.py` — single-file overlay (bind-mounted) for the **sglang** MoRI backend. +- `decode_tp_queue_agree.patch` — reference-only unified diff for the local + TP-rank decode queue agreement fix. Its runtime invocation is currently + disabled in `../setup_deps.sh`. > Note: the vLLM MoRIIO `minimax-m3` overlay (`moriio/`) was retired once the > upstream fixes (vLLM #46039 / #46290 / #46332) shipped in the ROCm nightly diff --git a/benchmarks/multi_node/amd_utils/patches/decode_tp_queue_agree.patch b/benchmarks/multi_node/amd_utils/patches/decode_tp_queue_agree.patch new file mode 100644 index 0000000000..e09c02112f --- /dev/null +++ b/benchmarks/multi_node/amd_utils/patches/decode_tp_queue_agree.patch @@ -0,0 +1,107 @@ +--- a/python/sglang/srt/disaggregation/decode.py 2026-06-25 02:44:11.274035348 +0000 ++++ b/python/sglang/srt/disaggregation/decode.py 2026-06-25 02:44:11.276035310 +0000 +@@ -1616,8 +1616,57 @@ + ) + kv_manager._staging_handler = self.staging_handler + ++ def _agree_and_order_queue(self) -> Tuple[List["DecodeRequest"], List["DecodeRequest"]]: ++ """Split self.queue into (gated, deferred) using a tp-group agreement. ++ ++ The metadata gate (utils.poll_and_all_reduce) issues a tp-group all_reduce ++ whose tensor shape == len(self.queue) and whose per-index meaning is the ++ i-th queued request. That is only correct if every tp rank polls an ++ IDENTICAL queue (same requests, same order). Per-rank prealloc/enqueue ++ timing can leave queues with different membership or order, which desyncs ++ the collective -> deadlock (the JID-17417 decode hang). ++ ++ We all_gather the local request ids, keep only requests present on EVERY ++ rank, and order them deterministically (request ids are rank-invariant), so ++ the subsequent gate runs a matching collective on all ranks. Requests not ++ yet on every rank are returned as `deferred` and retried on a later call. ++ ++ NOTE: every rank that reaches pop_transferred must call this (it contains a ++ collective). That holds for the same reason the existing gate is safe: ++ pop_transferred is entered rank-symmetrically over self.gloo_group. ++ """ ++ tp_size = torch.distributed.get_world_size(self.gloo_group) ++ if tp_size <= 1: ++ return self.queue, [] ++ ++ local_rids = [dr.req.rid for dr in self.queue] ++ gathered: List[Optional[List[str]]] = [None] * tp_size ++ torch.distributed.all_gather_object( ++ gathered, local_rids, group=self.gloo_group ++ ) ++ ++ common = set(gathered[0] or []) ++ for rids in gathered[1:]: ++ common &= set(rids or []) ++ ++ if not common: ++ return [], list(self.queue) ++ ++ by_rid = {dr.req.rid: dr for dr in self.queue} ++ # sorted() yields an identical order on every rank (rids are rank-invariant). ++ gated = [by_rid[rid] for rid in sorted(common)] ++ deferred = [dr for dr in self.queue if dr.req.rid not in common] ++ return gated, deferred ++ + def pop_transferred(self, rids_to_check: Optional[List[str]] = None) -> List[Req]: ++ # Agree on a tp-rank-identical, identically-ordered subset BEFORE issuing any ++ # metadata-gate collective. Do NOT add a local `if not self.queue` early ++ # return here: an empty-queue rank must still join the agreement collective, ++ # otherwise it skips it and desyncs the tp group (root cause of the hang). ++ self.queue, _deferred = self._agree_and_order_queue() + if not self.queue: ++ # Empty agreement is rank-symmetric: all ranks skip the gate together. ++ self.queue = _deferred + return [] + + if self.scheduler.enable_decode_hicache: +@@ -1731,6 +1780,10 @@ + self.queue = [ + entry for i, entry in enumerate(self.queue) if i not in indices_to_remove + ] ++ # Re-attach requests that were not yet present on every tp rank this ++ # iteration; they are gated again on a later call once all ranks have them. ++ if _deferred: ++ self.queue.extend(_deferred) + + return transferred_reqs + +@@ -1946,8 +1999,33 @@ + # try to resume retracted requests if there are enough space for another `num_reserved_decode_tokens` decode steps + resumed_reqs = self.disagg_decode_prealloc_queue.resume_retracted_reqs() + self.waiting_queue.extend(resumed_reqs) +- if len(self.disagg_decode_prealloc_queue.retracted_queue) > 0: +- # if there are still retracted requests, we do not allocate new requests ++ # PATCH(call-site tp-agreement): the retracted-queue early return below is ++ # rank-divergent — retracted_queue length is per-rank (per-rank KV-cache ++ # pressure). A divergent return skips the polling_count increment and the ++ # pop_transferred() collectives on some ranks, permanently desyncing the tp ++ # group: one rank ends up a full collective ahead, so pop_transferred's ++ # _agree_and_order_queue all_gather_object (decode.py:1644) on the lagging ++ # ranks lines up against the metadata-gate all_reduce (decode.py:1591) on ++ # the leading rank -> mismatched-collective deadlock (GPU 0%, detokenizer ++ # heartbeat freeze; JID-17445). process_decode_queue is called ++ # unconditionally every event-loop iteration on every rank, so an all_reduce ++ # here (before any divergent return) is rank-symmetric. Hold off new ++ # allocation iff ANY rank still has retracted reqs, so all ranks branch ++ # identically every iteration and pop_transferred is entered symmetrically. ++ _agree_gg = self.disagg_decode_transfer_queue.gloo_group ++ _local_retracted = ( ++ 1 if len(self.disagg_decode_prealloc_queue.retracted_queue) > 0 else 0 ++ ) ++ if torch.distributed.get_world_size(_agree_gg) > 1: ++ _retracted_t = torch.tensor([_local_retracted], dtype=torch.int32) ++ torch.distributed.all_reduce( ++ _retracted_t, op=torch.distributed.ReduceOp.MAX, group=_agree_gg ++ ) ++ _any_retracted = bool(_retracted_t.item()) ++ else: ++ _any_retracted = bool(_local_retracted) ++ if _any_retracted: ++ # if any rank still has retracted requests, no rank allocates new ones + return + + if not hasattr(self, "polling_count"): diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh index 34351b1e43..47ccfee927 100755 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ b/benchmarks/multi_node/amd_utils/server_sglang.sh @@ -119,12 +119,14 @@ def parse_range(cuda_range, default_start, default_end): print(f'MODEL_BASE_FLAGS=\"{m.get(\"base_flags\", \"\")}\"') print(f'MODEL_MTP_FLAGS=\"{m.get(\"mtp_flags\", \"\")}\"') print(f'MODEL_DP_FLAGS=\"{m.get(\"dp_flags\", \"\")}\"') +print(f'MODEL_EP_FLAGS=\"{m.get(\"ep_flags\", \"\")}\"') prefill = m.get('prefill', {}) decode = m.get('decode', {}) print(f'PREFILL_MEM_FRACTION_STATIC=\"{prefill.get(\"mem_fraction_static\", 0.8)}\"') print(f'PREFILL_DISABLE_RADIX_CACHE=\"{prefill.get(\"disable_radix_cache\", True)}\"') +print(f'PREFILL_DISABLE_CUDA_GRAPH=\"{prefill.get(\"disable_cuda_graph\", False)}\"') dp = prefill.get('dp', {}) no_dp = prefill.get('no_dp', {}) @@ -136,12 +138,15 @@ print(f'PREFILL_MAX_TOTAL_TOKENS_DP=\"{dp.get(\"max_total_tokens\", \"\")}\"') print(f'PREFILL_ENABLE_TWO_BATCH_OVERLAP_DP=\"{dp.get(\"enable_two_batch_overlap\", False)}\"') print(f'PREFILL_MAX_RUNNING_REQUESTS_NO_DP=\"{no_dp.get(\"max_running_requests\", 128)}\"') print(f'PREFILL_CHUNKED_PREFILL_SIZE_NO_DP=\"{eval_formula(no_dp.get(\"chunked_prefill_size\", 262144))}\"') +print(f'PREFILL_CONTEXT_LENGTH_NO_DP=\"{no_dp.get(\"context_length\", \"\")}\"') +print(f'PREFILL_MAX_TOTAL_TOKENS_NO_DP=\"{no_dp.get(\"max_total_tokens\", \"\")}\"') s, e = parse_range(no_dp.get('cuda_graph_bs_range', '1-128'), 1, 128) print(f'PREFILL_CUDA_GRAPH_BS_NO_DP_START=\"{s}\"') print(f'PREFILL_CUDA_GRAPH_BS_NO_DP_END=\"{e}\"') print(f'DECODE_MEM_FRACTION_STATIC=\"{decode.get(\"mem_fraction_static\", 0.85)}\"') print(f'DECODE_PREFILL_ROUND_ROBIN_BALANCE=\"{decode.get(\"prefill_round_robin_balance\", True)}\"') +print(f'DECODE_DISAGG_ENABLE_RADIX_CACHE=\"{decode.get(\"disagg_decode_enable_radix_cache\", False)}\"') dp = decode.get('dp', {}) ep_only = decode.get('ep_only', {}) @@ -183,8 +188,8 @@ else prefill_cuda_graph_bs=($(seq $PREFILL_CUDA_GRAPH_BS_NO_DP_START $PREFILL_CUDA_GRAPH_BS_NO_DP_END)) prefill_max_running_requests=$PREFILL_MAX_RUNNING_REQUESTS_NO_DP prefill_chunked_prefill_size=$PREFILL_CHUNKED_PREFILL_SIZE_NO_DP - prefill_context_length="" - prefill_max_total_tokens="" + prefill_context_length=$PREFILL_CONTEXT_LENGTH_NO_DP + prefill_max_total_tokens=$PREFILL_MAX_TOTAL_TOKENS_NO_DP prefill_enable_two_batch_overlap="false" fi @@ -193,7 +198,7 @@ if [[ "$PREFILL_ENABLE_DP" == "true" ]] && [[ "$PREFILL_ENABLE_EP" == "true" ]]; prefill_max_running_requests=$BENCH_MAX_CONC_VALUE prefill_dp_ranks=$PREFILL_TP_SIZE # MORI_MAX_DISPATCH_TOKENS_PREFILL stays at 8192 (no change) - MORI_MOE_MAX_INPUT_TOKENS_PREFILL=$((MORI_MAX_DISPATCH_TOKENS_PREFILL * prefill_dp_ranks / 2)) + # MORI_MOE_MAX_INPUT_TOKENS_PREFILL=$((MORI_MAX_DISPATCH_TOKENS_PREFILL * prefill_dp_ranks / 2)) echo "[DP+EP override] Prefill: max-running-requests=$prefill_max_running_requests, MOE_MAX_INPUT=$MORI_MOE_MAX_INPUT_TOKENS_PREFILL" fi @@ -214,7 +219,7 @@ if [[ "$DECODE_ENABLE_DP" == "true" ]] && [[ "$DECODE_ENABLE_EP" == "true" ]]; t decode_max_running_requests=$BENCH_MAX_CONC_VALUE decode_dp_ranks=$DECODE_TP_SIZE MORI_MAX_DISPATCH_TOKENS_DECODE=$((BENCH_MAX_CONC_VALUE / decode_dp_ranks)) - MORI_MOE_MAX_INPUT_TOKENS_DECODE=$((MORI_MAX_DISPATCH_TOKENS_DECODE * decode_dp_ranks * 7 / 10)) + # MORI_MOE_MAX_INPUT_TOKENS_DECODE=$((MORI_MAX_DISPATCH_TOKENS_DECODE * decode_dp_ranks * 7 / 10)) # Update derived variable SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD=$((MORI_MAX_DISPATCH_TOKENS_DECODE * 2)) export SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD @@ -222,10 +227,20 @@ if [[ "$DECODE_ENABLE_DP" == "true" ]] && [[ "$DECODE_ENABLE_EP" == "true" ]]; t fi # Build the composed config strings (equivalent to the old MODEL_PREFILL_CONFIGS / MODEL_DECODE_CONFIGS) -PREFILL_MODE_FLAGS="--mem-fraction-static ${PREFILL_MEM_FRACTION_STATIC} --max-running-requests ${prefill_max_running_requests} --chunked-prefill-size ${prefill_chunked_prefill_size} --cuda-graph-bs ${prefill_cuda_graph_bs[*]} " +# disable_cuda_graph (model-level) routes prefill to --disable-cuda-graph instead of --cuda-graph-bs. +if [[ "$PREFILL_DISABLE_CUDA_GRAPH" == "True" ]] || [[ "$PREFILL_DISABLE_CUDA_GRAPH" == "true" ]]; then + PREFILL_MODE_FLAGS="--mem-fraction-static ${PREFILL_MEM_FRACTION_STATIC} --max-running-requests ${prefill_max_running_requests} --chunked-prefill-size ${prefill_chunked_prefill_size} --disable-cuda-graph " +else + PREFILL_MODE_FLAGS="--mem-fraction-static ${PREFILL_MEM_FRACTION_STATIC} --max-running-requests ${prefill_max_running_requests} --chunked-prefill-size ${prefill_chunked_prefill_size} --cuda-graph-bs ${prefill_cuda_graph_bs[*]} " +fi + if [[ "$PREFILL_DISABLE_RADIX_CACHE" == "True" ]] || [[ "$PREFILL_DISABLE_RADIX_CACHE" == "true" ]]; then PREFILL_MODE_FLAGS="$PREFILL_MODE_FLAGS --disable-radix-cache" fi +# Agentic runs: keep radix/prefix cache enabled by replacing --disable-radix-cache with empty. +if [[ "${IS_AGENTIC:-0}" == "1" || "${IS_AGENTIC:-}" == "true" ]]; then + PREFILL_MODE_FLAGS="${PREFILL_MODE_FLAGS//--disable-radix-cache/}" +fi if [[ -n "$prefill_context_length" ]]; then PREFILL_MODE_FLAGS="$PREFILL_MODE_FLAGS --context-length ${prefill_context_length}" fi @@ -242,10 +257,13 @@ DECODE_MODE_FLAGS="--mem-fraction-static ${DECODE_MEM_FRACTION_STATIC} --max-run if [[ "$DECODE_PREFILL_ROUND_ROBIN_BALANCE" == "True" ]] || [[ "$DECODE_PREFILL_ROUND_ROBIN_BALANCE" == "true" ]]; then DECODE_MODE_FLAGS="$DECODE_MODE_FLAGS --prefill-round-robin-balance" fi +if [[ "$DECODE_DISAGG_ENABLE_RADIX_CACHE" == "True" ]] || [[ "$DECODE_DISAGG_ENABLE_RADIX_CACHE" == "true" ]]; then + DECODE_MODE_FLAGS="$DECODE_MODE_FLAGS --disaggregation-decode-enable-radix-cache" +fi if [[ "$DECODE_MTP_SIZE" -gt 0 ]]; then MORI_MAX_DISPATCH_TOKENS_DECODE=$((MORI_MAX_DISPATCH_TOKENS_DECODE * (DECODE_MTP_SIZE + 1))) - MORI_MOE_MAX_INPUT_TOKENS_DECODE=$((MORI_MOE_MAX_INPUT_TOKENS_DECODE * (DECODE_MTP_SIZE + 1))) + # MORI_MOE_MAX_INPUT_TOKENS_DECODE=$((MORI_MOE_MAX_INPUT_TOKENS_DECODE * (DECODE_MTP_SIZE + 1))) fi # ============================================================================= @@ -261,10 +279,20 @@ NODE_OFFSET=$((PREFILL_NODES_PER_WORKER * xP)) # Build prefill arguments dynamically based on xP PREFILL_HEADNODE_URLS=() PREFILL_ARGS="" +# Per-worker Prometheus /metrics endpoints (port 8000) for aiperf's +# --server-metrics scrape. The router on :30000 does not serve Prometheus, so +# aiperf must scrape each prefill/decode worker directly (see ENABLE_METRICS). +SERVER_METRICS_URLS=() +# Per-worker base URLs (port 8000) for direct cache flushing between +# concurrency points. The router (:30000) does not fan /flush_cache out, so +# trace_replay.sh must POST to each prefill/decode worker directly. +SERVER_FLUSH_URLS=() for i in $(seq 0 $((xP - 1))); do prefill_idx=$((i * PREFILL_NODES_PER_WORKER)) PREFILL_HEADNODE_URLS[$i]="${IP_ARRAY[$prefill_idx]}:${HEADNODE_PORT}" PREFILL_ARGS="$PREFILL_ARGS --prefill http://${IP_ARRAY[$prefill_idx]}:8000" + SERVER_METRICS_URLS+=("http://${IP_ARRAY[$prefill_idx]}:8000/metrics") + SERVER_FLUSH_URLS+=("http://${IP_ARRAY[$prefill_idx]}:8000") done # Build decode arguments dynamically based on yD @@ -274,10 +302,14 @@ for i in $(seq 0 $((yD - 1))); do decode_idx=$((i * DECODE_NODES_PER_WORKER + NODE_OFFSET)) DECODE_HEADNODE_URLS[$i]="${IP_ARRAY[$decode_idx]}:${HEADNODE_PORT}" DECODE_ARGS="$DECODE_ARGS --decode http://${IP_ARRAY[$decode_idx]}:8000" + SERVER_METRICS_URLS+=("http://${IP_ARRAY[$decode_idx]}:8000/metrics") + SERVER_FLUSH_URLS+=("http://${IP_ARRAY[$decode_idx]}:8000") done echo "Prefill worker headnode list: ${PREFILL_HEADNODE_URLS[@]}" echo "Decode worker headnode list: ${DECODE_HEADNODE_URLS[@]}" +echo "Server metrics endpoints: ${SERVER_METRICS_URLS[@]}" +echo "Server flush endpoints: ${SERVER_FLUSH_URLS[@]}" # ============================================================================= # Configuration Builder Functions @@ -318,6 +350,7 @@ build_server_config() { local base_config="$MODEL_BASE_FLAGS" local mtp_config="" local dp_config="" + local ep_config="" local specific_config="" # MTP config (only if MTP is enabled and mode is decode) @@ -330,6 +363,13 @@ build_server_config() { dp_config="$MODEL_DP_FLAGS" fi + # EP config (only if EP is enabled): a2a backend, deepep mode, ep-dispatch algo. + # With ep=1 (EP disabled) these are dropped, so the MoE runs tensor-parallel (TP) + # instead of expert-parallel — even when dp-attention is on. + if [[ "$enable_ep" == "true" ]]; then + ep_config="$MODEL_EP_FLAGS" + fi + # Mode-specific config if [[ "$mode" == "prefill" ]]; then specific_config="$PREFILL_MODE_FLAGS" @@ -337,11 +377,14 @@ build_server_config() { specific_config="$DECODE_MODE_FLAGS" fi - # Combine: parallel args + base config + mtp config (decode only) + dp config + specific config + # Combine: parallel args + base config + ep config + mtp config (decode only) + dp config + specific config local full_config="$parallel_args" if [[ -n "$base_config" ]]; then full_config="$full_config $base_config" fi + if [[ -n "$ep_config" ]]; then + full_config="$full_config $ep_config" + fi if [[ -n "$mtp_config" ]] && [[ "$mode" == "decode" ]]; then full_config="$full_config $mtp_config" fi @@ -359,10 +402,101 @@ build_server_config() { PREFILL_SERVER_CONFIG=$(build_server_config "prefill" "$MODEL_NAME" "$PREFILL_TP_SIZE" "$PREFILL_ENABLE_EP" "$PREFILL_ENABLE_DP" "$DECODE_MTP_SIZE") DECODE_SERVER_CONFIG=$(build_server_config "decode" "$MODEL_NAME" "$DECODE_TP_SIZE" "$DECODE_ENABLE_EP" "$DECODE_ENABLE_DP" "$DECODE_MTP_SIZE") +# Expose Prometheus /metrics on the servers when requested (ENABLE_METRICS=1). +if [[ "${ENABLE_METRICS:-0}" == "1" ]]; then + [[ "$PREFILL_SERVER_CONFIG" != *"--enable-metrics"* ]] && PREFILL_SERVER_CONFIG="$PREFILL_SERVER_CONFIG --enable-metrics" + [[ "$DECODE_SERVER_CONFIG" != *"--enable-metrics"* ]] && DECODE_SERVER_CONFIG="$DECODE_SERVER_CONFIG --enable-metrics" +fi + if [[ -n "$MODEL_NAME" ]]; then echo "Using model-specific configuration for: $MODEL_NAME" fi +# ============================================================================= +# Optional KV cache offloading (HiCache) — enabled when +# KV_OFFLOADING != none AND KV_OFFLOAD_BACKEND == hicache. +# HiCache extends RadixAttention, so radix cache MUST stay on (drop +# --disable-radix-cache). The --hicache-* flags are appended to BOTH the +# prefill and decode server configs. +# ============================================================================= +KV_OFFLOADING="${KV_OFFLOADING:-none}" +KV_OFFLOAD_BACKEND="${KV_OFFLOAD_BACKEND:-}" +if [[ "$KV_OFFLOADING" != "none" && "$KV_OFFLOAD_BACKEND" == "hicache" ]]; then + HICACHE_TOTAL_CPU_DRAM_GB="${HICACHE_TOTAL_CPU_DRAM_GB:-2000}" + HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-1}" + HICACHE_PAGE_SIZE="${HICACHE_PAGE_SIZE:-1}" + HICACHE_PREFETCH_POLICY="${HICACHE_PREFETCH_POLICY:-wait_complete}" + + # Optional L3 storage tier behind the CPU-DRAM (L2) cache. + # "" -> CPU DRAM only (default) + # "mooncake"-> Mooncake distributed KV store (needs a mooncake_master) + HICACHE_STORAGE_BACKEND="${HICACHE_STORAGE_BACKEND:-}" + + # Layout / IO backend / write policy are backend-specific: + # mooncake L3: page_first_direct + the "direct" IO backend (the Mooncake + # store maps a page-contiguous segment for RDMA/zero-copy). This layout + # asserts host_pool > device_pool, so it needs a large CPU-DRAM budget. + # L2-only (CPU DRAM): layer_first + the "kernel" IO backend. layer_first + # has no host>device constraint (the "direct" IO backend REQUIRES a + # page_first layout, so it cannot be paired with layer_first). + if [[ "$HICACHE_STORAGE_BACKEND" == "mooncake" ]]; then + HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first}" + HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" + HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" + else + HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first_direct}" + HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" + HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" + fi + + # Mooncake master/connection settings (used only when storage=mooncake). + # The master runs once on node 0; every prefill/decode server connects to + # it via NODE0_ADDR so it is reachable across nodes. + MC_MASTER_PORT="${MC_MASTER_PORT:-50061}" + MC_METADATA_PORT="${MC_METADATA_PORT:-8080}" + MC_METRICS_PORT="${MC_METRICS_PORT:-9003}" + MC_MASTER_THREADS="${MC_MASTER_THREADS:-64}" + MC_EVICTION_HIGH_WATERMARK="${MC_EVICTION_HIGH_WATERMARK:-0.95}" + MC_PROTOCOL="${MC_PROTOCOL:-tcp}" + MC_GLOBAL_SEG="${MC_GLOBAL_SEG:-64gb}" + MC_DEVICE="${MC_DEVICE:-$IBDEVICES}" + MC_MASTER_ADDR="${MC_MASTER_ADDR:-${NODE0_ADDR}:${MC_MASTER_PORT}}" + MC_METADATA_SERVER="${MC_METADATA_SERVER:-http://${NODE0_ADDR}:${MC_METADATA_PORT}/metadata}" + + # Emit the --hicache-storage-backend flags (empty unless mooncake). The + # extra-config JSON is single-quoted so it survives the later `eval` of the + # launch command as a single argument. + build_storage_flags() { + [[ "$HICACHE_STORAGE_BACKEND" != "mooncake" ]] && return 0 + local extra="{\"master_server_address\": \"${MC_MASTER_ADDR}\", \"protocol\": \"${MC_PROTOCOL}\", \"device_name\": \"${MC_DEVICE}\", \"local_hostname\": \"${host_ip}\", \"global_segment_size\": \"${MC_GLOBAL_SEG}\", \"metadata_server\": \"${MC_METADATA_SERVER}\", \"check_server\": false}" + echo "--hicache-storage-backend mooncake --hicache-storage-backend-extra-config '${extra}' --enable-metrics --enable-cache-report" + } + + # HiCache capacity via --hicache-ratio (scales with GPU KV pool). + HICACHE_RATIO="${HICACHE_RATIO:-16}" + + build_hicache_flags() { + echo "--page-size ${HICACHE_PAGE_SIZE} --enable-hierarchical-cache --hicache-ratio ${HICACHE_RATIO} --hicache-io-backend ${HICACHE_IO_BACKEND} --hicache-mem-layout ${HICACHE_MEM_LAYOUT} --hicache-write-policy ${HICACHE_WRITE_POLICY} --hicache-storage-prefetch-policy ${HICACHE_PREFETCH_POLICY} $(build_storage_flags)" + } + + # HiCache requires RadixAttention; strip any --disable-radix-cache. + PREFILL_SERVER_CONFIG="${PREFILL_SERVER_CONFIG//--disable-radix-cache/}" + DECODE_SERVER_CONFIG="${DECODE_SERVER_CONFIG//--disable-radix-cache/}" + + # Prefill always gets HiCache. + PREFILL_SERVER_CONFIG="$PREFILL_SERVER_CONFIG $(build_hicache_flags "$PREFILL_TP_SIZE")" + + + DECODE_SERVER_CONFIG="$DECODE_SERVER_CONFIG --page-size ${HICACHE_PAGE_SIZE}" + echo "[HiCache] KV_OFFLOADING=${KV_OFFLOADING} backend=${KV_OFFLOAD_BACKEND} applied to prefill only; decode mirrors --page-size ${HICACHE_PAGE_SIZE} for transfer compatibility (chunk cache under the mori transfer backend)" + echo "[HiCache] params: io_backend=${HICACHE_IO_BACKEND}, mem_layout=${HICACHE_MEM_LAYOUT}, page_size=${HICACHE_PAGE_SIZE}, write_policy=${HICACHE_WRITE_POLICY}, prefetch_policy=${HICACHE_PREFETCH_POLICY}, storage_backend=${HICACHE_STORAGE_BACKEND:-none}" + if [[ "$HICACHE_STORAGE_BACKEND" == "mooncake" ]]; then + echo "[HiCache] Mooncake store: master=${MC_MASTER_ADDR} metadata=${MC_METADATA_SERVER} protocol=${MC_PROTOCOL} device=${MC_DEVICE} segment=${MC_GLOBAL_SEG} threads=${MC_MASTER_THREADS} eviction_watermark=${MC_EVICTION_HIGH_WATERMARK}" + fi +else + echo "[HiCache] KV_OFFLOADING=${KV_OFFLOADING} backend=${KV_OFFLOAD_BACKEND:-none} (HiCache disabled)" +fi + if [[ "${EVAL_ONLY:-false}" == "true" ]] || [[ "${RUN_EVAL:-false}" == "true" ]]; then PREFILL_SERVER_CONFIG=$(echo "$PREFILL_SERVER_CONFIG" | sed 's/--ep-dispatch-algorithm fake//g') DECODE_SERVER_CONFIG=$(echo "$DECODE_SERVER_CONFIG" | sed 's/--ep-dispatch-algorithm fake//g') @@ -411,6 +545,63 @@ if [ "$NODE_RANK" -eq 0 ]; then echo "================================================" + # Dump all resolved commands to a text file for debugging / reproducibility. + CMD_DUMP="/run_logs/slurm_job-${SLURM_JOB_ID}/commands_${host_name}.txt" + dump_cmd() { echo -e "\n# ── $1 ──\n$2" >> "$CMD_DUMP"; } + echo "# Commands dump — $(date -u '+%Y-%m-%d %H:%M:%S UTC')" > "$CMD_DUMP" + echo "# Host: ${host_name} (${host_ip}) Node rank: ${NODE_RANK}" >> "$CMD_DUMP" + echo "# Model: ${MODEL_NAME} Image: ${DOCKER_IMAGE_NAME:-unknown}" >> "$CMD_DUMP" + + # Start the Mooncake store master (L3 HiCache backend) on node 0 only. + # All prefill/decode servers connect to it via NODE0_ADDR:MC_MASTER_PORT. + if [[ "${KV_OFFLOADING:-none}" != "none" && "${KV_OFFLOAD_BACKEND:-}" == "hicache" && "${HICACHE_STORAGE_BACKEND:-}" == "mooncake" ]]; then + echo "Starting Mooncake master on ${host_ip}:${MC_MASTER_PORT} (metadata :${MC_METADATA_PORT}, metrics :${MC_METRICS_PORT})" + MC_MASTER_CMD="mooncake_master \ + --enable_http_metadata_server=true \ + --http_metadata_server_host=0.0.0.0 \ + --http_metadata_server_port=${MC_METADATA_PORT} \ + --rpc_port=${MC_MASTER_PORT} \ + --rpc_thread_num=${MC_MASTER_THREADS} \ + --metrics_port=${MC_METRICS_PORT} \ + --enable_metric_reporting=true \ + --eviction_high_watermark_ratio=${MC_EVICTION_HIGH_WATERMARK}" + dump_cmd "MOONCAKE MASTER" "$MC_MASTER_CMD" + if [[ "$DRY_RUN" -eq 1 ]]; then + echo "DRY RUN: $MC_MASTER_CMD" + else + MC_MASTER_LOG="/run_logs/slurm_job-${SLURM_JOB_ID}/mooncake_master_${host_name}.log" + mooncake_master \ + --enable_http_metadata_server=true \ + --http_metadata_server_host=0.0.0.0 \ + --http_metadata_server_port="${MC_METADATA_PORT}" \ + --rpc_port="${MC_MASTER_PORT}" \ + --rpc_thread_num="${MC_MASTER_THREADS}" \ + --metrics_port="${MC_METRICS_PORT}" \ + --enable_metric_reporting=true \ + --eviction_high_watermark_ratio="${MC_EVICTION_HIGH_WATERMARK}" \ + > "${MC_MASTER_LOG}" 2>&1 & + mc_master_pid=$! + sleep 3 + # Fail loudly on a port collision. On shared nodes the Mooncake RPC + # port may already be taken by another user's master; in that case the + # metrics-port health check below can still pass against the foreign + # master while our RPC port is dead, and the prefill then hangs. + if grep -qiE "Address already in use|bind .*error" "${MC_MASTER_LOG}" 2>/dev/null; then + echo "ERROR: mooncake_master failed to bind port ${MC_MASTER_PORT} (already in use)." + echo " Set MC_MASTER_PORT/MC_METRICS_PORT to free ports and resubmit." + grep -iE "Address already in use|bind .*error" "${MC_MASTER_LOG}" | tail -3 + exit 1 + fi + for ((i=3; i<=60; i+=3)); do + if curl -sf "http://127.0.0.1:${MC_METRICS_PORT}/get_all_segments" >/dev/null 2>&1; then + echo " mooncake master OK at ${i}s" + break + fi + sleep 3 + done + fi + fi + # start the head prefill server PREFILL_MORI_MOE_ENV="" set -x @@ -418,7 +609,7 @@ if [ "$NODE_RANK" -eq 0 ]; then PREFILL_MORI_MOE_ENV="SGLANG_MORI_MOE_MAX_INPUT_TOKENS=${MORI_MOE_MAX_INPUT_TOKENS_PREFILL}" fi set +x - PREFILL_CMD="SGLANG_MORI_COMBINE_DTYPE=${MORI_COMBINE_DTYPE_PREFILL} ${PREFILL_SDMA_ENV} ${PREFILL_MORI_MOE_ENV} SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_MAX_DISPATCH_TOKENS_PREFILL} python3 -m sglang.launch_server \ + PREFILL_CMD="SGLANG_MORI_COMBINE_DTYPE=${MORI_COMBINE_DTYPE_PREFILL} ${PREFILL_SDMA_ENV} ${PREFILL_MORI_MOE_ENV} SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_PREFILL:-${MORI_MAX_DISPATCH_TOKENS_PREFILL}} MORI_IO_SQ_BACKOFF_TIMEOUT_US=${MORI_IO_SQ_BACKOFF_TIMEOUT_US} MORI_IO_QP_MAX_SEND_WR=${MORI_IO_QP_MAX_SEND_WR} ${LAUNCH_PREFIX:-} python3 -m sglang.launch_server \ --model-path $MODEL_DIR/$MODEL_NAME \ --disaggregation-mode prefill \ --disaggregation-ib-device ${IBDEVICES} \ @@ -433,6 +624,7 @@ if [ "$NODE_RANK" -eq 0 ]; then fi + dump_cmd "PREFILL (node 0)" "$PREFILL_CMD" if [[ "$DRY_RUN" -eq 1 ]]; then echo "DRY RUN: $PREFILL_CMD" else @@ -460,20 +652,41 @@ if [ "$NODE_RANK" -eq 0 ]; then fi echo "Congratulations!!! All prefill and decode servers are up . . ." + # Router resilience: a single prefill worker doing huge long-context prefills + # (256K+ token agentic prompts in 65280-token chunks) can be slow to drain a + # concurrent burst. With defaults the circuit breaker opens after 10 failures + # and short-circuits the whole worker, so the aiperf profiling burst sees + # "No available prefill workers (all circuits open or unhealthy)" and aborts + # with 100% errors. Disable the breaker and relax health-check sensitivity so + # a busy-but-alive worker is not ejected. Override via ROUTER_RESILIENCE_FLAGS. + ROUTER_RESILIENCE_FLAGS="${ROUTER_RESILIENCE_FLAGS:---disable-circuit-breaker --health-failure-threshold 100 --health-check-timeout-secs 600 --health-check-interval-secs 30}" + + # Router scheduling policy. cache_aware prefill routing exploits HiCache/radix + # prefix reuse across the agentic trace; round_robin decode keeps the single + # decode worker fed evenly. cache_threshold / balance_*_threshold tune the + # cache_aware load-balancing (router defaults are 0.5 / 64 / 1.5). Override any + # of these via env. + ROUTER_PREFILL_POLICY="${ROUTER_PREFILL_POLICY:-cache_aware}" + ROUTER_DECODE_POLICY="${ROUTER_DECODE_POLICY:-round_robin}" + ROUTER_CACHE_THRESHOLD="${ROUTER_CACHE_THRESHOLD:-0.3}" + ROUTER_BALANCE_ABS_THRESHOLD="${ROUTER_BALANCE_ABS_THRESHOLD:-2}" + ROUTER_BALANCE_REL_THRESHOLD="${ROUTER_BALANCE_REL_THRESHOLD:-1.1}" + ROUTER_POLICY_FLAGS="${ROUTER_POLICY_FLAGS:---policy ${ROUTER_PREFILL_POLICY} --prefill-policy ${ROUTER_PREFILL_POLICY} --decode-policy ${ROUTER_DECODE_POLICY} --cache-threshold ${ROUTER_CACHE_THRESHOLD} --balance-abs-threshold ${ROUTER_BALANCE_ABS_THRESHOLD} --balance-rel-threshold ${ROUTER_BALANCE_REL_THRESHOLD}}" + ROUTER_CMD="python -m sglang_router.launch_router \ --pd-disaggregation \ --port 30000 \ - --policy random \ - --prefill-policy random \ - --decode-policy random \ + ${ROUTER_POLICY_FLAGS} \ + ${ROUTER_RESILIENCE_FLAGS} \ ${PREFILL_ARGS} \ ${DECODE_ARGS}" + dump_cmd "ROUTER" "$ROUTER_CMD" if [[ "$DRY_RUN" -eq 1 ]]; then echo "DRY RUN: $ROUTER_CMD" else - ROUTER_LOG_FILE="/tmp/slurm_job-${SLURM_JOB_ID}_proxy_${host_name}.log" + ROUTER_LOG_FILE="/run_logs/slurm_job-${SLURM_JOB_ID}/router_${host_name}.log" set -x if [[ "${SGLANG_ROUTER_STDOUT_LOGS:-0}" == "1" ]]; then eval "$ROUTER_CMD" 2>&1 | tee "$ROUTER_LOG_FILE" & @@ -513,11 +726,41 @@ if [ "$NODE_RANK" -eq 0 ]; then export IS_MTP=false fi - # n_prefill n_decode prefill_gpus decode_gpus model_dir model_name log_path isl osl concurrency_list req_rate random_range_ratio num_prompts_multiplier - BENCH_CMD="bash $SGLANG_WS_PATH/bench.sh ${xP} ${yD} $((PREFILL_TP_SIZE*xP)) $((DECODE_TP_SIZE*yD)) \ - $MODEL_DIR $MODEL_NAME /run_logs/slurm_job-${SLURM_JOB_ID} ${BENCH_INPUT_LEN} \ - ${BENCH_OUTPUT_LEN} "${BENCH_MAX_CONCURRENCY}" ${BENCH_REQUEST_RATE} \ - ${BENCH_RANDOM_RANGE_RATIO} ${BENCH_NUM_PROMPTS_MULTIPLIER}" + # Select the benchmark runner. + # IS_AGENTIC=1/true → agentic trace replay (trace_replay.sh) + # IS_AGENTIC unset/0 → fixed-seq-len throughput benchmark (bench.sh) + if [[ "${IS_AGENTIC:-0}" == "1" || "${IS_AGENTIC:-}" == "true" ]]; then + # Point aiperf's server-metrics scrape at the per-worker Prometheus + # /metrics endpoints. The router (:30000) that aiperf auto-detects from + # --url does not expose Prometheus, so without this the scrape finds no + # reachable endpoint and all server-side cache/KV fields come out null. + # Only set it when the workers were actually started with --enable-metrics. + if [[ "${ENABLE_METRICS:-0}" == "1" && "${#SERVER_METRICS_URLS[@]}" -gt 0 ]]; then + AIPERF_SERVER_METRICS_URLS=$(IFS=,; echo "${SERVER_METRICS_URLS[*]}") + export AIPERF_SERVER_METRICS_URLS + echo "AIPERF_SERVER_METRICS_URLS=${AIPERF_SERVER_METRICS_URLS}" + fi + # Per-worker base URLs for cache flushing between concurrency points. + # trace_replay.sh consults these when CLEAR_CACHE_BETWEEN_CONC=1. + if [[ "${#SERVER_FLUSH_URLS[@]}" -gt 0 ]]; then + SERVER_FLUSH_URLS_CSV=$(IFS=,; echo "${SERVER_FLUSH_URLS[*]}") + export SERVER_FLUSH_URLS_CSV + echo "SERVER_FLUSH_URLS_CSV=${SERVER_FLUSH_URLS_CSV}" + fi + # trace_replay.sh signature: model_path model_name concurrency_list log_path + BENCH_CMD="bash $SGLANG_WS_PATH/trace_replay.sh \ + $MODEL_DIR $MODEL_NAME $BENCH_MAX_CONCURRENCY /run_logs/slurm_job-${SLURM_JOB_ID}" + echo "Benchmark runner: trace_replay.sh (agentic, KV_OFFLOADING=${KV_OFFLOADING:-none}, backend=${KV_OFFLOAD_BACKEND:-none}, CONC=${BENCH_MAX_CONCURRENCY})" + else + # bench.sh signature: + # n_prefill n_decode prefill_gpus decode_gpus model_dir model_name log_path + # isl osl concurrency_list req_rate random_range_ratio num_prompts_multiplier + BENCH_CMD="bash $SGLANG_WS_PATH/bench.sh ${xP} ${yD} $((PREFILL_TP_SIZE*xP)) $((DECODE_TP_SIZE*yD)) \ + $MODEL_DIR $MODEL_NAME /run_logs/slurm_job-${SLURM_JOB_ID} ${BENCH_INPUT_LEN} \ + ${BENCH_OUTPUT_LEN} \"${BENCH_MAX_CONCURRENCY}\" ${BENCH_REQUEST_RATE} \ + ${BENCH_RANDOM_RANGE_RATIO} ${BENCH_NUM_PROMPTS_MULTIPLIER}" + echo "Benchmark runner: bench.sh (fixed-seq-len)" + fi if [[ "${EVAL_ONLY:-false}" == "true" ]]; then echo "EVAL_ONLY mode: skipping throughput benchmark" @@ -645,13 +888,18 @@ elif [ "$NODE_RANK" -gt 0 ] && [ "$NODE_RANK" -lt "$NODE_OFFSET" ]; then echo "Using prefill config: $PREFILL_SERVER_CONFIG" echo "Prefill parallelism: TP=${PREFILL_TP_SIZE}, EP enabled: ${PREFILL_ENABLE_EP}, DP enabled: ${PREFILL_ENABLE_DP}" + CMD_DUMP="/run_logs/slurm_job-${SLURM_JOB_ID}/commands_${host_name}.txt" + dump_cmd() { echo -e "\n# ── $1 ──\n$2" >> "$CMD_DUMP"; } + echo "# Commands dump — $(date -u '+%Y-%m-%d %H:%M:%S UTC')" > "$CMD_DUMP" + echo "# Host: ${host_name} (${host_ip}) Node rank: ${NODE_RANK}" >> "$CMD_DUMP" + PREFILL_MORI_MOE_ENV="" set -x if [[ -n "$MORI_MOE_MAX_INPUT_TOKENS_PREFILL" ]]; then PREFILL_MORI_MOE_ENV="SGLANG_MORI_MOE_MAX_INPUT_TOKENS=${MORI_MOE_MAX_INPUT_TOKENS_PREFILL}" fi set +x - PREFILL_CMD="SGLANG_MORI_COMBINE_DTYPE=${MORI_COMBINE_DTYPE_PREFILL} ${PREFILL_SDMA_ENV} ${PREFILL_MORI_MOE_ENV} SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_MAX_DISPATCH_TOKENS_PREFILL} python3 -m sglang.launch_server \ + PREFILL_CMD="SGLANG_MORI_COMBINE_DTYPE=${MORI_COMBINE_DTYPE_PREFILL} ${PREFILL_SDMA_ENV} ${PREFILL_MORI_MOE_ENV} SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_PREFILL:-${MORI_MAX_DISPATCH_TOKENS_PREFILL}} MORI_IO_SQ_BACKOFF_TIMEOUT_US=${MORI_IO_SQ_BACKOFF_TIMEOUT_US} MORI_IO_QP_MAX_SEND_WR=${MORI_IO_QP_MAX_SEND_WR} ${LAUNCH_PREFIX:-} python3 -m sglang.launch_server \ --model-path $MODEL_DIR/${MODEL_NAME} \ --disaggregation-mode prefill \ --disaggregation-ib-device ${IBDEVICES} \ @@ -667,6 +915,7 @@ elif [ "$NODE_RANK" -gt 0 ] && [ "$NODE_RANK" -lt "$NODE_OFFSET" ]; then PREFILL_CMD="$PREFILL_CMD --dist-init-addr ${PREFILL_HEADNODE_URLS[$prefill_idx]} --nnodes ${PREFILL_NODES_PER_WORKER} --node-rank $rank" fi + dump_cmd "PREFILL (rank ${NODE_RANK})" "$PREFILL_CMD" if [[ "$DRY_RUN" -eq 1 ]]; then echo "DRY RUN: $PREFILL_CMD" else @@ -714,13 +963,18 @@ else echo "Decode node rank: $RANK" echo "Decode parallelism: TP=${DECODE_TP_SIZE}, EP enabled: ${DECODE_ENABLE_EP}, DP enabled: ${DECODE_ENABLE_DP}" + CMD_DUMP="/run_logs/slurm_job-${SLURM_JOB_ID}/commands_${host_name}.txt" + dump_cmd() { echo -e "\n# ── $1 ──\n$2" >> "$CMD_DUMP"; } + echo "# Commands dump — $(date -u '+%Y-%m-%d %H:%M:%S UTC')" > "$CMD_DUMP" + echo "# Host: ${host_name} (${host_ip}) Node rank: ${NODE_RANK}" >> "$CMD_DUMP" + DECODE_MORI_MOE_ENV="" set -x if [[ -n "$MORI_MOE_MAX_INPUT_TOKENS_DECODE" ]]; then DECODE_MORI_MOE_ENV="SGLANG_MORI_MOE_MAX_INPUT_TOKENS=${MORI_MOE_MAX_INPUT_TOKENS_DECODE}" fi set +x - DECODE_CMD="SGLANG_MORI_COMBINE_DTYPE=${MORI_COMBINE_DTYPE_DECODE} ${DECODE_MORI_MOE_ENV} SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_MAX_DISPATCH_TOKENS_DECODE} python3 -m sglang.launch_server \ + DECODE_CMD="SGLANG_MORI_COMBINE_DTYPE=${MORI_COMBINE_DTYPE_DECODE} ${DECODE_MORI_MOE_ENV} SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_DECODE:-${MORI_MAX_DISPATCH_TOKENS_DECODE}} MORI_IO_SQ_BACKOFF_TIMEOUT_US=${MORI_IO_SQ_BACKOFF_TIMEOUT_US} MORI_IO_QP_MAX_SEND_WR=${MORI_IO_QP_MAX_SEND_WR} ${LAUNCH_PREFIX:-} python3 -m sglang.launch_server \ --model-path ${MODEL_DIR}/${MODEL_NAME} \ --disaggregation-mode decode \ --disaggregation-ib-device ${IBDEVICES} \ @@ -736,6 +990,7 @@ else DECODE_CMD="$DECODE_CMD --dist-init-addr ${DECODE_HEADNODE_URLS[$decode_idx]} --nnodes ${DECODE_NODES_PER_WORKER} --node-rank $rank" fi + dump_cmd "DECODE (rank ${NODE_RANK})" "$DECODE_CMD" if [[ "$DRY_RUN" -eq 1 ]]; then echo "DRY RUN: $DECODE_CMD" else diff --git a/benchmarks/multi_node/amd_utils/setup_deps.sh b/benchmarks/multi_node/amd_utils/setup_deps.sh index 35eaf17dc0..5ee7078467 100644 --- a/benchmarks/multi_node/amd_utils/setup_deps.sh +++ b/benchmarks/multi_node/amd_utils/setup_deps.sh @@ -34,7 +34,6 @@ git_clone_retry() { return 1 } - # --------------------------------------------------------------------------- # 5. Container RDMA/net tools # - ibv_devinfo comes from ibverbs-utils @@ -80,86 +79,202 @@ install_amd_quark() { } # --------------------------------------------------------------------------- -# SGLang: Patch aiter gluon pa_mqa_logits — fix 2D → 3D instr_shape for -# Triton ≥ 3.5. -# -# Bug: _gluon_deepgemm_fp8_paged_mqa_logits (the non-preshuffle variant) -# hardcodes AMDMFMALayout(instr_shape=[16, 16]) which fails on Triton -# builds where AMDMFMALayout requires 3D (M, N, K) format. -# -# The two preshuffle variants already conditionally select 2D vs 3D via -# the module-level _Use_2d_instr_shape_mfma_layout flag, but the base -# variant was missed. This patch brings it in line. +# SGLang: prevent TP-rank collective desync deadlock in disaggregation prefill. # -# Affects: GLM-5 (NSA attention) and any future model that uses -# deepgemm_fp8_paged_mqa_logits with Preshuffle=False. +# resolve_waiting_queue_bootstrap() runs poll_and_all_reduce_attn_cp_tp_group() +# over `candidates`. The upstream candidate set (all non-aborted waiting reqs) +# can differ across TP ranks, so some ranks enter the all_reduce while others +# skip it -> hang. Narrow candidates to optimistic (pending_bootstrap) requests, +# which is consistent across ranks and is the only set finalize_bootstrap acts on. # --------------------------------------------------------------------------- -patch_gluon_pa_mqa_logits_instr_shape() { +patch_disagg_prefill_bootstrap_desync() { python3 -c ' import os, sys -target = "/sgl-workspace/aiter/aiter/ops/triton/gluon/pa_mqa_logits.py" +target = "/sgl-workspace/sglang/python/sglang/srt/disaggregation/prefill.py" if not os.path.isfile(target): - print("[SETUP] gluon pa_mqa_logits.py not found, skipping") + print("[SETUP] disaggregation/prefill.py not found, skipping") sys.exit(0) src = open(target).read() -if "[PATCHED] 3D instr_shape for base gluon variant" in src: - print("[SETUP] gluon pa_mqa_logits 3D instr_shape patch already applied") - sys.exit(0) +old = " candidates = [req for req in self.waiting_queue if not is_aborted(req)]" +new = ( + " candidates = [\n" + " req\n" + " for req in self.waiting_queue\n" + " if req.pending_bootstrap and not is_aborted(req)\n" + " ]" +) -# The buggy code: the base _gluon_deepgemm_fp8_paged_mqa_logits uses 2D -# instr_shape unconditionally. We replace it with a conditional that -# mirrors the preshuffle variants. -old = """\ - mfma_layout: gl.constexpr = gl.amd.AMDMFMALayout( - version=CDNA_VERSION, - instr_shape=[16, 16], - transposed=False, - warps_per_cta=[1, NumWarps], - ) - mfma_layout_a: gl.constexpr = gl.DotOperandLayout( - operand_index=0, parent=mfma_layout, k_width=16 - ) - mfma_layout_b: gl.constexpr = gl.DotOperandLayout( - operand_index=1, parent=mfma_layout, k_width=16 - )""" - -new = """\ - # [PATCHED] 3D instr_shape for base gluon variant - if _Use_2d_instr_shape_mfma_layout: - mfma_layout: gl.constexpr = gl.amd.AMDMFMALayout( - version=CDNA_VERSION, - instr_shape=[16, 16], - transposed=False, - warps_per_cta=[1, NumWarps], - ) - else: - mfma_layout: gl.constexpr = gl.amd.AMDMFMALayout( - version=CDNA_VERSION, - instr_shape=[16, 16, 32], - transposed=False, - warps_per_cta=[1, NumWarps], - ) - mfma_layout_a: gl.constexpr = gl.DotOperandLayout( - operand_index=0, parent=mfma_layout, k_width=16 - ) - mfma_layout_b: gl.constexpr = gl.DotOperandLayout( - operand_index=1, parent=mfma_layout, k_width=16 - )""" +if new in src: + print("[SETUP] prefill bootstrap-desync patch already applied") + sys.exit(0) if old not in src: - print("[SETUP] WARN: gluon pa_mqa_logits pattern not found — aiter version may have changed") + print("[SETUP] WARN: resolve_waiting_queue_bootstrap pattern not found — sglang version may have changed") sys.exit(0) -# Only replace the FIRST occurrence (the base variant, not preshuffle ones) -new_src = src.replace(old, new, 1) - -open(target, "w").write(new_src) -print("[SETUP] Patched: gluon pa_mqa_logits 3D instr_shape for base variant") +open(target, "w").write(src.replace(old, new)) +print("[SETUP] Patched: disaggregation/prefill.py resolve_waiting_queue_bootstrap candidates") ' - _SETUP_INSTALLED+=("gluon-instr-shape-fix") + _SETUP_INSTALLED+=("prefill-bootstrap-desync-fix") +} + +# --------------------------------------------------------------------------- +# SGLang: tp-group agreement for disagg decode queue (decode_tp_queue_agree). +# +# The metadata gate (poll_and_all_reduce) issues a tp-group collective whose +# shape/order == self.queue. Per-rank prealloc/enqueue timing can leave ranks +# with different queue membership/order, desyncing the collective -> decode +# hang (JID-17417 / JID-17445). This inserts _agree_and_order_queue() so every +# rank polls an identical, identically-ordered subset, and replaces the +# rank-divergent retracted-queue early return in process_decode_queue with a +# rank-symmetric MAX all_reduce. Mirrors patches/decode_tp_queue_agree.patch. +# --------------------------------------------------------------------------- +patch_decode_tp_queue_agree() { + python3 - <<'PYEOF' +import os, sys + +target = "/sgl-workspace/sglang/python/sglang/srt/disaggregation/decode.py" +if not os.path.isfile(target): + print("[SETUP] disaggregation/decode.py not found, skipping") + sys.exit(0) + +src = open(target).read() + +if "_agree_and_order_queue" in src: + print("[SETUP] decode tp-queue-agree patch already applied") + sys.exit(0) + +# --- Hunk 1+2: insert agreement method + gate pop_transferred ------------- +old1 = ( + " def pop_transferred(self, rids_to_check: Optional[List[str]] = None) -> List[Req]:\n" + " if not self.queue:\n" + " return []\n" +) +new1 = ''' def _agree_and_order_queue(self) -> Tuple[List["DecodeRequest"], List["DecodeRequest"]]: + """Split self.queue into (gated, deferred) using a tp-group agreement. + + The metadata gate (utils.poll_and_all_reduce) issues a tp-group all_reduce + whose tensor shape == len(self.queue) and whose per-index meaning is the + i-th queued request. That is only correct if every tp rank polls an + IDENTICAL queue (same requests, same order). Per-rank prealloc/enqueue + timing can leave queues with different membership or order, which desyncs + the collective -> deadlock (the JID-17417 decode hang). + + We all_gather the local request ids, keep only requests present on EVERY + rank, and order them deterministically (request ids are rank-invariant), so + the subsequent gate runs a matching collective on all ranks. Requests not + yet on every rank are returned as `deferred` and retried on a later call. + + NOTE: every rank that reaches pop_transferred must call this (it contains a + collective). That holds for the same reason the existing gate is safe: + pop_transferred is entered rank-symmetrically over self.gloo_group. + """ + tp_size = torch.distributed.get_world_size(self.gloo_group) + if tp_size <= 1: + return self.queue, [] + + local_rids = [dr.req.rid for dr in self.queue] + gathered: List[Optional[List[str]]] = [None] * tp_size + torch.distributed.all_gather_object( + gathered, local_rids, group=self.gloo_group + ) + + common = set(gathered[0] or []) + for rids in gathered[1:]: + common &= set(rids or []) + + if not common: + return [], list(self.queue) + + by_rid = {dr.req.rid: dr for dr in self.queue} + # sorted() yields an identical order on every rank (rids are rank-invariant). + gated = [by_rid[rid] for rid in sorted(common)] + deferred = [dr for dr in self.queue if dr.req.rid not in common] + return gated, deferred + + def pop_transferred(self, rids_to_check: Optional[List[str]] = None) -> List[Req]: + # Agree on a tp-rank-identical, identically-ordered subset BEFORE issuing any + # metadata-gate collective. Do NOT add a local `if not self.queue` early + # return here: an empty-queue rank must still join the agreement collective, + # otherwise it skips it and desyncs the tp group (root cause of the hang). + self.queue, _deferred = self._agree_and_order_queue() + if not self.queue: + # Empty agreement is rank-symmetric: all ranks skip the gate together. + self.queue = _deferred + return [] +''' + +# --- Hunk 3: re-attach deferred reqs after removal ------------------------ +old3 = ( + " entry for i, entry in enumerate(self.queue) if i not in indices_to_remove\n" + " ]\n" + "\n" + " return transferred_reqs\n" +) +new3 = ( + " entry for i, entry in enumerate(self.queue) if i not in indices_to_remove\n" + " ]\n" + " # Re-attach requests that were not yet present on every tp rank this\n" + " # iteration; they are gated again on a later call once all ranks have them.\n" + " if _deferred:\n" + " self.queue.extend(_deferred)\n" + "\n" + " return transferred_reqs\n" +) + +# --- Hunk 4: rank-symmetric retracted-queue gate -------------------------- +old4 = ( + " if len(self.disagg_decode_prealloc_queue.retracted_queue) > 0:\n" + " # if there are still retracted requests, we do not allocate new requests\n" + " return\n" +) +new4 = ''' # PATCH(call-site tp-agreement): the retracted-queue early return below is + # rank-divergent — retracted_queue length is per-rank (per-rank KV-cache + # pressure). A divergent return skips the polling_count increment and the + # pop_transferred() collectives on some ranks, permanently desyncing the tp + # group: one rank ends up a full collective ahead, so pop_transferred's + # _agree_and_order_queue all_gather_object on the lagging ranks lines up + # against the metadata-gate all_reduce on the leading rank -> mismatched- + # collective deadlock (GPU 0%, detokenizer heartbeat freeze; JID-17445). + # process_decode_queue is called unconditionally every event-loop iteration + # on every rank, so an all_reduce here (before any divergent return) is + # rank-symmetric. Hold off new allocation iff ANY rank still has retracted + # reqs, so all ranks branch identically every iteration and pop_transferred + # is entered symmetrically. + _agree_gg = self.disagg_decode_transfer_queue.gloo_group + _local_retracted = ( + 1 if len(self.disagg_decode_prealloc_queue.retracted_queue) > 0 else 0 + ) + if torch.distributed.get_world_size(_agree_gg) > 1: + _retracted_t = torch.tensor([_local_retracted], dtype=torch.int32) + torch.distributed.all_reduce( + _retracted_t, op=torch.distributed.ReduceOp.MAX, group=_agree_gg + ) + _any_retracted = bool(_retracted_t.item()) + else: + _any_retracted = bool(_local_retracted) + if _any_retracted: + # if any rank still has retracted requests, no rank allocates new ones + return +''' + +for label, old, new in ( + ("agreement-method+pop_transferred", old1, new1), + ("deferred-reattach", old3, new3), + ("retracted-gate", old4, new4), +): + if old not in src: + print(f"[SETUP] WARN: decode.py anchor for '{label}' not found — sglang version may have changed") + sys.exit(0) + src = src.replace(old, new, 1) + +open(target, "w").write(src) +print("[SETUP] Patched: disaggregation/decode.py tp-group decode queue agreement") +PYEOF + _SETUP_INSTALLED+=("decode-tp-queue-agree-fix") } # --------------------------------------------------------------------------- @@ -202,7 +317,9 @@ if [[ "$ENGINE" == "vllm-disagg" ]]; then export PATH="${UCX_HOME}/bin:/usr/local/bin/etcd:/root/.cargo/bin:${PATH}" export LD_LIBRARY_PATH="${UCX_HOME}/lib:${RIXL_HOME}/lib:${RIXL_HOME}/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH:-}" else - patch_gluon_pa_mqa_logits_instr_shape + patch_disagg_prefill_bootstrap_desync + # patch_decode_tp_queue_agree + install_transformers_glm5 fi diff --git a/benchmarks/multi_node/amd_utils/submit.sh b/benchmarks/multi_node/amd_utils/submit.sh index fc91a78e8e..262fd25aa0 100755 --- a/benchmarks/multi_node/amd_utils/submit.sh +++ b/benchmarks/multi_node/amd_utils/submit.sh @@ -47,6 +47,10 @@ Required environment variables: MODEL_NAME Model name directory CONTAINER_IMAGE Docker image name (e.g., vllm_disagg_pd:latest) RUNNER_NAME Runner identifier (for job name) + +Optional environment variables: + DRY_RUN 1 = echo composed server/router launch commands instead of + running them (preview a recipe against a real allocation). USAGE } @@ -124,6 +128,12 @@ export BENCH_MAX_CONCURRENCY=${CONCURRENCIES} export BENCH_REQUEST_RATE=${REQUEST_RATE} export BENCH_RANDOM_RANGE_RATIO=${RANDOM_RANGE_RATIO:-0.8} +# DRY_RUN=1 makes server_sglang.sh echo the composed prefill/decode/router launch +# commands instead of executing them (useful for previewing a recipe against a real +# allocation). Threaded here → job.slurm → Docker (-e DRY_RUN) → server_sglang.sh. +# sbatch defaults to --export=ALL, so exporting it is what carries it into the job. +export DRY_RUN="${DRY_RUN:-0}" + # Eval-related env vars (threaded from workflow → runner → here → job.slurm → Docker) export RUN_EVAL="${RUN_EVAL:-false}" export EVAL_ONLY="${EVAL_ONLY:-false}" diff --git a/benchmarks/multi_node/amd_utils/trace_replay.sh b/benchmarks/multi_node/amd_utils/trace_replay.sh new file mode 100644 index 0000000000..b3eb0b8fd7 --- /dev/null +++ b/benchmarks/multi_node/amd_utils/trace_replay.sh @@ -0,0 +1,160 @@ +#!/bin/bash +# Dual-Engine Disaggregated Benchmark Runner +# +# ENGINE=sglang (default): SGLang benchmark +# ENGINE=vllm: vLLM benchmark +# +# Produces JSON result files via benchmark_serving.py so that the CI pipeline +# can collect and process results. +# +# Usage: bash bench.sh \ +# \ +# + +ENGINE="${ENGINE:-sglang-disagg}" + +model_path=$1 +model_name=$2 +concurrency_list=${3:-"1"} +MODEL_PATH="${MODEL_PATH:-${model_path}/${model_name}}" +# vllm-disagg uses --served-model-name MODEL_NAME; sglang defaults to MODEL_PATH +if [[ "$ENGINE" == "vllm-disagg" ]]; then + MODEL="${MODEL_NAME:-${MODEL_PATH}}" +else + MODEL="${MODEL_PATH}" +fi +log_path=${4:-/run_logs} + +# Split BENCH_MAX_CONCURRENCY (x-delimited, e.g. "8x16x32") into an array. +# Falls back to 1 if unset so the loop always runs at least once. +IFS='x' read -r -a chosen_concurrencies <<< "${concurrency_list}" + + +ROUTER_PORT="${ROUTER_PORT:-30000}" + +export TRANSFORMERS_VERBOSITY=error +export TOKENIZERS_PARALLELISM=false + +# echo "Config ${chosen_isl}; ${chosen_osl}; ${chosen_concurrencies[0]}; ${chosen_req_rate}" + +RESULT_DIR="${RESULT_DIR:-${log_path}/agentic}" +mkdir -p "$RESULT_DIR" + +source "$(dirname "$0")/../../benchmark_lib.sh" + +# clear_kv_caches — wipe all KV cache tiers on every backend worker before a +# concurrency point, so each conc is measured cold (no prefix reuse bleeding in +# from the previous conc). Mirrors mori-scheduler/scripts/benchmark/lib/ +# clear_caches.sh, but the worker base URLs are already resolved by +# server_sglang.sh (SERVER_FLUSH_URLS_CSV) so no SSH/IP lookup is needed. +# +# Tiers (SGLang server APIs), hit on EACH worker directly (the router does not +# fan /flush_cache out): +# L1 (GPU radix) + L2 (host hicache): POST /flush_cache — NO-OP while any +# request is in flight, so we drain-retry until "Cache flushed" or +# FLUSH_DRAIN_TIMEOUT (default 120s) elapses. +# L3 (umbp / mooncake store): POST /hicache/storage-backend/clear +# — HTTP != 200 when L3 is off, tolerated. +# Best-effort: logs WARN, never hard-fails the sweep. +clear_kv_caches() { + local drain_tmo="${FLUSH_DRAIN_TIMEOUT:-120}" + local urls_csv="${SERVER_FLUSH_URLS_CSV:-}" + if [[ -z "$urls_csv" ]]; then + echo "[clear_caches] WARN: SERVER_FLUSH_URLS_CSV unset; skipping cache flush" >&2 + return 0 + fi + local -a urls + IFS=',' read -r -a urls <<< "$urls_csv" + local url start ok resp code + for url in "${urls[@]}"; do + [[ -n "$url" ]] || continue + # L1 + L2: drain-retry until flushed (no-op while requests in flight). + start=$(date +%s); ok=0; resp="" + while :; do + resp=$(curl -sf -m 10 -X POST "${url}/flush_cache" 2>/dev/null || true) + echo "$resp" | grep -qi "Cache flushed" && { ok=1; break; } + (( $(date +%s) - start >= drain_tmo )) && break + sleep 3 + done + if [[ "$ok" == 1 ]]; then + echo "[clear_caches] ${url}: L1+L2 flushed" + else + echo "[clear_caches] WARN ${url}: L1+L2 flush NOT confirmed after ${drain_tmo}s (resp='${resp:0:80}')" >&2 + fi + # L3: storage-backend clear (umbp / mooncake). 200 when a backend is attached. + code=$(curl -s -m 60 -o /dev/null -w '%{http_code}' -X POST "${url}/hicache/storage-backend/clear" 2>/dev/null || echo 000) + if [[ "$code" == 200 ]]; then + echo "[clear_caches] ${url}: L3 store cleared" + else + echo "[clear_caches] ${url}: L3 clear http=${code} (no storage backend / L3 off — ok)" + fi + done +} + +# REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" + +PORT="${ROUTER_PORT}" +MODEL="${MODEL:-${BENCH_MODEL}}" +DURATION="${DURATION:-1800}" +export MODEL DURATION MAX_MODEL_LEN +RESULT_DIR="${RESULT_DIR:-${profile_folder}}" +# Base name for the per-conc aggregate written by the existing +# utils.agentic.aggregation.process_agentic_result module. +# The workflow guard / upload steps expect a "${RESULT_FILENAME}_conc.json" +# file per concurrency, so each concurrency below is always suffixed with +# _conc (matching agentic_srt.sh on the gb200 path). +RESULT_FILENAME_BASE="${RESULT_FILENAME:-agentic_bench}" + +mkdir -p "$RESULT_DIR" + +resolve_trace_source +install_agentic_deps + +ANY_FAILED=0 +for max_concurrency in "${chosen_concurrencies[@]}"; do + + echo "==========================================" + echo "Agentic trace replay: conc=$max_concurrency" + echo "==========================================" + + # Clear all KV cache tiers on every backend before this conc point so it is + # measured cold (no prefix reuse from the previous conc). Default on; set + # CLEAR_CACHE_BETWEEN_CONC=0 to disable. Best-effort — never fails the run. + if [[ "${CLEAR_CACHE_BETWEEN_CONC:-1}" == "1" ]]; then + echo "conc=$max_concurrency: clearing L1/L2/L3 on all backends (no server restart)" + clear_kv_caches || echo "WARNING: cache clear had issues for conc=$max_concurrency" >&2 + fi + + # Mirror agentic_srt.sh (the srtctl/gb200 path): every concurrency writes + # its artifacts into a conc_/ subdir of RESULT_DIR. The CI matrix explodes + # agentic runs to one concurrency per job, but benchmark-multinode-tmpl.yml + # still expects the per-conc nesting (LOGS/agentic/conc_*/...) and the + # _conc result-file suffix, so we always nest to keep the layout identical + # across runners and avoid overwriting earlier runs in local multi-conc sweeps. + CONC_RESULT_DIR="$RESULT_DIR/conc_${max_concurrency}" + mkdir -p "$CONC_RESULT_DIR" + + CONC="$max_concurrency" + USERS="$max_concurrency" + export CONC USERS + build_replay_cmd "$CONC_RESULT_DIR" + + # Per-conc result name consumed by write_agentic_result_json. Always suffix + # with _conc so the file matches + # the workflow guard's "${RESULT_FILENAME}_conc*.json" glob (and the agg / + # checkpoint upload steps) for both single-conc CI runs and multi-conc sweeps. + export RESULT_FILENAME="${RESULT_FILENAME_BASE}_conc${max_concurrency}" + if ! run_agentic_replay_and_write_outputs "$CONC_RESULT_DIR"; then + echo "WARNING: agentic trace replay for conc=$max_concurrency failed (replay or validation) after writing available results" >&2 + ANY_FAILED=1 + fi + + echo "-----------------------------------------" + +done + +export RESULT_FILENAME="$RESULT_FILENAME_BASE" + +if [ "$ANY_FAILED" -ne 0 ]; then + echo "WARNING: at least one conc had a non-zero exit; per-conc result files were still written when possible." >&2 +fi diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 687439a98f..cad7cf4eef 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -3029,3 +3029,36 @@ minimaxm3-fp8-mi325x-vllm-agentic: - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: native, conc-list: [3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] } - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: native, conc-list: [10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 32] } - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: native, conc-list: [24, 32, 36, 40, 44, 48, 52, 56, 60, 64, 72, 80, 96] } + +dsv4-fp4-mi355x-sglang-disagg-agentic-hicache: + image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260709 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:mi355x-amds + precision: fp4 + framework: sglang-disagg + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - spec-decoding: "none" + conc-list: [ 16,32,48,64 ] + kv-offloading: dram + kv-offload-backend: hicache + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MTP_SIZE=0" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c1a3c3aee8..58f98a529c 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4699,3 +4699,10 @@ - "Clean the export envs" - "Enable two batch overlap" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2093 + +- config-keys: + - dsv4-fp4-mi355x-sglang-disagg-agentic-hicache + description: + - "Bump the SGLang ROCm image from 20260706 to 20260709 and retire the DeepSeek-V4 compress-state, DSA paged-MQA, AITER instruction-shape, and HiCache host-pool shims that are already included upstream." + - "Remove the SWA re-prefill and unified-KV HiCache overlays after their upstream PRs merged; retain only patch code without a merged upstream equivalent." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2127 diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index 8cb92b7a16..1ff0dffd59 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -78,7 +78,15 @@ if [[ "$IS_MULTINODE" == "true" ]]; then SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_mi355x_${FRAMEWORK}.sh" if [[ "$FRAMEWORK" == "sglang-disagg" ]] || [[ "$FRAMEWORK" == "vllm-disagg" ]] || [[ "$FRAMEWORK" == "atom-disagg" ]]; then - BENCHMARK_SUBDIR="multi_node" + # Agentic recipes live under multi_node/agentic/ and export the + # HiCache tunables (page-size, io-backend, ...); fixed-seq-len recipes + # live at the multi_node/ root. Honor SCENARIO_SUBDIR so agentic-coding + # configs pick the agentic recipe instead of the root one. + if [[ "${SCENARIO_SUBDIR}" == "agentic/" ]]; then + BENCHMARK_SUBDIR="multi_node/agentic" + else + BENCHMARK_SUBDIR="multi_node" + fi else BENCHMARK_SUBDIR="single_node/fixed_seq_len" fi @@ -126,7 +134,7 @@ if [[ "$IS_MULTINODE" == "true" ]]; then # search for "FRAMEWORK_DIFF_IF_STATEMENT #3" for this if-statement # Find the latest log directory that contains the data - if [[ "${EVAL_ONLY:-false}" != "true" ]]; then + if [[ "${EVAL_ONLY:-false}" != "true" && "${IS_AGENTIC:-0}" != "1" ]]; then cat > collect_latest_results.py <<'PY' import os, sys job_dir, isl, osl, nexp, framework = sys.argv[1], int(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4]), sys.argv[5] @@ -181,6 +189,58 @@ PY fi fi + # Stage agentic raw artifacts + server logs for the CI upload steps. + # server_sglang.sh copies /run_logs/slurm_job- to + # $BENCHMARK_LOGS_DIR/logs/slurm_job- on shared storage, and + # trace_replay.sh writes each concurrency's aiperf artifacts under + # agentic/conc_/ (mirroring agentic_srt.sh). benchmark-multinode-tmpl.yml + # uploads them from $GITHUB_WORKSPACE/LOGS/agentic/conc_*/... plus a + # multinode_server_logs.tar.gz, so preserve the conc_/ nesting here + # before the logs dir is removed below. The agg result JSON is already + # written straight to the mounted workspace by the existing agentic + # aggregation module. + if [[ "${IS_AGENTIC:-0}" == "1" ]]; then + JOB_LOGS_DIR="$BENCHMARK_LOGS_DIR/logs/slurm_job-${JOB_ID}" + if [ -d "$JOB_LOGS_DIR" ]; then + # trace_replay.sh always nests artifacts under agentic/conc_/. + # Copy the whole agentic/ tree so the conc_/ subdirs are + # preserved for the LOGS/agentic/conc_*/... upload globs. + AGENTIC_SRC="$JOB_LOGS_DIR/agentic" + if [ -d "$AGENTIC_SRC" ] && find "$AGENTIC_SRC" -mindepth 1 -maxdepth 1 -type d -name 'conc_*' -print -quit 2>/dev/null | grep -q .; then + echo "Staging agentic raw artifacts from $AGENTIC_SRC" + mkdir -p "$GITHUB_WORKSPACE/LOGS/agentic" + cp -r "$AGENTIC_SRC"/. "$GITHUB_WORKSPACE/LOGS/agentic/" + # The source artifacts are created inside the container as root + # (--container-remap-root), so depending on how the runner + # invokes this script the copies can end up root-owned and/or + # read-only (aiperf/server_sglang make some dirs mode 0555). If + # the staged tree isn't owned+writable by the runner user, the + # next checkout's `git clean` fails with + # EACCES: permission denied, rmdir '.../LOGS/agentic'. + # chown to the invoking user (the same one that runs git clean) + # via sudo (already passwordless here for rm -rf). The follow-up + # chmod uses a+rwX (not just u+rwX): the *next* job against this + # same $GITHUB_WORKSPACE may be picked up by a different runner + # process running as a different OS user, which would otherwise + # fall outside the owner bits and still fail the same + # `git clean` with EACCES despite the chown above. + sudo chown -R "$(id -u):$(id -g)" "$GITHUB_WORKSPACE/LOGS" 2>/dev/null || true + chmod -R a+rwX "$GITHUB_WORKSPACE/LOGS" 2>/dev/null || true + ls -laR "$GITHUB_WORKSPACE/LOGS/agentic" + else + echo "WARNING: no agentic conc_*/ artifacts found under $JOB_LOGS_DIR/agentic" + fi + # Server/router/prefill/decode logs for the multinode_server_logs_* artifact. + if tar czf "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz" -C "$JOB_LOGS_DIR" . 2>/dev/null; then + echo "Created multinode_server_logs.tar.gz" + else + echo "WARNING: failed to create multinode_server_logs.tar.gz" + fi + else + echo "WARNING: agentic staging skipped; $JOB_LOGS_DIR not found" + fi + fi + echo "All result files processed" # Use sync scancel to ensure nfs file handle is released in time set +x diff --git a/utils/agentic/aggregation/backends/sglang.py b/utils/agentic/aggregation/backends/sglang.py index 2f560f3bf7..acbbc12b02 100644 --- a/utils/agentic/aggregation/backends/sglang.py +++ b/utils/agentic/aggregation/backends/sglang.py @@ -61,6 +61,8 @@ def populate( flat["server_gpu_cache_hit_rate"] = rate(device_hits, prompt_total) flat["server_cpu_cache_hit_rate"] = rate(host_hits, prompt_total) + # HiCache host hits are external to the GPU cache. + flat["server_external_cache_hit_rate"] = flat["server_cpu_cache_hit_rate"] flat["server_overall_cache_hit_rate"] = rate(total_cached, prompt_total) if flat["server_overall_cache_hit_rate"] is None: diff --git a/utils/agentic/aggregation/test_process_agentic_result.py b/utils/agentic/aggregation/test_process_agentic_result.py index 70bd16a172..db4fdb2e8a 100644 --- a/utils/agentic/aggregation/test_process_agentic_result.py +++ b/utils/agentic/aggregation/test_process_agentic_result.py @@ -1024,6 +1024,7 @@ def test_processor_normalizes_sglang_server_metrics(tmp_path: Path): _assert_stable_server_metrics_schema(agg) assert agg["server_metrics"]["cache"]["gpu_cache_hit_rate"] == pytest.approx(0.4) assert agg["server_metrics"]["cache"]["cpu_cache_hit_rate"] == pytest.approx(0.1) + assert agg["server_metrics"]["cache"]["external_cache_hit_rate"] == pytest.approx(0.1) assert agg["server_metrics"]["cache"]["overall_cache_hit_rate"] == pytest.approx(0.5) assert agg["server_metrics"]["kv_cache"]["gpu_usage_pct"] == pytest.approx(0.75) assert agg["server_metrics"]["kv_cache"]["cpu_usage_pct"] == pytest.approx(0.3) From c024d9e23b488d1d572fcbef91abd6619fa3b986 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Thu, 9 Jul 2026 06:31:15 +0000 Subject: [PATCH 02/15] run-sweep: fix broken conc-list interpolation for multi-node agentic matrix.config.conc for multi-node agentic is already a JSON array (e.g. [16,32,64] from generate_sweep_configs.py). Wrapping it in a string literal as '[${{ matrix.config.conc }}]' can't correctly interpolate an array into a string, producing a malformed conc-list so fromJson(inputs.conc-list) in the reusable template ended up with an empty/wrong CONC_LIST. Use toJson(matrix.config.conc) instead, matching the pattern already used for the other two conc-list inputs in this file. Co-authored-by: Cursor --- .github/workflows/run-sweep.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index 1f5e1e7bc0..507f795934 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -536,7 +536,7 @@ jobs: model-prefix: ${{ matrix.config.model-prefix }} framework: ${{ matrix.config.framework }} precision: ${{ matrix.config.precision }} - conc-list: '[${{ matrix.config.conc }}]' + conc-list: ${{ toJson(matrix.config.conc) }} spec-decoding: ${{ matrix.config.spec-decoding }} disagg: ${{ matrix.config.disagg }} prefill-hardware: ${{ matrix.config.prefill.hardware }} From 82b4eff311da26344b9dfda6a7d71d99a3ff8215 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Fri, 10 Jul 2026 03:02:32 +0000 Subject: [PATCH 03/15] bump image to rocm/sgl-dev:sglang-0.5.14-rocm720-mi35x-mori-0706 Signed-off-by: Theresa Shan --- configs/amd-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index cad7cf4eef..5d5b5b772b 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -3031,7 +3031,7 @@ minimaxm3-fp8-mi325x-vllm-agentic: - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: native, conc-list: [24, 32, 36, 40, 44, 48, 52, 56, 60, 64, 72, 80, 96] } dsv4-fp4-mi355x-sglang-disagg-agentic-hicache: - image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260709 + image: rocm/sgl-dev:sglang-0.5.14-rocm720-mi35x-mori-0706 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:mi355x-amds From 55a79833a210f32098aae538392e3618692e4092 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Fri, 10 Jul 2026 03:14:15 +0000 Subject: [PATCH 04/15] run-sweep: fix sequence-typed conc input for multi-node agentic matrix.config.conc for multi-node agentic entries is a JSON array (chunked concurrencies per allocation), but sweep-multi-node-agentic passed it directly into benchmark-multinode-tmpl.yml's `conc` input, which is declared `type: string` ("First concurrency for agentic-coding scenarios; CONC_LIST carries the full batch"). GitHub Actions' reusable-workflow input validator rejects a sequence value for a string-typed input at evaluation time, so the whole job failed to even load: evaluate reusable workflow inputs: .github/workflows/run-sweep.yml (Line: 554, Col: 19): A sequence was not expected Since the job never materializes when this happens, sweep-multi-node- agentic silently disappeared from run summaries entirely instead of showing as failed. Slice to the first element (matching the intended "first concurrency" semantics and the same fix already applied elsewhere, e.g. PR #2122) to restore a scalar value. Co-authored-by: Cursor --- .github/workflows/run-sweep.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index 507f795934..9fa7db41a1 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -551,7 +551,7 @@ jobs: decode-ep: ${{ matrix.config.decode.ep }} decode-dp-attn: ${{ matrix.config.decode.dp-attn }} decode-additional-settings: ${{ toJson(matrix.config.decode.additional-settings) }} - conc: ${{ matrix.config.conc }} + conc: ${{ matrix.config.conc[0] }} kv-offloading: ${{ matrix.config.kv-offloading }} kv-offload-backend: ${{ matrix.config.kv-offload-backend }} duration: ${{ matrix.config.duration }} From 9a52ee2f0eca95df8d0ddc5f1c5a5a1df97bc893 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Fri, 10 Jul 2026 03:36:55 +0000 Subject: [PATCH 05/15] agentic: run one task per concurrency for multi-node sweeps Multi-node agentic sweeps batched up to 4 concurrencies per SLURM allocation, running them sequentially against one shared server session. A slow/hung conc could block the rest of the batch from ever producing results, which is why run #6719 only reported c16 despite a 16/32/48/64 conc-list. Drop the batch size to 1 so each concurrency gets its own task/allocation, matching the granularity already used for single-node agentic sweeps. Cherry-picked from backup/agentx-v1.0-rebase-pre-upstream-rewrite-20260710 (082a59de0), adapted for the current test suite. Co-authored-by: Cursor --- utils/matrix_logic/generate_sweep_configs.py | 3 ++- .../test_generate_sweep_configs.py | 18 +++++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index ca03bf9a83..266169c08d 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -24,7 +24,8 @@ MIN_EVAL_CONC = 16 # Bound how many multinode agentic conc points share one server allocation. -MAX_MULTINODE_AGENTIC_CONCURRENCIES_PER_ALLOCATION = 4 +# 1 = one task/SLURM allocation per concurrency (matches single-node agentic). +MAX_MULTINODE_AGENTIC_CONCURRENCIES_PER_ALLOCATION = 1 BYTES_PER_MIB = 1024 * 1024 BYTES_PER_GB = 1_000_000_000 # 3 TB decimal DRAM cap, expressed in MiB, before utilization scaling. diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index fcc15fac2e..0f83cd3ae4 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -2057,7 +2057,7 @@ def test_agentic_node_dram_rejects_tp_above_runner_gpus(self, sample_runner_conf generate_test_config_sweep(args, config, runner_config) def test_multinode_agentic_groups_concurrencies_per_search_entry(self): - """One server allocation should run the selected concurrency batch.""" + """One server allocation should run exactly one concurrency (one task per conc).""" config = { "dsv4-agentic-2p1d": { "image": "vllm/vllm-openai:v0.23.0", @@ -2093,11 +2093,15 @@ def test_multinode_agentic_groups_concurrencies_per_search_entry(self): result = generate_test_config_sweep(args, config) - assert len(result) == 2 - assert result[0]["conc"] == [16, 32, 64, 128] - assert result[0]["exp-name"] == "dsv4_p2x8_d1x8_conc16x32x64x128" - assert result[1]["conc"] == [256] - assert result[1]["exp-name"] == "dsv4_p2x8_d1x8_conc256" + assert len(result) == 5 + assert [entry["conc"] for entry in result] == [[16], [32], [64], [128], [256]] + assert [entry["exp-name"] for entry in result] == [ + "dsv4_p2x8_d1x8_conc16", + "dsv4_p2x8_d1x8_conc32", + "dsv4_p2x8_d1x8_conc64", + "dsv4_p2x8_d1x8_conc128", + "dsv4_p2x8_d1x8_conc256", + ] def test_multinode_agentic_preserves_kv_offload_fields(self): config = { @@ -2259,7 +2263,7 @@ def test_node_type_filters_apply_to_agentic_configs( "agentic-coding": [{ "search-space": [ { - "conc-list": [16, 32], + "conc-list": [16], "prefill": {"hardware": "gb200", "num-worker": 2, "tp": 8, "ep": 8, "dp-attn": False}, "decode": {"hardware": "h100", "num-worker": 1, "tp": 8, "ep": 1, "dp-attn": False}, }, From c96cf8160342ff90a872e65b057257ce8e7ebe8a Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Fri, 10 Jul 2026 15:55:51 +0800 Subject: [PATCH 06/15] Bump image to latest upstream image with 4 PR fixes included. --- configs/amd-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 5d5b5b772b..f61881de2d 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -3031,7 +3031,7 @@ minimaxm3-fp8-mi325x-vllm-agentic: - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: native, conc-list: [24, 32, 36, 40, 44, 48, 52, 56, 60, 64, 72, 80, 96] } dsv4-fp4-mi355x-sglang-disagg-agentic-hicache: - image: rocm/sgl-dev:sglang-0.5.14-rocm720-mi35x-mori-0706 + image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:mi355x-amds From 8040648a47804f85b98aa7d2d242dbaec1b986d7 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Fri, 10 Jul 2026 11:11:33 +0000 Subject: [PATCH 07/15] update con-list Signed-off-by: Theresa Shan --- configs/amd-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index f61881de2d..4ae6fd9232 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -3044,7 +3044,7 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache: - dram-utilization: 0.80 search-space: - spec-decoding: "none" - conc-list: [ 16,32,48,64 ] + conc-list: [ 2,4,8,16,32 ] kv-offloading: dram kv-offload-backend: hicache prefill: From bb95016b61068fd405e29de2e6c0cf72fe3cb1fb Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Fri, 10 Jul 2026 11:18:26 +0000 Subject: [PATCH 08/15] setup_deps: remove disabled decode_tp_queue_agree patch patch_decode_tp_queue_agree()'s invocation was already commented out (dead code with no runtime effect), and the reference-only patches/decode_tp_queue_agree.patch it mirrored had no other callers. Drop both plus the README bullet pointing at it. Co-authored-by: Cursor --- .../multi_node/amd_utils/patches/README.md | 3 - .../patches/decode_tp_queue_agree.patch | 107 ------------ benchmarks/multi_node/amd_utils/setup_deps.sh | 157 ------------------ 3 files changed, 267 deletions(-) delete mode 100644 benchmarks/multi_node/amd_utils/patches/decode_tp_queue_agree.patch diff --git a/benchmarks/multi_node/amd_utils/patches/README.md b/benchmarks/multi_node/amd_utils/patches/README.md index 55abad7a5f..765d571b27 100644 --- a/benchmarks/multi_node/amd_utils/patches/README.md +++ b/benchmarks/multi_node/amd_utils/patches/README.md @@ -8,9 +8,6 @@ block our benchmark + accuracy configs — so we can keep reusing the - `mori_conn.py` — single-file overlay (bind-mounted) for the **sglang** MoRI backend. -- `decode_tp_queue_agree.patch` — reference-only unified diff for the local - TP-rank decode queue agreement fix. Its runtime invocation is currently - disabled in `../setup_deps.sh`. > Note: the vLLM MoRIIO `minimax-m3` overlay (`moriio/`) was retired once the > upstream fixes (vLLM #46039 / #46290 / #46332) shipped in the ROCm nightly diff --git a/benchmarks/multi_node/amd_utils/patches/decode_tp_queue_agree.patch b/benchmarks/multi_node/amd_utils/patches/decode_tp_queue_agree.patch deleted file mode 100644 index e09c02112f..0000000000 --- a/benchmarks/multi_node/amd_utils/patches/decode_tp_queue_agree.patch +++ /dev/null @@ -1,107 +0,0 @@ ---- a/python/sglang/srt/disaggregation/decode.py 2026-06-25 02:44:11.274035348 +0000 -+++ b/python/sglang/srt/disaggregation/decode.py 2026-06-25 02:44:11.276035310 +0000 -@@ -1616,8 +1616,57 @@ - ) - kv_manager._staging_handler = self.staging_handler - -+ def _agree_and_order_queue(self) -> Tuple[List["DecodeRequest"], List["DecodeRequest"]]: -+ """Split self.queue into (gated, deferred) using a tp-group agreement. -+ -+ The metadata gate (utils.poll_and_all_reduce) issues a tp-group all_reduce -+ whose tensor shape == len(self.queue) and whose per-index meaning is the -+ i-th queued request. That is only correct if every tp rank polls an -+ IDENTICAL queue (same requests, same order). Per-rank prealloc/enqueue -+ timing can leave queues with different membership or order, which desyncs -+ the collective -> deadlock (the JID-17417 decode hang). -+ -+ We all_gather the local request ids, keep only requests present on EVERY -+ rank, and order them deterministically (request ids are rank-invariant), so -+ the subsequent gate runs a matching collective on all ranks. Requests not -+ yet on every rank are returned as `deferred` and retried on a later call. -+ -+ NOTE: every rank that reaches pop_transferred must call this (it contains a -+ collective). That holds for the same reason the existing gate is safe: -+ pop_transferred is entered rank-symmetrically over self.gloo_group. -+ """ -+ tp_size = torch.distributed.get_world_size(self.gloo_group) -+ if tp_size <= 1: -+ return self.queue, [] -+ -+ local_rids = [dr.req.rid for dr in self.queue] -+ gathered: List[Optional[List[str]]] = [None] * tp_size -+ torch.distributed.all_gather_object( -+ gathered, local_rids, group=self.gloo_group -+ ) -+ -+ common = set(gathered[0] or []) -+ for rids in gathered[1:]: -+ common &= set(rids or []) -+ -+ if not common: -+ return [], list(self.queue) -+ -+ by_rid = {dr.req.rid: dr for dr in self.queue} -+ # sorted() yields an identical order on every rank (rids are rank-invariant). -+ gated = [by_rid[rid] for rid in sorted(common)] -+ deferred = [dr for dr in self.queue if dr.req.rid not in common] -+ return gated, deferred -+ - def pop_transferred(self, rids_to_check: Optional[List[str]] = None) -> List[Req]: -+ # Agree on a tp-rank-identical, identically-ordered subset BEFORE issuing any -+ # metadata-gate collective. Do NOT add a local `if not self.queue` early -+ # return here: an empty-queue rank must still join the agreement collective, -+ # otherwise it skips it and desyncs the tp group (root cause of the hang). -+ self.queue, _deferred = self._agree_and_order_queue() - if not self.queue: -+ # Empty agreement is rank-symmetric: all ranks skip the gate together. -+ self.queue = _deferred - return [] - - if self.scheduler.enable_decode_hicache: -@@ -1731,6 +1780,10 @@ - self.queue = [ - entry for i, entry in enumerate(self.queue) if i not in indices_to_remove - ] -+ # Re-attach requests that were not yet present on every tp rank this -+ # iteration; they are gated again on a later call once all ranks have them. -+ if _deferred: -+ self.queue.extend(_deferred) - - return transferred_reqs - -@@ -1946,8 +1999,33 @@ - # try to resume retracted requests if there are enough space for another `num_reserved_decode_tokens` decode steps - resumed_reqs = self.disagg_decode_prealloc_queue.resume_retracted_reqs() - self.waiting_queue.extend(resumed_reqs) -- if len(self.disagg_decode_prealloc_queue.retracted_queue) > 0: -- # if there are still retracted requests, we do not allocate new requests -+ # PATCH(call-site tp-agreement): the retracted-queue early return below is -+ # rank-divergent — retracted_queue length is per-rank (per-rank KV-cache -+ # pressure). A divergent return skips the polling_count increment and the -+ # pop_transferred() collectives on some ranks, permanently desyncing the tp -+ # group: one rank ends up a full collective ahead, so pop_transferred's -+ # _agree_and_order_queue all_gather_object (decode.py:1644) on the lagging -+ # ranks lines up against the metadata-gate all_reduce (decode.py:1591) on -+ # the leading rank -> mismatched-collective deadlock (GPU 0%, detokenizer -+ # heartbeat freeze; JID-17445). process_decode_queue is called -+ # unconditionally every event-loop iteration on every rank, so an all_reduce -+ # here (before any divergent return) is rank-symmetric. Hold off new -+ # allocation iff ANY rank still has retracted reqs, so all ranks branch -+ # identically every iteration and pop_transferred is entered symmetrically. -+ _agree_gg = self.disagg_decode_transfer_queue.gloo_group -+ _local_retracted = ( -+ 1 if len(self.disagg_decode_prealloc_queue.retracted_queue) > 0 else 0 -+ ) -+ if torch.distributed.get_world_size(_agree_gg) > 1: -+ _retracted_t = torch.tensor([_local_retracted], dtype=torch.int32) -+ torch.distributed.all_reduce( -+ _retracted_t, op=torch.distributed.ReduceOp.MAX, group=_agree_gg -+ ) -+ _any_retracted = bool(_retracted_t.item()) -+ else: -+ _any_retracted = bool(_local_retracted) -+ if _any_retracted: -+ # if any rank still has retracted requests, no rank allocates new ones - return - - if not hasattr(self, "polling_count"): diff --git a/benchmarks/multi_node/amd_utils/setup_deps.sh b/benchmarks/multi_node/amd_utils/setup_deps.sh index 5ee7078467..9f10958de1 100644 --- a/benchmarks/multi_node/amd_utils/setup_deps.sh +++ b/benchmarks/multi_node/amd_utils/setup_deps.sh @@ -121,162 +121,6 @@ print("[SETUP] Patched: disaggregation/prefill.py resolve_waiting_queue_bootstra _SETUP_INSTALLED+=("prefill-bootstrap-desync-fix") } -# --------------------------------------------------------------------------- -# SGLang: tp-group agreement for disagg decode queue (decode_tp_queue_agree). -# -# The metadata gate (poll_and_all_reduce) issues a tp-group collective whose -# shape/order == self.queue. Per-rank prealloc/enqueue timing can leave ranks -# with different queue membership/order, desyncing the collective -> decode -# hang (JID-17417 / JID-17445). This inserts _agree_and_order_queue() so every -# rank polls an identical, identically-ordered subset, and replaces the -# rank-divergent retracted-queue early return in process_decode_queue with a -# rank-symmetric MAX all_reduce. Mirrors patches/decode_tp_queue_agree.patch. -# --------------------------------------------------------------------------- -patch_decode_tp_queue_agree() { - python3 - <<'PYEOF' -import os, sys - -target = "/sgl-workspace/sglang/python/sglang/srt/disaggregation/decode.py" -if not os.path.isfile(target): - print("[SETUP] disaggregation/decode.py not found, skipping") - sys.exit(0) - -src = open(target).read() - -if "_agree_and_order_queue" in src: - print("[SETUP] decode tp-queue-agree patch already applied") - sys.exit(0) - -# --- Hunk 1+2: insert agreement method + gate pop_transferred ------------- -old1 = ( - " def pop_transferred(self, rids_to_check: Optional[List[str]] = None) -> List[Req]:\n" - " if not self.queue:\n" - " return []\n" -) -new1 = ''' def _agree_and_order_queue(self) -> Tuple[List["DecodeRequest"], List["DecodeRequest"]]: - """Split self.queue into (gated, deferred) using a tp-group agreement. - - The metadata gate (utils.poll_and_all_reduce) issues a tp-group all_reduce - whose tensor shape == len(self.queue) and whose per-index meaning is the - i-th queued request. That is only correct if every tp rank polls an - IDENTICAL queue (same requests, same order). Per-rank prealloc/enqueue - timing can leave queues with different membership or order, which desyncs - the collective -> deadlock (the JID-17417 decode hang). - - We all_gather the local request ids, keep only requests present on EVERY - rank, and order them deterministically (request ids are rank-invariant), so - the subsequent gate runs a matching collective on all ranks. Requests not - yet on every rank are returned as `deferred` and retried on a later call. - - NOTE: every rank that reaches pop_transferred must call this (it contains a - collective). That holds for the same reason the existing gate is safe: - pop_transferred is entered rank-symmetrically over self.gloo_group. - """ - tp_size = torch.distributed.get_world_size(self.gloo_group) - if tp_size <= 1: - return self.queue, [] - - local_rids = [dr.req.rid for dr in self.queue] - gathered: List[Optional[List[str]]] = [None] * tp_size - torch.distributed.all_gather_object( - gathered, local_rids, group=self.gloo_group - ) - - common = set(gathered[0] or []) - for rids in gathered[1:]: - common &= set(rids or []) - - if not common: - return [], list(self.queue) - - by_rid = {dr.req.rid: dr for dr in self.queue} - # sorted() yields an identical order on every rank (rids are rank-invariant). - gated = [by_rid[rid] for rid in sorted(common)] - deferred = [dr for dr in self.queue if dr.req.rid not in common] - return gated, deferred - - def pop_transferred(self, rids_to_check: Optional[List[str]] = None) -> List[Req]: - # Agree on a tp-rank-identical, identically-ordered subset BEFORE issuing any - # metadata-gate collective. Do NOT add a local `if not self.queue` early - # return here: an empty-queue rank must still join the agreement collective, - # otherwise it skips it and desyncs the tp group (root cause of the hang). - self.queue, _deferred = self._agree_and_order_queue() - if not self.queue: - # Empty agreement is rank-symmetric: all ranks skip the gate together. - self.queue = _deferred - return [] -''' - -# --- Hunk 3: re-attach deferred reqs after removal ------------------------ -old3 = ( - " entry for i, entry in enumerate(self.queue) if i not in indices_to_remove\n" - " ]\n" - "\n" - " return transferred_reqs\n" -) -new3 = ( - " entry for i, entry in enumerate(self.queue) if i not in indices_to_remove\n" - " ]\n" - " # Re-attach requests that were not yet present on every tp rank this\n" - " # iteration; they are gated again on a later call once all ranks have them.\n" - " if _deferred:\n" - " self.queue.extend(_deferred)\n" - "\n" - " return transferred_reqs\n" -) - -# --- Hunk 4: rank-symmetric retracted-queue gate -------------------------- -old4 = ( - " if len(self.disagg_decode_prealloc_queue.retracted_queue) > 0:\n" - " # if there are still retracted requests, we do not allocate new requests\n" - " return\n" -) -new4 = ''' # PATCH(call-site tp-agreement): the retracted-queue early return below is - # rank-divergent — retracted_queue length is per-rank (per-rank KV-cache - # pressure). A divergent return skips the polling_count increment and the - # pop_transferred() collectives on some ranks, permanently desyncing the tp - # group: one rank ends up a full collective ahead, so pop_transferred's - # _agree_and_order_queue all_gather_object on the lagging ranks lines up - # against the metadata-gate all_reduce on the leading rank -> mismatched- - # collective deadlock (GPU 0%, detokenizer heartbeat freeze; JID-17445). - # process_decode_queue is called unconditionally every event-loop iteration - # on every rank, so an all_reduce here (before any divergent return) is - # rank-symmetric. Hold off new allocation iff ANY rank still has retracted - # reqs, so all ranks branch identically every iteration and pop_transferred - # is entered symmetrically. - _agree_gg = self.disagg_decode_transfer_queue.gloo_group - _local_retracted = ( - 1 if len(self.disagg_decode_prealloc_queue.retracted_queue) > 0 else 0 - ) - if torch.distributed.get_world_size(_agree_gg) > 1: - _retracted_t = torch.tensor([_local_retracted], dtype=torch.int32) - torch.distributed.all_reduce( - _retracted_t, op=torch.distributed.ReduceOp.MAX, group=_agree_gg - ) - _any_retracted = bool(_retracted_t.item()) - else: - _any_retracted = bool(_local_retracted) - if _any_retracted: - # if any rank still has retracted requests, no rank allocates new ones - return -''' - -for label, old, new in ( - ("agreement-method+pop_transferred", old1, new1), - ("deferred-reattach", old3, new3), - ("retracted-gate", old4, new4), -): - if old not in src: - print(f"[SETUP] WARN: decode.py anchor for '{label}' not found — sglang version may have changed") - sys.exit(0) - src = src.replace(old, new, 1) - -open(target, "w").write(src) -print("[SETUP] Patched: disaggregation/decode.py tp-group decode queue agreement") -PYEOF - _SETUP_INSTALLED+=("decode-tp-queue-agree-fix") -} - # --------------------------------------------------------------------------- # SGLang: Install latest transformers for GLM-5 model type support. # @@ -318,7 +162,6 @@ if [[ "$ENGINE" == "vllm-disagg" ]]; then export LD_LIBRARY_PATH="${UCX_HOME}/lib:${RIXL_HOME}/lib:${RIXL_HOME}/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH:-}" else patch_disagg_prefill_bootstrap_desync - # patch_decode_tp_queue_agree install_transformers_glm5 fi From 66ae1924c1bcf878f6b6485d2d0c3f1eb2e6c718 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Fri, 10 Jul 2026 12:54:34 +0000 Subject: [PATCH 09/15] setup_deps: broaden GLM transformers gate; disable prefill bootstrap-desync patch install_transformers_glm5() was gated on an exact MODEL_NAME == "GLM-5-FP8" match; broaden to any model name containing "GLM" so other GLM variants pick up the same glm_moe_dsa transformers fix. Also disable patch_disagg_prefill_bootstrap_desync's invocation (commented out, matching the already-disabled decode_tp_queue_agree pattern removed earlier). Co-authored-by: Cursor --- benchmarks/multi_node/amd_utils/setup_deps.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/benchmarks/multi_node/amd_utils/setup_deps.sh b/benchmarks/multi_node/amd_utils/setup_deps.sh index 9f10958de1..f1ae30b925 100644 --- a/benchmarks/multi_node/amd_utils/setup_deps.sh +++ b/benchmarks/multi_node/amd_utils/setup_deps.sh @@ -122,14 +122,15 @@ print("[SETUP] Patched: disaggregation/prefill.py resolve_waiting_queue_bootstra } # --------------------------------------------------------------------------- -# SGLang: Install latest transformers for GLM-5 model type support. +# SGLang: Install latest transformers for GLM model type support. # # GLM-5 (zai-org/GLM-5-FP8) requires a transformers build that includes -# the glm_moe_dsa model type. The mori images do not ship it. -# Only install if GLM-5 is the active model (avoid overhead otherwise). +# the glm_moe_dsa model type. The mori images do not ship it. Gated on any +# GLM model name (not just GLM-5-FP8) so other GLM variants pick up the same +# fix; only installs when a GLM model is active (avoid overhead otherwise). # --------------------------------------------------------------------------- install_transformers_glm5() { - if [[ "$MODEL_NAME" != "GLM-5-FP8" ]]; then + if [[ "$MODEL_NAME" != *GLM* ]]; then return 0 fi @@ -161,7 +162,7 @@ if [[ "$ENGINE" == "vllm-disagg" ]]; then export PATH="${UCX_HOME}/bin:/usr/local/bin/etcd:/root/.cargo/bin:${PATH}" export LD_LIBRARY_PATH="${UCX_HOME}/lib:${RIXL_HOME}/lib:${RIXL_HOME}/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH:-}" else - patch_disagg_prefill_bootstrap_desync + # patch_disagg_prefill_bootstrap_desync install_transformers_glm5 fi From 0002e515b216f83538c77bc36b6d9790c33a298a Mon Sep 17 00:00:00 2001 From: AMD-yanfeiwang Date: Fri, 10 Jul 2026 21:06:08 +0800 Subject: [PATCH 10/15] agentic: node-0 sibling benchmark-client container for DSv4 sweeps (#2147) * agentic: add node-0 sibling benchmark-client container for DSv4 sweeps Port the "same-node sibling container" client mode from ROCm/InferenceY: when CLIENT_IMAGE is set (and no CLIENT_NODES), node 0 launches the aiperf trace replay in its own pre-baked container via the host docker socket, instead of rebuilding the aiperf venv inside the server container and running it co-located. This keeps the client's CPU-heavy tokenize/aggregate work off the sglang scheduler + router, which inflates TTFT/E2E and lowers throughput under agentic concurrency. - server_sglang.sh: add IS_AGENTIC_RUN and a CLIENT_IMAGE sibling-container branch that writes client.env and docker-runs the client against the local router (--network host). - job.slurm: define CLIENT_CONT_NAME; when CLIENT_IMAGE is set, mount the host docker socket + CLI into the server container, forward HOST_REPO_DIR/HOST_MODEL_DIR/HOST_BENCH_LOGS/CLIENT_CONT_NAME, pre-pull the client image, and clean up the client container on teardown. - amd-master.yaml: enable the sibling client on dsv4-fp4-mi355x-sglang-disagg-agentic-hicache via CLIENT_IMAGE. The separate-client-NODE mode is intentionally not ported. * agentic: use server image for sibling client so upstream CI can pull it The pre-baked rocm/pytorch-private aiperf client image is not pullable by upstream CI runners. Reuse the (public) server image as CLIENT_IMAGE and build aiperf on the fly from /workspace/utils/aiperf, matching the co-located path. Gate the pre-baked-venv env (AIPERF_USE_PREBUILT / AIPERF_VENV) behind an optional CLIENT_AIPERF_VENV so a real pre-baked client image can still opt in. --- benchmarks/multi_node/amd_utils/job.slurm | 27 +++++++- .../multi_node/amd_utils/server_sglang.sh | 61 +++++++++++++++++++ configs/amd-master.yaml | 7 +++ 3 files changed, 94 insertions(+), 1 deletion(-) diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm index 72f4d9dc9d..3dff39b0d0 100755 --- a/benchmarks/multi_node/amd_utils/job.slurm +++ b/benchmarks/multi_node/amd_utils/job.slurm @@ -349,6 +349,8 @@ export DOCKER_CONT_NAME="container_${ENGINE}_${SANITIZED_USER}_${MODEL_NAME}_${S # dated tags are garbage-collected (manifest unknown) VLLM_ROUTER_IMAGE="${VLLM_ROUTER_IMAGE:-vllm/vllm-router:nightly-20260629-e667ebb}" ROUTER_CONT_NAME="router_vllm_${SANITIZED_USER}_${SLURM_JOB_ID}" +# Separate agentic benchmark-client container (see CLIENT_IMAGE handling below). +CLIENT_CONT_NAME="container_${ENGINE}_${SANITIZED_USER}_client_${SLURM_JOB_ID}" export RUN_FILE_FULL="$WS_PATH/${RUN_FILE}" SELECTED_NODELIST_SRUN=$(echo "$SELECTED_NODES" | paste -sd,) @@ -518,6 +520,27 @@ echo "[config] wrote HiCache/Mooncake settings -> $HICACHE_MC_CONFIG" # Engine-specific container filter for pre-clean CONT_FILTER="name=^container_${ENGINE}_" +# ============================================================================= +# Optional: separate benchmark-client image (agentic runs) — node-0 sibling +# ============================================================================= +# When CLIENT_IMAGE is set, node 0 runs the aiperf trace replay in its own +# sibling container built from CLIENT_IMAGE (which ships a pre-baked aiperf + +# deps), instead of rebuilding the aiperf venv inside the server container every +# run. Give the server container access to the host docker socket + CLI and the +# host paths the sibling needs for its bind mounts. These fragments are expanded +# at submit time and injected into the server `docker run` below; empty (no-op) +# when CLIENT_IMAGE is unset, so the in-container aiperf path is unchanged. +CLIENT_DOCKER_MOUNTS="" +CLIENT_DOCKER_ENV="" +if [[ -n "${CLIENT_IMAGE:-}" ]]; then + HOST_DOCKER_BIN="$(command -v docker || echo /usr/bin/docker)" + CLIENT_DOCKER_MOUNTS="-v /var/run/docker.sock:/var/run/docker.sock -v ${HOST_DOCKER_BIN}:/usr/bin/docker" + CLIENT_DOCKER_ENV="-e CLIENT_IMAGE=${CLIENT_IMAGE} -e HOST_REPO_DIR=${DI_REPO_DIR} -e HOST_MODEL_DIR=${MODEL_DIR} -e HOST_BENCH_LOGS=${BENCHMARK_LOGS_DIR} -e CLIENT_CONT_NAME=${CLIENT_CONT_NAME}" + echo "[client] node-0 sibling benchmark-client image enabled: ${CLIENT_IMAGE}" + # Best-effort pre-pull on all nodes so node 0's sibling launch doesn't stall. + srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c 'eval "$DOCKER_CMD_DETECT"; $DOCKER_CMD pull '"$CLIENT_IMAGE"' >/dev/null 2>&1 || true' 2>/dev/null || true +fi + srun \ --nodelist="$SELECTED_NODELIST_SRUN" \ --kill-on-bad-exit=1 \ @@ -678,9 +701,11 @@ fi -v ${DI_REPO_DIR}:${DOCKER_MOUNT_PATH} \ -v ${HICACHE_MC_CONFIG}:/config/hicache_mc.env:ro \ ${EXTRA_DOCKER_MOUNTS:-} \ + ${CLIENT_DOCKER_MOUNTS} \ \${RDMA_MOUNTS[@]+"\${RDMA_MOUNTS[@]}"} \ ${DOCKER_ENV_COMMON[*]} \ ${DOCKER_ENV_ENGINE[*]} \ + ${CLIENT_DOCKER_ENV} \ --name \"$DOCKER_CONT_NAME\" \ --entrypoint \"\" \ \"$DOCKER_IMAGE_NAME\" bash -lc ' @@ -697,7 +722,7 @@ exit \$DOCKER_EXIT_CODE " if [[ "${KEEP_CONTAINERS}" != "1" ]]; then - srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c 'eval "$DOCKER_CMD_DETECT"; $DOCKER_CMD rm -f '"$DOCKER_CONT_NAME"' 2>/dev/null || true' + srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c 'eval "$DOCKER_CMD_DETECT"; $DOCKER_CMD rm -f '"$DOCKER_CONT_NAME"' '"$CLIENT_CONT_NAME"' 2>/dev/null || true' # Clean up vLLM external router container on node 0 if [[ "$ENGINE" == "vllm-disagg" && "$ROUTER_TYPE" == "vllm-router" ]]; then diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh index 47ccfee927..d4da07548a 100755 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ b/benchmarks/multi_node/amd_utils/server_sglang.sh @@ -762,10 +762,71 @@ if [ "$NODE_RANK" -eq 0 ]; then echo "Benchmark runner: bench.sh (fixed-seq-len)" fi + IS_AGENTIC_RUN=0 + if [[ "${IS_AGENTIC:-0}" == "1" || "${IS_AGENTIC:-}" == "true" ]]; then + IS_AGENTIC_RUN=1 + fi + if [[ "${EVAL_ONLY:-false}" == "true" ]]; then echo "EVAL_ONLY mode: skipping throughput benchmark" elif [[ "$DRY_RUN" -eq 1 ]]; then echo "DRY RUN: $BENCH_CMD" + elif [[ -n "${CLIENT_IMAGE:-}" && "$IS_AGENTIC_RUN" == "1" ]]; then + # Separate client image (node-0 sibling container): run the aiperf trace + # replay in its own sibling container built from CLIENT_IMAGE (which ships + # a pre-baked aiperf + deps) instead of rebuilding the aiperf venv inside + # this server container. The server/router stay up in this container while + # the client container drives the benchmark against the router on + # localhost (--network host). job.slurm mounts the host docker socket + CLI + # into this container and forwards HOST_REPO_DIR / HOST_MODEL_DIR / + # HOST_BENCH_LOGS / CLIENT_CONT_NAME so the sibling can be launched here. + CLIENT_ENV_FILE="/run_logs/slurm_job-${SLURM_JOB_ID}/client.env" + mkdir -p "/run_logs/slurm_job-${SLURM_JOB_ID}" + # Forward the benchmark-relevant env (incl. runtime-computed metrics/flush + # URLs) to the client container; override the few paths/flags that differ + # inside the pre-baked image. Unset vars are skipped, so the client keeps + # its own defaults for anything not exported here. + { + for _v in ENGINE MODEL_NAME MODEL_PREFIX PRECISION FRAMEWORK SPEC_DECODING \ + DURATION MAX_MODEL_LEN RESULT_FILENAME RUNNER_NAME \ + AIPERF_SERVER_METRICS_URLS SERVER_FLUSH_URLS_CSV \ + ENABLE_METRICS IS_AGENTIC CLEAR_CACHE_BETWEEN_CONC \ + KV_OFFLOADING KV_OFFLOAD_BACKEND TOTAL_CPU_DRAM_GB \ + WEKA_LOADER_OVERRIDE AIPERF_FAILED_REQUEST_THRESHOLD \ + AIPERF_AGENTIC_CACHE_WARMUP_DURATION AIPERF_UNSAFE_OVERRIDE \ + AIPERF_TRAJECTORY_START_MIN_RATIO AIPERF_TRAJECTORY_START_MAX_RATIO \ + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES ROUTER_PORT TQDM_MININTERVAL; do + if [[ -n "${!_v+x}" ]]; then printf '%s=%s\n' "$_v" "${!_v}"; fi + done + echo "INFMAX_CONTAINER_WORKSPACE=/workspace" + echo "AGENTIC_OUTPUT_DIR=/run_logs/slurm_job-${SLURM_JOB_ID}" + echo "HF_HOME=/run_logs/hf_cache" + echo "MODEL_DIR=/models" + # A pre-baked client image ships aiperf at CLIENT_AIPERF_VENV; when + # unset (e.g. reusing the server image, which carries no pre-baked + # venv), trace_replay builds aiperf on the fly from + # /workspace/utils/aiperf — same as the co-located path. + if [[ -n "${CLIENT_AIPERF_VENV:-}" ]]; then + echo "AIPERF_USE_PREBUILT=1" + echo "AIPERF_VENV=${CLIENT_AIPERF_VENV}" + fi + } > "$CLIENT_ENV_FILE" + + echo "Launching agentic benchmark in separate client container: ${CLIENT_IMAGE}" + docker rm -f "${CLIENT_CONT_NAME}" 2>/dev/null || true + set -x + docker run --rm --network host \ + --name "${CLIENT_CONT_NAME}" \ + --shm-size 32G \ + -v "${HOST_REPO_DIR}:/workspace" \ + -v "${HOST_MODEL_DIR}:/models" \ + -v /tmp:/run_logs \ + -v "${HOST_BENCH_LOGS}:/benchmark_logs" \ + --env-file "${CLIENT_ENV_FILE}" \ + --entrypoint "" \ + "${CLIENT_IMAGE}" \ + bash -lc "cd /workspace/benchmarks/multi_node/amd_utils && bash trace_replay.sh /models ${MODEL_NAME} \"${BENCH_MAX_CONCURRENCY}\" /run_logs/slurm_job-${SLURM_JOB_ID}" + set +x else set -x eval "$BENCH_CMD" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 4ae6fd9232..4de7f497a1 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -3054,6 +3054,13 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache: dp-attn: false additional-settings: - "PREFILL_NODES=1" + # Node-0 sibling client: run the aiperf trace-replay in its own sibling + # container on node 0 (via the host docker socket) instead of inside the + # server container. Reuse the (publicly pullable) server image so + # upstream CI can fetch it; aiperf is built on the fly from + # /workspace/utils/aiperf. Setting only CLIENT_IMAGE (no CLIENT_NODES) + # selects the sibling-container mode. + - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710" decode: num-worker: 1 tp: 8 From 98156ef6f0c7f36c77cf28a5f693284529894700 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Fri, 10 Jul 2026 13:10:07 +0000 Subject: [PATCH 11/15] enable log info and extend timeout Signed-off-by: Theresa Shan --- benchmarks/multi_node/amd_utils/models.yaml | 2 +- .../multi_node/amd_utils/server_sglang.sh | 27 ++++++++++++++----- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/benchmarks/multi_node/amd_utils/models.yaml b/benchmarks/multi_node/amd_utils/models.yaml index 698fcbc5eb..d88d143e86 100644 --- a/benchmarks/multi_node/amd_utils/models.yaml +++ b/benchmarks/multi_node/amd_utils/models.yaml @@ -359,7 +359,7 @@ DeepSeek-R1-0528-MXFP4-v2: # prefill.disable_cuda_graph routes prefill to --disable-cuda-graph; decode keeps # --cuda-graph-bs. See dsv4_mi355x_sglang_disagg_plan.md. hf_dir: "models--deepseek-ai--DeepSeek-V4-Pro" DeepSeek-V4-Pro: - base_flags: "--watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend dsv4 --page-size 256 --swa-full-tokens-ratio 0.1 --disable-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --disaggregation-transfer-backend mori --log-level error --log-level-http error" + base_flags: "--watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend dsv4 --page-size 256 --swa-full-tokens-ratio 0.1 --disable-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --disaggregation-transfer-backend mori --log-level info --log-level-http error" dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" prefill: diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh index d4da07548a..f892c812aa 100755 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ b/benchmarks/multi_node/amd_utils/server_sglang.sh @@ -508,15 +508,28 @@ fi # Container Synchronization # ============================================================================= +# sync.py barrier/health-barrier exits 1 on timeout (and prints which +# node/port never became ready), but without an explicit check here the +# script would silently continue past a timed-out barrier -- printing a +# misleading "success" message and launching the next stage against +# servers/routers that never actually came up, instead of failing fast. +run_barrier_or_die() { + local desc="$1" cmd="$2" + if ! eval "$cmd"; then + echo "FATAL: ${desc} failed — see the sync.py timeout output above for which node/port never became ready." >&2 + exit 1 + fi +} + echo "Waiting at the container creation barrier on $host_name" -python3 $SGLANG_WS_PATH/sync.py barrier \ +run_barrier_or_die "container creation barrier" "python3 $SGLANG_WS_PATH/sync.py barrier \ --local-ip ${host_ip} \ --local-port 5000 \ --enable-port \ --node-ips ${IPADDRS} \ --node-ports 5000 \ --wait-for-all-ports \ - --timeout 300 + --timeout 300" # ============================================================================= @@ -643,12 +656,12 @@ if [ "$NODE_RANK" -eq 0 ]; then --node-ips ${IPADDRS} \ --node-ports 8000 \ --wait-for-all-ports \ - --timeout 1800" + --timeout 2400" if [[ "$DRY_RUN" -eq 1 ]]; then echo "DRY RUN: $BARRIER_CMD" else - eval "$BARRIER_CMD" + run_barrier_or_die "prefill/decode server ports barrier" "$BARRIER_CMD" fi echo "Congratulations!!! All prefill and decode servers are up . . ." @@ -707,7 +720,7 @@ if [ "$NODE_RANK" -eq 0 ]; then if [[ "$DRY_RUN" -eq 1 ]]; then echo "DRY RUN: $HEALTH_BARRIER_CMD" else - eval "$HEALTH_BARRIER_CMD" + run_barrier_or_die "router readiness barrier" "$HEALTH_BARRIER_CMD" fi echo "Router is ready for benchmarking" @@ -997,7 +1010,7 @@ elif [ "$NODE_RANK" -gt 0 ] && [ "$NODE_RANK" -lt "$NODE_OFFSET" ]; then if [[ "$DRY_RUN" -eq 1 ]]; then echo "DRY RUN: $BARRIER_CMD" else - eval "$BARRIER_CMD" + run_barrier_or_die "proxy server barrier (prefill node)" "$BARRIER_CMD" fi echo "Waiting until proxy server closes..." @@ -1074,7 +1087,7 @@ else if [[ "$DRY_RUN" -eq 1 ]]; then echo "DRY RUN: $BARRIER_CMD" else - eval "$BARRIER_CMD" + run_barrier_or_die "proxy server barrier (decode node)" "$BARRIER_CMD" fi From b0b4b046e476ab89e9f436147fcd8641f963b9a9 Mon Sep 17 00:00:00 2001 From: AMD-yanfeiwang Date: Sat, 11 Jul 2026 09:40:57 +0800 Subject: [PATCH 12/15] agentic(sibling): write result JSON to /workspace so the guard finds it (#2165) The node-0 sibling client pinned AGENTIC_OUTPUT_DIR=/run_logs/slurm_job-* (host /tmp), so the aggregated ${RESULT_FILENAME}_conc.json landed outside GITHUB_WORKSPACE and the workflow result-count guard failed with "expected 1 agentic results, found 0" (run 29095134929, dsv4 c32). The co-located path leaves AGENTIC_OUTPUT_DIR unset, defaulting to INFMAX_CONTAINER_WORKSPACE=/workspace (host repo bind-mount == GITHUB_WORKSPACE). Drop the override so the sibling matches: raw artifacts still go under the trace_replay log dir (/run_logs), only the top-level result JSON moves to /workspace where the guard/upload steps expect it. --- benchmarks/multi_node/amd_utils/server_sglang.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh index f892c812aa..59c8d8ee12 100755 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ b/benchmarks/multi_node/amd_utils/server_sglang.sh @@ -812,7 +812,10 @@ if [ "$NODE_RANK" -eq 0 ]; then if [[ -n "${!_v+x}" ]]; then printf '%s=%s\n' "$_v" "${!_v}"; fi done echo "INFMAX_CONTAINER_WORKSPACE=/workspace" - echo "AGENTIC_OUTPUT_DIR=/run_logs/slurm_job-${SLURM_JOB_ID}" + # Do NOT pin AGENTIC_OUTPUT_DIR: it must default to /workspace (the + # host repo mount == GITHUB_WORKSPACE) so the aggregated + # ${RESULT_FILENAME}_conc.json lands where the workflow guard globs + # it. /workspace is bind-mounted writable, same as the co-located path. echo "HF_HOME=/run_logs/hf_cache" echo "MODEL_DIR=/models" # A pre-baked client image ships aiperf at CLIENT_AIPERF_VENV; when From ac2f9afe9825677007659e7788fa1a2cee15beda Mon Sep 17 00:00:00 2001 From: AMD-yanfeiwang Date: Sat, 11 Jul 2026 10:27:52 +0800 Subject: [PATCH 13/15] agentic(sibling): forward GPU-shape env so per-GPU throughput isn't 16x inflated (#2168) The sibling client.env omitted IS_MULTINODE / PREFILL_* / DECODE_* / TP, so process_agentic_result._gpu_shape() fell back to the single-node branch (num_gpus=1) in the client container. That left "Throughput per GPU" undivided -> ~16x too high (157484 vs the co-located 9669 tok/s at c32) and recorded wrong tp/ep/num_gpus/is_multinode/disagg in the aggregated JSON. Forward the GPU-shape + aggregation-metadata env (IS_MULTINODE, DISAGG, RUNNER_TYPE, IMAGE, TP/EP_SIZE/DP_ATTENTION/DCP_SIZE/PCP_SIZE, and the PREFILL_*/DECODE_* worker/tp/ep/dp/hardware vars) so the sibling computes the same num_gpus (16) and metadata as the co-located path. Unset vars are skipped by the existing guard. --- benchmarks/multi_node/amd_utils/server_sglang.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh index 59c8d8ee12..91ddff0304 100755 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ b/benchmarks/multi_node/amd_utils/server_sglang.sh @@ -801,9 +801,13 @@ if [ "$NODE_RANK" -eq 0 ]; then # its own defaults for anything not exported here. { for _v in ENGINE MODEL_NAME MODEL_PREFIX PRECISION FRAMEWORK SPEC_DECODING \ - DURATION MAX_MODEL_LEN RESULT_FILENAME RUNNER_NAME \ + DURATION MAX_MODEL_LEN RESULT_FILENAME RUNNER_NAME RUNNER_TYPE IMAGE \ AIPERF_SERVER_METRICS_URLS SERVER_FLUSH_URLS_CSV \ ENABLE_METRICS IS_AGENTIC CLEAR_CACHE_BETWEEN_CONC \ + DISAGG IS_MULTINODE \ + TP EP_SIZE DP_ATTENTION DCP_SIZE PCP_SIZE \ + PREFILL_NUM_WORKERS PREFILL_TP PREFILL_EP PREFILL_DP_ATTN PREFILL_HARDWARE \ + DECODE_NUM_WORKERS DECODE_TP DECODE_EP DECODE_DP_ATTN DECODE_HARDWARE \ KV_OFFLOADING KV_OFFLOAD_BACKEND TOTAL_CPU_DRAM_GB \ WEKA_LOADER_OVERRIDE AIPERF_FAILED_REQUEST_THRESHOLD \ AIPERF_AGENTIC_CACHE_WARMUP_DURATION AIPERF_UNSAFE_OVERRIDE \ From 5da22201862f453b408765375b02f6c99f04b919 Mon Sep 17 00:00:00 2001 From: Duyi-Wang Date: Sat, 11 Jul 2026 13:20:40 +0800 Subject: [PATCH 14/15] fix: address PR 2170 review comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update the DSv4 MI355X agentic perf-changelog entry to point at PR 2170 with an accurate description, and move hf_dir into the DeepSeek-V4-Pro YAML block so the SGLang model path extractor can read it. 中文：修复 PR 2170 的 review 反馈：将 DSv4 MI355X agentic 的 perf-changelog 条目改为指向 PR 2170 并更新为准确描述，同时把 hf_dir 移入 DeepSeek-V4-Pro YAML 配置块，确保 SGLang 模型路径提取逻辑可以读取。 --- benchmarks/multi_node/amd_utils/models.yaml | 3 ++- perf-changelog.yaml | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/benchmarks/multi_node/amd_utils/models.yaml b/benchmarks/multi_node/amd_utils/models.yaml index 5fe4e76c1d..775c4bb2b6 100644 --- a/benchmarks/multi_node/amd_utils/models.yaml +++ b/benchmarks/multi_node/amd_utils/models.yaml @@ -370,8 +370,9 @@ DeepSeek-R1-0528-MXFP4-v2: # thinking/reasoning-effort, dispatch dtypes, per-role PER_RANK dispatch tokens) is set # in env.sh's DeepSeek-V4-Pro block. The bench client uses --dsv4 framing (bench.sh). # prefill.disable_cuda_graph routes prefill to --disable-cuda-graph; decode keeps -# --cuda-graph-bs. See dsv4_mi355x_sglang_disagg_plan.md. hf_dir: "models--deepseek-ai--DeepSeek-V4-Pro" +# --cuda-graph-bs. See dsv4_mi355x_sglang_disagg_plan.md. DeepSeek-V4-Pro: + hf_dir: "models--deepseek-ai--DeepSeek-V4-Pro" base_flags: "--decode-log-interval 100 --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend dsv4 --page-size 256 --swa-full-tokens-ratio 0.1 --disable-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --disaggregation-transfer-backend mori --log-level info --log-level-http error" dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 028631183e..9fba3d61e8 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4720,6 +4720,7 @@ - config-keys: - dsv4-fp4-mi355x-sglang-disagg-agentic-hicache description: - - "Bump the SGLang ROCm image from 20260706 to 20260709 and retire the DeepSeek-V4 compress-state, DSA paged-MQA, AITER instruction-shape, and HiCache host-pool shims that are already included upstream." - - "Remove the SWA re-prefill and unified-KV HiCache overlays after their upstream PRs merged; retain only patch code without a merged upstream equivalent." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2127 + - "Add DeepSeek-V4-Pro FP4 MI355X SGLang-disagg agentic-coding benchmark with HiCache DRAM KV offloading." + - "Add the multi-node agentic runtime plumbing: node-0 sibling benchmark-client container, per-worker metrics scrape, per-concurrency cache flushing, trace replay client, and one allocation per concurrency." + - "Image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2170 From 5ec0195446c78b6ecf11965f03b6e79c94b288e4 Mon Sep 17 00:00:00 2001 From: Duyi-Wang Date: Sat, 11 Jul 2026 13:26:08 +0800 Subject: [PATCH 15/15] fix: drop unused sglang hf_dir field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the DeepSeek-V4-Pro hf_dir key from the SGLang models.yaml block; the MI355X recipe resolves through MODEL_PATH/MODEL_DIR and only needs the misleading commented hf_dir fragment removed. 中文：删除 SGLang models.yaml 中未使用的 DeepSeek-V4-Pro hf_dir 字段；MI355X 配方通过 MODEL_PATH/MODEL_DIR 解析模型路径，只需要移除误导性的注释片段。 --- benchmarks/multi_node/amd_utils/models.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/benchmarks/multi_node/amd_utils/models.yaml b/benchmarks/multi_node/amd_utils/models.yaml index 775c4bb2b6..b5c9d6b540 100644 --- a/benchmarks/multi_node/amd_utils/models.yaml +++ b/benchmarks/multi_node/amd_utils/models.yaml @@ -372,7 +372,6 @@ DeepSeek-R1-0528-MXFP4-v2: # prefill.disable_cuda_graph routes prefill to --disable-cuda-graph; decode keeps # --cuda-graph-bs. See dsv4_mi355x_sglang_disagg_plan.md. DeepSeek-V4-Pro: - hf_dir: "models--deepseek-ai--DeepSeek-V4-Pro" base_flags: "--decode-log-interval 100 --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend dsv4 --page-size 256 --swa-full-tokens-ratio 0.1 --disable-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --disaggregation-transfer-backend mori --log-level info --log-level-http error" dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal"