diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index 86930487cb..003ac2eb49 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -545,7 +545,7 @@ jobs: model-prefix: ${{ matrix.config.model-prefix }} framework: ${{ matrix.config.framework }} precision: ${{ matrix.config.precision }} - conc-list: '[${{ matrix.config.conc }}]' + conc-list: ${{ toJson(matrix.config.conc) }} spec-decoding: ${{ matrix.config.spec-decoding }} disagg: ${{ matrix.config.disagg }} prefill-hardware: ${{ matrix.config.prefill.hardware }} @@ -566,7 +566,7 @@ jobs: decode-ep: ${{ matrix.config.decode.ep }} decode-dp-attn: ${{ matrix.config.decode.dp-attn }} decode-additional-settings: ${{ toJson(matrix.config.decode.additional-settings) }} - conc: ${{ matrix.config.conc }} + conc: ${{ matrix.config.conc[0] }} kv-offloading: ${{ matrix.config.kv-offloading }} kv-offload-backend: ${{ matrix.config.kv-offload-backend }} duration: ${{ matrix.config.duration }} diff --git a/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh b/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh new file mode 100755 index 0000000000..e5dde8d120 --- /dev/null +++ b/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash + +# Agentic trace-replay recipe for a disaggregated SGLang server on MI355X +# (DeepSeek-V4-Pro FP4, 1P1D TP8). +# +# CI-style sibling of dsr1_fp4_mi355x_sglang-disagg.sh: driven entirely by +# environment variables and submits a SLURM job via submit.sh. The agentic / +# HiCache-offload configuration mirrors the DSR1 recipe but uses DSV4-Pro +# specific flags (dsv4 attention backend, page-size 256, SWA settings). + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/../../benchmark_lib.sh" + +check_env_vars \ + CONC_LIST \ + ISL \ + OSL \ + IMAGE \ + SPEC_DECODING \ + MODEL_PATH \ + PREFILL_NUM_WORKERS \ + PREFILL_TP \ + PREFILL_EP \ + PREFILL_DP_ATTN \ + DECODE_NUM_WORKERS \ + DECODE_TP \ + DECODE_EP \ + DECODE_DP_ATTN \ + PREFILL_NODES \ + DECODE_NODES \ + RANDOM_RANGE_RATIO \ + DURATION \ + KV_OFFLOADING \ + IS_AGENTIC \ + FRAMEWORK + +if [[ -n "$SLURM_JOB_ID" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +fi + +set -x + +# Use upstreamed multi_node scripts (no external clone needed) +cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 + +# Set up SGL launch script-specific environment variables +export TIME_LIMIT="${TIME_LIMIT:-08:00:00}" +export MODEL_PATH=$MODEL_PATH +export MODEL_NAME=$MODEL_NAME +export CONTAINER_IMAGE=$IMAGE + +# ── Identity / result naming ── +export MODEL_PREFIX="${MODEL_PREFIX:-dsv4}" +export PRECISION="${PRECISION:-fp4}" +export RESULT_FILENAME="${RESULT_FILENAME:-${RUNNER_NAME:-dsv4-fp4-agentic}}" + +# ── Agentic benchmark params ── +export DURATION="${DURATION:-1800}" +# DSV4-Pro max model len for agentic traces (matches single-node recipe). +export MAX_MODEL_LEN="${MAX_MODEL_LEN:-1000000}" + +# ── In-tree sglang patches ── +# mori_conn.py targets hybrid-state bugs (GLM-5, Qwen3.5). DSV4-Pro uses a +# pure MoE/DSA architecture without hybrid state; skip to avoid interference. +export MORI_CONN_PATCH="${MORI_CONN_PATCH:-skip}" + +# ── Aiter fault mitigation ── +# --disable-custom-all-reduce avoids a known aiter fault on MI355X. +export DISABLE_CUSTOM_ALL_REDUCE="${DISABLE_CUSTOM_ALL_REDUCE:-0}" + +# ── KV cache offloading (HiCache) ── +# KV_OFFLOADING=none | dram (passed from YAML; default none for disagg). +# KV_OFFLOAD_BACKEND selects the backend when offloading is on; this recipe +# only implements HiCache, so "hicache" is the only supported value. +# HICACHE_TIER: L2 -> GPU + CPU-DRAM host pool. L3 -> + Mooncake store. +export KV_OFFLOADING="${KV_OFFLOADING:-none}" +if [[ "$KV_OFFLOADING" != "none" ]]; then + export KV_OFFLOAD_BACKEND="${KV_OFFLOAD_BACKEND:-hicache}" +fi +# HiCache/Mooncake tunables only matter when KV offloading is enabled. +if [[ "$KV_OFFLOADING" != "none" && "${KV_OFFLOAD_BACKEND:-}" == "hicache" ]]; then + export HICACHE_TIER="${HICACHE_TIER:-L2}" + export HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-1}" + # DSV4 uses page-size 256 (set in models.yaml); HiCache must match. + export HICACHE_PAGE_SIZE="${HICACHE_PAGE_SIZE:-256}" + # HiCache ratio (host pool = ratio * GPU KV pool). Default derived in server_sglang.sh. + export HICACHE_RATIO="${HICACHE_RATIO:-}" + + # ── HiCache layout/backend by tier ── + # L3 (Mooncake): page_first + direct + write_through + storage=mooncake + # L2 (CPU DRAM): layer_first + direct + write_through_selective + storage=none + # NOTE: write_through_selective evicts only under GPU memory pressure, avoiding + # the mori RDMA race that causes GPU memory access faults with write_through. + if [[ "${HICACHE_TIER^^}" == "L3" ]]; then + export HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first}" + export HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" + export HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" + export HICACHE_STORAGE_BACKEND="${HICACHE_STORAGE_BACKEND:-mooncake}" + else + export HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first}" + export HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" + export HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" + export HICACHE_STORAGE_BACKEND="${HICACHE_STORAGE_BACKEND:-}" + fi + export HICACHE_PREFETCH_POLICY="${HICACHE_PREFETCH_POLICY:-best_effort}" + # Shared nodes: use non-default Mooncake ports to avoid collisions. + export MC_MASTER_PORT="${MC_MASTER_PORT:-58137}" + export MC_METADATA_PORT="${MC_METADATA_PORT:-8080}" + export MC_METRICS_PORT="${MC_METRICS_PORT:-19003}" + export MC_MASTER_THREADS="${MC_MASTER_THREADS:-64}" + export MC_EVICTION_HIGH_WATERMARK="${MC_EVICTION_HIGH_WATERMARK:-0.95}" + export MC_PATCH_HOSTPOOL="${MC_PATCH_HOSTPOOL:-1}" + export MC_PROTOCOL="${MC_PROTOCOL:-tcp}" + export MC_GLOBAL_SEG="${MC_GLOBAL_SEG:-64gb}" + export MC_DEVICE="${MC_DEVICE:-}" + export MC_MASTER_ADDR="${MC_MASTER_ADDR:-}" + export MC_METADATA_SERVER="${MC_METADATA_SERVER:-}" +fi + +# ── MoRIIO RDMA Send Queue tuning ── +export MORI_IO_SQ_BACKOFF_TIMEOUT_US="${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-500000}" +export MORI_IO_QP_MAX_SEND_WR="${MORI_IO_QP_MAX_SEND_WR:-32768}" + +# ── SGLang PD router policy + server metrics ── +export PREFILL_ROUTER_POLICY="${PREFILL_ROUTER_POLICY:-cache_aware}" +export ENABLE_METRICS="${ENABLE_METRICS:-1}" + +# ── MTP ── +export DECODE_MTP_SIZE="${DECODE_MTP_SIZE:-0}" + +# Derive EP/DP enable flags from the topology inputs. +if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then +export PREFILL_ENABLE_EP=false +else +export PREFILL_ENABLE_EP=true +fi + +if [[ "$PREFILL_DP_ATTN" == "true" ]]; then +export PREFILL_ENABLE_DP=true +else +export PREFILL_ENABLE_DP=false +fi + +if [[ "${DECODE_EP:-1}" -eq 1 ]]; then +export DECODE_ENABLE_EP=false +else +export DECODE_ENABLE_EP=true +fi + +if [[ "$DECODE_DP_ATTN" == "true" ]]; then +export DECODE_ENABLE_DP=true +else +export DECODE_ENABLE_DP=false +fi + +# Launch the job. CONC_LIST is space-delimited in YAML; submit.sh wants 'x'. +JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ + $PREFILL_NUM_WORKERS \ + $DECODE_NODES \ + $DECODE_NUM_WORKERS \ + $ISL $OSL "${CONC_LIST// /x}" inf \ + ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ + ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ + ${PREFILL_TP} ${DECODE_TP} \ + ${RANDOM_RANGE_RATIO}) + +if [[ $? -ne 0 ]]; then + echo "Failed to submit job" >&2 + exit 1 +fi + +echo "$JOB_ID" diff --git a/benchmarks/multi_node/amd_utils/env.sh b/benchmarks/multi_node/amd_utils/env.sh index c75675cb1c..966cde4589 100755 --- a/benchmarks/multi_node/amd_utils/env.sh +++ b/benchmarks/multi_node/amd_utils/env.sh @@ -12,6 +12,23 @@ set -x ENGINE="${ENGINE:-sglang-disagg}" export PYTHONDONTWRITEBYTECODE=1 +# ============================================================================= +# HiCache / Mooncake settings from job.slurm +# ============================================================================= +# job.slurm writes the recipe-provided HiCache/Mooncake tunables to +# hicache_mc_.env and mounts it read-only at /config/hicache_mc.env. Source +# it here (auto-export) so values like HICACHE_PAGE_SIZE=256 reach the container +# before server_sglang.sh applies its "${VAR:-default}" fallbacks. Without this +# the vars arrive unset and server_sglang.sh defaults HICACHE_PAGE_SIZE to 1, +# overriding the recipe's --page-size. Empty values in the file are harmless: +# the "${VAR:-default}" fallbacks still treat "" as unset. +if [[ -f /config/hicache_mc.env ]]; then + set -a + source /config/hicache_mc.env + set +a + echo "[env.sh] sourced HiCache config from /config/hicache_mc.env (HICACHE_PAGE_SIZE=${HICACHE_PAGE_SIZE:-unset})" +fi + # ============================================================================= # Shared: IBDEVICES detection # ============================================================================= @@ -50,11 +67,11 @@ export NCCL_IB_HCA=${NCCL_IB_HCA:-$IBDEVICES} # ============================================================================= # Shared by the vLLM MoRIIOConnector and the SGLang/MoRI KV-transfer path. -export MORI_IO_SQ_BACKOFF_TIMEOUT_US=50000 -export MORI_IO_QP_MAX_SEND_WR=16384 -export MORI_IO_QP_MAX_CQE=32768 -export MORI_IO_QP_MAX_SGE=2 -export MORI_IO_TC_DISABLE=0 +export MORI_IO_SQ_BACKOFF_TIMEOUT_US="${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-50000}" +export MORI_IO_QP_MAX_SEND_WR="${MORI_IO_QP_MAX_SEND_WR:-16384}" +export MORI_IO_QP_MAX_CQE="${MORI_IO_QP_MAX_CQE:-32768}" +export MORI_IO_QP_MAX_SGE="${MORI_IO_QP_MAX_SGE:-2}" +export MORI_IO_TC_DISABLE="${MORI_IO_TC_DISABLE:-0}" # QoS/DSCP configuration # Priority order: 1) Set by runner, 2) Detect via nicctl, 3) Detect from hostname @@ -197,14 +214,27 @@ else export AITER_LOG_LEVEL=ERROR export SGLANG_MORI_DISPATCH_DTYPE=auto - export MORI_COMBINE_DTYPE_PREFILL=fp8_direct_cast - export MORI_COMBINE_DTYPE_DECODE=fp8 + # export MORI_COMBINE_DTYPE_PREFILL=fp8_direct_cast + # export MORI_COMBINE_DTYPE_DECODE=fp8 + export MORI_COMBINE_DTYPE_PREFILL="" + export MORI_COMBINE_DTYPE_DECODE="" export SGLANG_MORI_QP_PER_TRANSFER=4 export SGLANG_MORI_NUM_WORKERS=4 + # Keep these as overridable defaults (not hard assignments), otherwise + # later tuning blocks cannot raise them for high-concurrency runs. + # export MORI_IO_SQ_BACKOFF_TIMEOUT_US="${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-500000}" + + # export MORI_IO_QP_MAX_SEND_WR="${MORI_IO_QP_MAX_SEND_WR:-16384}" + # export MORI_IO_QP_MAX_CQE=32768 + # export MORI_IO_QP_MAX_SGE=1 + + # export MORI_IO_TC_DISABLE=0 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600 export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600 + export SGLANG_HEALTH_CHECK_TIMEOUT=600 + # GLM-5: uses NSA (not MLA), needs fused-decode-MLA disabled + fast loading if [[ "$MODEL_NAME" == "GLM-5-FP8" ]]; then export SGLANG_ROCM_FUSED_DECODE_MLA=0 @@ -300,4 +330,80 @@ else # FIXME: WA for latest upstream 0305 image export PYTHONPATH=/sgl-workspace/aiter:${PYTHONPATH} -fi + # ========================================================================= + # DeepSeek-V4-Pro PD recipe overrides + # Placed at the end of the SGLang env block so it wins over the global + # MoRI/SGLang defaults set above. Mirrors the validated DSv4 manual PD + # commands (ported from InferenceX amd/dsv4_sgl_di). These SGLANG_OPT_* / + # AITER_* kernel-routing knobs steer DSv4 away from the default aiter CK + # fused-MoE path, which raises "Unsupported kernel config for moe heuristic + # dispatch" at decode time on this fp4 model (job 19034 crash). Only the + # SGLang/MoRI env knobs are pinned here; CLI flags live in models.yaml and + # the cluster NIC/socket vars stay runner-derived. + # ========================================================================= + if [[ "$MODEL_NAME" == "DeepSeek-V4-Pro" ]]; then + # MoRI RDMA send-queue depth for DSv4 (overrides the global default above). + export MORI_IO_QP_MAX_SEND_WR=32767 + # Unified radix tree: cache impl with per-component (full-attn / SWA) + # management for hybrid-attention models. Set unconditionally (not gated on + # hicache) so all SGLang runs use it. + export SGLANG_ENABLE_UNIFIED_RADIX_TREE=1 + # Proactively free out-of-window SWA KV slots during chunked prefill. + # Without it, in-flight requests pin SWA KV for their whole context, keeping + # the SWA pool under constant eviction pressure; under LRU the trailing + # window of cached sessions gets flushed, making prefix-cache hits bimodal + # and collapsing the effective hit rate on multi-turn agentic workloads. + export SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS=1 + + # MoRI dispatch/combine dtypes: auto for both roles (not the fp8 split default) + export SGLANG_MORI_DISPATCH_DTYPE=auto + export MORI_COMBINE_DTYPE_PREFILL=auto + export MORI_COMBINE_DTYPE_DECODE=auto + + # Per-role MoRI dispatch sizing (used by the harness chunked/MoE math) + export MORI_MAX_DISPATCH_TOKENS_PREFILL=8192 + export MORI_MAX_DISPATCH_TOKENS_DECODE=64 + unset MORI_MOE_MAX_INPUT_TOKENS_PREFILL + unset MORI_MOE_MAX_INPUT_TOKENS_DECODE + + # PER_RANK dispatch tokens pinned independently (16384 prefill / 128 + # decode); server_sglang.sh prefers these over the MORI_MAX_DISPATCH_* + # coupling when set. + export MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_PREFILL=16384 + export MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_DECODE=128 + + # Fixed inter-kernel switch threshold (not derived). + export SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD=4096 + + # Overlap plan stream on for DSv4 (global default is 0) + # export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 + + # DSv4 model kernel routing (mirrors the single-node / manual PD recipe) + export SGLANG_DEFAULT_THINKING=1 + export SGLANG_DSV4_REASONING_EFFORT=max + export SGLANG_OPT_DEEPGEMM_HC_PRENORM=false + export SGLANG_USE_AITER=1 + export SGLANG_USE_ROCM700A=0 + export SGLANG_OPT_USE_FUSED_COMPRESS=true + export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton + export SGLANG_OPT_FP8_WO_A_GEMM=false + export SGLANG_OPT_USE_JIT_INDEXER_METADATA=false + export SGLANG_OPT_USE_TOPK_V2=false + export SGLANG_OPT_USE_AITER_INDEXER=${SGLANG_OPT_USE_AITER_INDEXER:-true} + export SGLANG_OPT_USE_TILELANG_INDEXER=false + export SGLANG_OPT_USE_TILELANG_MHC_PRE=false + export SGLANG_OPT_USE_TILELANG_MHC_POST=false + export SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1 + export SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true + export SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false + export SGLANG_ROCM_USE_MULTI_STREAM=false + export AITER_BF16_FP8_MOE_BOUND=0 + export SGLANG_EAGER_INPUT_NO_COPY=true + export SGLANG_SHARED_EXPERT_TP1=1 + export SGLANG_DP_SHARED_EXPERT_LOCAL=1 + export SGLANG_DP_USE_GATHERV=1 + export SGLANG_DP_USE_REDUCE_SCATTER=1 + export GPU_MAX_HW_QUEUES=5 + fi + +fi \ No newline at end of file diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm index 4638e5cadb..404260ae83 100755 --- a/benchmarks/multi_node/amd_utils/job.slurm +++ b/benchmarks/multi_node/amd_utils/job.slurm @@ -179,11 +179,41 @@ else fi } - if check_model_path "$MODEL_DIR/$MODEL_NAME" "$MODEL_DIR"; then + # Extract hf_dir from models.yaml (same as vllm-disagg path above) + SGL_DISK_DIR_NAME=$(awk '/^'"$MODEL_NAME"':/{found=1; next} + found && /^[^ ]/{exit} + found && /hf_dir:/{gsub(/[" ]/, "", $2); print $2; exit}' "$MODELS_YAML") + SGL_DISK_DIR_NAME="${SGL_DISK_DIR_NAME:-$MODEL_NAME}" + + # Prefer the caller-supplied MODEL_PATH (recipe scripts set this explicitly); + # fall back to MODEL_DIR/hf_dir then MODEL_DIR/MODEL_NAME. + if [[ -n "${MODEL_PATH:-}" && "$MODEL_PATH" != "$MODEL_DIR" ]]; then + # Caller already resolved the path (e.g. MODEL_PATH=/it-share/hf_cache/models--...) + # Use it directly if it exists on all nodes, otherwise try subdirectory combos. + if check_model_path "$MODEL_PATH" "MODEL_PATH (caller-supplied)"; then + echo "Selected MODEL_PATH: $MODEL_PATH (caller-supplied, available on all nodes)" + elif check_model_path "$MODEL_PATH/$SGL_DISK_DIR_NAME" "$MODEL_PATH/$SGL_DISK_DIR_NAME"; then + MODEL_PATH="$MODEL_PATH/$SGL_DISK_DIR_NAME" + echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" + elif check_model_path "$MODEL_PATH/$MODEL_NAME" "$MODEL_PATH/$MODEL_NAME"; then + MODEL_PATH="$MODEL_PATH/$MODEL_NAME" + echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" + else + echo "FATAL ERROR: Model '$MODEL_NAME' not found on ALL allocated nodes in:" + echo " - $MODEL_PATH" + echo " - $MODEL_PATH/$SGL_DISK_DIR_NAME" + echo " - $MODEL_PATH/$MODEL_NAME" + exit 1 + fi + elif check_model_path "$MODEL_DIR/$SGL_DISK_DIR_NAME" "$MODEL_DIR/$SGL_DISK_DIR_NAME"; then + MODEL_PATH="$MODEL_DIR/$SGL_DISK_DIR_NAME" + echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" + elif check_model_path "$MODEL_DIR/$MODEL_NAME" "$MODEL_DIR"; then MODEL_PATH="$MODEL_DIR/$MODEL_NAME" echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" else echo "FATAL ERROR: Model '$MODEL_NAME' not found on ALL allocated nodes in:" + echo " - $MODEL_DIR/$SGL_DISK_DIR_NAME" echo " - $MODEL_DIR/$MODEL_NAME" exit 1 fi @@ -296,6 +326,8 @@ export DOCKER_CONT_NAME="container_${ENGINE}_${SANITIZED_USER}_${MODEL_NAME}_${S # dated tags are garbage-collected (manifest unknown) VLLM_ROUTER_IMAGE="${VLLM_ROUTER_IMAGE:-vllm/vllm-router:nightly-20260629-e667ebb}" ROUTER_CONT_NAME="router_vllm_${SANITIZED_USER}_${SLURM_JOB_ID}" +# Separate agentic benchmark-client container (see CLIENT_IMAGE handling below). +CLIENT_CONT_NAME="container_${ENGINE}_${SANITIZED_USER}_client_${SLURM_JOB_ID}" export RUN_FILE_FULL="$WS_PATH/${RUN_FILE}" SELECTED_NODELIST_SRUN=$(echo "$SELECTED_NODES" | paste -sd,) @@ -350,7 +382,6 @@ DOCKER_ENV_COMMON=( -e BENCH_MAX_CONCURRENCY=\$BENCH_MAX_CONCURRENCY -e BENCH_REQUEST_RATE=\$BENCH_REQUEST_RATE -e TQDM_MININTERVAL=\$TQDM_MININTERVAL - -e DRY_RUN=\$DRY_RUN -e BENCHMARK_LOGS_DIR=/benchmark_logs -e ENGINE=\$ENGINE -e WS_PATH=${WS_PATH} @@ -363,14 +394,44 @@ DOCKER_ENV_COMMON=( -e RUNNER_TYPE=\$RUNNER_TYPE -e RESULT_FILENAME=\$RESULT_FILENAME -e SPEC_DECODING=\$SPEC_DECODING + # DISAGG was never forwarded into the container at all (not even under a + # different name), so process_agentic_result.py's env_bool("DISAGG") always + # defaulted to false in the result JSON regardless of the actual topology. + -e DISAGG=\${DISAGG:-false} -e PREFILL_TP_SIZE=\$PREFILL_TP_SIZE + # PREFILL_TP/DECODE_TP/*_NUM_WORKERS (below, undecorated -- distinct from the + # *_SIZE vars server_sglang.sh uses for launch args) are what + # process_agentic_result.py's _gpu_shape() reads for multinode runs. Without + # these the container never sees them (only *_SIZE was passed), so agentic + # result JSONs silently recorded tp=0 / prefill_tp=0 / prefill_num_workers=0 + # for every multinode run. + -e PREFILL_TP=\$PREFILL_TP + -e PREFILL_NUM_WORKERS=\$PREFILL_NUM_WORKERS -e PREFILL_ENABLE_EP=\$PREFILL_ENABLE_EP -e PREFILL_ENABLE_DP=\$PREFILL_ENABLE_DP + -e PREFILL_CONTEXT_LENGTH=\${PREFILL_CONTEXT_LENGTH:-} + -e PREFILL_CHUNKED_PREFILL_SIZE=\${PREFILL_CHUNKED_PREFILL_SIZE:-} + -e SGLANG_AITER_MLA_PERSIST=\${SGLANG_AITER_MLA_PERSIST:-0} + -e DISABLE_CUSTOM_ALL_REDUCE=\${DISABLE_CUSTOM_ALL_REDUCE:-0} + -e MAX_MODEL_LEN=\${MAX_MODEL_LEN:-} + -e DURATION=\${DURATION:-1800} + -e IS_AGENTIC=\${IS_AGENTIC:-0} + -e KV_OFFLOADING=\${KV_OFFLOADING:-none} + -e KV_OFFLOAD_BACKEND=\${KV_OFFLOAD_BACKEND:-} + -e TOTAL_CPU_DRAM_GB=\${TOTAL_CPU_DRAM_GB:-} + -e ENABLE_METRICS=\${ENABLE_METRICS:-0} + -e PREFILL_ROUTER_POLICY=\${PREFILL_ROUTER_POLICY:-random} + -e DECODE_ROUTER_POLICY=\${DECODE_ROUTER_POLICY:-random} + -e MORI_IO_SQ_BACKOFF_TIMEOUT_US=\${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-} + -e MORI_IO_QP_MAX_SEND_WR=\${MORI_IO_QP_MAX_SEND_WR:-} -e DECODE_TP_SIZE=\$DECODE_TP_SIZE + -e DECODE_TP=\$DECODE_TP + -e DECODE_NUM_WORKERS=\$DECODE_NUM_WORKERS -e DECODE_ENABLE_EP=\$DECODE_ENABLE_EP -e DECODE_ENABLE_DP=\$DECODE_ENABLE_DP -e DECODE_MTP_SIZE=\$DECODE_MTP_SIZE -e IS_MULTINODE=\$IS_MULTINODE + -e DRY_RUN=\${DRY_RUN:-0} ) # Engine-specific env vars @@ -411,9 +472,59 @@ else ) fi +# HiCache / Mooncake settings are delivered via a bind-mounted config file rather +# than a long list of docker -e flags. Write it once to the shared benchmark-logs +# dir (already a host path, visible on every node) and mount it read-only at +# /config/hicache_mc.env, where env.sh sources it before applying its defaults. +# Empty values are preserved so env.sh's "${VAR:-default}" fallbacks still apply. +HICACHE_MC_CONFIG="${BENCHMARK_LOGS_DIR}/hicache_mc_${SLURM_JOB_ID}.env" +cat > "$HICACHE_MC_CONFIG" < $HICACHE_MC_CONFIG" + # Engine-specific container filter for pre-clean CONT_FILTER="name=^container_${ENGINE}_" +# ============================================================================= +# Optional: separate benchmark-client image (agentic runs) — node-0 sibling +# ============================================================================= +# When CLIENT_IMAGE is set, node 0 runs the aiperf trace replay in its own +# sibling container built from CLIENT_IMAGE (which ships a pre-baked aiperf + +# deps), instead of rebuilding the aiperf venv inside the server container every +# run. Give the server container access to the host docker socket + CLI and the +# host paths the sibling needs for its bind mounts. These fragments are expanded +# at submit time and injected into the server `docker run` below; empty (no-op) +# when CLIENT_IMAGE is unset, so the in-container aiperf path is unchanged. +CLIENT_DOCKER_MOUNTS="" +CLIENT_DOCKER_ENV="" +if [[ -n "${CLIENT_IMAGE:-}" ]]; then + HOST_DOCKER_BIN="$(command -v docker || echo /usr/bin/docker)" + CLIENT_DOCKER_MOUNTS="-v /var/run/docker.sock:/var/run/docker.sock -v ${HOST_DOCKER_BIN}:/usr/bin/docker" + CLIENT_DOCKER_ENV="-e CLIENT_IMAGE=${CLIENT_IMAGE} -e HOST_REPO_DIR=${DI_REPO_DIR} -e HOST_MODEL_DIR=${MODEL_DIR} -e HOST_BENCH_LOGS=${BENCHMARK_LOGS_DIR} -e CLIENT_CONT_NAME=${CLIENT_CONT_NAME}" + echo "[client] node-0 sibling benchmark-client image enabled: ${CLIENT_IMAGE}" + # Best-effort pre-pull on all nodes so node 0's sibling launch doesn't stall. + srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c 'eval "$DOCKER_CMD_DETECT"; $DOCKER_CMD pull '"$CLIENT_IMAGE"' >/dev/null 2>&1 || true' 2>/dev/null || true +fi + srun \ --nodelist="$SELECTED_NODELIST_SRUN" \ --kill-on-bad-exit=1 \ @@ -572,10 +683,13 @@ fi -v /tmp:/run_logs \ -v ${BENCHMARK_LOGS_DIR}:/benchmark_logs \ -v ${DI_REPO_DIR}:${DOCKER_MOUNT_PATH} \ + -v ${HICACHE_MC_CONFIG}:/config/hicache_mc.env:ro \ ${EXTRA_DOCKER_MOUNTS:-} \ + ${CLIENT_DOCKER_MOUNTS} \ \${RDMA_MOUNTS[@]+"\${RDMA_MOUNTS[@]}"} \ ${DOCKER_ENV_COMMON[*]} \ ${DOCKER_ENV_ENGINE[*]} \ + ${CLIENT_DOCKER_ENV} \ --name \"$DOCKER_CONT_NAME\" \ --entrypoint \"\" \ \"$DOCKER_IMAGE_NAME\" bash -lc ' @@ -592,7 +706,7 @@ exit \$DOCKER_EXIT_CODE " if [[ "${KEEP_CONTAINERS}" != "1" ]]; then - srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c 'eval "$DOCKER_CMD_DETECT"; $DOCKER_CMD rm -f '"$DOCKER_CONT_NAME"' 2>/dev/null || true' + srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c 'eval "$DOCKER_CMD_DETECT"; $DOCKER_CMD rm -f '"$DOCKER_CONT_NAME"' '"$CLIENT_CONT_NAME"' 2>/dev/null || true' # Clean up vLLM external router container on node 0 if [[ "$ENGINE" == "vllm-disagg" && "$ROUTER_TYPE" == "vllm-router" ]]; then diff --git a/benchmarks/multi_node/amd_utils/models.yaml b/benchmarks/multi_node/amd_utils/models.yaml index bc2b39aa07..b5c9d6b540 100644 --- a/benchmarks/multi_node/amd_utils/models.yaml +++ b/benchmarks/multi_node/amd_utils/models.yaml @@ -372,7 +372,7 @@ DeepSeek-R1-0528-MXFP4-v2: # prefill.disable_cuda_graph routes prefill to --disable-cuda-graph; decode keeps # --cuda-graph-bs. See dsv4_mi355x_sglang_disagg_plan.md. DeepSeek-V4-Pro: - base_flags: "--decode-log-interval 100 --log-level info --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend dsv4 --page-size 256 --swa-full-tokens-ratio 0.1 --disable-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --disaggregation-transfer-backend mori" + base_flags: "--decode-log-interval 100 --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend dsv4 --page-size 256 --swa-full-tokens-ratio 0.1 --disable-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --disaggregation-transfer-backend mori --log-level info --log-level-http error" dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" prefill: @@ -381,17 +381,24 @@ DeepSeek-V4-Pro: disable_cuda_graph: true dp: max_running_requests: 1024 - chunked_prefill_size: 131072 - context_length: 9217 - max_total_tokens: 262144 + chunked_prefill_size: 65280 # dsv4 compressor kernel uint16 token cap (255*256) + context_length: 1048576 + max_total_tokens: 1048576 no_dp: max_running_requests: 128 - chunked_prefill_size: 131072 - context_length: 9217 - max_total_tokens: 262144 + # Small prefill chunks interleave long-context agentic prefills across + # requests instead of letting one ~100K-token prefill monopolize the + # engine (the conc>=16 queue-saturation / decode-stall failure mode). + # Mirrors the single-node DSv4 agentic recipe (dsv4_fp4_mi355x.sh=8192). + # Was 65280 (255*256, the dsv4 compressor kernel uint16 token cap); 8192 + # (32*256) stays a page-size multiple well under that cap. + chunked_prefill_size: 8192 + context_length: 1048576 + max_total_tokens: 1048576 decode: mem_fraction_static: 0.85 prefill_round_robin_balance: true + disagg_decode_enable_radix_cache: false dp: max_running_requests: 1024 cuda_graph_bs_range: "1-128" diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh index ec5805eb0e..840d6931aa 100755 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ b/benchmarks/multi_node/amd_utils/server_sglang.sh @@ -146,6 +146,7 @@ print(f'PREFILL_CUDA_GRAPH_BS_NO_DP_END=\"{e}\"') print(f'DECODE_MEM_FRACTION_STATIC=\"{decode.get(\"mem_fraction_static\", 0.85)}\"') print(f'DECODE_PREFILL_ROUND_ROBIN_BALANCE=\"{decode.get(\"prefill_round_robin_balance\", True)}\"') +print(f'DECODE_DISAGG_ENABLE_RADIX_CACHE=\"{decode.get(\"disagg_decode_enable_radix_cache\", False)}\"') dp = decode.get('dp', {}) ep_only = decode.get('ep_only', {}) @@ -232,9 +233,14 @@ if [[ "$PREFILL_DISABLE_CUDA_GRAPH" == "True" ]] || [[ "$PREFILL_DISABLE_CUDA_GR else PREFILL_MODE_FLAGS="--mem-fraction-static ${PREFILL_MEM_FRACTION_STATIC} --max-running-requests ${prefill_max_running_requests} --chunked-prefill-size ${prefill_chunked_prefill_size} --cuda-graph-bs ${prefill_cuda_graph_bs[*]} " fi + if [[ "$PREFILL_DISABLE_RADIX_CACHE" == "True" ]] || [[ "$PREFILL_DISABLE_RADIX_CACHE" == "true" ]]; then PREFILL_MODE_FLAGS="$PREFILL_MODE_FLAGS --disable-radix-cache" fi +# Agentic runs: keep radix/prefix cache enabled by replacing --disable-radix-cache with empty. +if [[ "${IS_AGENTIC:-0}" == "1" || "${IS_AGENTIC:-}" == "true" ]]; then + PREFILL_MODE_FLAGS="${PREFILL_MODE_FLAGS//--disable-radix-cache/}" +fi if [[ -n "$prefill_context_length" ]]; then PREFILL_MODE_FLAGS="$PREFILL_MODE_FLAGS --context-length ${prefill_context_length}" fi @@ -251,6 +257,9 @@ DECODE_MODE_FLAGS="--mem-fraction-static ${DECODE_MEM_FRACTION_STATIC} --max-run if [[ "$DECODE_PREFILL_ROUND_ROBIN_BALANCE" == "True" ]] || [[ "$DECODE_PREFILL_ROUND_ROBIN_BALANCE" == "true" ]]; then DECODE_MODE_FLAGS="$DECODE_MODE_FLAGS --prefill-round-robin-balance" fi +if [[ "$DECODE_DISAGG_ENABLE_RADIX_CACHE" == "True" ]] || [[ "$DECODE_DISAGG_ENABLE_RADIX_CACHE" == "true" ]]; then + DECODE_MODE_FLAGS="$DECODE_MODE_FLAGS --disaggregation-decode-enable-radix-cache" +fi if [[ "$DECODE_MTP_SIZE" -gt 0 ]]; then MORI_MAX_DISPATCH_TOKENS_DECODE=$((MORI_MAX_DISPATCH_TOKENS_DECODE * (DECODE_MTP_SIZE + 1))) @@ -270,10 +279,20 @@ NODE_OFFSET=$((PREFILL_NODES_PER_WORKER * xP)) # Build prefill arguments dynamically based on xP PREFILL_HEADNODE_URLS=() PREFILL_ARGS="" +# Per-worker Prometheus /metrics endpoints (port 8000) for aiperf's +# --server-metrics scrape. The router on :30000 does not serve Prometheus, so +# aiperf must scrape each prefill/decode worker directly (see ENABLE_METRICS). +SERVER_METRICS_URLS=() +# Per-worker base URLs (port 8000) for direct cache flushing between +# concurrency points. The router (:30000) does not fan /flush_cache out, so +# trace_replay.sh must POST to each prefill/decode worker directly. +SERVER_FLUSH_URLS=() for i in $(seq 0 $((xP - 1))); do prefill_idx=$((i * PREFILL_NODES_PER_WORKER)) PREFILL_HEADNODE_URLS[$i]="${IP_ARRAY[$prefill_idx]}:${HEADNODE_PORT}" PREFILL_ARGS="$PREFILL_ARGS --prefill http://${IP_ARRAY[$prefill_idx]}:8000" + SERVER_METRICS_URLS+=("http://${IP_ARRAY[$prefill_idx]}:8000/metrics") + SERVER_FLUSH_URLS+=("http://${IP_ARRAY[$prefill_idx]}:8000") done # Build decode arguments dynamically based on yD @@ -283,10 +302,14 @@ for i in $(seq 0 $((yD - 1))); do decode_idx=$((i * DECODE_NODES_PER_WORKER + NODE_OFFSET)) DECODE_HEADNODE_URLS[$i]="${IP_ARRAY[$decode_idx]}:${HEADNODE_PORT}" DECODE_ARGS="$DECODE_ARGS --decode http://${IP_ARRAY[$decode_idx]}:8000" + SERVER_METRICS_URLS+=("http://${IP_ARRAY[$decode_idx]}:8000/metrics") + SERVER_FLUSH_URLS+=("http://${IP_ARRAY[$decode_idx]}:8000") done echo "Prefill worker headnode list: ${PREFILL_HEADNODE_URLS[@]}" echo "Decode worker headnode list: ${DECODE_HEADNODE_URLS[@]}" +echo "Server metrics endpoints: ${SERVER_METRICS_URLS[@]}" +echo "Server flush endpoints: ${SERVER_FLUSH_URLS[@]}" # ============================================================================= # Configuration Builder Functions @@ -354,7 +377,7 @@ build_server_config() { specific_config="$DECODE_MODE_FLAGS" fi - # Combine: parallel args + base config + mtp config (decode only) + dp config + specific config + # Combine: parallel args + base config + ep config + mtp config (decode only) + dp config + specific config local full_config="$parallel_args" if [[ -n "$base_config" ]]; then full_config="$full_config $base_config" @@ -379,10 +402,101 @@ build_server_config() { PREFILL_SERVER_CONFIG=$(build_server_config "prefill" "$MODEL_NAME" "$PREFILL_TP_SIZE" "$PREFILL_ENABLE_EP" "$PREFILL_ENABLE_DP" "$DECODE_MTP_SIZE") DECODE_SERVER_CONFIG=$(build_server_config "decode" "$MODEL_NAME" "$DECODE_TP_SIZE" "$DECODE_ENABLE_EP" "$DECODE_ENABLE_DP" "$DECODE_MTP_SIZE") +# Expose Prometheus /metrics on the servers when requested (ENABLE_METRICS=1). +if [[ "${ENABLE_METRICS:-0}" == "1" ]]; then + [[ "$PREFILL_SERVER_CONFIG" != *"--enable-metrics"* ]] && PREFILL_SERVER_CONFIG="$PREFILL_SERVER_CONFIG --enable-metrics" + [[ "$DECODE_SERVER_CONFIG" != *"--enable-metrics"* ]] && DECODE_SERVER_CONFIG="$DECODE_SERVER_CONFIG --enable-metrics" +fi + if [[ -n "$MODEL_NAME" ]]; then echo "Using model-specific configuration for: $MODEL_NAME" fi +# ============================================================================= +# Optional KV cache offloading (HiCache) — enabled when +# KV_OFFLOADING != none AND KV_OFFLOAD_BACKEND == hicache. +# HiCache extends RadixAttention, so radix cache MUST stay on (drop +# --disable-radix-cache). The --hicache-* flags are appended to BOTH the +# prefill and decode server configs. +# ============================================================================= +KV_OFFLOADING="${KV_OFFLOADING:-none}" +KV_OFFLOAD_BACKEND="${KV_OFFLOAD_BACKEND:-}" +if [[ "$KV_OFFLOADING" != "none" && "$KV_OFFLOAD_BACKEND" == "hicache" ]]; then + HICACHE_TOTAL_CPU_DRAM_GB="${HICACHE_TOTAL_CPU_DRAM_GB:-2000}" + HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-1}" + HICACHE_PAGE_SIZE="${HICACHE_PAGE_SIZE:-1}" + HICACHE_PREFETCH_POLICY="${HICACHE_PREFETCH_POLICY:-wait_complete}" + + # Optional L3 storage tier behind the CPU-DRAM (L2) cache. + # "" -> CPU DRAM only (default) + # "mooncake"-> Mooncake distributed KV store (needs a mooncake_master) + HICACHE_STORAGE_BACKEND="${HICACHE_STORAGE_BACKEND:-}" + + # Layout / IO backend / write policy are backend-specific: + # mooncake L3: page_first_direct + the "direct" IO backend (the Mooncake + # store maps a page-contiguous segment for RDMA/zero-copy). This layout + # asserts host_pool > device_pool, so it needs a large CPU-DRAM budget. + # L2-only (CPU DRAM): layer_first + the "kernel" IO backend. layer_first + # has no host>device constraint (the "direct" IO backend REQUIRES a + # page_first layout, so it cannot be paired with layer_first). + if [[ "$HICACHE_STORAGE_BACKEND" == "mooncake" ]]; then + HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first}" + HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" + HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" + else + HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first_direct}" + HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" + HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" + fi + + # Mooncake master/connection settings (used only when storage=mooncake). + # The master runs once on node 0; every prefill/decode server connects to + # it via NODE0_ADDR so it is reachable across nodes. + MC_MASTER_PORT="${MC_MASTER_PORT:-50061}" + MC_METADATA_PORT="${MC_METADATA_PORT:-8080}" + MC_METRICS_PORT="${MC_METRICS_PORT:-9003}" + MC_MASTER_THREADS="${MC_MASTER_THREADS:-64}" + MC_EVICTION_HIGH_WATERMARK="${MC_EVICTION_HIGH_WATERMARK:-0.95}" + MC_PROTOCOL="${MC_PROTOCOL:-tcp}" + MC_GLOBAL_SEG="${MC_GLOBAL_SEG:-64gb}" + MC_DEVICE="${MC_DEVICE:-$IBDEVICES}" + MC_MASTER_ADDR="${MC_MASTER_ADDR:-${NODE0_ADDR}:${MC_MASTER_PORT}}" + MC_METADATA_SERVER="${MC_METADATA_SERVER:-http://${NODE0_ADDR}:${MC_METADATA_PORT}/metadata}" + + # Emit the --hicache-storage-backend flags (empty unless mooncake). The + # extra-config JSON is single-quoted so it survives the later `eval` of the + # launch command as a single argument. + build_storage_flags() { + [[ "$HICACHE_STORAGE_BACKEND" != "mooncake" ]] && return 0 + local extra="{\"master_server_address\": \"${MC_MASTER_ADDR}\", \"protocol\": \"${MC_PROTOCOL}\", \"device_name\": \"${MC_DEVICE}\", \"local_hostname\": \"${host_ip}\", \"global_segment_size\": \"${MC_GLOBAL_SEG}\", \"metadata_server\": \"${MC_METADATA_SERVER}\", \"check_server\": false}" + echo "--hicache-storage-backend mooncake --hicache-storage-backend-extra-config '${extra}' --enable-metrics --enable-cache-report" + } + + # HiCache capacity via --hicache-ratio (scales with GPU KV pool). + HICACHE_RATIO="${HICACHE_RATIO:-16}" + + build_hicache_flags() { + echo "--page-size ${HICACHE_PAGE_SIZE} --enable-hierarchical-cache --hicache-ratio ${HICACHE_RATIO} --hicache-io-backend ${HICACHE_IO_BACKEND} --hicache-mem-layout ${HICACHE_MEM_LAYOUT} --hicache-write-policy ${HICACHE_WRITE_POLICY} --hicache-storage-prefetch-policy ${HICACHE_PREFETCH_POLICY} $(build_storage_flags)" + } + + # HiCache requires RadixAttention; strip any --disable-radix-cache. + PREFILL_SERVER_CONFIG="${PREFILL_SERVER_CONFIG//--disable-radix-cache/}" + DECODE_SERVER_CONFIG="${DECODE_SERVER_CONFIG//--disable-radix-cache/}" + + # Prefill always gets HiCache. + PREFILL_SERVER_CONFIG="$PREFILL_SERVER_CONFIG $(build_hicache_flags "$PREFILL_TP_SIZE")" + + + DECODE_SERVER_CONFIG="$DECODE_SERVER_CONFIG --page-size ${HICACHE_PAGE_SIZE}" + echo "[HiCache] KV_OFFLOADING=${KV_OFFLOADING} backend=${KV_OFFLOAD_BACKEND} applied to prefill only; decode mirrors --page-size ${HICACHE_PAGE_SIZE} for transfer compatibility (chunk cache under the mori transfer backend)" + echo "[HiCache] params: io_backend=${HICACHE_IO_BACKEND}, mem_layout=${HICACHE_MEM_LAYOUT}, page_size=${HICACHE_PAGE_SIZE}, write_policy=${HICACHE_WRITE_POLICY}, prefetch_policy=${HICACHE_PREFETCH_POLICY}, storage_backend=${HICACHE_STORAGE_BACKEND:-none}" + if [[ "$HICACHE_STORAGE_BACKEND" == "mooncake" ]]; then + echo "[HiCache] Mooncake store: master=${MC_MASTER_ADDR} metadata=${MC_METADATA_SERVER} protocol=${MC_PROTOCOL} device=${MC_DEVICE} segment=${MC_GLOBAL_SEG} threads=${MC_MASTER_THREADS} eviction_watermark=${MC_EVICTION_HIGH_WATERMARK}" + fi +else + echo "[HiCache] KV_OFFLOADING=${KV_OFFLOADING} backend=${KV_OFFLOAD_BACKEND:-none} (HiCache disabled)" +fi + if [[ "${EVAL_ONLY:-false}" == "true" ]] || [[ "${RUN_EVAL:-false}" == "true" ]]; then PREFILL_SERVER_CONFIG=$(echo "$PREFILL_SERVER_CONFIG" | sed 's/--ep-dispatch-algorithm fake//g') DECODE_SERVER_CONFIG=$(echo "$DECODE_SERVER_CONFIG" | sed 's/--ep-dispatch-algorithm fake//g') @@ -394,15 +508,28 @@ fi # Container Synchronization # ============================================================================= +# sync.py barrier/health-barrier exits 1 on timeout (and prints which +# node/port never became ready), but without an explicit check here the +# script would silently continue past a timed-out barrier -- printing a +# misleading "success" message and launching the next stage against +# servers/routers that never actually came up, instead of failing fast. +run_barrier_or_die() { + local desc="$1" cmd="$2" + if ! eval "$cmd"; then + echo "FATAL: ${desc} failed — see the sync.py timeout output above for which node/port never became ready." >&2 + exit 1 + fi +} + echo "Waiting at the container creation barrier on $host_name" -python3 $SGLANG_WS_PATH/sync.py barrier \ +run_barrier_or_die "container creation barrier" "python3 $SGLANG_WS_PATH/sync.py barrier \ --local-ip ${host_ip} \ --local-port 5000 \ --enable-port \ --node-ips ${IPADDRS} \ --node-ports 5000 \ --wait-for-all-ports \ - --timeout 300 + --timeout 300" # ============================================================================= @@ -449,6 +576,63 @@ if [ "$NODE_RANK" -eq 0 ]; then echo "================================================" + # Dump all resolved commands to a text file for debugging / reproducibility. + CMD_DUMP="/run_logs/slurm_job-${SLURM_JOB_ID}/commands_${host_name}.txt" + dump_cmd() { echo -e "\n# ── $1 ──\n$2" >> "$CMD_DUMP"; } + echo "# Commands dump — $(date -u '+%Y-%m-%d %H:%M:%S UTC')" > "$CMD_DUMP" + echo "# Host: ${host_name} (${host_ip}) Node rank: ${NODE_RANK}" >> "$CMD_DUMP" + echo "# Model: ${MODEL_NAME} Image: ${DOCKER_IMAGE_NAME:-unknown}" >> "$CMD_DUMP" + + # Start the Mooncake store master (L3 HiCache backend) on node 0 only. + # All prefill/decode servers connect to it via NODE0_ADDR:MC_MASTER_PORT. + if [[ "${KV_OFFLOADING:-none}" != "none" && "${KV_OFFLOAD_BACKEND:-}" == "hicache" && "${HICACHE_STORAGE_BACKEND:-}" == "mooncake" ]]; then + echo "Starting Mooncake master on ${host_ip}:${MC_MASTER_PORT} (metadata :${MC_METADATA_PORT}, metrics :${MC_METRICS_PORT})" + MC_MASTER_CMD="mooncake_master \ + --enable_http_metadata_server=true \ + --http_metadata_server_host=0.0.0.0 \ + --http_metadata_server_port=${MC_METADATA_PORT} \ + --rpc_port=${MC_MASTER_PORT} \ + --rpc_thread_num=${MC_MASTER_THREADS} \ + --metrics_port=${MC_METRICS_PORT} \ + --enable_metric_reporting=true \ + --eviction_high_watermark_ratio=${MC_EVICTION_HIGH_WATERMARK}" + dump_cmd "MOONCAKE MASTER" "$MC_MASTER_CMD" + if [[ "$DRY_RUN" -eq 1 ]]; then + echo "DRY RUN: $MC_MASTER_CMD" + else + MC_MASTER_LOG="/run_logs/slurm_job-${SLURM_JOB_ID}/mooncake_master_${host_name}.log" + mooncake_master \ + --enable_http_metadata_server=true \ + --http_metadata_server_host=0.0.0.0 \ + --http_metadata_server_port="${MC_METADATA_PORT}" \ + --rpc_port="${MC_MASTER_PORT}" \ + --rpc_thread_num="${MC_MASTER_THREADS}" \ + --metrics_port="${MC_METRICS_PORT}" \ + --enable_metric_reporting=true \ + --eviction_high_watermark_ratio="${MC_EVICTION_HIGH_WATERMARK}" \ + > "${MC_MASTER_LOG}" 2>&1 & + mc_master_pid=$! + sleep 3 + # Fail loudly on a port collision. On shared nodes the Mooncake RPC + # port may already be taken by another user's master; in that case the + # metrics-port health check below can still pass against the foreign + # master while our RPC port is dead, and the prefill then hangs. + if grep -qiE "Address already in use|bind .*error" "${MC_MASTER_LOG}" 2>/dev/null; then + echo "ERROR: mooncake_master failed to bind port ${MC_MASTER_PORT} (already in use)." + echo " Set MC_MASTER_PORT/MC_METRICS_PORT to free ports and resubmit." + grep -iE "Address already in use|bind .*error" "${MC_MASTER_LOG}" | tail -3 + exit 1 + fi + for ((i=3; i<=60; i+=3)); do + if curl -sf "http://127.0.0.1:${MC_METRICS_PORT}/get_all_segments" >/dev/null 2>&1; then + echo " mooncake master OK at ${i}s" + break + fi + sleep 3 + done + fi + fi + # start the head prefill server PREFILL_MORI_MOE_ENV="" set -x @@ -456,7 +640,7 @@ if [ "$NODE_RANK" -eq 0 ]; then PREFILL_MORI_MOE_ENV="SGLANG_MORI_MOE_MAX_INPUT_TOKENS=${MORI_MOE_MAX_INPUT_TOKENS_PREFILL}" fi set +x - PREFILL_CMD="SGLANG_MORI_COMBINE_DTYPE=${MORI_COMBINE_DTYPE_PREFILL} ${PREFILL_SDMA_ENV} ${PREFILL_MORI_MOE_ENV} SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_PREFILL:-${MORI_MAX_DISPATCH_TOKENS_PREFILL}} python3 -m sglang.launch_server \ + PREFILL_CMD="SGLANG_MORI_COMBINE_DTYPE=${MORI_COMBINE_DTYPE_PREFILL} ${PREFILL_SDMA_ENV} ${PREFILL_MORI_MOE_ENV} SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_PREFILL:-${MORI_MAX_DISPATCH_TOKENS_PREFILL}} MORI_IO_SQ_BACKOFF_TIMEOUT_US=${MORI_IO_SQ_BACKOFF_TIMEOUT_US} MORI_IO_QP_MAX_SEND_WR=${MORI_IO_QP_MAX_SEND_WR} ${LAUNCH_PREFIX:-} python3 -m sglang.launch_server \ --model-path $MODEL_DIR/$MODEL_NAME \ --disaggregation-mode prefill \ --disaggregation-ib-device ${IBDEVICES} \ @@ -470,6 +654,7 @@ if [ "$NODE_RANK" -eq 0 ]; then fi + dump_cmd "PREFILL (node 0)" "$PREFILL_CMD" if [[ "$DRY_RUN" -eq 1 ]]; then echo "DRY RUN: $PREFILL_CMD" else @@ -497,7 +682,7 @@ if [ "$NODE_RANK" -eq 0 ]; then --node-ips ${IPADDRS} \ --node-ports 8000 \ --wait-for-all-ports \ - --timeout 1800" + --timeout 2400" if [[ "$DRY_RUN" -eq 1 ]]; then echo "DRY RUN: $BARRIER_CMD" @@ -506,20 +691,41 @@ if [ "$NODE_RANK" -eq 0 ]; then fi echo "Congratulations!!! All prefill and decode servers are up . . ." + # Router resilience: a single prefill worker doing huge long-context prefills + # (256K+ token agentic prompts in 65280-token chunks) can be slow to drain a + # concurrent burst. With defaults the circuit breaker opens after 10 failures + # and short-circuits the whole worker, so the aiperf profiling burst sees + # "No available prefill workers (all circuits open or unhealthy)" and aborts + # with 100% errors. Disable the breaker and relax health-check sensitivity so + # a busy-but-alive worker is not ejected. Override via ROUTER_RESILIENCE_FLAGS. + ROUTER_RESILIENCE_FLAGS="${ROUTER_RESILIENCE_FLAGS:---disable-circuit-breaker --health-failure-threshold 100 --health-check-timeout-secs 600 --health-check-interval-secs 30}" + + # Router scheduling policy. cache_aware prefill routing exploits HiCache/radix + # prefix reuse across the agentic trace; round_robin decode keeps the single + # decode worker fed evenly. cache_threshold / balance_*_threshold tune the + # cache_aware load-balancing (router defaults are 0.5 / 64 / 1.5). Override any + # of these via env. + ROUTER_PREFILL_POLICY="${ROUTER_PREFILL_POLICY:-cache_aware}" + ROUTER_DECODE_POLICY="${ROUTER_DECODE_POLICY:-round_robin}" + ROUTER_CACHE_THRESHOLD="${ROUTER_CACHE_THRESHOLD:-0.3}" + ROUTER_BALANCE_ABS_THRESHOLD="${ROUTER_BALANCE_ABS_THRESHOLD:-2}" + ROUTER_BALANCE_REL_THRESHOLD="${ROUTER_BALANCE_REL_THRESHOLD:-1.1}" + ROUTER_POLICY_FLAGS="${ROUTER_POLICY_FLAGS:---policy ${ROUTER_PREFILL_POLICY} --prefill-policy ${ROUTER_PREFILL_POLICY} --decode-policy ${ROUTER_DECODE_POLICY} --cache-threshold ${ROUTER_CACHE_THRESHOLD} --balance-abs-threshold ${ROUTER_BALANCE_ABS_THRESHOLD} --balance-rel-threshold ${ROUTER_BALANCE_REL_THRESHOLD}}" + ROUTER_CMD="python -m sglang_router.launch_router \ --pd-disaggregation \ --port 30000 \ - --policy random \ - --prefill-policy random \ - --decode-policy random \ + ${ROUTER_POLICY_FLAGS} \ + ${ROUTER_RESILIENCE_FLAGS} \ ${PREFILL_ARGS} \ ${DECODE_ARGS}" + dump_cmd "ROUTER" "$ROUTER_CMD" if [[ "$DRY_RUN" -eq 1 ]]; then echo "DRY RUN: $ROUTER_CMD" else - ROUTER_LOG_FILE="/run_logs/slurm_job-${SLURM_JOB_ID}/proxy_${host_name}.log" + ROUTER_LOG_FILE="/run_logs/slurm_job-${SLURM_JOB_ID}/router_${host_name}.log" # sgl-router (Rust/tracing) emits ANSI color codes. NO_COLOR asks it to # skip them at the source; the sed strip guarantees a clean file even if # it doesn't honor NO_COLOR. Both branches use process substitution so @@ -573,16 +779,114 @@ if [ "$NODE_RANK" -eq 0 ]; then export IS_MTP=false fi - # n_prefill n_decode prefill_gpus decode_gpus model_dir model_name log_path isl osl concurrency_list req_rate random_range_ratio num_prompts_multiplier - BENCH_CMD="bash $SGLANG_WS_PATH/bench.sh ${xP} ${yD} $((PREFILL_TP_SIZE*xP)) $((DECODE_TP_SIZE*yD)) \ - $MODEL_DIR $MODEL_NAME /run_logs/slurm_job-${SLURM_JOB_ID} ${BENCH_INPUT_LEN} \ - ${BENCH_OUTPUT_LEN} "${BENCH_MAX_CONCURRENCY}" ${BENCH_REQUEST_RATE} \ - ${BENCH_RANDOM_RANGE_RATIO} ${BENCH_NUM_PROMPTS_MULTIPLIER}" + # Select the benchmark runner. + # IS_AGENTIC=1/true → agentic trace replay (trace_replay.sh) + # IS_AGENTIC unset/0 → fixed-seq-len throughput benchmark (bench.sh) + if [[ "${IS_AGENTIC:-0}" == "1" || "${IS_AGENTIC:-}" == "true" ]]; then + # Point aiperf's server-metrics scrape at the per-worker Prometheus + # /metrics endpoints. The router (:30000) that aiperf auto-detects from + # --url does not expose Prometheus, so without this the scrape finds no + # reachable endpoint and all server-side cache/KV fields come out null. + # Only set it when the workers were actually started with --enable-metrics. + if [[ "${ENABLE_METRICS:-0}" == "1" && "${#SERVER_METRICS_URLS[@]}" -gt 0 ]]; then + AIPERF_SERVER_METRICS_URLS=$(IFS=,; echo "${SERVER_METRICS_URLS[*]}") + export AIPERF_SERVER_METRICS_URLS + echo "AIPERF_SERVER_METRICS_URLS=${AIPERF_SERVER_METRICS_URLS}" + fi + # Per-worker base URLs for cache flushing between concurrency points. + # trace_replay.sh consults these when CLEAR_CACHE_BETWEEN_CONC=1. + if [[ "${#SERVER_FLUSH_URLS[@]}" -gt 0 ]]; then + SERVER_FLUSH_URLS_CSV=$(IFS=,; echo "${SERVER_FLUSH_URLS[*]}") + export SERVER_FLUSH_URLS_CSV + echo "SERVER_FLUSH_URLS_CSV=${SERVER_FLUSH_URLS_CSV}" + fi + # trace_replay.sh signature: model_path model_name concurrency_list log_path + BENCH_CMD="bash $SGLANG_WS_PATH/trace_replay.sh \ + $MODEL_DIR $MODEL_NAME $BENCH_MAX_CONCURRENCY /run_logs/slurm_job-${SLURM_JOB_ID}" + echo "Benchmark runner: trace_replay.sh (agentic, KV_OFFLOADING=${KV_OFFLOADING:-none}, backend=${KV_OFFLOAD_BACKEND:-none}, CONC=${BENCH_MAX_CONCURRENCY})" + else + # bench.sh signature: + # n_prefill n_decode prefill_gpus decode_gpus model_dir model_name log_path + # isl osl concurrency_list req_rate random_range_ratio num_prompts_multiplier + BENCH_CMD="bash $SGLANG_WS_PATH/bench.sh ${xP} ${yD} $((PREFILL_TP_SIZE*xP)) $((DECODE_TP_SIZE*yD)) \ + $MODEL_DIR $MODEL_NAME /run_logs/slurm_job-${SLURM_JOB_ID} ${BENCH_INPUT_LEN} \ + ${BENCH_OUTPUT_LEN} \"${BENCH_MAX_CONCURRENCY}\" ${BENCH_REQUEST_RATE} \ + ${BENCH_RANDOM_RANGE_RATIO} ${BENCH_NUM_PROMPTS_MULTIPLIER}" + echo "Benchmark runner: bench.sh (fixed-seq-len)" + fi + + IS_AGENTIC_RUN=0 + if [[ "${IS_AGENTIC:-0}" == "1" || "${IS_AGENTIC:-}" == "true" ]]; then + IS_AGENTIC_RUN=1 + fi if [[ "${EVAL_ONLY:-false}" == "true" ]]; then echo "EVAL_ONLY mode: skipping throughput benchmark" elif [[ "$DRY_RUN" -eq 1 ]]; then echo "DRY RUN: $BENCH_CMD" + elif [[ -n "${CLIENT_IMAGE:-}" && "$IS_AGENTIC_RUN" == "1" ]]; then + # Separate client image (node-0 sibling container): run the aiperf trace + # replay in its own sibling container built from CLIENT_IMAGE (which ships + # a pre-baked aiperf + deps) instead of rebuilding the aiperf venv inside + # this server container. The server/router stay up in this container while + # the client container drives the benchmark against the router on + # localhost (--network host). job.slurm mounts the host docker socket + CLI + # into this container and forwards HOST_REPO_DIR / HOST_MODEL_DIR / + # HOST_BENCH_LOGS / CLIENT_CONT_NAME so the sibling can be launched here. + CLIENT_ENV_FILE="/run_logs/slurm_job-${SLURM_JOB_ID}/client.env" + mkdir -p "/run_logs/slurm_job-${SLURM_JOB_ID}" + # Forward the benchmark-relevant env (incl. runtime-computed metrics/flush + # URLs) to the client container; override the few paths/flags that differ + # inside the pre-baked image. Unset vars are skipped, so the client keeps + # its own defaults for anything not exported here. + { + for _v in ENGINE MODEL_NAME MODEL_PREFIX PRECISION FRAMEWORK SPEC_DECODING \ + DURATION MAX_MODEL_LEN RESULT_FILENAME RUNNER_NAME RUNNER_TYPE IMAGE \ + AIPERF_SERVER_METRICS_URLS SERVER_FLUSH_URLS_CSV \ + ENABLE_METRICS IS_AGENTIC CLEAR_CACHE_BETWEEN_CONC \ + DISAGG IS_MULTINODE \ + TP EP_SIZE DP_ATTENTION DCP_SIZE PCP_SIZE \ + PREFILL_NUM_WORKERS PREFILL_TP PREFILL_EP PREFILL_DP_ATTN PREFILL_HARDWARE \ + DECODE_NUM_WORKERS DECODE_TP DECODE_EP DECODE_DP_ATTN DECODE_HARDWARE \ + KV_OFFLOADING KV_OFFLOAD_BACKEND TOTAL_CPU_DRAM_GB \ + WEKA_LOADER_OVERRIDE AIPERF_FAILED_REQUEST_THRESHOLD \ + AIPERF_AGENTIC_CACHE_WARMUP_DURATION AIPERF_UNSAFE_OVERRIDE \ + AIPERF_TRAJECTORY_START_MIN_RATIO AIPERF_TRAJECTORY_START_MAX_RATIO \ + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES ROUTER_PORT TQDM_MININTERVAL; do + if [[ -n "${!_v+x}" ]]; then printf '%s=%s\n' "$_v" "${!_v}"; fi + done + echo "INFMAX_CONTAINER_WORKSPACE=/workspace" + # Do NOT pin AGENTIC_OUTPUT_DIR: it must default to /workspace (the + # host repo mount == GITHUB_WORKSPACE) so the aggregated + # ${RESULT_FILENAME}_conc.json lands where the workflow guard globs + # it. /workspace is bind-mounted writable, same as the co-located path. + echo "HF_HOME=/run_logs/hf_cache" + echo "MODEL_DIR=/models" + # A pre-baked client image ships aiperf at CLIENT_AIPERF_VENV; when + # unset (e.g. reusing the server image, which carries no pre-baked + # venv), trace_replay builds aiperf on the fly from + # /workspace/utils/aiperf — same as the co-located path. + if [[ -n "${CLIENT_AIPERF_VENV:-}" ]]; then + echo "AIPERF_USE_PREBUILT=1" + echo "AIPERF_VENV=${CLIENT_AIPERF_VENV}" + fi + } > "$CLIENT_ENV_FILE" + + echo "Launching agentic benchmark in separate client container: ${CLIENT_IMAGE}" + docker rm -f "${CLIENT_CONT_NAME}" 2>/dev/null || true + set -x + docker run --rm --network host \ + --name "${CLIENT_CONT_NAME}" \ + --shm-size 32G \ + -v "${HOST_REPO_DIR}:/workspace" \ + -v "${HOST_MODEL_DIR}:/models" \ + -v /tmp:/run_logs \ + -v "${HOST_BENCH_LOGS}:/benchmark_logs" \ + --env-file "${CLIENT_ENV_FILE}" \ + --entrypoint "" \ + "${CLIENT_IMAGE}" \ + bash -lc "cd /workspace/benchmarks/multi_node/amd_utils && bash trace_replay.sh /models ${MODEL_NAME} \"${BENCH_MAX_CONCURRENCY}\" /run_logs/slurm_job-${SLURM_JOB_ID}" + set +x else set -x eval "$BENCH_CMD" @@ -713,13 +1017,18 @@ elif [ "$NODE_RANK" -gt 0 ] && [ "$NODE_RANK" -lt "$NODE_OFFSET" ]; then echo "Using prefill config: $PREFILL_SERVER_CONFIG" echo "Prefill parallelism: TP=${PREFILL_TP_SIZE}, EP enabled: ${PREFILL_ENABLE_EP}, DP enabled: ${PREFILL_ENABLE_DP}" + CMD_DUMP="/run_logs/slurm_job-${SLURM_JOB_ID}/commands_${host_name}.txt" + dump_cmd() { echo -e "\n# ── $1 ──\n$2" >> "$CMD_DUMP"; } + echo "# Commands dump — $(date -u '+%Y-%m-%d %H:%M:%S UTC')" > "$CMD_DUMP" + echo "# Host: ${host_name} (${host_ip}) Node rank: ${NODE_RANK}" >> "$CMD_DUMP" + PREFILL_MORI_MOE_ENV="" set -x if [[ -n "$MORI_MOE_MAX_INPUT_TOKENS_PREFILL" ]]; then PREFILL_MORI_MOE_ENV="SGLANG_MORI_MOE_MAX_INPUT_TOKENS=${MORI_MOE_MAX_INPUT_TOKENS_PREFILL}" fi set +x - PREFILL_CMD="SGLANG_MORI_COMBINE_DTYPE=${MORI_COMBINE_DTYPE_PREFILL} ${PREFILL_SDMA_ENV} ${PREFILL_MORI_MOE_ENV} SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_PREFILL:-${MORI_MAX_DISPATCH_TOKENS_PREFILL}} python3 -m sglang.launch_server \ + PREFILL_CMD="SGLANG_MORI_COMBINE_DTYPE=${MORI_COMBINE_DTYPE_PREFILL} ${PREFILL_SDMA_ENV} ${PREFILL_MORI_MOE_ENV} SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_PREFILL:-${MORI_MAX_DISPATCH_TOKENS_PREFILL}} MORI_IO_SQ_BACKOFF_TIMEOUT_US=${MORI_IO_SQ_BACKOFF_TIMEOUT_US} MORI_IO_QP_MAX_SEND_WR=${MORI_IO_QP_MAX_SEND_WR} ${LAUNCH_PREFIX:-} python3 -m sglang.launch_server \ --model-path $MODEL_DIR/${MODEL_NAME} \ --disaggregation-mode prefill \ --disaggregation-ib-device ${IBDEVICES} \ @@ -734,6 +1043,7 @@ elif [ "$NODE_RANK" -gt 0 ] && [ "$NODE_RANK" -lt "$NODE_OFFSET" ]; then PREFILL_CMD="$PREFILL_CMD --dist-init-addr ${PREFILL_HEADNODE_URLS[$prefill_idx]} --nnodes ${PREFILL_NODES_PER_WORKER} --node-rank $rank" fi + dump_cmd "PREFILL (rank ${NODE_RANK})" "$PREFILL_CMD" if [[ "$DRY_RUN" -eq 1 ]]; then echo "DRY RUN: $PREFILL_CMD" else @@ -788,13 +1098,18 @@ else echo "Decode node rank: $RANK" echo "Decode parallelism: TP=${DECODE_TP_SIZE}, EP enabled: ${DECODE_ENABLE_EP}, DP enabled: ${DECODE_ENABLE_DP}" + CMD_DUMP="/run_logs/slurm_job-${SLURM_JOB_ID}/commands_${host_name}.txt" + dump_cmd() { echo -e "\n# ── $1 ──\n$2" >> "$CMD_DUMP"; } + echo "# Commands dump — $(date -u '+%Y-%m-%d %H:%M:%S UTC')" > "$CMD_DUMP" + echo "# Host: ${host_name} (${host_ip}) Node rank: ${NODE_RANK}" >> "$CMD_DUMP" + DECODE_MORI_MOE_ENV="" set -x if [[ -n "$MORI_MOE_MAX_INPUT_TOKENS_DECODE" ]]; then DECODE_MORI_MOE_ENV="SGLANG_MORI_MOE_MAX_INPUT_TOKENS=${MORI_MOE_MAX_INPUT_TOKENS_DECODE}" fi set +x - DECODE_CMD="SGLANG_MORI_COMBINE_DTYPE=${MORI_COMBINE_DTYPE_DECODE} ${DECODE_MORI_MOE_ENV} SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_DECODE:-${MORI_MAX_DISPATCH_TOKENS_DECODE}} python3 -m sglang.launch_server \ + DECODE_CMD="SGLANG_MORI_COMBINE_DTYPE=${MORI_COMBINE_DTYPE_DECODE} ${DECODE_MORI_MOE_ENV} SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_DECODE:-${MORI_MAX_DISPATCH_TOKENS_DECODE}} MORI_IO_SQ_BACKOFF_TIMEOUT_US=${MORI_IO_SQ_BACKOFF_TIMEOUT_US} MORI_IO_QP_MAX_SEND_WR=${MORI_IO_QP_MAX_SEND_WR} ${LAUNCH_PREFIX:-} python3 -m sglang.launch_server \ --model-path ${MODEL_DIR}/${MODEL_NAME} \ --disaggregation-mode decode \ --disaggregation-ib-device ${IBDEVICES} \ @@ -809,6 +1124,7 @@ else DECODE_CMD="$DECODE_CMD --dist-init-addr ${DECODE_HEADNODE_URLS[$decode_idx]} --nnodes ${DECODE_NODES_PER_WORKER} --node-rank $rank" fi + dump_cmd "DECODE (rank ${NODE_RANK})" "$DECODE_CMD" if [[ "$DRY_RUN" -eq 1 ]]; then echo "DRY RUN: $DECODE_CMD" else diff --git a/benchmarks/multi_node/amd_utils/setup_deps.sh b/benchmarks/multi_node/amd_utils/setup_deps.sh index 35eaf17dc0..f1ae30b925 100644 --- a/benchmarks/multi_node/amd_utils/setup_deps.sh +++ b/benchmarks/multi_node/amd_utils/setup_deps.sh @@ -34,7 +34,6 @@ git_clone_retry() { return 1 } - # --------------------------------------------------------------------------- # 5. Container RDMA/net tools # - ibv_devinfo comes from ibverbs-utils @@ -80,97 +79,58 @@ install_amd_quark() { } # --------------------------------------------------------------------------- -# SGLang: Patch aiter gluon pa_mqa_logits — fix 2D → 3D instr_shape for -# Triton ≥ 3.5. -# -# Bug: _gluon_deepgemm_fp8_paged_mqa_logits (the non-preshuffle variant) -# hardcodes AMDMFMALayout(instr_shape=[16, 16]) which fails on Triton -# builds where AMDMFMALayout requires 3D (M, N, K) format. -# -# The two preshuffle variants already conditionally select 2D vs 3D via -# the module-level _Use_2d_instr_shape_mfma_layout flag, but the base -# variant was missed. This patch brings it in line. +# SGLang: prevent TP-rank collective desync deadlock in disaggregation prefill. # -# Affects: GLM-5 (NSA attention) and any future model that uses -# deepgemm_fp8_paged_mqa_logits with Preshuffle=False. +# resolve_waiting_queue_bootstrap() runs poll_and_all_reduce_attn_cp_tp_group() +# over `candidates`. The upstream candidate set (all non-aborted waiting reqs) +# can differ across TP ranks, so some ranks enter the all_reduce while others +# skip it -> hang. Narrow candidates to optimistic (pending_bootstrap) requests, +# which is consistent across ranks and is the only set finalize_bootstrap acts on. # --------------------------------------------------------------------------- -patch_gluon_pa_mqa_logits_instr_shape() { +patch_disagg_prefill_bootstrap_desync() { python3 -c ' import os, sys -target = "/sgl-workspace/aiter/aiter/ops/triton/gluon/pa_mqa_logits.py" +target = "/sgl-workspace/sglang/python/sglang/srt/disaggregation/prefill.py" if not os.path.isfile(target): - print("[SETUP] gluon pa_mqa_logits.py not found, skipping") + print("[SETUP] disaggregation/prefill.py not found, skipping") sys.exit(0) src = open(target).read() -if "[PATCHED] 3D instr_shape for base gluon variant" in src: - print("[SETUP] gluon pa_mqa_logits 3D instr_shape patch already applied") +old = " candidates = [req for req in self.waiting_queue if not is_aborted(req)]" +new = ( + " candidates = [\n" + " req\n" + " for req in self.waiting_queue\n" + " if req.pending_bootstrap and not is_aborted(req)\n" + " ]" +) + +if new in src: + print("[SETUP] prefill bootstrap-desync patch already applied") sys.exit(0) -# The buggy code: the base _gluon_deepgemm_fp8_paged_mqa_logits uses 2D -# instr_shape unconditionally. We replace it with a conditional that -# mirrors the preshuffle variants. -old = """\ - mfma_layout: gl.constexpr = gl.amd.AMDMFMALayout( - version=CDNA_VERSION, - instr_shape=[16, 16], - transposed=False, - warps_per_cta=[1, NumWarps], - ) - mfma_layout_a: gl.constexpr = gl.DotOperandLayout( - operand_index=0, parent=mfma_layout, k_width=16 - ) - mfma_layout_b: gl.constexpr = gl.DotOperandLayout( - operand_index=1, parent=mfma_layout, k_width=16 - )""" - -new = """\ - # [PATCHED] 3D instr_shape for base gluon variant - if _Use_2d_instr_shape_mfma_layout: - mfma_layout: gl.constexpr = gl.amd.AMDMFMALayout( - version=CDNA_VERSION, - instr_shape=[16, 16], - transposed=False, - warps_per_cta=[1, NumWarps], - ) - else: - mfma_layout: gl.constexpr = gl.amd.AMDMFMALayout( - version=CDNA_VERSION, - instr_shape=[16, 16, 32], - transposed=False, - warps_per_cta=[1, NumWarps], - ) - mfma_layout_a: gl.constexpr = gl.DotOperandLayout( - operand_index=0, parent=mfma_layout, k_width=16 - ) - mfma_layout_b: gl.constexpr = gl.DotOperandLayout( - operand_index=1, parent=mfma_layout, k_width=16 - )""" - if old not in src: - print("[SETUP] WARN: gluon pa_mqa_logits pattern not found — aiter version may have changed") + print("[SETUP] WARN: resolve_waiting_queue_bootstrap pattern not found — sglang version may have changed") sys.exit(0) -# Only replace the FIRST occurrence (the base variant, not preshuffle ones) -new_src = src.replace(old, new, 1) - -open(target, "w").write(new_src) -print("[SETUP] Patched: gluon pa_mqa_logits 3D instr_shape for base variant") +open(target, "w").write(src.replace(old, new)) +print("[SETUP] Patched: disaggregation/prefill.py resolve_waiting_queue_bootstrap candidates") ' - _SETUP_INSTALLED+=("gluon-instr-shape-fix") + _SETUP_INSTALLED+=("prefill-bootstrap-desync-fix") } # --------------------------------------------------------------------------- -# SGLang: Install latest transformers for GLM-5 model type support. +# SGLang: Install latest transformers for GLM model type support. # # GLM-5 (zai-org/GLM-5-FP8) requires a transformers build that includes -# the glm_moe_dsa model type. The mori images do not ship it. -# Only install if GLM-5 is the active model (avoid overhead otherwise). +# the glm_moe_dsa model type. The mori images do not ship it. Gated on any +# GLM model name (not just GLM-5-FP8) so other GLM variants pick up the same +# fix; only installs when a GLM model is active (avoid overhead otherwise). # --------------------------------------------------------------------------- install_transformers_glm5() { - if [[ "$MODEL_NAME" != "GLM-5-FP8" ]]; then + if [[ "$MODEL_NAME" != *GLM* ]]; then return 0 fi @@ -202,7 +162,8 @@ if [[ "$ENGINE" == "vllm-disagg" ]]; then export PATH="${UCX_HOME}/bin:/usr/local/bin/etcd:/root/.cargo/bin:${PATH}" export LD_LIBRARY_PATH="${UCX_HOME}/lib:${RIXL_HOME}/lib:${RIXL_HOME}/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH:-}" else - patch_gluon_pa_mqa_logits_instr_shape + # patch_disagg_prefill_bootstrap_desync + install_transformers_glm5 fi diff --git a/benchmarks/multi_node/amd_utils/trace_replay.sh b/benchmarks/multi_node/amd_utils/trace_replay.sh new file mode 100644 index 0000000000..b3eb0b8fd7 --- /dev/null +++ b/benchmarks/multi_node/amd_utils/trace_replay.sh @@ -0,0 +1,160 @@ +#!/bin/bash +# Dual-Engine Disaggregated Benchmark Runner +# +# ENGINE=sglang (default): SGLang benchmark +# ENGINE=vllm: vLLM benchmark +# +# Produces JSON result files via benchmark_serving.py so that the CI pipeline +# can collect and process results. +# +# Usage: bash bench.sh \ +# \ +# + +ENGINE="${ENGINE:-sglang-disagg}" + +model_path=$1 +model_name=$2 +concurrency_list=${3:-"1"} +MODEL_PATH="${MODEL_PATH:-${model_path}/${model_name}}" +# vllm-disagg uses --served-model-name MODEL_NAME; sglang defaults to MODEL_PATH +if [[ "$ENGINE" == "vllm-disagg" ]]; then + MODEL="${MODEL_NAME:-${MODEL_PATH}}" +else + MODEL="${MODEL_PATH}" +fi +log_path=${4:-/run_logs} + +# Split BENCH_MAX_CONCURRENCY (x-delimited, e.g. "8x16x32") into an array. +# Falls back to 1 if unset so the loop always runs at least once. +IFS='x' read -r -a chosen_concurrencies <<< "${concurrency_list}" + + +ROUTER_PORT="${ROUTER_PORT:-30000}" + +export TRANSFORMERS_VERBOSITY=error +export TOKENIZERS_PARALLELISM=false + +# echo "Config ${chosen_isl}; ${chosen_osl}; ${chosen_concurrencies[0]}; ${chosen_req_rate}" + +RESULT_DIR="${RESULT_DIR:-${log_path}/agentic}" +mkdir -p "$RESULT_DIR" + +source "$(dirname "$0")/../../benchmark_lib.sh" + +# clear_kv_caches — wipe all KV cache tiers on every backend worker before a +# concurrency point, so each conc is measured cold (no prefix reuse bleeding in +# from the previous conc). Mirrors mori-scheduler/scripts/benchmark/lib/ +# clear_caches.sh, but the worker base URLs are already resolved by +# server_sglang.sh (SERVER_FLUSH_URLS_CSV) so no SSH/IP lookup is needed. +# +# Tiers (SGLang server APIs), hit on EACH worker directly (the router does not +# fan /flush_cache out): +# L1 (GPU radix) + L2 (host hicache): POST /flush_cache — NO-OP while any +# request is in flight, so we drain-retry until "Cache flushed" or +# FLUSH_DRAIN_TIMEOUT (default 120s) elapses. +# L3 (umbp / mooncake store): POST /hicache/storage-backend/clear +# — HTTP != 200 when L3 is off, tolerated. +# Best-effort: logs WARN, never hard-fails the sweep. +clear_kv_caches() { + local drain_tmo="${FLUSH_DRAIN_TIMEOUT:-120}" + local urls_csv="${SERVER_FLUSH_URLS_CSV:-}" + if [[ -z "$urls_csv" ]]; then + echo "[clear_caches] WARN: SERVER_FLUSH_URLS_CSV unset; skipping cache flush" >&2 + return 0 + fi + local -a urls + IFS=',' read -r -a urls <<< "$urls_csv" + local url start ok resp code + for url in "${urls[@]}"; do + [[ -n "$url" ]] || continue + # L1 + L2: drain-retry until flushed (no-op while requests in flight). + start=$(date +%s); ok=0; resp="" + while :; do + resp=$(curl -sf -m 10 -X POST "${url}/flush_cache" 2>/dev/null || true) + echo "$resp" | grep -qi "Cache flushed" && { ok=1; break; } + (( $(date +%s) - start >= drain_tmo )) && break + sleep 3 + done + if [[ "$ok" == 1 ]]; then + echo "[clear_caches] ${url}: L1+L2 flushed" + else + echo "[clear_caches] WARN ${url}: L1+L2 flush NOT confirmed after ${drain_tmo}s (resp='${resp:0:80}')" >&2 + fi + # L3: storage-backend clear (umbp / mooncake). 200 when a backend is attached. + code=$(curl -s -m 60 -o /dev/null -w '%{http_code}' -X POST "${url}/hicache/storage-backend/clear" 2>/dev/null || echo 000) + if [[ "$code" == 200 ]]; then + echo "[clear_caches] ${url}: L3 store cleared" + else + echo "[clear_caches] ${url}: L3 clear http=${code} (no storage backend / L3 off — ok)" + fi + done +} + +# REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" + +PORT="${ROUTER_PORT}" +MODEL="${MODEL:-${BENCH_MODEL}}" +DURATION="${DURATION:-1800}" +export MODEL DURATION MAX_MODEL_LEN +RESULT_DIR="${RESULT_DIR:-${profile_folder}}" +# Base name for the per-conc aggregate written by the existing +# utils.agentic.aggregation.process_agentic_result module. +# The workflow guard / upload steps expect a "${RESULT_FILENAME}_conc.json" +# file per concurrency, so each concurrency below is always suffixed with +# _conc (matching agentic_srt.sh on the gb200 path). +RESULT_FILENAME_BASE="${RESULT_FILENAME:-agentic_bench}" + +mkdir -p "$RESULT_DIR" + +resolve_trace_source +install_agentic_deps + +ANY_FAILED=0 +for max_concurrency in "${chosen_concurrencies[@]}"; do + + echo "==========================================" + echo "Agentic trace replay: conc=$max_concurrency" + echo "==========================================" + + # Clear all KV cache tiers on every backend before this conc point so it is + # measured cold (no prefix reuse from the previous conc). Default on; set + # CLEAR_CACHE_BETWEEN_CONC=0 to disable. Best-effort — never fails the run. + if [[ "${CLEAR_CACHE_BETWEEN_CONC:-1}" == "1" ]]; then + echo "conc=$max_concurrency: clearing L1/L2/L3 on all backends (no server restart)" + clear_kv_caches || echo "WARNING: cache clear had issues for conc=$max_concurrency" >&2 + fi + + # Mirror agentic_srt.sh (the srtctl/gb200 path): every concurrency writes + # its artifacts into a conc_/ subdir of RESULT_DIR. The CI matrix explodes + # agentic runs to one concurrency per job, but benchmark-multinode-tmpl.yml + # still expects the per-conc nesting (LOGS/agentic/conc_*/...) and the + # _conc result-file suffix, so we always nest to keep the layout identical + # across runners and avoid overwriting earlier runs in local multi-conc sweeps. + CONC_RESULT_DIR="$RESULT_DIR/conc_${max_concurrency}" + mkdir -p "$CONC_RESULT_DIR" + + CONC="$max_concurrency" + USERS="$max_concurrency" + export CONC USERS + build_replay_cmd "$CONC_RESULT_DIR" + + # Per-conc result name consumed by write_agentic_result_json. Always suffix + # with _conc so the file matches + # the workflow guard's "${RESULT_FILENAME}_conc*.json" glob (and the agg / + # checkpoint upload steps) for both single-conc CI runs and multi-conc sweeps. + export RESULT_FILENAME="${RESULT_FILENAME_BASE}_conc${max_concurrency}" + if ! run_agentic_replay_and_write_outputs "$CONC_RESULT_DIR"; then + echo "WARNING: agentic trace replay for conc=$max_concurrency failed (replay or validation) after writing available results" >&2 + ANY_FAILED=1 + fi + + echo "-----------------------------------------" + +done + +export RESULT_FILENAME="$RESULT_FILENAME_BASE" + +if [ "$ANY_FAILED" -ne 0 ]; then + echo "WARNING: at least one conc had a non-zero exit; per-conc result files were still written when possible." >&2 +fi diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 03c7a296ca..213ace2d80 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -3122,3 +3122,43 @@ minimaxm3-fp8-mi325x-vllm-agentic: - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: native, conc-list: [3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] } - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: native, conc-list: [10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 32] } - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: native, conc-list: [24, 32, 36, 40, 44, 48, 52, 56, 60, 64, 72, 80, 96] } + +dsv4-fp4-mi355x-sglang-disagg-agentic-hicache: + image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:mi355x-amds + precision: fp4 + framework: sglang-disagg + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - spec-decoding: "none" + conc-list: [ 2,4,8,16,32 ] + kv-offloading: dram + kv-offload-backend: hicache + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + # Node-0 sibling client: run the aiperf trace-replay in its own sibling + # container on node 0 (via the host docker socket) instead of inside the + # server container. Reuse the (publicly pullable) server image so + # upstream CI can fetch it; aiperf is built on the fly from + # /workspace/utils/aiperf. Setting only CLIENT_IMAGE (no CLIENT_NODES) + # selects the sibling-container mode. + - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MTP_SIZE=0" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index be28208ab4..9fba3d61e8 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4716,3 +4716,11 @@ - "Clean the export envs" - "Enable two batch overlap" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2093 + +- config-keys: + - dsv4-fp4-mi355x-sglang-disagg-agentic-hicache + description: + - "Add DeepSeek-V4-Pro FP4 MI355X SGLang-disagg agentic-coding benchmark with HiCache DRAM KV offloading." + - "Add the multi-node agentic runtime plumbing: node-0 sibling benchmark-client container, per-worker metrics scrape, per-concurrency cache flushing, trace replay client, and one allocation per concurrency." + - "Image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2170 diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index 8cb92b7a16..1ff0dffd59 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -78,7 +78,15 @@ if [[ "$IS_MULTINODE" == "true" ]]; then SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_mi355x_${FRAMEWORK}.sh" if [[ "$FRAMEWORK" == "sglang-disagg" ]] || [[ "$FRAMEWORK" == "vllm-disagg" ]] || [[ "$FRAMEWORK" == "atom-disagg" ]]; then - BENCHMARK_SUBDIR="multi_node" + # Agentic recipes live under multi_node/agentic/ and export the + # HiCache tunables (page-size, io-backend, ...); fixed-seq-len recipes + # live at the multi_node/ root. Honor SCENARIO_SUBDIR so agentic-coding + # configs pick the agentic recipe instead of the root one. + if [[ "${SCENARIO_SUBDIR}" == "agentic/" ]]; then + BENCHMARK_SUBDIR="multi_node/agentic" + else + BENCHMARK_SUBDIR="multi_node" + fi else BENCHMARK_SUBDIR="single_node/fixed_seq_len" fi @@ -126,7 +134,7 @@ if [[ "$IS_MULTINODE" == "true" ]]; then # search for "FRAMEWORK_DIFF_IF_STATEMENT #3" for this if-statement # Find the latest log directory that contains the data - if [[ "${EVAL_ONLY:-false}" != "true" ]]; then + if [[ "${EVAL_ONLY:-false}" != "true" && "${IS_AGENTIC:-0}" != "1" ]]; then cat > collect_latest_results.py <<'PY' import os, sys job_dir, isl, osl, nexp, framework = sys.argv[1], int(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4]), sys.argv[5] @@ -181,6 +189,58 @@ PY fi fi + # Stage agentic raw artifacts + server logs for the CI upload steps. + # server_sglang.sh copies /run_logs/slurm_job- to + # $BENCHMARK_LOGS_DIR/logs/slurm_job- on shared storage, and + # trace_replay.sh writes each concurrency's aiperf artifacts under + # agentic/conc_/ (mirroring agentic_srt.sh). benchmark-multinode-tmpl.yml + # uploads them from $GITHUB_WORKSPACE/LOGS/agentic/conc_*/... plus a + # multinode_server_logs.tar.gz, so preserve the conc_/ nesting here + # before the logs dir is removed below. The agg result JSON is already + # written straight to the mounted workspace by the existing agentic + # aggregation module. + if [[ "${IS_AGENTIC:-0}" == "1" ]]; then + JOB_LOGS_DIR="$BENCHMARK_LOGS_DIR/logs/slurm_job-${JOB_ID}" + if [ -d "$JOB_LOGS_DIR" ]; then + # trace_replay.sh always nests artifacts under agentic/conc_/. + # Copy the whole agentic/ tree so the conc_/ subdirs are + # preserved for the LOGS/agentic/conc_*/... upload globs. + AGENTIC_SRC="$JOB_LOGS_DIR/agentic" + if [ -d "$AGENTIC_SRC" ] && find "$AGENTIC_SRC" -mindepth 1 -maxdepth 1 -type d -name 'conc_*' -print -quit 2>/dev/null | grep -q .; then + echo "Staging agentic raw artifacts from $AGENTIC_SRC" + mkdir -p "$GITHUB_WORKSPACE/LOGS/agentic" + cp -r "$AGENTIC_SRC"/. "$GITHUB_WORKSPACE/LOGS/agentic/" + # The source artifacts are created inside the container as root + # (--container-remap-root), so depending on how the runner + # invokes this script the copies can end up root-owned and/or + # read-only (aiperf/server_sglang make some dirs mode 0555). If + # the staged tree isn't owned+writable by the runner user, the + # next checkout's `git clean` fails with + # EACCES: permission denied, rmdir '.../LOGS/agentic'. + # chown to the invoking user (the same one that runs git clean) + # via sudo (already passwordless here for rm -rf). The follow-up + # chmod uses a+rwX (not just u+rwX): the *next* job against this + # same $GITHUB_WORKSPACE may be picked up by a different runner + # process running as a different OS user, which would otherwise + # fall outside the owner bits and still fail the same + # `git clean` with EACCES despite the chown above. + sudo chown -R "$(id -u):$(id -g)" "$GITHUB_WORKSPACE/LOGS" 2>/dev/null || true + chmod -R a+rwX "$GITHUB_WORKSPACE/LOGS" 2>/dev/null || true + ls -laR "$GITHUB_WORKSPACE/LOGS/agentic" + else + echo "WARNING: no agentic conc_*/ artifacts found under $JOB_LOGS_DIR/agentic" + fi + # Server/router/prefill/decode logs for the multinode_server_logs_* artifact. + if tar czf "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz" -C "$JOB_LOGS_DIR" . 2>/dev/null; then + echo "Created multinode_server_logs.tar.gz" + else + echo "WARNING: failed to create multinode_server_logs.tar.gz" + fi + else + echo "WARNING: agentic staging skipped; $JOB_LOGS_DIR not found" + fi + fi + echo "All result files processed" # Use sync scancel to ensure nfs file handle is released in time set +x diff --git a/utils/agentic/aggregation/backends/sglang.py b/utils/agentic/aggregation/backends/sglang.py index 2f560f3bf7..acbbc12b02 100644 --- a/utils/agentic/aggregation/backends/sglang.py +++ b/utils/agentic/aggregation/backends/sglang.py @@ -61,6 +61,8 @@ def populate( flat["server_gpu_cache_hit_rate"] = rate(device_hits, prompt_total) flat["server_cpu_cache_hit_rate"] = rate(host_hits, prompt_total) + # HiCache host hits are external to the GPU cache. + flat["server_external_cache_hit_rate"] = flat["server_cpu_cache_hit_rate"] flat["server_overall_cache_hit_rate"] = rate(total_cached, prompt_total) if flat["server_overall_cache_hit_rate"] is None: diff --git a/utils/agentic/aggregation/test_process_agentic_result.py b/utils/agentic/aggregation/test_process_agentic_result.py index 6296a4f308..0aed14e132 100644 --- a/utils/agentic/aggregation/test_process_agentic_result.py +++ b/utils/agentic/aggregation/test_process_agentic_result.py @@ -1071,6 +1071,7 @@ def test_processor_normalizes_sglang_server_metrics(tmp_path: Path): _assert_stable_server_metrics_schema(agg) assert agg["server_metrics"]["cache"]["gpu_cache_hit_rate"] == pytest.approx(0.4) assert agg["server_metrics"]["cache"]["cpu_cache_hit_rate"] == pytest.approx(0.1) + assert agg["server_metrics"]["cache"]["external_cache_hit_rate"] == pytest.approx(0.1) assert agg["server_metrics"]["cache"]["overall_cache_hit_rate"] == pytest.approx(0.5) assert agg["server_metrics"]["kv_cache"]["gpu_usage_pct"] == pytest.approx(0.75) assert agg["server_metrics"]["kv_cache"]["cpu_usage_pct"] == pytest.approx(0.3) diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index 5728580cb1..33bed0b8e3 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -24,7 +24,8 @@ MIN_EVAL_CONC = 16 # Bound how many multinode agentic conc points share one server allocation. -MAX_MULTINODE_AGENTIC_CONCURRENCIES_PER_ALLOCATION = 4 +# 1 = one task/SLURM allocation per concurrency (matches single-node agentic). +MAX_MULTINODE_AGENTIC_CONCURRENCIES_PER_ALLOCATION = 1 BYTES_PER_MIB = 1024 * 1024 BYTES_PER_GB = 1_000_000_000 # 3 TB decimal DRAM cap, expressed in MiB, before utilization scaling. diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index 0d385b4024..b77a1cdc48 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -2133,7 +2133,7 @@ def test_agentic_node_dram_rejects_tp_above_runner_gpus(self, sample_runner_conf generate_test_config_sweep(args, config, runner_config) def test_multinode_agentic_groups_concurrencies_per_search_entry(self): - """One server allocation should run the selected concurrency batch.""" + """One server allocation should run exactly one concurrency (one task per conc).""" config = { "dsv4-agentic-2p1d": { "image": "vllm/vllm-openai:v0.23.0", @@ -2169,17 +2169,21 @@ def test_multinode_agentic_groups_concurrencies_per_search_entry(self): result = generate_test_config_sweep(args, config) - assert len(result) == 2 - assert result[0]["conc"] == [16, 32, 64, 128] - assert result[0]["exp-name"] == "dsv4_p2x4_d1x4_conc16x32x64x128" + assert len(result) == 5 + assert [entry["conc"] for entry in result] == [[16], [32], [64], [128], [256]] + assert [entry["exp-name"] for entry in result] == [ + "dsv4_p2x4_d1x4_conc16", + "dsv4_p2x4_d1x4_conc32", + "dsv4_p2x4_d1x4_conc64", + "dsv4_p2x4_d1x4_conc128", + "dsv4_p2x4_d1x4_conc256", + ] assert result[0]["prefill"]["pp"] == 2 assert result[0]["prefill"]["dcp-size"] == 2 assert result[0]["prefill"]["pcp-size"] == 2 assert result[0]["decode"]["pp"] == 2 assert result[0]["decode"]["dcp-size"] == 2 assert result[0]["decode"]["pcp-size"] == 1 - assert result[1]["conc"] == [256] - assert result[1]["exp-name"] == "dsv4_p2x4_d1x4_conc256" def test_multinode_agentic_preserves_kv_offload_fields(self): config = { @@ -2341,7 +2345,7 @@ def test_node_type_filters_apply_to_agentic_configs( "agentic-coding": [{ "search-space": [ { - "conc-list": [16, 32], + "conc-list": [16], "prefill": {"hardware": "gb200", "num-worker": 2, "tp": 4, "pp": 2, "dcp-size": 2, "pcp-size": 2, "ep": 4, "dp-attn": False}, "decode": {"hardware": "h100", "num-worker": 1, "tp": 4, "pp": 2, "dcp-size": 2, "pcp-size": 1, "ep": 1, "dp-attn": False}, },