diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index 1f5e1e7bc0..e67a0ec766 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -536,7 +536,7 @@ jobs: model-prefix: ${{ matrix.config.model-prefix }} framework: ${{ matrix.config.framework }} precision: ${{ matrix.config.precision }} - conc-list: '[${{ matrix.config.conc }}]' + conc-list: ${{ toJson(matrix.config.conc) }} spec-decoding: ${{ matrix.config.spec-decoding }} disagg: ${{ matrix.config.disagg }} prefill-hardware: ${{ matrix.config.prefill.hardware }} @@ -551,7 +551,6 @@ jobs: decode-ep: ${{ matrix.config.decode.ep }} decode-dp-attn: ${{ matrix.config.decode.dp-attn }} decode-additional-settings: ${{ toJson(matrix.config.decode.additional-settings) }} - conc: ${{ matrix.config.conc }} kv-offloading: ${{ matrix.config.kv-offloading }} kv-offload-backend: ${{ matrix.config.kv-offload-backend }} duration: ${{ matrix.config.duration }} diff --git a/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh b/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh new file mode 100755 index 0000000000..a45daba41a --- /dev/null +++ b/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash + +# Agentic trace-replay recipe for a disaggregated SGLang server on MI355X +# (DeepSeek-V4-Pro FP4, 1P1D TP8). +# +# CI-style sibling of dsr1_fp4_mi355x_sglang-disagg.sh: driven entirely by +# environment variables and submits a SLURM job via submit.sh. The agentic / +# HiCache-offload configuration mirrors the DSR1 recipe but uses DSV4-Pro +# specific flags (dsv4 attention backend, page-size 256, SWA settings). + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/../../benchmark_lib.sh" + +check_env_vars \ + CONC_LIST \ + ISL \ + OSL \ + IMAGE \ + SPEC_DECODING \ + MODEL_PATH \ + PREFILL_NUM_WORKERS \ + PREFILL_TP \ + PREFILL_EP \ + PREFILL_DP_ATTN \ + DECODE_NUM_WORKERS \ + DECODE_TP \ + DECODE_EP \ + DECODE_DP_ATTN \ + PREFILL_NODES \ + DECODE_NODES \ + RANDOM_RANGE_RATIO \ + DURATION \ + KV_OFFLOADING \ + IS_AGENTIC \ + FRAMEWORK + +if [[ -n "$SLURM_JOB_ID" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +fi + +set -x + +# Use upstreamed multi_node scripts (no external clone needed) +cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 + +# Set up SGL launch script-specific environment variables +export TIME_LIMIT="${TIME_LIMIT:-08:00:00}" +export MODEL_PATH=$MODEL_PATH +export MODEL_NAME=$MODEL_NAME +export CONTAINER_IMAGE=$IMAGE + +# ── Identity / result naming ── +export MODEL_PREFIX="${MODEL_PREFIX:-dsv4}" +export PRECISION="${PRECISION:-fp4}" +export RESULT_FILENAME="${RESULT_FILENAME:-${RUNNER_NAME:-dsv4-fp4-agentic}}" + +# ── Agentic benchmark params ── +export DURATION="${DURATION:-1800}" +# DSV4-Pro max model len for agentic traces (matches single-node recipe). +export MAX_MODEL_LEN="${MAX_MODEL_LEN:-1000000}" + +# ── In-tree sglang patches ── +# mori_conn.py targets hybrid-state bugs (GLM-5, Qwen3.5). DSV4-Pro uses a +# pure MoE/DSA architecture without hybrid state; skip to avoid interference. +export MORI_CONN_PATCH="${MORI_CONN_PATCH:-skip}" + +# ── Aiter fault mitigation ── +# --disable-custom-all-reduce avoids a known aiter fault on MI355X. +export DISABLE_CUSTOM_ALL_REDUCE="${DISABLE_CUSTOM_ALL_REDUCE:-0}" + +# ── KV cache offloading (HiCache) ── +# KV_OFFLOADING=none | dram (passed from YAML; default none for disagg). +# KV_OFFLOAD_BACKEND selects the backend when offloading is on; this recipe +# only implements HiCache, so "hicache" is the only supported value. +# HICACHE_TIER: L2 -> GPU + CPU-DRAM host pool. L3 -> + Mooncake store. +export KV_OFFLOADING="${KV_OFFLOADING:-none}" +if [[ "$KV_OFFLOADING" != "none" ]]; then + export KV_OFFLOAD_BACKEND="${KV_OFFLOAD_BACKEND:-hicache}" +fi +# HiCache/Mooncake tunables only matter when KV offloading is enabled. +if [[ "$KV_OFFLOADING" != "none" && "${KV_OFFLOAD_BACKEND:-}" == "hicache" ]]; then + export HICACHE_TIER="${HICACHE_TIER:-L2}" + export HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-1}" + # DSV4 uses page-size 256 (set in models.yaml); HiCache must match. + export HICACHE_PAGE_SIZE="${HICACHE_PAGE_SIZE:-256}" + # HiCache ratio (host pool = ratio * GPU KV pool). Default derived in server_sglang.sh. + export HICACHE_RATIO="${HICACHE_RATIO:-}" + + # ── HiCache layout/backend by tier ── + # L3 (Mooncake): page_first + direct + write_through + storage=mooncake + # L2 (CPU DRAM): layer_first + direct + write_through_selective + storage=none + # NOTE: write_through_selective evicts only under GPU memory pressure, avoiding + # the mori RDMA race that causes GPU memory access faults with write_through. + if [[ "${HICACHE_TIER^^}" == "L3" ]]; then + export HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first}" + export HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" + export HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" + export HICACHE_STORAGE_BACKEND="${HICACHE_STORAGE_BACKEND:-mooncake}" + else + export HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first}" + export HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" + export HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" + export HICACHE_STORAGE_BACKEND="${HICACHE_STORAGE_BACKEND:-}" + fi + export HICACHE_PREFETCH_POLICY="${HICACHE_PREFETCH_POLICY:-best_effort}" + # Shared nodes: use non-default Mooncake ports to avoid collisions. + export MC_MASTER_PORT="${MC_MASTER_PORT:-58137}" + export MC_METADATA_PORT="${MC_METADATA_PORT:-8080}" + export MC_METRICS_PORT="${MC_METRICS_PORT:-19003}" + export MC_MASTER_THREADS="${MC_MASTER_THREADS:-64}" + export MC_EVICTION_HIGH_WATERMARK="${MC_EVICTION_HIGH_WATERMARK:-0.95}" + export MC_PATCH_HOSTPOOL="${MC_PATCH_HOSTPOOL:-1}" + export MC_PROTOCOL="${MC_PROTOCOL:-tcp}" + export MC_GLOBAL_SEG="${MC_GLOBAL_SEG:-64gb}" + export MC_DEVICE="${MC_DEVICE:-}" + export MC_MASTER_ADDR="${MC_MASTER_ADDR:-}" + export MC_METADATA_SERVER="${MC_METADATA_SERVER:-}" +fi + +# ── MoRIIO RDMA Send Queue tuning ── +export MORI_IO_SQ_BACKOFF_TIMEOUT_US="${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-500000}" +export MORI_IO_QP_MAX_SEND_WR="${MORI_IO_QP_MAX_SEND_WR:-32768}" + +# ── SGLang PD router policy + server metrics ── +export PREFILL_ROUTER_POLICY="${PREFILL_ROUTER_POLICY:-cache_aware}" +export ENABLE_METRICS="${ENABLE_METRICS:-1}" + +# ── MTP ── +export DECODE_MTP_SIZE="${DECODE_MTP_SIZE:-0}" + +# Derive EP/DP enable flags from the topology inputs. +if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then +export PREFILL_ENABLE_EP=false +else +export PREFILL_ENABLE_EP=true +fi + +if [[ "$PREFILL_DP_ATTN" == "true" ]]; then +export PREFILL_ENABLE_DP=true +else +export PREFILL_ENABLE_DP=false +fi + +if [[ "${DECODE_EP:-1}" -eq 1 ]]; then +export DECODE_ENABLE_EP=false +else +export DECODE_ENABLE_EP=true +fi + +if [[ "$DECODE_DP_ATTN" == "true" ]]; then +export DECODE_ENABLE_DP=true +else +export DECODE_ENABLE_DP=false +fi + +# Launch the job. CONC_LIST is space-delimited in YAML; submit.sh wants 'x'. +JOB_ID=$(bash ./submit_agentic.sh $PREFILL_NODES \ + $PREFILL_NUM_WORKERS \ + $DECODE_NODES \ + $DECODE_NUM_WORKERS \ + $ISL $OSL "${CONC_LIST// /x}" inf \ + ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ + ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ + ${PREFILL_TP} ${DECODE_TP} \ + ${RANDOM_RANGE_RATIO}) + +if [[ $? -ne 0 ]]; then + echo "Failed to submit job" >&2 + exit 1 +fi + +echo "$JOB_ID" diff --git a/benchmarks/multi_node/amd_utils/env_agentic.sh b/benchmarks/multi_node/amd_utils/env_agentic.sh new file mode 100755 index 0000000000..2a3005c644 --- /dev/null +++ b/benchmarks/multi_node/amd_utils/env_agentic.sh @@ -0,0 +1,354 @@ +#!/bin/bash +# Dual-engine environment setup for multi-node disaggregated serving. +# +# ENGINE=sglang (default): SGLang/MoRI environment +# ENGINE=vllm: vLLM/Nixl environment +# +# REQUIRED ENVIRONMENT VARIABLES: +# IBDEVICES - RDMA/InfiniBand device names (e.g., ionic_0,ionic_1,... or mlx5_0,mlx5_1,...) +# Set by runner or auto-detected from hostname. +set -x + +ENGINE="${ENGINE:-sglang-disagg}" +export PYTHONDONTWRITEBYTECODE=1 + +# ============================================================================= +# HiCache / Mooncake settings from job.slurm +# ============================================================================= +# job.slurm writes the recipe-provided HiCache/Mooncake tunables to +# hicache_mc_.env and mounts it read-only at /config/hicache_mc.env. Source +# it here (auto-export) so values like HICACHE_PAGE_SIZE=256 reach the container +# before server_sglang.sh applies its "${VAR:-default}" fallbacks. Without this +# the vars arrive unset and server_sglang.sh defaults HICACHE_PAGE_SIZE to 1, +# overriding the recipe's --page-size. Empty values in the file are harmless: +# the "${VAR:-default}" fallbacks still treat "" as unset. +if [[ -f /config/hicache_mc.env ]]; then + set -a + source /config/hicache_mc.env + set +a + echo "[env.sh] sourced HiCache config from /config/hicache_mc.env (HICACHE_PAGE_SIZE=${HICACHE_PAGE_SIZE:-unset})" +fi + +# ============================================================================= +# Shared: IBDEVICES detection +# ============================================================================= + +# Prefer IBDEVICES set by runner (runners/launch_mi355x-amds.sh) +# Fall back to hostname detection if not set (for direct script execution) +if [[ -z "$IBDEVICES" ]]; then + DETECTED=$(ibv_devinfo 2>/dev/null | grep "hca_id:" | awk '{print $2}' | paste -sd',') + if [[ -n "$DETECTED" ]]; then + export IBDEVICES="$DETECTED" + echo "[INFO] Auto-detected IBDEVICES=$IBDEVICES via ibv_devinfo on $(hostname -s)" + else + echo "ERROR: Unable to detect RDMA devices. Set IBDEVICES explicitly." >&2 + exit 1 + fi +else + echo "[INFO] Using IBDEVICES=$IBDEVICES (set by runner or environment)" +fi +export IBDEVICES + +# Shared: Auto-detect default network interface (portable across clusters) +# Only auto-detect if not already set by the runner/environment +if [[ -z "$GLOO_SOCKET_IFNAME" ]]; then + export GLOO_SOCKET_IFNAME=$(ip route 2>/dev/null | grep '^default' | awk '{print $5}' | head -n 1) +fi +if [[ -z "$NCCL_SOCKET_IFNAME" ]]; then + export NCCL_SOCKET_IFNAME=$(ip route 2>/dev/null | grep '^default' | awk '{print $5}' | head -n 1) +fi + +set +x + +export NCCL_IB_HCA=${NCCL_IB_HCA:-$IBDEVICES} + +# ============================================================================= +# MoRI-specific environment +# ============================================================================= +# Shared by the vLLM MoRIIOConnector and the SGLang/MoRI KV-transfer path. + +export MORI_IO_SQ_BACKOFF_TIMEOUT_US="${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-50000}" +export MORI_IO_QP_MAX_SEND_WR="${MORI_IO_QP_MAX_SEND_WR:-16384}" +export MORI_IO_QP_MAX_CQE="${MORI_IO_QP_MAX_CQE:-32768}" +export MORI_IO_QP_MAX_SGE="${MORI_IO_QP_MAX_SGE:-2}" +export MORI_IO_TC_DISABLE="${MORI_IO_TC_DISABLE:-0}" + +# QoS/DSCP configuration +# Priority order: 1) Set by runner, 2) Detect via nicctl, 3) Detect from hostname +if [[ -n "$MORI_RDMA_TC" ]]; then + echo "[INFO] Using MORI_RDMA_TC=$MORI_RDMA_TC (set by runner or environment)" +elif command -v nicctl &> /dev/null; then + ND_PRIO=$(nicctl show qos 2>/dev/null | awk '/PFC no-drop priorities/ {print $NF; exit}') + ND_DSCP=$(nicctl show qos 2>/dev/null| awk -v p="$ND_PRIO" ' +$1 == "DSCP" && $2 == ":" && $NF == p { + print $3; exit +}') + # nicctl may emit trailing commas (e.g. "24,"); keep the leading integer so the + # arithmetic can't choke and unparseable output falls back to hostname detection. + ND_PRIO="${ND_PRIO%%,*}"; ND_PRIO="${ND_PRIO//[!0-9]/}" + ND_DSCP="${ND_DSCP%%,*}"; ND_DSCP="${ND_DSCP//[!0-9]/}" + + if [[ "$ND_DSCP" =~ ^[0-9]+$ ]] && [[ "$ND_PRIO" =~ ^[0-9]+$ ]]; then + TC=$(( 4 * ND_DSCP )) + export MORI_RDMA_SL=$ND_PRIO + export MORI_IO_SL=$ND_PRIO + export MORI_RDMA_TC=$TC + export MORI_IO_TC=$TC + echo "[INFO] Detected QoS config from nicctl: MORI_RDMA_TC=$MORI_RDMA_TC, MORI_RDMA_SL=$MORI_RDMA_SL, MORI_IO_TC=$MORI_IO_TC, MORI_IO_SL=$MORI_IO_SL" + else + echo "[WARN] nicctl available but QoS data unavailable; trying hostname detection." + # Fall back to hostname-based detection + NODENAME=$(hostname -s) + if [[ $NODENAME == GPU* ]] || [[ $NODENAME == smci355-ccs-aus* ]]; then + export MORI_RDMA_TC=96 + export MORI_IO_TC=96 + echo "[INFO] Auto-detected MORI_RDMA_TC=$MORI_RDMA_TC from hostname $NODENAME" + elif [[ $NODENAME == mia1* ]]; then + export MORI_RDMA_TC=104 + export MORI_IO_TC=104 + echo "[INFO] Auto-detected MORI_RDMA_TC=$MORI_RDMA_TC from hostname $NODENAME" + else + echo "[INFO] Unable to detect MORI_RDMA_TC from hostname. Skipping RDMA QoS configuration." + fi + fi +else + # nicctl not available, try hostname-based detection + NODENAME=$(hostname -s) + if [[ $NODENAME == GPU* ]] || [[ $NODENAME == smci355-ccs-aus* ]]; then + export MORI_RDMA_TC=96 + export MORI_IO_TC=96 + echo "[INFO] Auto-detected MORI_RDMA_TC=$MORI_RDMA_TC from hostname $NODENAME" + elif [[ $NODENAME == mia1* ]]; then + export MORI_RDMA_TC=104 + export MORI_IO_TC=104 + echo "[INFO] Auto-detected MORI_RDMA_TC=$MORI_RDMA_TC from hostname $NODENAME" + else + echo "[INFO] nicctl not found and unable to detect from hostname. Skipping RDMA QoS configuration." + echo " This is normal for clusters without QoS or outside Docker containers." + fi +fi + +# ============================================================================= +# Engine-specific environment +# ============================================================================= + +if [[ "$ENGINE" == "vllm-disagg" ]]; then + # ========================================================================= + # vLLM/Nixl-specific environment + # ========================================================================= + export VLLM_USE_V1=1 + export VLLM_SERVER_DEV_MODE=0 + export VLLM_DISABLE_REQUEST_ID_RANDOMIZATION=1 + + set -x + + # UCX_NET_DEVICES: Use the first tw-eth interface for UCX TCP transport + if [[ -z "$UCX_NET_DEVICES" ]]; then + UCX_NET_DEV=$(ip -o link show 2>/dev/null | awk -F': ' '/tw-eth/{print $2}' | head -1) + if [[ -n "$UCX_NET_DEV" ]]; then + export UCX_NET_DEVICES="$UCX_NET_DEV" + else + FIRST_IB=$(echo "$IBDEVICES" | cut -d',' -f1) + if [[ -n "$FIRST_IB" ]]; then + export UCX_NET_DEVICES="${FIRST_IB}:1" + fi + fi + echo "[INFO] Auto-set UCX_NET_DEVICES=$UCX_NET_DEVICES" + else + echo "[INFO] Using UCX_NET_DEVICES=$UCX_NET_DEVICES (set by environment)" + fi + + # RoCEv2: use IPv4-mapped GID (index 1) for inter-node RDMA routing + export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-1} + + # QoS/DSCP configuration for lossless RoCEv2 fabric. + if [[ -n "$UCX_IB_TRAFFIC_CLASS" ]]; then + echo "[INFO] Using UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS (set by environment)" + elif command -v nicctl &> /dev/null; then + ND_PRIO=$(nicctl show qos 2>/dev/null | awk '/PFC no-drop priorities/ {print $NF; exit}') + ND_DSCP=$(nicctl show qos 2>/dev/null | awk -v p="$ND_PRIO" ' +$1 == "DSCP" && $2 == ":" && $NF == p { + print $3; exit +}') + # nicctl may emit trailing commas (e.g. "24,"); keep the leading integer so the + # arithmetic can't choke and unparseable output falls back to hostname detection. + ND_PRIO="${ND_PRIO%%,*}"; ND_PRIO="${ND_PRIO//[!0-9]/}" + ND_DSCP="${ND_DSCP%%,*}"; ND_DSCP="${ND_DSCP//[!0-9]/}" + if [[ "$ND_DSCP" =~ ^[0-9]+$ ]] && [[ "$ND_PRIO" =~ ^[0-9]+$ ]]; then + export UCX_IB_TRAFFIC_CLASS=$(( 4 * ND_DSCP )) + export UCX_IB_SL=$ND_PRIO + echo "[INFO] Detected QoS from nicctl: UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS, UCX_IB_SL=$UCX_IB_SL" + else + echo "[WARN] nicctl available but QoS data unavailable; trying hostname detection." + NODENAME=$(hostname -s) + if [[ $NODENAME == GPU* ]] || [[ $NODENAME == smci355-ccs-aus* ]]; then + export UCX_IB_TRAFFIC_CLASS=96 + echo "[INFO] Auto-detected UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS from hostname $NODENAME" + elif [[ $NODENAME == mia1* ]]; then + export UCX_IB_TRAFFIC_CLASS=104 + echo "[INFO] Auto-detected UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS from hostname $NODENAME" + fi + fi + else + NODENAME=$(hostname -s) + if [[ $NODENAME == GPU* ]] || [[ $NODENAME == smci355-ccs-aus* ]]; then + export UCX_IB_TRAFFIC_CLASS=96 + echo "[INFO] Auto-detected UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS from hostname $NODENAME" + elif [[ $NODENAME == mia1* ]]; then + export UCX_IB_TRAFFIC_CLASS=104 + echo "[INFO] Auto-detected UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS from hostname $NODENAME" + else + echo "[INFO] No nicctl and unable to detect from hostname. Skipping QoS configuration." + fi + fi + + set +x + echo "[INFO] IBDEVICES=$IBDEVICES UCX_NET_DEVICES=$UCX_NET_DEVICES NCCL_SOCKET_IFNAME=$NCCL_SOCKET_IFNAME UCX_IB_GID_INDEX=$UCX_IB_GID_INDEX UCX_IB_TRAFFIC_CLASS=${UCX_IB_TRAFFIC_CLASS:-unset}" + +else + # ========================================================================= + # SGLang-specific environment + # ========================================================================= + + export SGLANG_USE_AITER=1 + export AITER_LOG_LEVEL=ERROR + + export SGLANG_MORI_DISPATCH_DTYPE=auto + # export MORI_COMBINE_DTYPE_PREFILL=fp8_direct_cast + # export MORI_COMBINE_DTYPE_DECODE=fp8 + export MORI_COMBINE_DTYPE_PREFILL="" + export MORI_COMBINE_DTYPE_DECODE="" + export SGLANG_MORI_QP_PER_TRANSFER=4 + export SGLANG_MORI_NUM_WORKERS=4 + # Keep these as overridable defaults (not hard assignments), otherwise + # later tuning blocks cannot raise them for high-concurrency runs. + # export MORI_IO_SQ_BACKOFF_TIMEOUT_US="${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-500000}" + + # export MORI_IO_QP_MAX_SEND_WR="${MORI_IO_QP_MAX_SEND_WR:-16384}" + # export MORI_IO_QP_MAX_CQE=32768 + # export MORI_IO_QP_MAX_SGE=1 + + # export MORI_IO_TC_DISABLE=0 + + export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600 + export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600 + + export SGLANG_HEALTH_CHECK_TIMEOUT=600 + + # GLM-5: uses NSA (not MLA), needs fused-decode-MLA disabled + fast loading + if [[ "$MODEL_NAME" == "GLM-5-FP8" ]]; then + export SGLANG_ROCM_FUSED_DECODE_MLA=0 + export ROCM_QUICK_REDUCE_QUANTIZATION=INT4 + export SAFETENSORS_FAST_GPU=1 + fi + + # Disable allocating memory in one pass + export MORI_SHMEM_MODE=ISOLATION + + # Enable spec v2 + export SGLANG_ENABLE_SPEC_V2=1 + export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 + + export SGLANG_LOG_MS=true + export SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS=32 + + export MORI_MAX_DISPATCH_TOKENS_PREFILL=8192 + export MORI_MAX_DISPATCH_TOKENS_DECODE=512 + + export MORI_MOE_MAX_INPUT_TOKENS_PREFILL=32768 + export MORI_MOE_MAX_INPUT_TOKENS_DECODE=2703 + + # set MTP size=1 when EP16 + export SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD=$((MORI_MAX_DISPATCH_TOKENS_DECODE * 2)) + + export MORI_EP_LAUNCH_CONFIG_MODE=AUTO + + # Default to WARNING to cut per-op MoRI log spam on long multinode/eval + # runs; override with MORI_APP_LOG_LEVEL=INFO when debugging. + export MORI_APP_LOG_LEVEL="${MORI_APP_LOG_LEVEL:-WARNING}" + + # Router logging control: + # 0 (default) keeps noisy per-request access logs out of stdout while still logging to file. + # 1 mirrors router logs to stdout via tee (useful for live debugging). + export SGLANG_ROUTER_STDOUT_LOGS="${SGLANG_ROUTER_STDOUT_LOGS:-0}" + + # FIXME: WA for latest upstream 0305 image + export PYTHONPATH=/sgl-workspace/aiter:${PYTHONPATH} + + # ========================================================================= + # DeepSeek-V4-Pro PD recipe overrides + # Placed at the end of the SGLang env block so it wins over the global + # MoRI/SGLang defaults set above. Mirrors the validated DSv4 manual PD + # commands (ported from InferenceX amd/dsv4_sgl_di). These SGLANG_OPT_* / + # AITER_* kernel-routing knobs steer DSv4 away from the default aiter CK + # fused-MoE path, which raises "Unsupported kernel config for moe heuristic + # dispatch" at decode time on this fp4 model (job 19034 crash). Only the + # SGLang/MoRI env knobs are pinned here; CLI flags live in models.yaml and + # the cluster NIC/socket vars stay runner-derived. + # ========================================================================= + if [[ "$MODEL_NAME" == "DeepSeek-V4-Pro" ]]; then + # MoRI RDMA send-queue depth for DSv4 (overrides the global default above). + export MORI_IO_QP_MAX_SEND_WR=32767 + # Unified radix tree: cache impl with per-component (full-attn / SWA) + # management for hybrid-attention models. Set unconditionally (not gated on + # hicache) so all SGLang runs use it. + export SGLANG_ENABLE_UNIFIED_RADIX_TREE=1 + # Proactively free out-of-window SWA KV slots during chunked prefill. + # Without it, in-flight requests pin SWA KV for their whole context, keeping + # the SWA pool under constant eviction pressure; under LRU the trailing + # window of cached sessions gets flushed, making prefix-cache hits bimodal + # and collapsing the effective hit rate on multi-turn agentic workloads. + export SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS=1 + + # MoRI dispatch/combine dtypes: auto for both roles (not the fp8 split default) + export SGLANG_MORI_DISPATCH_DTYPE=auto + export MORI_COMBINE_DTYPE_PREFILL=auto + export MORI_COMBINE_DTYPE_DECODE=auto + + # Per-role MoRI dispatch sizing (used by the harness chunked/MoE math) + export MORI_MAX_DISPATCH_TOKENS_PREFILL=8192 + export MORI_MAX_DISPATCH_TOKENS_DECODE=64 + unset MORI_MOE_MAX_INPUT_TOKENS_PREFILL + unset MORI_MOE_MAX_INPUT_TOKENS_DECODE + + # PER_RANK dispatch tokens pinned independently (16384 prefill / 128 + # decode); server_sglang.sh prefers these over the MORI_MAX_DISPATCH_* + # coupling when set. + export MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_PREFILL=16384 + export MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_DECODE=128 + + # Fixed inter-kernel switch threshold (not derived). + export SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD=4096 + + # Overlap plan stream on for DSv4 (global default is 0) + # export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 + + # DSv4 model kernel routing (mirrors the single-node / manual PD recipe) + export SGLANG_DEFAULT_THINKING=1 + export SGLANG_DSV4_REASONING_EFFORT=max + export SGLANG_OPT_DEEPGEMM_HC_PRENORM=false + export SGLANG_USE_AITER=1 + export SGLANG_USE_ROCM700A=0 + export SGLANG_OPT_USE_FUSED_COMPRESS=true + export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton + export SGLANG_OPT_FP8_WO_A_GEMM=false + export SGLANG_OPT_USE_JIT_INDEXER_METADATA=false + export SGLANG_OPT_USE_TOPK_V2=false + export SGLANG_OPT_USE_AITER_INDEXER=${SGLANG_OPT_USE_AITER_INDEXER:-true} + export SGLANG_OPT_USE_TILELANG_INDEXER=false + export SGLANG_OPT_USE_TILELANG_MHC_PRE=false + export SGLANG_OPT_USE_TILELANG_MHC_POST=false + export SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1 + export SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true + export SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false + export SGLANG_ROCM_USE_MULTI_STREAM=false + export AITER_BF16_FP8_MOE_BOUND=0 + export SGLANG_EAGER_INPUT_NO_COPY=true + export SGLANG_SHARED_EXPERT_TP1=1 + export SGLANG_DP_SHARED_EXPERT_LOCAL=1 + export SGLANG_DP_USE_GATHERV=1 + export SGLANG_DP_USE_REDUCE_SCATTER=1 + export GPU_MAX_HW_QUEUES=5 + fi + +fi \ No newline at end of file diff --git a/benchmarks/multi_node/amd_utils/job_agentic.slurm b/benchmarks/multi_node/amd_utils/job_agentic.slurm new file mode 100755 index 0000000000..f859b86365 --- /dev/null +++ b/benchmarks/multi_node/amd_utils/job_agentic.slurm @@ -0,0 +1,708 @@ +#!/bin/bash +#SBATCH --job-name=disagg-bench +#SBATCH -N 3 # Overridden by submit.sh -N flag +#SBATCH -n 3 # Overridden by submit.sh -n flag +#SBATCH --ntasks-per-node=1 +#SBATCH --spread-job +#SBATCH --gres=gpu:8 +#SBATCH --time=24:00:00 +# --output and --error are set by submit.sh via BENCHMARK_LOGS_DIR + +ENGINE="${ENGINE:-sglang-disagg}" + +echo "=== Job Start Time ===" +echo "UTC Time: $(TZ=UTC date '+%Y-%m-%d %H:%M:%S %Z')" +echo "PST Time: $(TZ=America/Los_Angeles date '+%Y-%m-%d %H:%M:%S %Z')" +echo "ENGINE: $ENGINE" +echo "=======================" +echo "" + +# ============================================================================= +# Model Validation +# ============================================================================= + +# Use $(pwd) not BASH_SOURCE — sbatch copies the script to /var/spool/slurmd/ +# at runtime, but the CWD remains the submit-time directory (amd_utils/). +if [[ "$ENGINE" == "vllm-disagg" ]]; then + MODELS_YAML="$(pwd)/models_vllm.yaml" +elif [[ "$ENGINE" == "atom-disagg" ]]; then + MODELS_YAML="$(pwd)/models_atom.yaml" +else + MODELS_YAML="$(pwd)/models_agentic.yaml" +fi + +if [[ ! -f "$MODELS_YAML" ]]; then + echo "Error: models YAML not found at $MODELS_YAML" + exit 1 +fi + +if [[ -z "${DOCKER_IMAGE_NAME:-}" ]]; then + echo "Error: DOCKER_IMAGE_NAME is not set." + exit 1 +fi + +MODEL_NAME="${MODEL_NAME:-None}" +if ! grep -q "^${MODEL_NAME}:" "$MODELS_YAML"; then + echo "Error: Model '$MODEL_NAME' not found in $MODELS_YAML" + echo "Available models:" + grep -E '^[A-Za-z]' "$MODELS_YAML" | sed 's/:.*$//' | sed 's/^/ - /' + exit 1 +fi +echo "Model found: $MODEL_NAME" + +RUN_FILE="server_agentic.sh" +echo "Runfile set: $RUN_FILE" + +# DI_REPO_DIR points to the repo root. +# $(pwd) is amd_utils/ (the sbatch submit dir); go up 3 levels to reach the repo root. +export DI_REPO_DIR=$(cd "$(pwd)/../../.." && pwd) + +# ── In-tree sglang patches: auto-apply for known-affected images ────── +# sglang v0.5.12.post1 ships a known-broken MoRI PD-disaggregation +# backend that crashes hybrid-attention models (GLM-5, Qwen3.5-MoE, +# anything with state_types: List[StateType]) at startup. We carry an +# in-tree overlay of mori/conn.py that fixes the wire format + the +# legacy state_type fallback (see patches/README.md for the bug +# analysis and patch detail). +# +# Auto-applied when the image tag contains "v0.5.12.post1", unless the +# caller sets MORI_CONN_PATCH=skip. The overlay is appended to +# ${EXTRA_DOCKER_MOUNTS:-} so callers can still inject other mounts. +# Dedup guard avoids double-mounting if EXTRA_DOCKER_MOUNTS already +# contains the target path (docker rejects duplicate destinations). +_MORI_PATCH_FILE="$DI_REPO_DIR/benchmarks/multi_node/amd_utils/patches/mori_conn.py" +_MORI_PATCH_TARGET="/sgl-workspace/sglang/python/sglang/srt/disaggregation/mori/conn.py" +if [[ "${MORI_CONN_PATCH:-auto}" != "skip" ]] \ + && [[ -f "$_MORI_PATCH_FILE" ]] \ + && [[ "${DOCKER_IMAGE_NAME:-}" == *"v0.5.12.post1"* ]] \ + && [[ "${EXTRA_DOCKER_MOUNTS:-}" != *"$_MORI_PATCH_TARGET"* ]]; then + EXTRA_DOCKER_MOUNTS="${EXTRA_DOCKER_MOUNTS:-} -v ${_MORI_PATCH_FILE}:${_MORI_PATCH_TARGET}:ro" + export EXTRA_DOCKER_MOUNTS + echo "[job.slurm] auto-applied MoRI conn.py overlay: ${_MORI_PATCH_FILE}" +fi +xP="${xP:-1}" +yD="${yD:-1}" + +# Benchmark configuration +BENCH_INPUT_LEN="${BENCH_INPUT_LEN:-1024}" +BENCH_OUTPUT_LEN="${BENCH_OUTPUT_LEN:-1024}" +BENCH_RANDOM_RANGE_RATIO="${BENCH_RANDOM_RANGE_RATIO:-1}" +BENCH_NUM_PROMPTS_MULTIPLIER="${BENCH_NUM_PROMPTS_MULTIPLIER:-10}" +BENCH_MAX_CONCURRENCY="${BENCH_MAX_CONCURRENCY:-512}" +BENCH_REQUEST_RATE="${BENCH_REQUEST_RATE:-inf}" + +GPUS_PER_NODE="${GPUS_PER_NODE:-8}" + +# Engine-specific defaults +PREFILL_ENABLE_EP="${PREFILL_ENABLE_EP:-false}" +PREFILL_ENABLE_DP="${PREFILL_ENABLE_DP:-false}" +DECODE_ENABLE_EP="${DECODE_ENABLE_EP:-false}" +DECODE_ENABLE_DP="${DECODE_ENABLE_DP:-false}" +PREFILL_TP_SIZE="${PREFILL_TP_SIZE:-8}" +DECODE_TP_SIZE="${DECODE_TP_SIZE:-8}" +DECODE_MTP_SIZE=${DECODE_MTP_SIZE:-0} + +# Router selection: "vllm-router" (external container) or "moriio" (in-container proxy) +ROUTER_TYPE="${ROUTER_TYPE:-vllm-router}" +ROUTER_PORT="${ROUTER_PORT:-30000}" +PROXY_PING_PORT="${PROXY_PING_PORT:-36367}" + +# ============================================================================= +# Model Path Resolution +# ============================================================================= + +# MODEL_DIR detection: prefer env var, fall back to hostname detection +if [[ -z "$MODEL_DIR" ]]; then + NODENAME=$(hostname -s) + if [[ $NODENAME == GPU* ]] || [[ $NODENAME == smci355-ccs-aus* ]]; then + MODEL_DIR="/nfsdata" + elif [[ $NODENAME == mia1* ]]; then + MODEL_DIR="/it-share/data" + else + MODEL_DIR="/nfsdata" + fi + echo "[INFO] Auto-detected MODEL_DIR=$MODEL_DIR from hostname $(hostname -s)" +fi +export MODEL_DIR + +if [[ "$ENGINE" == "vllm-disagg" ]]; then + # vLLM: Extract hf_dir from models.yaml, search multiple paths, resolve HF cache snapshots + DISK_DIR_NAME=$(awk '/^'"$MODEL_NAME"':/{found=1; next} + found && /^[^ ]/{exit} + found && /hf_dir:/{gsub(/[" ]/, "", $2); print $2; exit}' "$MODELS_YAML") + DISK_DIR_NAME="${DISK_DIR_NAME:-$MODEL_NAME}" + echo "Looking for model: $MODEL_NAME (disk dir: $DISK_DIR_NAME)" + + resolve_hf_cache_path() { + local base_path=$1 + if [[ -d "${base_path}/snapshots" ]]; then + local snapshot=$(ls -1 "${base_path}/snapshots" 2>/dev/null | head -1) + if [[ -n "$snapshot" ]]; then + echo "${base_path}/snapshots/${snapshot}" + return 0 + fi + fi + echo "$base_path" + return 1 + } + + MODEL_PATH="" + SEARCH_PATHS=( + "${MODEL_DIR}/${DISK_DIR_NAME}" + "${MODEL_DIR}/${MODEL_NAME}" + "/nfsdata/hf_hub_cache-0/${DISK_DIR_NAME}" + "/nfsdata/hf_hub_cache-0/${MODEL_NAME}" + ) + + for search_path in "${SEARCH_PATHS[@]}"; do + if [[ -d "$search_path" ]]; then + RESOLVED=$(resolve_hf_cache_path "$search_path") + MODEL_PATH="$RESOLVED" + echo "Found MODEL_PATH: $MODEL_PATH" + break + fi + done + + if [[ -z "$MODEL_PATH" ]]; then + echo "FATAL: Model '$MODEL_NAME' not found. Searched:" + for p in "${SEARCH_PATHS[@]}"; do echo " - $p"; done + exit 1 + fi + echo "Final MODEL_PATH: $MODEL_PATH" +else + # SGLang: Validate model path across all allocated nodes + echo "Looking for model: $MODEL_NAME" + echo "Checking model availability across all allocated nodes..." + + ALL_NODES=$(scontrol show hostnames "$SLURM_JOB_NODELIST") + TOTAL_NODES=$(echo "$ALL_NODES" | wc -l) + echo "Total allocated nodes: $TOTAL_NODES" + echo "Nodes: $(echo "$ALL_NODES" | tr '\n' ' ')" + + check_model_path() { + local path=$1 + local check_name=$2 + echo "Checking $check_name: $path" + srun --nodes=$SLURM_NNODES --ntasks=$SLURM_NNODES /bin/bash -c " + if [ -d '$path' ]; then + echo \"\$(hostname): Found $path\" + exit 0 + else + echo \"\$(hostname): Missing $path\" + exit 1 + fi + " + local exit_code=$? + if [ $exit_code -eq 0 ]; then + echo "$check_name available on ALL nodes" + return 0 + else + echo "$check_name NOT available on all nodes" + return 1 + fi + } + + # Extract hf_dir from models.yaml (same as vllm-disagg path above) + SGL_DISK_DIR_NAME=$(awk '/^'"$MODEL_NAME"':/{found=1; next} + found && /^[^ ]/{exit} + found && /hf_dir:/{gsub(/[" ]/, "", $2); print $2; exit}' "$MODELS_YAML") + SGL_DISK_DIR_NAME="${SGL_DISK_DIR_NAME:-$MODEL_NAME}" + + # Prefer the caller-supplied MODEL_PATH (recipe scripts set this explicitly); + # fall back to MODEL_DIR/hf_dir then MODEL_DIR/MODEL_NAME. + if [[ -n "${MODEL_PATH:-}" && "$MODEL_PATH" != "$MODEL_DIR" ]]; then + # Caller already resolved the path (e.g. MODEL_PATH=/it-share/hf_cache/models--...) + # Use it directly if it exists on all nodes, otherwise try subdirectory combos. + if check_model_path "$MODEL_PATH" "MODEL_PATH (caller-supplied)"; then + echo "Selected MODEL_PATH: $MODEL_PATH (caller-supplied, available on all nodes)" + elif check_model_path "$MODEL_PATH/$SGL_DISK_DIR_NAME" "$MODEL_PATH/$SGL_DISK_DIR_NAME"; then + MODEL_PATH="$MODEL_PATH/$SGL_DISK_DIR_NAME" + echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" + elif check_model_path "$MODEL_PATH/$MODEL_NAME" "$MODEL_PATH/$MODEL_NAME"; then + MODEL_PATH="$MODEL_PATH/$MODEL_NAME" + echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" + else + echo "FATAL ERROR: Model '$MODEL_NAME' not found on ALL allocated nodes in:" + echo " - $MODEL_PATH" + echo " - $MODEL_PATH/$SGL_DISK_DIR_NAME" + echo " - $MODEL_PATH/$MODEL_NAME" + exit 1 + fi + elif check_model_path "$MODEL_DIR/$SGL_DISK_DIR_NAME" "$MODEL_DIR/$SGL_DISK_DIR_NAME"; then + MODEL_PATH="$MODEL_DIR/$SGL_DISK_DIR_NAME" + echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" + elif check_model_path "$MODEL_DIR/$MODEL_NAME" "$MODEL_DIR"; then + MODEL_PATH="$MODEL_DIR/$MODEL_NAME" + echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" + else + echo "FATAL ERROR: Model '$MODEL_NAME' not found on ALL allocated nodes in:" + echo " - $MODEL_DIR/$SGL_DISK_DIR_NAME" + echo " - $MODEL_DIR/$MODEL_NAME" + exit 1 + fi + echo "Final MODEL_PATH: $MODEL_PATH" +fi + +# ============================================================================= +# Node Selection +# ============================================================================= + +NUM_NODES=$((xP + yD)) +echo "NUM_NODES: $NUM_NODES (xP=$xP + yD=$yD)" + +FULL_NODELIST=$(scontrol show hostnames "$SLURM_JOB_NODELIST") +SELECTED_NODES=$(echo "$FULL_NODELIST" | head -n $NUM_NODES) +SELECTED_NODELIST_STR=$(echo "$SELECTED_NODES" | tr '\n' ',' | sed 's/,$//') + +# Docker privilege detection — evaluated per-node since group membership varies. +# Exported as a snippet so every srun participant resolves it locally. +export DOCKER_CMD_DETECT='if docker ps &>/dev/null 2>&1; then DOCKER_CMD=docker; else DOCKER_CMD="sudo docker"; fi' + +# Update SLURM environment variables +export SLURM_NNODES=$NUM_NODES +export SLURM_NTASKS=$NUM_NODES +export SLURM_JOB_NUM_NODES=$NUM_NODES +export SLURM_NPROCS=$NUM_NODES +export SLURM_JOB_NODELIST="$SELECTED_NODELIST_STR" +export SLURM_NODELIST="$SELECTED_NODELIST_STR" +export SLURM_TASKS_PER_NODE="1(x$NUM_NODES)" +export SLURM_NTASKS_PER_NODE=1 + +echo "" +echo "Selected nodes: $SELECTED_NODELIST_STR" + +# ============================================================================= +# IP Resolution +# ============================================================================= + +USER_NAME=$(whoami) +MASTER_NODE=$(echo "$SELECTED_NODES" | head -n 1) +NODE0_ADDR=$(srun --nodes=1 --ntasks=1 --time=00:20:00 --nodelist="$MASTER_NODE" bash -c 'ip route get 1.1.1.1') +NODE0_ADDR=$(echo "$NODE0_ADDR" | awk '/src/ {print $7}') + +IPS=() +for NODE in $SELECTED_NODES; do + IP=$(srun --nodes=1 --ntasks=1 --time=00:20:00 --nodelist="$NODE" bash -c 'ip route get 1.1.1.1') + IP=$(echo "$IP" | awk '/src/ {print $7}') + IPS+=("$IP") +done + +echo "Node IPs: ${IPS[*]}" + +DOCKER_MOUNT_PATH="/workspace" +WS_PATH="${DOCKER_MOUNT_PATH}/benchmarks/multi_node/amd_utils" + +NNODES=$NUM_NODES + +echo "MASTER_NODE: ${MASTER_NODE}" +echo "NODE0_ADDR: ${NODE0_ADDR}" +echo "NNODES: ${NNODES}" +echo "REPO DIR: ${DI_REPO_DIR}" +echo "USER: ${USER_NAME}" + +# Reduce log spam +export TQDM_MININTERVAL=20 + +# Translate the host-resolved MODEL_PATH to the Docker mount namespace +DOCKER_MODEL_PATH="${MODEL_PATH/#$MODEL_DIR//models}" + +export DI_REPO_DIR=$DI_REPO_DIR +export WS_PATH=$WS_PATH +export NNODES=$NNODES +export NODE0_ADDR=$NODE0_ADDR +export MODEL_PATH=$MODEL_PATH +export MODEL_DIR=$MODEL_DIR +export xP=$xP +export yD=$yD +export MODEL_NAME=$MODEL_NAME +export USER_NAME=$USER_NAME +export IPADDRS="$(echo "${IPS[*]}" | sed 's/ /,/g')" +export GPUS_PER_NODE=$GPUS_PER_NODE +export BENCH_INPUT_LEN=$BENCH_INPUT_LEN +export BENCH_OUTPUT_LEN=$BENCH_OUTPUT_LEN +export BENCH_RANDOM_RANGE_RATIO=$BENCH_RANDOM_RANGE_RATIO +export BENCH_NUM_PROMPTS_MULTIPLIER=$BENCH_NUM_PROMPTS_MULTIPLIER +export BENCH_MAX_CONCURRENCY=$BENCH_MAX_CONCURRENCY +export BENCH_REQUEST_RATE=$BENCH_REQUEST_RATE +export DRY_RUN="${DRY_RUN:-0}" +export BENCHMARK_LOGS_DIR="${BENCHMARK_LOGS_DIR:-$(pwd)/benchmark_logs}" +export KEEP_CONTAINERS="${KEEP_CONTAINERS:-0}" +export ENGINE=$ENGINE + +# Eval-related env vars (threaded from submit.sh) +export RUN_EVAL="${RUN_EVAL:-false}" +export EVAL_ONLY="${EVAL_ONLY:-false}" +export EVAL_CONC="${EVAL_CONC:-}" +export FRAMEWORK="${FRAMEWORK:-}" +export PRECISION="${PRECISION:-}" +export MODEL_PREFIX="${MODEL_PREFIX:-}" +export RUNNER_TYPE="${RUNNER_TYPE:-}" +export RESULT_FILENAME="${RESULT_FILENAME:-}" +export SPEC_DECODING="${SPEC_DECODING:-}" +export IS_MULTINODE="${IS_MULTINODE:-false}" + +SANITIZED_USER=$(echo "$USER_NAME" | tr -c 'a-zA-Z0-9_.-' '_') +export DOCKER_CONT_NAME="container_${ENGINE}_${SANITIZED_USER}_${MODEL_NAME}_${SLURM_JOB_ID}" + +# vLLM external router container. +# NOTE: vllm/vllm-router only retains ~16 recent nightlies on Docker Hub; older +# dated tags are garbage-collected (manifest unknown) +VLLM_ROUTER_IMAGE="${VLLM_ROUTER_IMAGE:-vllm/vllm-router:nightly-20260629-e667ebb}" +ROUTER_CONT_NAME="router_vllm_${SANITIZED_USER}_${SLURM_JOB_ID}" +export RUN_FILE_FULL="$WS_PATH/${RUN_FILE}" + +SELECTED_NODELIST_SRUN=$(echo "$SELECTED_NODES" | paste -sd,) + +cleanup() { + echo "[${SLURM_JOB_ID}] termination received on $(hostname); cleaning up..." + rm -rf ${SLURM_SUBMIT_DIR}/logs 2>/dev/null || true + echo "[${SLURM_JOB_ID}] cleanup done." +} + +trap cleanup INT TERM HUP + +# Force NFS cache refresh on all nodes +echo "Refreshing NFS caches on all nodes..." +srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c ' + sync + ls -la '"$DI_REPO_DIR"'/benchmarks/multi_node/amd_utils > /dev/null 2>&1 + stat '"$DI_REPO_DIR"'/benchmarks/multi_node/amd_utils/server_agentic.sh > /dev/null 2>&1 + cat '"$DI_REPO_DIR"'/benchmarks/multi_node/amd_utils/server_agentic.sh > /dev/null 2>&1 + echo 3 | sudo tee /proc/sys/vm/drop_caches > /dev/null 2>&1 || true + echo "NFS cache refreshed on $(hostname)" +' + +# ============================================================================= +# Build engine-specific Docker environment variables +# ============================================================================= + +# Common env vars (always passed) +DOCKER_ENV_COMMON=( + -e SLURM_JOB_ID=\$SLURM_JOB_ID + -e SLURM_JOB_NODELIST=\$SLURM_JOB_NODELIST + -e NNODES=\$NNODES + -e NODE_RANK=\$SLURM_PROCID + -e NODE0_ADDR=\$NODE0_ADDR + -e MODEL_DIR=/models + -e MODEL_NAME=\$MODEL_NAME + -e GPUS_PER_NODE=\$GPUS_PER_NODE + -e xP=\$xP + -e yD=\$yD + -e IPADDRS=\$IPADDRS + -e BENCH_INPUT_LEN=\$BENCH_INPUT_LEN + -e BENCH_OUTPUT_LEN=\$BENCH_OUTPUT_LEN + -e BENCH_RANDOM_RANGE_RATIO=\$BENCH_RANDOM_RANGE_RATIO + -e BENCH_NUM_PROMPTS_MULTIPLIER=\$BENCH_NUM_PROMPTS_MULTIPLIER + -e BENCH_MAX_CONCURRENCY=\$BENCH_MAX_CONCURRENCY + -e BENCH_REQUEST_RATE=\$BENCH_REQUEST_RATE + -e TQDM_MININTERVAL=\$TQDM_MININTERVAL + -e BENCHMARK_LOGS_DIR=/benchmark_logs + -e ENGINE=\$ENGINE + -e WS_PATH=${WS_PATH} + -e RUN_EVAL=\$RUN_EVAL + -e EVAL_ONLY=\$EVAL_ONLY + -e \"EVAL_CONC=\$EVAL_CONC\" + -e FRAMEWORK=\$FRAMEWORK + -e PRECISION=\$PRECISION + -e MODEL_PREFIX=\$MODEL_PREFIX + -e RUNNER_TYPE=\$RUNNER_TYPE + -e RESULT_FILENAME=\$RESULT_FILENAME + -e SPEC_DECODING=\$SPEC_DECODING + # DISAGG was never forwarded into the container at all (not even under a + # different name), so process_agentic_result.py's env_bool("DISAGG") always + # defaulted to false in the result JSON regardless of the actual topology. + -e DISAGG=\${DISAGG:-false} + -e PREFILL_TP_SIZE=\$PREFILL_TP_SIZE + # PREFILL_TP/DECODE_TP/*_NUM_WORKERS (below, undecorated -- distinct from the + # *_SIZE vars server_sglang.sh uses for launch args) are what + # process_agentic_result.py's _gpu_shape() reads for multinode runs. Without + # these the container never sees them (only *_SIZE was passed), so agentic + # result JSONs silently recorded tp=0 / prefill_tp=0 / prefill_num_workers=0 + # for every multinode run. + -e PREFILL_TP=\$PREFILL_TP + -e PREFILL_NUM_WORKERS=\$PREFILL_NUM_WORKERS + -e PREFILL_ENABLE_EP=\$PREFILL_ENABLE_EP + -e PREFILL_ENABLE_DP=\$PREFILL_ENABLE_DP + -e PREFILL_CONTEXT_LENGTH=\${PREFILL_CONTEXT_LENGTH:-} + -e PREFILL_CHUNKED_PREFILL_SIZE=\${PREFILL_CHUNKED_PREFILL_SIZE:-} + -e SGLANG_AITER_MLA_PERSIST=\${SGLANG_AITER_MLA_PERSIST:-0} + -e DISABLE_CUSTOM_ALL_REDUCE=\${DISABLE_CUSTOM_ALL_REDUCE:-0} + -e MAX_MODEL_LEN=\${MAX_MODEL_LEN:-} + -e DURATION=\${DURATION:-1800} + -e IS_AGENTIC=\${IS_AGENTIC:-0} + -e KV_OFFLOADING=\${KV_OFFLOADING:-none} + -e KV_OFFLOAD_BACKEND=\${KV_OFFLOAD_BACKEND:-} + -e TOTAL_CPU_DRAM_GB=\${TOTAL_CPU_DRAM_GB:-} + -e ENABLE_METRICS=\${ENABLE_METRICS:-0} + -e PREFILL_ROUTER_POLICY=\${PREFILL_ROUTER_POLICY:-random} + -e DECODE_ROUTER_POLICY=\${DECODE_ROUTER_POLICY:-random} + -e MORI_IO_SQ_BACKOFF_TIMEOUT_US=\${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-} + -e MORI_IO_QP_MAX_SEND_WR=\${MORI_IO_QP_MAX_SEND_WR:-} + -e DECODE_TP_SIZE=\$DECODE_TP_SIZE + -e DECODE_TP=\$DECODE_TP + -e DECODE_NUM_WORKERS=\$DECODE_NUM_WORKERS + -e DECODE_ENABLE_EP=\$DECODE_ENABLE_EP + -e DECODE_ENABLE_DP=\$DECODE_ENABLE_DP + -e DECODE_MTP_SIZE=\$DECODE_MTP_SIZE + -e IS_MULTINODE=\$IS_MULTINODE + -e DRY_RUN=\${DRY_RUN:-0} +) + +# Engine-specific env vars +if [[ "$ENGINE" == "vllm-disagg" ]]; then + DOCKER_ENV_ENGINE=( + -e VLLM_WS_PATH=${WS_PATH} + -e MODEL_PATH=$DOCKER_MODEL_PATH + -e UCX_TLS=tcp,self,shm,rocm_ipc,rocm_copy,cma + -e UCX_SOCKADDR_TLS_PRIORITY=tcp + -e UCX_MEMTYPE_CACHE=y + -e UCX_RNDV_SCHEME=get_zcopy + -e UCX_RNDV_THRESH=4k + -e UCX_ROCM_IPC_MIN_ZCOPY=0 + -e UCX_LOG_LEVEL=warn + -e HSA_ENABLE_SDMA=1 + -e PROXY_STREAM_IDLE_TIMEOUT=\${PROXY_STREAM_IDLE_TIMEOUT:-300} + -e PYTHONPYCACHEPREFIX=/tmp/pycache + ) +elif [[ "$ENGINE" == "atom-disagg" ]]; then + DOCKER_ENV_ENGINE=( + -e ATOM_WS_PATH=${WS_PATH} + -e PREFILL_PORT=${PREFILL_PORT:-8010} + -e DECODE_PORT=${DECODE_PORT:-8020} + -e ROUTER_PORT=${ROUTER_PORT:-30000} + -e HANDSHAKE_PORT=${HANDSHAKE_PORT:-6301} + -e MEM_FRAC_STATIC=${MEM_FRAC_STATIC:-0.85} + -e KV_CACHE_DTYPE=${KV_CACHE_DTYPE:-fp8} + -e BLOCK_SIZE=${BLOCK_SIZE:-16} + -e MAX_NUM_SEQS=${MAX_NUM_SEQS:-256} + -e MAX_MODEL_LEN=${MAX_MODEL_LEN:-} + -e MAX_NUM_BATCHED_TOKENS=${MAX_NUM_BATCHED_TOKENS:-} + -e EXTRA_SERVER_ARGS=\${EXTRA_SERVER_ARGS:-} + -e IBDEVICES=${IBDEVICES:-} + ) +else + DOCKER_ENV_ENGINE=( + -e SGLANG_WS_PATH=${WS_PATH} + ) +fi + +# HiCache / Mooncake settings are delivered via a bind-mounted config file rather +# than a long list of docker -e flags. Write it once to the shared benchmark-logs +# dir (already a host path, visible on every node) and mount it read-only at +# /config/hicache_mc.env, where env.sh sources it before applying its defaults. +# Empty values are preserved so env.sh's "${VAR:-default}" fallbacks still apply. +HICACHE_MC_CONFIG="${BENCHMARK_LOGS_DIR}/hicache_mc_${SLURM_JOB_ID}.env" +cat > "$HICACHE_MC_CONFIG" < $HICACHE_MC_CONFIG" + +# Engine-specific container filter for pre-clean +CONT_FILTER="name=^container_${ENGINE}_" + +srun \ + --nodelist="$SELECTED_NODELIST_SRUN" \ + --kill-on-bad-exit=1 \ + --signal=TERM@30 \ + --unbuffered \ + bash -lc " +set -euo pipefail + +echo \"Rank \$SLURM_PROCID on \$(hostname)\" + +# Per-node docker privilege detection +eval \"\$DOCKER_CMD_DETECT\" +echo \"[docker-detect] rank \$SLURM_PROCID: DOCKER_CMD=\$DOCKER_CMD\" + +# Enable out-of-tree RDMA library mounts for atom-disagg (mooncake requires host RDMA stack) +RDMA_MOUNTS=() +if [[ "$ENGINE" == "atom-disagg" ]]; then + +# When the container base OS differs from the host (e.g. Ubuntu 24.04 image +# on a 22.04 host), the container's bundled libibverbs/libionic may be +# ABI-incompatible with the host kernel drivers. Detect the NIC type and +# bind-mount the host's out-of-tree RDMA userspace libraries into the +# container so the RDMA stack always matches the running kernel. +_detect_nic_type() { + if [[ -n \"\${MORI_NIC_TYPE:-}\" ]]; then echo \"\$MORI_NIC_TYPE\"; return; fi + local bnxt=0 mlx5=0 ionic=0 + if [[ -d /sys/class/infiniband ]]; then + for dev in /sys/class/infiniband/*; do + local name; name=\$(basename \"\$dev\") + case \"\$name\" in + bnxt_re*) ((bnxt++)) ;; mlx5*) ((mlx5++)) ;; ionic*) ((ionic++)) ;; + *) + local drv; drv=\$(basename \"\$(readlink -f \"\$dev/device/driver\" 2>/dev/null)\" 2>/dev/null || true) + case \"\$drv\" in bnxt*) ((bnxt++)) ;; mlx5*) ((mlx5++)) ;; ionic*) ((ionic++)) ;; esac ;; + esac + done + fi + if (( bnxt >= mlx5 && bnxt >= ionic && bnxt > 0 )); then echo bnxt + elif (( ionic >= mlx5 && ionic > 0 )); then echo ionic + else echo mlx5; fi +} + +_find_host_ibverbs() { + for c in /usr/lib64/libibverbs.so.1 /lib/x86_64-linux-gnu/libibverbs.so.1 /usr/lib/x86_64-linux-gnu/libibverbs.so.1.14.39.0 /usr/lib/x86_64-linux-gnu/libibverbs.so.1; do + local r; r=\$(readlink -f \"\$c\" 2>/dev/null || true) + [[ \"\$r\" == *libibverbs.so.1.14.57.0 ]] && continue + if [[ -f \"\$r\" ]]; then echo \"\$r\"; return; fi + done +} + +_NIC_TYPE=\$(_detect_nic_type) +echo \"[rdma] NIC type: \${_NIC_TYPE} on \$(hostname)\" + +if [[ \"\$_NIC_TYPE\" == \"ionic\" || \"\$_NIC_TYPE\" == \"bnxt\" ]]; then + _host_ibv=\$(_find_host_ibverbs) + if [[ -n \"\$_host_ibv\" ]]; then + RDMA_MOUNTS+=(-v \"\$_host_ibv:/lib/x86_64-linux-gnu/libibverbs.so.1\") + fi +fi + +if [[ \"\$_NIC_TYPE\" == \"ionic\" ]]; then + for _dir in /usr/local/lib /usr/lib/x86_64-linux-gnu; do + for _lib in \"\$_dir\"/libionic*.so; do + [[ -f \"\$_lib\" ]] || continue + _real=\$(readlink -f \"\$_lib\") + [[ -f \"\$_real\" ]] && RDMA_MOUNTS+=(-v \"\$_real:\$_real\") + RDMA_MOUNTS+=(-v \"\$_lib:/usr/lib/x86_64-linux-gnu/\$(basename \"\$_lib\")\") + done + done + if [[ -d /usr/lib/x86_64-linux-gnu/libibverbs ]]; then + for _lib in /usr/lib/x86_64-linux-gnu/libibverbs/libionic-rdmav*.so; do + [[ -f \"\$_lib\" ]] && RDMA_MOUNTS+=(-v \"\$_lib:\$_lib\") + done + fi + [[ -d /etc/libibverbs.d ]] && RDMA_MOUNTS+=(-v /etc/libibverbs.d:/etc/libibverbs.d:ro) +elif [[ \"\$_NIC_TYPE\" == \"bnxt\" ]]; then + for _lib in /usr/local/lib/libbnxt_re-rdmav*.so; do + [[ -f \"\$_lib\" ]] && RDMA_MOUNTS+=(-v \"\$_lib:/usr/lib/x86_64-linux-gnu/libibverbs/\$(basename \"\$_lib\")\") + done + for _lib in /usr/local/lib/libbnxt_re.so; do + [[ -f \"\$_lib\" ]] && RDMA_MOUNTS+=(-v \"\$_lib:/usr/lib/x86_64-linux-gnu/\$(basename \"\$_lib\")\") + done + [[ -d /etc/libibverbs.d ]] && RDMA_MOUNTS+=(-v /etc/libibverbs.d:/etc/libibverbs.d:ro) +fi + +if [[ \${#RDMA_MOUNTS[@]} -gt 0 ]]; then + echo \"[rdma] bind-mounts: \${RDMA_MOUNTS[*]}\" +else + echo \"[rdma] no out-of-tree RDMA mounts needed\" +fi +fi # end: if ENGINE == atom-disagg + +# Pre-clean (idempotent) +\$DOCKER_CMD ps -aq --filter \"$CONT_FILTER\" | xargs -r \$DOCKER_CMD rm -f || true +\$DOCKER_CMD ps -aq | xargs -r \$DOCKER_CMD stop || true + +# Start vLLM external router container on node 0 +if [[ \"$ENGINE\" == \"vllm-disagg\" && \"$ROUTER_TYPE\" == \"vllm-router\" && \"\$SLURM_PROCID\" == \"0\" ]]; then + \$DOCKER_CMD rm -f \"$ROUTER_CONT_NAME\" 2>/dev/null || true + \$DOCKER_CMD run -d \ + --name \"$ROUTER_CONT_NAME\" \ + --network host \ + --ulimit nofile=1048576:1048576 \ + -v /tmp:/run_logs \ + \"$VLLM_ROUTER_IMAGE\" \ + bash -lc \"mkdir -p /run_logs/slurm_job-${SLURM_JOB_ID} && exec vllm-router \ + --vllm-pd-disaggregation \ + --kv-connector moriio \ + --vllm-discovery-address 0.0.0.0:${PROXY_PING_PORT} \ + --port ${ROUTER_PORT} \ + --host 0.0.0.0 \ + --policy consistent_hash \ + --prefill-policy consistent_hash \ + --decode-policy consistent_hash \ + --log-level info 2>&1 | tee /run_logs/slurm_job-${SLURM_JOB_ID}/vllm_router_\$(hostname).log \" +fi + +# Skip exec on vllm-disagg rank 0 so we can stop the router after the main +# container exits. Without this, decode nodes block forever waiting for the +# router port to close (the router is a separate container). +MAYBE_EXEC=exec +if [[ \"$ENGINE\" == \"vllm-disagg\" && \"$ROUTER_TYPE\" == \"vllm-router\" && \"\$SLURM_PROCID\" == \"0\" ]]; then + MAYBE_EXEC= + set +e +fi + +\$MAYBE_EXEC \$DOCKER_CMD run \ + --init \ + --stop-timeout 10 \ + --device /dev/dri \ + --device /dev/kfd \ + --device /dev/infiniband \ + --device=/dev/infiniband/rdma_cm \ + --device=/dev/infiniband/uverbs0 \ + --device=/dev/infiniband/uverbs1 \ + --device=/dev/infiniband/uverbs2 \ + --device=/dev/infiniband/uverbs3 \ + --device=/dev/infiniband/uverbs4 \ + --device=/dev/infiniband/uverbs5 \ + --device=/dev/infiniband/uverbs6 \ + --device=/dev/infiniband/uverbs7 \ + --ulimit memlock=-1 \ + --ulimit stack=67108864 \ + --ulimit nofile=1048576:1048576 \ + --network host \ + --ipc host \ + --group-add video \ + --cap-add SYS_PTRACE \ + --security-opt seccomp=unconfined \ + --privileged \ + -v /sys:/sys \ + $(command -v nicctl >/dev/null 2>&1 && echo "-v $(which nicctl):/usr/sbin/nicctl") \ + -v ${MODEL_DIR}:/models \ + -v \$HOME/.ssh:/root/.ssh \ + --shm-size 128G \ + -v /tmp:/run_logs \ + -v ${BENCHMARK_LOGS_DIR}:/benchmark_logs \ + -v ${DI_REPO_DIR}:${DOCKER_MOUNT_PATH} \ + -v ${HICACHE_MC_CONFIG}:/config/hicache_mc.env:ro \ + ${EXTRA_DOCKER_MOUNTS:-} \ + \${RDMA_MOUNTS[@]+"\${RDMA_MOUNTS[@]}"} \ + ${DOCKER_ENV_COMMON[*]} \ + ${DOCKER_ENV_ENGINE[*]} \ + --name \"$DOCKER_CONT_NAME\" \ + --entrypoint \"\" \ + \"$DOCKER_IMAGE_NAME\" bash -lc ' + set -o pipefail + mkdir -p /run_logs/slurm_job-'\"\$SLURM_JOB_ID\"' + '"$RUN_FILE_FULL"' 2>&1 | tee /run_logs/slurm_job-'\"\$SLURM_JOB_ID\"'/server_\$(hostname).log + ' + +# Only reached when exec was skipped (vllm-disagg rank 0) +DOCKER_EXIT_CODE=\$? +echo \"[rank 0] Main container exited (rc=\$DOCKER_EXIT_CODE). Stopping vllm-router...\" +\$DOCKER_CMD rm -f \"$ROUTER_CONT_NAME\" 2>/dev/null || true +exit \$DOCKER_EXIT_CODE +" + +if [[ "${KEEP_CONTAINERS}" != "1" ]]; then + srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c 'eval "$DOCKER_CMD_DETECT"; $DOCKER_CMD rm -f '"$DOCKER_CONT_NAME"' 2>/dev/null || true' + + # Clean up vLLM external router container on node 0 + if [[ "$ENGINE" == "vllm-disagg" && "$ROUTER_TYPE" == "vllm-router" ]]; then + srun --nodes=1 --ntasks=1 --nodelist="$MASTER_NODE" bash -c ' + eval "$DOCKER_CMD_DETECT"; $DOCKER_CMD rm -f '"$ROUTER_CONT_NAME"' 2>/dev/null || true + ' + fi +fi diff --git a/benchmarks/multi_node/amd_utils/models_agentic.yaml b/benchmarks/multi_node/amd_utils/models_agentic.yaml new file mode 100644 index 0000000000..698fcbc5eb --- /dev/null +++ b/benchmarks/multi_node/amd_utils/models_agentic.yaml @@ -0,0 +1,394 @@ +# Model-specific SGLang server configurations for disaggregated inference. +# +# Each top-level key is a MODEL_NAME value (must match the directory name under MODEL_DIR). +# +# To add a new model: add a new top-level entry following the same schema. +# No script changes are required. +# +# Schema: +# : +# base_flags: str # Common flags for both prefill and decode +# mtp_flags: str # Appended to decode when DECODE_MTP_SIZE > 0 +# dp_flags: str # Appended when DP is enabled (prefill or decode) +# prefill: +# mem_fraction_static: float +# disable_radix_cache: bool +# dp: # Config when data-parallel attention is enabled +# max_running_requests: int +# chunked_prefill_size: str # Can be integer or bash arithmetic expression +# cuda_graph_bs: str # Space-separated values +# no_dp: # Config when data-parallel attention is disabled +# max_running_requests: int +# chunked_prefill_size: int +# cuda_graph_bs_range: str # "start-end" expanded via seq +# decode: +# mem_fraction_static: float +# prefill_round_robin_balance: bool +# dp: +# max_running_requests: int +# chunked_prefill_size: str +# cuda_graph_bs_range: str +# ep_only: # Config when EP is enabled but DP is disabled +# max_running_requests: int +# chunked_prefill_size: int +# cuda_graph_bs_range: str +# no_dp: +# max_running_requests: int +# chunked_prefill_size: int +# cuda_graph_bs_range: str + +DeepSeek-V3: + base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --ep-dispatch-algorithm fake --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" + mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1" + dp_flags: "--moe-a2a-backend mori --deepep-mode normal --enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" + prefill: + mem_fraction_static: 0.8 + disable_radix_cache: true + dp: + max_running_requests: 24 + chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" + cuda_graph_bs: "1 2 3" + no_dp: + max_running_requests: 128 + chunked_prefill_size: 262144 + cuda_graph_bs_range: "1-128" + decode: + mem_fraction_static: 0.85 + prefill_round_robin_balance: true + dp: + max_running_requests: 4096 + chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" + cuda_graph_bs_range: "1-160" + ep_only: + max_running_requests: 256 + chunked_prefill_size: 262144 + cuda_graph_bs_range: "1-256" + no_dp: + max_running_requests: 128 + chunked_prefill_size: 262144 + cuda_graph_bs_range: "1-128" + +DeepSeek-V3-0324: + base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --ep-dispatch-algorithm fake --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" + mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1" + dp_flags: "--moe-a2a-backend mori --deepep-mode normal --enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" + prefill: + mem_fraction_static: 0.8 + disable_radix_cache: true + dp: + max_running_requests: 24 + chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" + cuda_graph_bs: "1 2 3" + no_dp: + max_running_requests: 128 + chunked_prefill_size: 262144 + cuda_graph_bs_range: "1-128" + decode: + mem_fraction_static: 0.85 + prefill_round_robin_balance: true + dp: + max_running_requests: 4096 + chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" + cuda_graph_bs_range: "1-160" + ep_only: + max_running_requests: 256 + chunked_prefill_size: 262144 + cuda_graph_bs_range: "1-256" + no_dp: + max_running_requests: 128 + chunked_prefill_size: 262144 + cuda_graph_bs_range: "1-128" + +DeepSeek-R1: + base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --ep-dispatch-algorithm fake --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" + mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1" + dp_flags: "--moe-a2a-backend mori --deepep-mode normal --enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" + prefill: + mem_fraction_static: 0.8 + disable_radix_cache: true + dp: + max_running_requests: 24 + chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" + cuda_graph_bs: "1 2 3" + no_dp: + max_running_requests: 128 + chunked_prefill_size: 262144 + cuda_graph_bs_range: "1-128" + decode: + mem_fraction_static: 0.85 + prefill_round_robin_balance: true + dp: + max_running_requests: 4096 + chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" + cuda_graph_bs_range: "1-160" + ep_only: + max_running_requests: 256 + chunked_prefill_size: 262144 + cuda_graph_bs_range: "1-256" + no_dp: + max_running_requests: 128 + chunked_prefill_size: 262144 + cuda_graph_bs_range: "1-128" + +DeepSeek-R1-0528: + base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --ep-dispatch-algorithm fake --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" + mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1" + dp_flags: "--moe-a2a-backend mori --deepep-mode normal --enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" + prefill: + mem_fraction_static: 0.8 + disable_radix_cache: true + dp: + max_running_requests: 24 + chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" + cuda_graph_bs: "1 2 3" + no_dp: + max_running_requests: 128 + chunked_prefill_size: 262144 + cuda_graph_bs_range: "1-128" + decode: + mem_fraction_static: 0.85 + prefill_round_robin_balance: true + dp: + max_running_requests: 4096 + chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" + cuda_graph_bs_range: "1-160" + ep_only: + max_running_requests: 256 + chunked_prefill_size: 262144 + cuda_graph_bs_range: "1-256" + no_dp: + max_running_requests: 128 + chunked_prefill_size: 262144 + cuda_graph_bs_range: "1-128" + +Qwen3.5-397B-A17B-MXFP4: + base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori --moe-dense-tp-size 1" + mtp_flags: "" + dp_flags: "--moe-a2a-backend mori --enable-dp-attention --enable-dp-lm-head" + prefill: + mem_fraction_static: 0.8 + disable_radix_cache: true + dp: + max_running_requests: 24 + chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" + cuda_graph_bs: "1 2 3" + no_dp: + max_running_requests: 128 + chunked_prefill_size: 262144 + cuda_graph_bs_range: "1-128" + decode: + mem_fraction_static: 0.85 + prefill_round_robin_balance: true + dp: + max_running_requests: 4096 + chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" + cuda_graph_bs_range: "1-160" + ep_only: + max_running_requests: 256 + chunked_prefill_size: 262144 + cuda_graph_bs_range: "1-256" + no_dp: + max_running_requests: 128 + chunked_prefill_size: 262144 + cuda_graph_bs_range: "1-128" + +Qwen3.5-397B-A17B-FP8: + base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori --moe-dense-tp-size 1" + mtp_flags: "" + dp_flags: "--moe-a2a-backend mori --enable-dp-attention --enable-dp-lm-head" + prefill: + mem_fraction_static: 0.8 + disable_radix_cache: true + dp: + max_running_requests: 24 + chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" + cuda_graph_bs: "1 2 3" + no_dp: + max_running_requests: 128 + chunked_prefill_size: 262144 + cuda_graph_bs_range: "1-128" + decode: + mem_fraction_static: 0.85 + prefill_round_robin_balance: true + dp: + max_running_requests: 4096 + chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" + cuda_graph_bs_range: "1-160" + ep_only: + max_running_requests: 256 + chunked_prefill_size: 262144 + cuda_graph_bs_range: "1-256" + no_dp: + max_running_requests: 128 + chunked_prefill_size: 262144 + cuda_graph_bs_range: "1-128" + +GLM-5-FP8: + base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --disaggregation-transfer-backend mori --tool-call-parser glm47 --reasoning-parser glm45 --model-loader-extra-config '{\\\"enable_multithread_load\\\": true, \\\"num_threads\\\": 8}'" + mtp_flags: "" + dp_flags: "--moe-a2a-backend mori --enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" + prefill: + mem_fraction_static: 0.8 + disable_radix_cache: true + dp: + max_running_requests: 24 + chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" + cuda_graph_bs: "1 2 3" + no_dp: + max_running_requests: 128 + chunked_prefill_size: 262144 + cuda_graph_bs_range: "1-128" + decode: + mem_fraction_static: 0.85 + prefill_round_robin_balance: true + dp: + max_running_requests: 4096 + chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" + cuda_graph_bs_range: "1-160" + ep_only: + max_running_requests: 256 + chunked_prefill_size: 262144 + cuda_graph_bs_range: "1-256" + no_dp: + max_running_requests: 128 + chunked_prefill_size: 262144 + cuda_graph_bs_range: "1-128" + +DeepSeek-R1-0528-MXFP4-Preview: + base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --ep-dispatch-algorithm fake --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" + mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1" + dp_flags: "--moe-a2a-backend mori --deepep-mode normal --enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" + prefill: + mem_fraction_static: 0.8 + disable_radix_cache: true + dp: + max_running_requests: 24 + chunked_prefill_size: 16384 + cuda_graph_bs: "1 2 3" + no_dp: + max_running_requests: 128 + chunked_prefill_size: 16384 + cuda_graph_bs_range: "1-128" + decode: + mem_fraction_static: 0.85 + prefill_round_robin_balance: true + dp: + max_running_requests: 4096 + chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" + cuda_graph_bs_range: "1-160" + ep_only: + max_running_requests: 256 + chunked_prefill_size: 262144 + cuda_graph_bs_range: "1-256" + no_dp: + max_running_requests: 128 + chunked_prefill_size: 262144 + cuda_graph_bs_range: "1-128" + +DeepSeek-R1-0528-MXFP4: + base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --ep-dispatch-algorithm fake --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" + mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1" + dp_flags: "--moe-a2a-backend mori --deepep-mode normal --enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" + prefill: + mem_fraction_static: 0.8 + disable_radix_cache: true + dp: + max_running_requests: 24 + chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" + cuda_graph_bs: "1 2 3" + no_dp: + max_running_requests: 128 + chunked_prefill_size: 16384 + cuda_graph_bs_range: "1-128" + decode: + mem_fraction_static: 0.85 + prefill_round_robin_balance: true + dp: + max_running_requests: 4096 + chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" + cuda_graph_bs_range: "1-160" + ep_only: + max_running_requests: 256 + chunked_prefill_size: 262144 + cuda_graph_bs_range: "1-256" + no_dp: + max_running_requests: 128 + chunked_prefill_size: 262144 + cuda_graph_bs_range: "1-128" + +DeepSeek-R1-0528-MXFP4-v2: + base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --ep-dispatch-algorithm fake --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" + mtp_flags: "--speculative-draft-model-path SGLang/DeepSeek-R1-NextN --speculative-algorithm NEXTN --speculative-eagle-topk 1 --speculative-attention-mode decode " + dp_flags: "--moe-a2a-backend mori --deepep-mode normal --enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head --stream-interval 100 --tokenizer-worker-num 32 " + prefill: + mem_fraction_static: 0.8 + disable_radix_cache: true + dp: + max_running_requests: 4096 + chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" + cuda_graph_bs: "1 2 3" + context_length: 9217 + max_total_tokens: 131072 + enable_two_batch_overlap: true + no_dp: + max_running_requests: 128 + chunked_prefill_size: 16384 + cuda_graph_bs_range: "1-128" + decode: + mem_fraction_static: 0.85 + prefill_round_robin_balance: true + dp: + max_running_requests: 4096 + chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" + cuda_graph_bs_range: "1-512" + ep_only: + max_running_requests: 256 + chunked_prefill_size: 262144 + cuda_graph_bs_range: "1-256" + no_dp: + max_running_requests: 128 + chunked_prefill_size: 262144 + cuda_graph_bs_range: "1-128" + +# DeepSeek-V4-Pro PD-disaggregation recipe (MI355X, SGLang + MoRI). +# KV transfer = mori for both topologies (pure-TP and DEP); the DP path additionally +# routes the MoE all-to-all through mori (--moe-a2a-backend mori) with dp-attention. +# DSv4-specific kernel routing (unified_kv_triton, AITER indexer, fp8 wo_a fallback, +# thinking/reasoning-effort, dispatch dtypes, per-role PER_RANK dispatch tokens) is set +# in env.sh's DeepSeek-V4-Pro block. The bench client uses --dsv4 framing (bench.sh). +# prefill.disable_cuda_graph routes prefill to --disable-cuda-graph; decode keeps +# --cuda-graph-bs. See dsv4_mi355x_sglang_disagg_plan.md. hf_dir: "models--deepseek-ai--DeepSeek-V4-Pro" +DeepSeek-V4-Pro: + base_flags: "--watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend dsv4 --page-size 256 --swa-full-tokens-ratio 0.1 --disable-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --disaggregation-transfer-backend mori --log-level error --log-level-http error" + dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" + ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" + prefill: + mem_fraction_static: 0.8 + disable_radix_cache: true + disable_cuda_graph: true + dp: + max_running_requests: 1024 + chunked_prefill_size: 65280 # dsv4 compressor kernel uint16 token cap (255*256) + context_length: 1048576 + max_total_tokens: 1048576 + no_dp: + max_running_requests: 128 + # Small prefill chunks interleave long-context agentic prefills across + # requests instead of letting one ~100K-token prefill monopolize the + # engine (the conc>=16 queue-saturation / decode-stall failure mode). + # Mirrors the single-node DSv4 agentic recipe (dsv4_fp4_mi355x.sh=8192). + # Was 65280 (255*256, the dsv4 compressor kernel uint16 token cap); 8192 + # (32*256) stays a page-size multiple well under that cap. + chunked_prefill_size: 8192 + context_length: 1048576 + max_total_tokens: 1048576 + decode: + mem_fraction_static: 0.85 + prefill_round_robin_balance: true + disagg_decode_enable_radix_cache: false + dp: + max_running_requests: 1024 + cuda_graph_bs_range: "1-128" + no_dp: + max_running_requests: 128 + cuda_graph_bs_range: "1-128" diff --git a/benchmarks/multi_node/amd_utils/server_agentic.sh b/benchmarks/multi_node/amd_utils/server_agentic.sh new file mode 100755 index 0000000000..c2ba9c3553 --- /dev/null +++ b/benchmarks/multi_node/amd_utils/server_agentic.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Multi-Engine Disaggregated Server Dispatcher +# ============================================================================= +# Dispatches to the engine-specific server launcher based on ENGINE env var. +# ENGINE=sglang-disagg (default) -> server_sglang_agentic.sh (SGLang + MoRI) +# ENGINE=vllm-disagg -> server_vllm.sh (vLLM + Nixl/MoRI-IO) +# ENGINE=atom-disagg -> server_atom.sh (ATOM + mooncake) +# ============================================================================= + +ENGINE="${ENGINE:-sglang-disagg}" +WS_PATH="${WS_PATH:-${SGLANG_WS_PATH:-${VLLM_WS_PATH:-${ATOM_WS_PATH:-$(dirname "${BASH_SOURCE[0]}")}}}}" +export WS_PATH ENGINE + +echo "[DISPATCHER] ENGINE=$ENGINE WS_PATH=$WS_PATH" + +if [[ "$ENGINE" == "vllm-disagg" ]]; then + source "$WS_PATH/server_vllm.sh" +elif [[ "$ENGINE" == "atom-disagg" ]]; then + export ATOM_WS_PATH="$WS_PATH" + source "$WS_PATH/server_atom.sh" +else + source "$WS_PATH/server_sglang_agentic.sh" +fi diff --git a/benchmarks/multi_node/amd_utils/server_sglang_agentic.sh b/benchmarks/multi_node/amd_utils/server_sglang_agentic.sh new file mode 100755 index 0000000000..d19691f428 --- /dev/null +++ b/benchmarks/multi_node/amd_utils/server_sglang_agentic.sh @@ -0,0 +1,1039 @@ +#!/bin/bash +# SGLang Disaggregated Server Launcher with Model-Specific Configurations +# ============================================================================= + +# ============================================================================= +# Environment Configuration +# ============================================================================= + +NODE0_ADDR="${NODE0_ADDR:-localhost}" +NODE_RANK="${NODE_RANK:-0}" +MODEL_DIR="${MODEL_DIR:-}" +MODEL_NAME="${MODEL_NAME:-}" + +xP="${xP:-1}" #-> Number of Prefill Workers +yD="${yD:-1}" #-> Number of Decode Workers + +IPADDRS="${IPADDRS:-localhost}" +HEADNODE_PORT="${HEADNODE_PORT:-20000}" +# Parallelism Configuration +PREFILL_TP_SIZE="${PREFILL_TP_SIZE:-8}" +PREFILL_ENABLE_EP="${PREFILL_ENABLE_EP:-true}" +PREFILL_ENABLE_DP="${PREFILL_ENABLE_DP:-true}" +DECODE_TP_SIZE="${DECODE_TP_SIZE:-8}" +DECODE_ENABLE_EP="${DECODE_ENABLE_EP:-true}" +DECODE_ENABLE_DP="${DECODE_ENABLE_DP:-true}" +DECODE_MTP_SIZE="${DECODE_MTP_SIZE:-0}" + +# Benchmark Configuration +BENCH_INPUT_LEN="${BENCH_INPUT_LEN:-1024}" +BENCH_OUTPUT_LEN="${BENCH_OUTPUT_LEN:-1024}" +BENCH_RANDOM_RANGE_RATIO="${BENCH_RANDOM_RANGE_RATIO:-1}" +BENCH_REQUEST_RATE="${BENCH_REQUEST_RATE:-inf}" +BENCH_NUM_PROMPTS_MULTIPLIER="${BENCH_NUM_PROMPTS_MULTIPLIER:-10}" +BENCH_MAX_CONCURRENCY="${BENCH_MAX_CONCURRENCY:-512}" + +# Extract the maximum concurrency from the x-delimited list +BENCH_MAX_CONC_VALUE=$(echo "$BENCH_MAX_CONCURRENCY" | tr 'x' '\n' | sort -n | tail -1) + +# Dry Run for debugging purpose +DRY_RUN="${DRY_RUN:-0}" + +# GPU count (expandable for different hardware) +GPUS_PER_NODE="${GPUS_PER_NODE:-8}" + + +# ============================================================================= +# Dependencies and Environment Setup +# ============================================================================= +source $SGLANG_WS_PATH/setup_deps_agentic.sh +source $SGLANG_WS_PATH/env_agentic.sh + +host_ip=$(ip route get 1.1.1.1 | awk '/src/ {print $7}') +host_name=$(hostname) + +# MORI_RDMA_TC configuration (optional) +# If set by runner, use it for RDMA traffic class configuration +# If not set, RDMA operations will proceed without QoS/traffic class settings +if [[ -n "${MORI_RDMA_TC}" ]]; then + echo "[INFO] Using MORI_RDMA_TC=$MORI_RDMA_TC for RDMA traffic class configuration" + echo "[INFO] Host '$host_name' configured with MORI_RDMA_TC=$MORI_RDMA_TC" +else + echo "[INFO] MORI_RDMA_TC not set. Skipping RDMA traffic class configuration." + echo "[INFO] This is normal for clusters without QoS requirements." +fi + +# ============================================================================= +# Model-Specific Configuration from YAML +# ============================================================================= +MODELS_YAML="${SGLANG_WS_PATH}/models_agentic.yaml" + +if [[ ! -f "$MODELS_YAML" ]]; then + echo "ERROR: models.yaml not found at $MODELS_YAML" + exit 1 +fi + +# Load model config via inline Python (PyYAML is available in SGLang containers) +# Formula evaluation (e.g. "SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK * TP * xP") +# is done here in Python to avoid bash glob-expanding the * characters. +eval "$(python3 -c " +import yaml, sys, os + +config_path = '${MODELS_YAML}' +model_name = '${MODEL_NAME}' + +with open(config_path) as f: + models = yaml.safe_load(f) + +if model_name not in models: + print(f'echo \"ERROR: Model {model_name} not in models.yaml\"; exit 1') + sys.exit(0) + +m = models[model_name] + +def eval_formula(val): + \"\"\"Evaluate chunked_prefill_size: if string, resolve variable names from env and compute.\"\"\" + if isinstance(val, (int, float)): + return int(val) + s = str(val) + # Build a namespace from env vars (convert numeric values to int) + ns = {} + for k, v in os.environ.items(): + try: + ns[k] = int(v) + except (ValueError, TypeError): + pass + try: + return int(eval(s, {'__builtins__': {}}, ns)) + except Exception as e: + print(f'echo \"WARNING: Cannot evaluate formula: {s} ({e})\"', file=sys.stderr) + return val + +def parse_range(cuda_range, default_start, default_end): + if '-' in str(cuda_range): + s, e = str(cuda_range).split('-') + return s, e + return str(default_start), str(default_end) + +# Output shell variables +print(f'MODEL_BASE_FLAGS=\"{m.get(\"base_flags\", \"\")}\"') +print(f'MODEL_MTP_FLAGS=\"{m.get(\"mtp_flags\", \"\")}\"') +print(f'MODEL_DP_FLAGS=\"{m.get(\"dp_flags\", \"\")}\"') +print(f'MODEL_EP_FLAGS=\"{m.get(\"ep_flags\", \"\")}\"') + +prefill = m.get('prefill', {}) +decode = m.get('decode', {}) + +print(f'PREFILL_MEM_FRACTION_STATIC=\"{prefill.get(\"mem_fraction_static\", 0.8)}\"') +print(f'PREFILL_DISABLE_RADIX_CACHE=\"{prefill.get(\"disable_radix_cache\", True)}\"') +print(f'PREFILL_DISABLE_CUDA_GRAPH=\"{prefill.get(\"disable_cuda_graph\", False)}\"') + +dp = prefill.get('dp', {}) +no_dp = prefill.get('no_dp', {}) +print(f'PREFILL_MAX_RUNNING_REQUESTS_DP=\"{dp.get(\"max_running_requests\", 24)}\"') +print(f'PREFILL_CHUNKED_PREFILL_SIZE_DP=\"{eval_formula(dp.get(\"chunked_prefill_size\", 262144))}\"') +print(f'PREFILL_CUDA_GRAPH_BS_DP=\"{dp.get(\"cuda_graph_bs\", \"1 2 3\")}\"') +print(f'PREFILL_CONTEXT_LENGTH_DP=\"{dp.get(\"context_length\", \"\")}\"') +print(f'PREFILL_MAX_TOTAL_TOKENS_DP=\"{dp.get(\"max_total_tokens\", \"\")}\"') +print(f'PREFILL_ENABLE_TWO_BATCH_OVERLAP_DP=\"{dp.get(\"enable_two_batch_overlap\", False)}\"') +print(f'PREFILL_MAX_RUNNING_REQUESTS_NO_DP=\"{no_dp.get(\"max_running_requests\", 128)}\"') +print(f'PREFILL_CHUNKED_PREFILL_SIZE_NO_DP=\"{eval_formula(no_dp.get(\"chunked_prefill_size\", 262144))}\"') +print(f'PREFILL_CONTEXT_LENGTH_NO_DP=\"{no_dp.get(\"context_length\", \"\")}\"') +print(f'PREFILL_MAX_TOTAL_TOKENS_NO_DP=\"{no_dp.get(\"max_total_tokens\", \"\")}\"') +s, e = parse_range(no_dp.get('cuda_graph_bs_range', '1-128'), 1, 128) +print(f'PREFILL_CUDA_GRAPH_BS_NO_DP_START=\"{s}\"') +print(f'PREFILL_CUDA_GRAPH_BS_NO_DP_END=\"{e}\"') + +print(f'DECODE_MEM_FRACTION_STATIC=\"{decode.get(\"mem_fraction_static\", 0.85)}\"') +print(f'DECODE_PREFILL_ROUND_ROBIN_BALANCE=\"{decode.get(\"prefill_round_robin_balance\", True)}\"') +print(f'DECODE_DISAGG_ENABLE_RADIX_CACHE=\"{decode.get(\"disagg_decode_enable_radix_cache\", False)}\"') + +dp = decode.get('dp', {}) +ep_only = decode.get('ep_only', {}) +no_dp = decode.get('no_dp', {}) + +# Decode DP config +print(f'DECODE_MAX_RUNNING_REQUESTS_DP=\"{dp.get(\"max_running_requests\", 4096)}\"') +print(f'DECODE_CHUNKED_PREFILL_SIZE_DP=\"{eval_formula(dp.get(\"chunked_prefill_size\", 262144))}\"') +s, e = parse_range(dp.get('cuda_graph_bs_range', '1-160'), 1, 160) +print(f'DECODE_CUDA_GRAPH_BS_DP_START=\"{s}\"') +print(f'DECODE_CUDA_GRAPH_BS_DP_END=\"{e}\"') + +# Decode EP-only config (EP enabled but DP disabled) +print(f'DECODE_MAX_RUNNING_REQUESTS_EP_ONLY=\"{ep_only.get(\"max_running_requests\", 256)}\"') +print(f'DECODE_CHUNKED_PREFILL_SIZE_EP_ONLY=\"{eval_formula(ep_only.get(\"chunked_prefill_size\", 262144))}\"') +s, e = parse_range(ep_only.get('cuda_graph_bs_range', '1-256'), 1, 256) +print(f'DECODE_CUDA_GRAPH_BS_EP_ONLY_START=\"{s}\"') +print(f'DECODE_CUDA_GRAPH_BS_EP_ONLY_END=\"{e}\"') + +# Decode no-DP config +print(f'DECODE_MAX_RUNNING_REQUESTS_NO_DP=\"{no_dp.get(\"max_running_requests\", 128)}\"') +print(f'DECODE_CHUNKED_PREFILL_SIZE_NO_DP=\"{eval_formula(no_dp.get(\"chunked_prefill_size\", 262144))}\"') +s, e = parse_range(no_dp.get('cuda_graph_bs_range', '1-128'), 1, 128) +print(f'DECODE_CUDA_GRAPH_BS_NO_DP_START=\"{s}\"') +print(f'DECODE_CUDA_GRAPH_BS_NO_DP_END=\"{e}\"') +")" + +echo "Loaded model configuration for: $MODEL_NAME" + +# Compute DP-dependent prefill parameters +if [[ "$PREFILL_ENABLE_DP" == "true" ]]; then + prefill_cuda_graph_bs=($PREFILL_CUDA_GRAPH_BS_DP) + prefill_max_running_requests=$PREFILL_MAX_RUNNING_REQUESTS_DP + prefill_chunked_prefill_size=$PREFILL_CHUNKED_PREFILL_SIZE_DP + prefill_context_length=$PREFILL_CONTEXT_LENGTH_DP + prefill_max_total_tokens=$PREFILL_MAX_TOTAL_TOKENS_DP + prefill_enable_two_batch_overlap=$PREFILL_ENABLE_TWO_BATCH_OVERLAP_DP +else + prefill_cuda_graph_bs=($(seq $PREFILL_CUDA_GRAPH_BS_NO_DP_START $PREFILL_CUDA_GRAPH_BS_NO_DP_END)) + prefill_max_running_requests=$PREFILL_MAX_RUNNING_REQUESTS_NO_DP + prefill_chunked_prefill_size=$PREFILL_CHUNKED_PREFILL_SIZE_NO_DP + prefill_context_length=$PREFILL_CONTEXT_LENGTH_NO_DP + prefill_max_total_tokens=$PREFILL_MAX_TOTAL_TOKENS_NO_DP + prefill_enable_two_batch_overlap="false" +fi + +# When both DP and EP are enabled, override max-running-requests with max bench concurrency +if [[ "$PREFILL_ENABLE_DP" == "true" ]] && [[ "$PREFILL_ENABLE_EP" == "true" ]]; then + prefill_max_running_requests=$BENCH_MAX_CONC_VALUE + prefill_dp_ranks=$PREFILL_TP_SIZE + # MORI_MAX_DISPATCH_TOKENS_PREFILL stays at 8192 (no change) + # MORI_MOE_MAX_INPUT_TOKENS_PREFILL=$((MORI_MAX_DISPATCH_TOKENS_PREFILL * prefill_dp_ranks / 2)) + echo "[DP+EP override] Prefill: max-running-requests=$prefill_max_running_requests, MOE_MAX_INPUT=$MORI_MOE_MAX_INPUT_TOKENS_PREFILL" +fi + +# Compute DP-dependent decode parameters (3-way: DP > EP-only > no_dp) +if [[ "$DECODE_ENABLE_DP" == "true" ]]; then + decode_cuda_graph_bs=($(seq $DECODE_CUDA_GRAPH_BS_DP_START $DECODE_CUDA_GRAPH_BS_DP_END)) + decode_max_running_requests=$((DECODE_CUDA_GRAPH_BS_DP_END * DECODE_TP_SIZE)) +elif [[ "$DECODE_ENABLE_EP" == "true" ]]; then + decode_cuda_graph_bs=($(seq $DECODE_CUDA_GRAPH_BS_EP_ONLY_START $DECODE_CUDA_GRAPH_BS_EP_ONLY_END)) + decode_max_running_requests=$DECODE_MAX_RUNNING_REQUESTS_EP_ONLY +else + decode_cuda_graph_bs=($(seq $DECODE_CUDA_GRAPH_BS_NO_DP_START $DECODE_CUDA_GRAPH_BS_NO_DP_END)) + decode_max_running_requests=$DECODE_MAX_RUNNING_REQUESTS_NO_DP +fi + +# When both DP and EP are enabled, override max-running-requests and dispatch tokens +if [[ "$DECODE_ENABLE_DP" == "true" ]] && [[ "$DECODE_ENABLE_EP" == "true" ]]; then + decode_max_running_requests=$BENCH_MAX_CONC_VALUE + decode_dp_ranks=$DECODE_TP_SIZE + MORI_MAX_DISPATCH_TOKENS_DECODE=$((BENCH_MAX_CONC_VALUE / decode_dp_ranks)) + # MORI_MOE_MAX_INPUT_TOKENS_DECODE=$((MORI_MAX_DISPATCH_TOKENS_DECODE * decode_dp_ranks * 7 / 10)) + # Update derived variable + SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD=$((MORI_MAX_DISPATCH_TOKENS_DECODE * 2)) + export SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD + echo "[DP+EP override] Decode: max-running-requests=$decode_max_running_requests, DISPATCH_TOKENS=$MORI_MAX_DISPATCH_TOKENS_DECODE, MOE_MAX_INPUT=$MORI_MOE_MAX_INPUT_TOKENS_DECODE, INTER_KERNEL_SWITCH=$SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD" +fi + +# Build the composed config strings (equivalent to the old MODEL_PREFILL_CONFIGS / MODEL_DECODE_CONFIGS) +# disable_cuda_graph (model-level) routes prefill to --disable-cuda-graph instead of --cuda-graph-bs. +if [[ "$PREFILL_DISABLE_CUDA_GRAPH" == "True" ]] || [[ "$PREFILL_DISABLE_CUDA_GRAPH" == "true" ]]; then + PREFILL_MODE_FLAGS="--mem-fraction-static ${PREFILL_MEM_FRACTION_STATIC} --max-running-requests ${prefill_max_running_requests} --chunked-prefill-size ${prefill_chunked_prefill_size} --disable-cuda-graph " +else + PREFILL_MODE_FLAGS="--mem-fraction-static ${PREFILL_MEM_FRACTION_STATIC} --max-running-requests ${prefill_max_running_requests} --chunked-prefill-size ${prefill_chunked_prefill_size} --cuda-graph-bs ${prefill_cuda_graph_bs[*]} " +fi + +if [[ "$PREFILL_DISABLE_RADIX_CACHE" == "True" ]] || [[ "$PREFILL_DISABLE_RADIX_CACHE" == "true" ]]; then + PREFILL_MODE_FLAGS="$PREFILL_MODE_FLAGS --disable-radix-cache" +fi +# Agentic runs: keep radix/prefix cache enabled by replacing --disable-radix-cache with empty. +if [[ "${IS_AGENTIC:-0}" == "1" || "${IS_AGENTIC:-}" == "true" ]]; then + PREFILL_MODE_FLAGS="${PREFILL_MODE_FLAGS//--disable-radix-cache/}" +fi +if [[ -n "$prefill_context_length" ]]; then + PREFILL_MODE_FLAGS="$PREFILL_MODE_FLAGS --context-length ${prefill_context_length}" +fi +if [[ -n "$prefill_max_total_tokens" ]]; then + PREFILL_MODE_FLAGS="$PREFILL_MODE_FLAGS --max-total-tokens ${prefill_max_total_tokens}" +fi +if [[ "$prefill_enable_two_batch_overlap" == "True" ]] || [[ "$prefill_enable_two_batch_overlap" == "true" ]]; then + PREFILL_MODE_FLAGS="$PREFILL_MODE_FLAGS --enable-two-batch-overlap" + PREFILL_SDMA_ENV="MORI_ENABLE_SDMA=true" +fi + +DECODE_MODE_FLAGS="--mem-fraction-static ${DECODE_MEM_FRACTION_STATIC} --max-running-requests ${decode_max_running_requests} --cuda-graph-bs ${decode_cuda_graph_bs[*]} " + +if [[ "$DECODE_PREFILL_ROUND_ROBIN_BALANCE" == "True" ]] || [[ "$DECODE_PREFILL_ROUND_ROBIN_BALANCE" == "true" ]]; then + DECODE_MODE_FLAGS="$DECODE_MODE_FLAGS --prefill-round-robin-balance" +fi +if [[ "$DECODE_DISAGG_ENABLE_RADIX_CACHE" == "True" ]] || [[ "$DECODE_DISAGG_ENABLE_RADIX_CACHE" == "true" ]]; then + DECODE_MODE_FLAGS="$DECODE_MODE_FLAGS --disaggregation-decode-enable-radix-cache" +fi + +if [[ "$DECODE_MTP_SIZE" -gt 0 ]]; then + MORI_MAX_DISPATCH_TOKENS_DECODE=$((MORI_MAX_DISPATCH_TOKENS_DECODE * (DECODE_MTP_SIZE + 1))) + # MORI_MOE_MAX_INPUT_TOKENS_DECODE=$((MORI_MOE_MAX_INPUT_TOKENS_DECODE * (DECODE_MTP_SIZE + 1))) +fi + +# ============================================================================= +# Cluster Topology Configuration +# ============================================================================= +IFS=',' read -ra IP_ARRAY <<< "$IPADDRS" + +# Ceiling division by GPUS_PER_NODE for nodes-per-worker +PREFILL_NODES_PER_WORKER=$(((PREFILL_TP_SIZE + 7) / GPUS_PER_NODE)) +DECODE_NODES_PER_WORKER=$(((DECODE_TP_SIZE + 7) / GPUS_PER_NODE)) +NODE_OFFSET=$((PREFILL_NODES_PER_WORKER * xP)) + +# Build prefill arguments dynamically based on xP +PREFILL_HEADNODE_URLS=() +PREFILL_ARGS="" +# Per-worker Prometheus /metrics endpoints (port 8000) for aiperf's +# --server-metrics scrape. The router on :30000 does not serve Prometheus, so +# aiperf must scrape each prefill/decode worker directly (see ENABLE_METRICS). +SERVER_METRICS_URLS=() +# Per-worker base URLs (port 8000) for direct cache flushing between +# concurrency points. The router (:30000) does not fan /flush_cache out, so +# trace_replay.sh must POST to each prefill/decode worker directly. +SERVER_FLUSH_URLS=() +for i in $(seq 0 $((xP - 1))); do + prefill_idx=$((i * PREFILL_NODES_PER_WORKER)) + PREFILL_HEADNODE_URLS[$i]="${IP_ARRAY[$prefill_idx]}:${HEADNODE_PORT}" + PREFILL_ARGS="$PREFILL_ARGS --prefill http://${IP_ARRAY[$prefill_idx]}:8000" + SERVER_METRICS_URLS+=("http://${IP_ARRAY[$prefill_idx]}:8000/metrics") + SERVER_FLUSH_URLS+=("http://${IP_ARRAY[$prefill_idx]}:8000") +done + +# Build decode arguments dynamically based on yD +DECODE_HEADNODE_URLS=() +DECODE_ARGS="" +for i in $(seq 0 $((yD - 1))); do + decode_idx=$((i * DECODE_NODES_PER_WORKER + NODE_OFFSET)) + DECODE_HEADNODE_URLS[$i]="${IP_ARRAY[$decode_idx]}:${HEADNODE_PORT}" + DECODE_ARGS="$DECODE_ARGS --decode http://${IP_ARRAY[$decode_idx]}:8000" + SERVER_METRICS_URLS+=("http://${IP_ARRAY[$decode_idx]}:8000/metrics") + SERVER_FLUSH_URLS+=("http://${IP_ARRAY[$decode_idx]}:8000") +done + +echo "Prefill worker headnode list: ${PREFILL_HEADNODE_URLS[@]}" +echo "Decode worker headnode list: ${DECODE_HEADNODE_URLS[@]}" +echo "Server metrics endpoints: ${SERVER_METRICS_URLS[@]}" +echo "Server flush endpoints: ${SERVER_FLUSH_URLS[@]}" + +# ============================================================================= +# Configuration Builder Functions +# ============================================================================= + +build_server_config() { + local mode="$1" + local model_name="$2" + local tp_size="$3" + local enable_ep="$4" + local enable_dp="$5" + local decode_mtp_size="$6" + + # Calculate EP and DP sizes based on enable flags + local ep_size=1 + local dp_size=1 + + if [[ "$enable_ep" == "true" ]]; then + ep_size=$tp_size + fi + + if [[ "$enable_dp" == "true" ]]; then + dp_size=$tp_size + fi + + # Build parallelism arguments + local parallel_args="--tp-size ${tp_size}" + + if [[ "$enable_ep" == "true" ]]; then + parallel_args="$parallel_args --ep-size ${ep_size}" + fi + + if [[ "$enable_dp" == "true" ]]; then + parallel_args="$parallel_args --dp-size ${dp_size}" + fi + + # Get model-specific configuration from YAML-loaded variables + local base_config="$MODEL_BASE_FLAGS" + local mtp_config="" + local dp_config="" + local ep_config="" + local specific_config="" + + # MTP config (only if MTP is enabled and mode is decode) + if [ "$decode_mtp_size" -gt 0 ]; then + mtp_config="${MODEL_MTP_FLAGS} --speculative-num-steps ${decode_mtp_size} --speculative-num-draft-tokens $((decode_mtp_size + 1))" + fi + + # DP config (only if DP is enabled) + if [[ "$enable_dp" == "true" ]]; then + dp_config="$MODEL_DP_FLAGS" + fi + + # EP config (only if EP is enabled): a2a backend, deepep mode, ep-dispatch algo. + # With ep=1 (EP disabled) these are dropped, so the MoE runs tensor-parallel (TP) + # instead of expert-parallel — even when dp-attention is on. + if [[ "$enable_ep" == "true" ]]; then + ep_config="$MODEL_EP_FLAGS" + fi + + # Mode-specific config + if [[ "$mode" == "prefill" ]]; then + specific_config="$PREFILL_MODE_FLAGS" + elif [[ "$mode" == "decode" ]]; then + specific_config="$DECODE_MODE_FLAGS" + fi + + # Combine: parallel args + base config + ep config + mtp config (decode only) + dp config + specific config + local full_config="$parallel_args" + if [[ -n "$base_config" ]]; then + full_config="$full_config $base_config" + fi + if [[ -n "$ep_config" ]]; then + full_config="$full_config $ep_config" + fi + if [[ -n "$mtp_config" ]] && [[ "$mode" == "decode" ]]; then + full_config="$full_config $mtp_config" + fi + if [[ -n "$dp_config" ]]; then + full_config="$full_config $dp_config" + fi + if [[ -n "$specific_config" ]]; then + full_config="$full_config $specific_config" + fi + + echo "$full_config" +} + +# Build complete server configurations +PREFILL_SERVER_CONFIG=$(build_server_config "prefill" "$MODEL_NAME" "$PREFILL_TP_SIZE" "$PREFILL_ENABLE_EP" "$PREFILL_ENABLE_DP" "$DECODE_MTP_SIZE") +DECODE_SERVER_CONFIG=$(build_server_config "decode" "$MODEL_NAME" "$DECODE_TP_SIZE" "$DECODE_ENABLE_EP" "$DECODE_ENABLE_DP" "$DECODE_MTP_SIZE") + +# Expose Prometheus /metrics on the servers when requested (ENABLE_METRICS=1). +if [[ "${ENABLE_METRICS:-0}" == "1" ]]; then + [[ "$PREFILL_SERVER_CONFIG" != *"--enable-metrics"* ]] && PREFILL_SERVER_CONFIG="$PREFILL_SERVER_CONFIG --enable-metrics" + [[ "$DECODE_SERVER_CONFIG" != *"--enable-metrics"* ]] && DECODE_SERVER_CONFIG="$DECODE_SERVER_CONFIG --enable-metrics" +fi + +if [[ -n "$MODEL_NAME" ]]; then + echo "Using model-specific configuration for: $MODEL_NAME" +fi + +# ============================================================================= +# Optional KV cache offloading (HiCache) — enabled when +# KV_OFFLOADING != none AND KV_OFFLOAD_BACKEND == hicache. +# HiCache extends RadixAttention, so radix cache MUST stay on (drop +# --disable-radix-cache). The --hicache-* flags are appended to BOTH the +# prefill and decode server configs. +# ============================================================================= +KV_OFFLOADING="${KV_OFFLOADING:-none}" +KV_OFFLOAD_BACKEND="${KV_OFFLOAD_BACKEND:-}" +if [[ "$KV_OFFLOADING" != "none" && "$KV_OFFLOAD_BACKEND" == "hicache" ]]; then + HICACHE_TOTAL_CPU_DRAM_GB="${HICACHE_TOTAL_CPU_DRAM_GB:-2000}" + HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-1}" + HICACHE_PAGE_SIZE="${HICACHE_PAGE_SIZE:-1}" + HICACHE_PREFETCH_POLICY="${HICACHE_PREFETCH_POLICY:-wait_complete}" + + # Optional L3 storage tier behind the CPU-DRAM (L2) cache. + # "" -> CPU DRAM only (default) + # "mooncake"-> Mooncake distributed KV store (needs a mooncake_master) + HICACHE_STORAGE_BACKEND="${HICACHE_STORAGE_BACKEND:-}" + + # Layout / IO backend / write policy are backend-specific: + # mooncake L3: page_first_direct + the "direct" IO backend (the Mooncake + # store maps a page-contiguous segment for RDMA/zero-copy). This layout + # asserts host_pool > device_pool, so it needs a large CPU-DRAM budget. + # L2-only (CPU DRAM): layer_first + the "kernel" IO backend. layer_first + # has no host>device constraint (the "direct" IO backend REQUIRES a + # page_first layout, so it cannot be paired with layer_first). + if [[ "$HICACHE_STORAGE_BACKEND" == "mooncake" ]]; then + HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first}" + HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" + HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" + else + HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first_direct}" + HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" + HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" + fi + + # Mooncake master/connection settings (used only when storage=mooncake). + # The master runs once on node 0; every prefill/decode server connects to + # it via NODE0_ADDR so it is reachable across nodes. + MC_MASTER_PORT="${MC_MASTER_PORT:-50061}" + MC_METADATA_PORT="${MC_METADATA_PORT:-8080}" + MC_METRICS_PORT="${MC_METRICS_PORT:-9003}" + MC_MASTER_THREADS="${MC_MASTER_THREADS:-64}" + MC_EVICTION_HIGH_WATERMARK="${MC_EVICTION_HIGH_WATERMARK:-0.95}" + MC_PROTOCOL="${MC_PROTOCOL:-tcp}" + MC_GLOBAL_SEG="${MC_GLOBAL_SEG:-64gb}" + MC_DEVICE="${MC_DEVICE:-$IBDEVICES}" + MC_MASTER_ADDR="${MC_MASTER_ADDR:-${NODE0_ADDR}:${MC_MASTER_PORT}}" + MC_METADATA_SERVER="${MC_METADATA_SERVER:-http://${NODE0_ADDR}:${MC_METADATA_PORT}/metadata}" + + # Emit the --hicache-storage-backend flags (empty unless mooncake). The + # extra-config JSON is single-quoted so it survives the later `eval` of the + # launch command as a single argument. + build_storage_flags() { + [[ "$HICACHE_STORAGE_BACKEND" != "mooncake" ]] && return 0 + local extra="{\"master_server_address\": \"${MC_MASTER_ADDR}\", \"protocol\": \"${MC_PROTOCOL}\", \"device_name\": \"${MC_DEVICE}\", \"local_hostname\": \"${host_ip}\", \"global_segment_size\": \"${MC_GLOBAL_SEG}\", \"metadata_server\": \"${MC_METADATA_SERVER}\", \"check_server\": false}" + echo "--hicache-storage-backend mooncake --hicache-storage-backend-extra-config '${extra}' --enable-metrics --enable-cache-report" + } + + # HiCache capacity via --hicache-ratio (scales with GPU KV pool). + HICACHE_RATIO="${HICACHE_RATIO:-16}" + + build_hicache_flags() { + echo "--page-size ${HICACHE_PAGE_SIZE} --enable-hierarchical-cache --hicache-ratio ${HICACHE_RATIO} --hicache-io-backend ${HICACHE_IO_BACKEND} --hicache-mem-layout ${HICACHE_MEM_LAYOUT} --hicache-write-policy ${HICACHE_WRITE_POLICY} --hicache-storage-prefetch-policy ${HICACHE_PREFETCH_POLICY} $(build_storage_flags)" + } + + # HiCache requires RadixAttention; strip any --disable-radix-cache. + PREFILL_SERVER_CONFIG="${PREFILL_SERVER_CONFIG//--disable-radix-cache/}" + DECODE_SERVER_CONFIG="${DECODE_SERVER_CONFIG//--disable-radix-cache/}" + + # Prefill always gets HiCache. + PREFILL_SERVER_CONFIG="$PREFILL_SERVER_CONFIG $(build_hicache_flags "$PREFILL_TP_SIZE")" + + + DECODE_SERVER_CONFIG="$DECODE_SERVER_CONFIG --page-size ${HICACHE_PAGE_SIZE}" + echo "[HiCache] KV_OFFLOADING=${KV_OFFLOADING} backend=${KV_OFFLOAD_BACKEND} applied to prefill only; decode mirrors --page-size ${HICACHE_PAGE_SIZE} for transfer compatibility (chunk cache under the mori transfer backend)" + echo "[HiCache] params: io_backend=${HICACHE_IO_BACKEND}, mem_layout=${HICACHE_MEM_LAYOUT}, page_size=${HICACHE_PAGE_SIZE}, write_policy=${HICACHE_WRITE_POLICY}, prefetch_policy=${HICACHE_PREFETCH_POLICY}, storage_backend=${HICACHE_STORAGE_BACKEND:-none}" + if [[ "$HICACHE_STORAGE_BACKEND" == "mooncake" ]]; then + echo "[HiCache] Mooncake store: master=${MC_MASTER_ADDR} metadata=${MC_METADATA_SERVER} protocol=${MC_PROTOCOL} device=${MC_DEVICE} segment=${MC_GLOBAL_SEG} threads=${MC_MASTER_THREADS} eviction_watermark=${MC_EVICTION_HIGH_WATERMARK}" + fi +else + echo "[HiCache] KV_OFFLOADING=${KV_OFFLOADING} backend=${KV_OFFLOAD_BACKEND:-none} (HiCache disabled)" +fi + +if [[ "${EVAL_ONLY:-false}" == "true" ]] || [[ "${RUN_EVAL:-false}" == "true" ]]; then + PREFILL_SERVER_CONFIG=$(echo "$PREFILL_SERVER_CONFIG" | sed 's/--ep-dispatch-algorithm fake//g') + DECODE_SERVER_CONFIG=$(echo "$DECODE_SERVER_CONFIG" | sed 's/--ep-dispatch-algorithm fake//g') + unset MORI_MOE_MAX_INPUT_TOKENS_PREFILL + unset MORI_MOE_MAX_INPUT_TOKENS_DECODE +fi + +# ============================================================================= +# Container Synchronization +# ============================================================================= + +echo "Waiting at the container creation barrier on $host_name" +python3 $SGLANG_WS_PATH/sync_agentic.py barrier \ + --local-ip ${host_ip} \ + --local-port 5000 \ + --enable-port \ + --node-ips ${IPADDRS} \ + --node-ports 5000 \ + --wait-for-all-ports \ + --timeout 300 + + +# ============================================================================= +# Node Role Assignment and Server Launch +# ============================================================================= + +if [ "$NODE_RANK" -eq 0 ]; then + echo "NODE INFO =======================================" + echo "================================================" + echo "Node List : ${SLURM_JOB_NODELIST}" + echo "Node IPs : ${IPADDRS}" + echo "Model Name : ${MODEL_NAME:-'Not specified'}" + echo "================================================" + + echo "CLUSTER INFO ====================================" + echo "================================================" + echo "${host_name}:${host_ip} is Proxy Node and Prefill Node" + echo "Using prefill config: $PREFILL_SERVER_CONFIG" + echo "Prefill parallelism: TP=${PREFILL_TP_SIZE}, EP enabled: ${PREFILL_ENABLE_EP}, DP enabled: ${PREFILL_ENABLE_DP}, MTP size=${DECODE_MTP_SIZE}" + echo "Decode parallelism: TP=${DECODE_TP_SIZE}, EP enabled: ${DECODE_ENABLE_EP}, DP enabled: ${DECODE_ENABLE_DP}, MTP size=${DECODE_MTP_SIZE}" + echo "Prefill servers ($((PREFILL_TP_SIZE/GPUS_PER_NODE)) nodes): ${PREFILL_ARGS}" + echo "Decode servers ($((DECODE_TP_SIZE/GPUS_PER_NODE)) nodes): ${DECODE_ARGS}" + echo "Prefill env: SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_MAX_DISPATCH_TOKENS_PREFILL}" + echo "Decode env: SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_MAX_DISPATCH_TOKENS_DECODE} " + echo "Decode env: SGLANG_MORI_MOE_MAX_INPUT_TOKENS=${MORI_MOE_MAX_INPUT_TOKENS_DECODE} " + + echo "================================================" + + # Dump all resolved commands to a text file for debugging / reproducibility. + CMD_DUMP="/run_logs/slurm_job-${SLURM_JOB_ID}/commands_${host_name}.txt" + dump_cmd() { echo -e "\n# ── $1 ──\n$2" >> "$CMD_DUMP"; } + echo "# Commands dump — $(date -u '+%Y-%m-%d %H:%M:%S UTC')" > "$CMD_DUMP" + echo "# Host: ${host_name} (${host_ip}) Node rank: ${NODE_RANK}" >> "$CMD_DUMP" + echo "# Model: ${MODEL_NAME} Image: ${DOCKER_IMAGE_NAME:-unknown}" >> "$CMD_DUMP" + + # Start the Mooncake store master (L3 HiCache backend) on node 0 only. + # All prefill/decode servers connect to it via NODE0_ADDR:MC_MASTER_PORT. + if [[ "${KV_OFFLOADING:-none}" != "none" && "${KV_OFFLOAD_BACKEND:-}" == "hicache" && "${HICACHE_STORAGE_BACKEND:-}" == "mooncake" ]]; then + echo "Starting Mooncake master on ${host_ip}:${MC_MASTER_PORT} (metadata :${MC_METADATA_PORT}, metrics :${MC_METRICS_PORT})" + MC_MASTER_CMD="mooncake_master \ + --enable_http_metadata_server=true \ + --http_metadata_server_host=0.0.0.0 \ + --http_metadata_server_port=${MC_METADATA_PORT} \ + --rpc_port=${MC_MASTER_PORT} \ + --rpc_thread_num=${MC_MASTER_THREADS} \ + --metrics_port=${MC_METRICS_PORT} \ + --enable_metric_reporting=true \ + --eviction_high_watermark_ratio=${MC_EVICTION_HIGH_WATERMARK}" + dump_cmd "MOONCAKE MASTER" "$MC_MASTER_CMD" + if [[ "$DRY_RUN" -eq 1 ]]; then + echo "DRY RUN: $MC_MASTER_CMD" + else + MC_MASTER_LOG="/run_logs/slurm_job-${SLURM_JOB_ID}/mooncake_master_${host_name}.log" + mooncake_master \ + --enable_http_metadata_server=true \ + --http_metadata_server_host=0.0.0.0 \ + --http_metadata_server_port="${MC_METADATA_PORT}" \ + --rpc_port="${MC_MASTER_PORT}" \ + --rpc_thread_num="${MC_MASTER_THREADS}" \ + --metrics_port="${MC_METRICS_PORT}" \ + --enable_metric_reporting=true \ + --eviction_high_watermark_ratio="${MC_EVICTION_HIGH_WATERMARK}" \ + > "${MC_MASTER_LOG}" 2>&1 & + mc_master_pid=$! + sleep 3 + # Fail loudly on a port collision. On shared nodes the Mooncake RPC + # port may already be taken by another user's master; in that case the + # metrics-port health check below can still pass against the foreign + # master while our RPC port is dead, and the prefill then hangs. + if grep -qiE "Address already in use|bind .*error" "${MC_MASTER_LOG}" 2>/dev/null; then + echo "ERROR: mooncake_master failed to bind port ${MC_MASTER_PORT} (already in use)." + echo " Set MC_MASTER_PORT/MC_METRICS_PORT to free ports and resubmit." + grep -iE "Address already in use|bind .*error" "${MC_MASTER_LOG}" | tail -3 + exit 1 + fi + for ((i=3; i<=60; i+=3)); do + if curl -sf "http://127.0.0.1:${MC_METRICS_PORT}/get_all_segments" >/dev/null 2>&1; then + echo " mooncake master OK at ${i}s" + break + fi + sleep 3 + done + fi + fi + + # start the head prefill server + PREFILL_MORI_MOE_ENV="" + set -x + if [[ -n "$MORI_MOE_MAX_INPUT_TOKENS_PREFILL" ]]; then + PREFILL_MORI_MOE_ENV="SGLANG_MORI_MOE_MAX_INPUT_TOKENS=${MORI_MOE_MAX_INPUT_TOKENS_PREFILL}" + fi + set +x + PREFILL_CMD="SGLANG_MORI_COMBINE_DTYPE=${MORI_COMBINE_DTYPE_PREFILL} ${PREFILL_SDMA_ENV} ${PREFILL_MORI_MOE_ENV} SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_PREFILL:-${MORI_MAX_DISPATCH_TOKENS_PREFILL}} MORI_IO_SQ_BACKOFF_TIMEOUT_US=${MORI_IO_SQ_BACKOFF_TIMEOUT_US} MORI_IO_QP_MAX_SEND_WR=${MORI_IO_QP_MAX_SEND_WR} ${LAUNCH_PREFIX:-} python3 -m sglang.launch_server \ + --model-path $MODEL_DIR/$MODEL_NAME \ + --disaggregation-mode prefill \ + --disaggregation-ib-device ${IBDEVICES} \ + --host 0.0.0.0 \ + --port 8000 \ + --trust-remote-code \ + --log-level ${SGLANG_SERVER_LOG_LEVEL:-warning} \ + ${PREFILL_SERVER_CONFIG} " + + if [ "$PREFILL_NODES_PER_WORKER" -gt 1 ]; then + PREFILL_CMD="$PREFILL_CMD --dist-init-addr ${PREFILL_HEADNODE_URLS[0]} --nnodes ${PREFILL_NODES_PER_WORKER} --node-rank 0" + fi + + + dump_cmd "PREFILL (node 0)" "$PREFILL_CMD" + if [[ "$DRY_RUN" -eq 1 ]]; then + echo "DRY RUN: $PREFILL_CMD" + else + set -x + eval "$PREFILL_CMD" \ + 2>&1 | tee /run_logs/slurm_job-${SLURM_JOB_ID}/prefill_${host_name}.log & + set +x + prefill0_pid=$! + fi + + + echo "Waiting for all prefill and decode servers to be up . . ." + + + BARRIER_CMD="python3 $SGLANG_WS_PATH/sync_agentic.py barrier \ + --node-ips ${IPADDRS} \ + --node-ports 8000 \ + --wait-for-all-ports \ + --timeout 1800" + + if [[ "$DRY_RUN" -eq 1 ]]; then + echo "DRY RUN: $BARRIER_CMD" + else + eval "$BARRIER_CMD" + fi + echo "Congratulations!!! All prefill and decode servers are up . . ." + + # Router resilience: a single prefill worker doing huge long-context prefills + # (256K+ token agentic prompts in 65280-token chunks) can be slow to drain a + # concurrent burst. With defaults the circuit breaker opens after 10 failures + # and short-circuits the whole worker, so the aiperf profiling burst sees + # "No available prefill workers (all circuits open or unhealthy)" and aborts + # with 100% errors. Disable the breaker and relax health-check sensitivity so + # a busy-but-alive worker is not ejected. Override via ROUTER_RESILIENCE_FLAGS. + ROUTER_RESILIENCE_FLAGS="${ROUTER_RESILIENCE_FLAGS:---disable-circuit-breaker --health-failure-threshold 100 --health-check-timeout-secs 600 --health-check-interval-secs 30}" + + # Router scheduling policy. cache_aware prefill routing exploits HiCache/radix + # prefix reuse across the agentic trace; round_robin decode keeps the single + # decode worker fed evenly. cache_threshold / balance_*_threshold tune the + # cache_aware load-balancing (router defaults are 0.5 / 64 / 1.5). Override any + # of these via env. + ROUTER_PREFILL_POLICY="${ROUTER_PREFILL_POLICY:-cache_aware}" + ROUTER_DECODE_POLICY="${ROUTER_DECODE_POLICY:-round_robin}" + ROUTER_CACHE_THRESHOLD="${ROUTER_CACHE_THRESHOLD:-0.3}" + ROUTER_BALANCE_ABS_THRESHOLD="${ROUTER_BALANCE_ABS_THRESHOLD:-2}" + ROUTER_BALANCE_REL_THRESHOLD="${ROUTER_BALANCE_REL_THRESHOLD:-1.1}" + ROUTER_POLICY_FLAGS="${ROUTER_POLICY_FLAGS:---policy ${ROUTER_PREFILL_POLICY} --prefill-policy ${ROUTER_PREFILL_POLICY} --decode-policy ${ROUTER_DECODE_POLICY} --cache-threshold ${ROUTER_CACHE_THRESHOLD} --balance-abs-threshold ${ROUTER_BALANCE_ABS_THRESHOLD} --balance-rel-threshold ${ROUTER_BALANCE_REL_THRESHOLD}}" + + ROUTER_CMD="python -m sglang_router.launch_router \ + --pd-disaggregation \ + --port 30000 \ + ${ROUTER_POLICY_FLAGS} \ + ${ROUTER_RESILIENCE_FLAGS} \ + ${PREFILL_ARGS} \ + ${DECODE_ARGS}" + + + dump_cmd "ROUTER" "$ROUTER_CMD" + if [[ "$DRY_RUN" -eq 1 ]]; then + echo "DRY RUN: $ROUTER_CMD" + else + ROUTER_LOG_FILE="/run_logs/slurm_job-${SLURM_JOB_ID}/router_${host_name}.log" + set -x + if [[ "${SGLANG_ROUTER_STDOUT_LOGS:-0}" == "1" ]]; then + eval "$ROUTER_CMD" 2>&1 | tee "$ROUTER_LOG_FILE" & + else + eval "$ROUTER_CMD" >"$ROUTER_LOG_FILE" 2>&1 & + fi + set +x + proxy_pid=$! + + # Wait for router to be ready via health endpoint + HEALTH_BARRIER_CMD="python3 $SGLANG_WS_PATH/sync_agentic.py barrier \ + --node-ips ${NODE0_ADDR} \ + --node-ports 30000 \ + --wait-for-all-health \ + --health-endpoint /readiness \ + --timeout 1800" + + if [[ "$DRY_RUN" -eq 1 ]]; then + echo "DRY RUN: $HEALTH_BARRIER_CMD" + else + eval "$HEALTH_BARRIER_CMD" + fi + + echo "Router is ready for benchmarking" + fi + + + echo "Ready for benchmarking on ${host_name}:${host_ip}" + + echo "Benchmarking on ${host_name}:${host_ip}" + cd $SGLANG_WS_PATH + + # Export IS_MTP based on whether MTP is enabled + if [ "$DECODE_MTP_SIZE" -gt 0 ]; then + export IS_MTP=true + else + export IS_MTP=false + fi + + # Select the benchmark runner. + # IS_AGENTIC=1/true → agentic trace replay (trace_replay.sh) + # IS_AGENTIC unset/0 → fixed-seq-len throughput benchmark (bench.sh) + if [[ "${IS_AGENTIC:-0}" == "1" || "${IS_AGENTIC:-}" == "true" ]]; then + # Point aiperf's server-metrics scrape at the per-worker Prometheus + # /metrics endpoints. The router (:30000) that aiperf auto-detects from + # --url does not expose Prometheus, so without this the scrape finds no + # reachable endpoint and all server-side cache/KV fields come out null. + # Only set it when the workers were actually started with --enable-metrics. + if [[ "${ENABLE_METRICS:-0}" == "1" && "${#SERVER_METRICS_URLS[@]}" -gt 0 ]]; then + AIPERF_SERVER_METRICS_URLS=$(IFS=,; echo "${SERVER_METRICS_URLS[*]}") + export AIPERF_SERVER_METRICS_URLS + echo "AIPERF_SERVER_METRICS_URLS=${AIPERF_SERVER_METRICS_URLS}" + fi + # Per-worker base URLs for cache flushing between concurrency points. + # trace_replay.sh consults these when CLEAR_CACHE_BETWEEN_CONC=1. + if [[ "${#SERVER_FLUSH_URLS[@]}" -gt 0 ]]; then + SERVER_FLUSH_URLS_CSV=$(IFS=,; echo "${SERVER_FLUSH_URLS[*]}") + export SERVER_FLUSH_URLS_CSV + echo "SERVER_FLUSH_URLS_CSV=${SERVER_FLUSH_URLS_CSV}" + fi + # trace_replay.sh signature: model_path model_name concurrency_list log_path + BENCH_CMD="bash $SGLANG_WS_PATH/trace_replay.sh \ + $MODEL_DIR $MODEL_NAME $BENCH_MAX_CONCURRENCY /run_logs/slurm_job-${SLURM_JOB_ID}" + echo "Benchmark runner: trace_replay.sh (agentic, KV_OFFLOADING=${KV_OFFLOADING:-none}, backend=${KV_OFFLOAD_BACKEND:-none}, CONC=${BENCH_MAX_CONCURRENCY})" + else + # bench.sh signature: + # n_prefill n_decode prefill_gpus decode_gpus model_dir model_name log_path + # isl osl concurrency_list req_rate random_range_ratio num_prompts_multiplier + BENCH_CMD="bash $SGLANG_WS_PATH/bench.sh ${xP} ${yD} $((PREFILL_TP_SIZE*xP)) $((DECODE_TP_SIZE*yD)) \ + $MODEL_DIR $MODEL_NAME /run_logs/slurm_job-${SLURM_JOB_ID} ${BENCH_INPUT_LEN} \ + ${BENCH_OUTPUT_LEN} \"${BENCH_MAX_CONCURRENCY}\" ${BENCH_REQUEST_RATE} \ + ${BENCH_RANDOM_RANGE_RATIO} ${BENCH_NUM_PROMPTS_MULTIPLIER}" + echo "Benchmark runner: bench.sh (fixed-seq-len)" + fi + + if [[ "${EVAL_ONLY:-false}" == "true" ]]; then + echo "EVAL_ONLY mode: skipping throughput benchmark" + elif [[ "$DRY_RUN" -eq 1 ]]; then + echo "DRY RUN: $BENCH_CMD" + else + set -x + eval "$BENCH_CMD" + set +x + fi + + # Run evaluation if requested (before killing router) + if [[ "${RUN_EVAL:-false}" == "true" ]]; then + echo "Running lm-eval evaluation on Node 0..." + + # Health check: verify the router is still serving before running eval. + # The throughput benchmark may have crashed/exhausted decode workers. + EVAL_HEALTH_OK=false + for _attempt in 1 2 3; do + if curl -sf --max-time 10 "http://0.0.0.0:30000/readiness" >/dev/null 2>&1; then + EVAL_HEALTH_OK=true + break + fi + echo "Eval health check attempt $_attempt failed, retrying in 10s..." + sleep 10 + done + + if [[ "$EVAL_HEALTH_OK" != "true" ]]; then + echo "WARNING: Router health check failed after 3 attempts. Skipping eval." + else + # Must run from repo root so utils/evals/${task}.yaml resolves + pushd /workspace + + # Source eval functions from benchmark_lib.sh + source /workspace/benchmarks/benchmark_lib.sh + + # Use EVAL_CONC from workflow if set, otherwise fall back to max of conc list + if [[ -n "${EVAL_CONC:-}" ]]; then + export EVAL_CONCURRENT_REQUESTS="${EVAL_CONC}" + else + export EVAL_CONCURRENT_REQUESTS=$(echo "$BENCH_MAX_CONCURRENCY" | tr 'x' '\n' | sort -n | tail -1) + fi + + # Override eval context length with model's configured context_length + if [[ -n "$prefill_context_length" ]]; then + export EVAL_MAX_MODEL_LEN="$prefill_context_length" + fi + + if [[ "$DRY_RUN" -eq 1 ]]; then + echo "DRY RUN: run_eval --framework lm-eval --port 30000 (conc=${EVAL_CONCURRENT_REQUESTS}, ctx=${EVAL_MAX_MODEL_LEN:-auto})" + else + # Run lm-eval against the router on port 30000 + run_eval --framework lm-eval --port 30000 + eval_rc=$? + + if [[ $eval_rc -ne 0 ]]; then + echo "ERROR: run_eval exited rc=$eval_rc; skipping metadata write and eval artifact staging" >&2 + EVAL_FAILED=1 + else + # Set metadata env vars for append_lm_eval_summary + export TP="${PREFILL_TP_SIZE}" + export CONC="${EVAL_CONCURRENT_REQUESTS}" + export EP_SIZE=1 + [[ "${PREFILL_ENABLE_EP}" == "true" ]] && EP_SIZE="${PREFILL_TP_SIZE}" + export PREFILL_TP="${PREFILL_TP_SIZE}" + export PREFILL_EP=1 + [[ "${PREFILL_ENABLE_EP}" == "true" ]] && PREFILL_EP="${PREFILL_TP_SIZE}" + export PREFILL_NUM_WORKERS="${xP}" + export DECODE_TP="${DECODE_TP_SIZE}" + export DECODE_EP=1 + [[ "${DECODE_ENABLE_EP}" == "true" ]] && DECODE_EP="${DECODE_TP_SIZE}" + export DECODE_NUM_WORKERS="${yD}" + export DP_ATTENTION="${PREFILL_ENABLE_DP}" + export PREFILL_DP_ATTENTION="${PREFILL_ENABLE_DP}" + export DECODE_DP_ATTENTION="${DECODE_ENABLE_DP}" + export ISL="${BENCH_INPUT_LEN}" + export OSL="${BENCH_OUTPUT_LEN}" + # IS_MULTINODE, FRAMEWORK, PRECISION, MODEL_PREFIX, RUNNER_TYPE, + # RESULT_FILENAME are already set via Docker -e flags from job.slurm + + append_lm_eval_summary + # Files (meta_env.json, results*.json, sample*.jsonl) are now in /workspace + + # Copy eval artifacts to run_logs for NFS extraction by runner + EVAL_COPY_DIR="/run_logs/slurm_job-${SLURM_JOB_ID}/eval_results" + mkdir -p "$EVAL_COPY_DIR" + for f in meta_env.json; do + [ -e "/workspace/$f" ] && cp -f "/workspace/$f" "$EVAL_COPY_DIR/" + done + # Use find for glob patterns to avoid "no match" errors + find /workspace -maxdepth 1 -name 'results*.json' -exec cp -f {} "$EVAL_COPY_DIR/" \; + find /workspace -maxdepth 1 -name 'sample*.jsonl' -exec cp -f {} "$EVAL_COPY_DIR/" \; + + echo "Eval completed. Artifacts staged in $EVAL_COPY_DIR" + fi + fi + + popd + fi + fi + + # Copy benchmark results to BENCHMARK_LOGS_DIR (mounted from host) + LOGS_OUTPUT="${BENCHMARK_LOGS_DIR:-/run_logs}/logs" + mkdir -p "$LOGS_OUTPUT" + + if [[ "$DRY_RUN" -eq 0 ]]; then + cp -r /run_logs/slurm_job-${SLURM_JOB_ID} "$LOGS_OUTPUT/" + echo "Copied results to $LOGS_OUTPUT/slurm_job-${SLURM_JOB_ID}" + fi + + echo "Killing the proxy server and prefill server" + + if [[ "$DRY_RUN" -eq 0 ]]; then + kill $proxy_pid + kill $prefill0_pid + fi + + if [[ "${EVAL_FAILED:-0}" -eq 1 ]]; then + echo "ERROR: eval failed; exiting node-0 with rc=1" + exit 1 + fi + +elif [ "$NODE_RANK" -gt 0 ] && [ "$NODE_RANK" -lt "$NODE_OFFSET" ]; then + echo "${host_name}:${host_ip} is Prefill Node (Model: ${MODEL_NAME:-'default'})" + echo "Using prefill config: $PREFILL_SERVER_CONFIG" + echo "Prefill parallelism: TP=${PREFILL_TP_SIZE}, EP enabled: ${PREFILL_ENABLE_EP}, DP enabled: ${PREFILL_ENABLE_DP}" + + CMD_DUMP="/run_logs/slurm_job-${SLURM_JOB_ID}/commands_${host_name}.txt" + dump_cmd() { echo -e "\n# ── $1 ──\n$2" >> "$CMD_DUMP"; } + echo "# Commands dump — $(date -u '+%Y-%m-%d %H:%M:%S UTC')" > "$CMD_DUMP" + echo "# Host: ${host_name} (${host_ip}) Node rank: ${NODE_RANK}" >> "$CMD_DUMP" + + PREFILL_MORI_MOE_ENV="" + set -x + if [[ -n "$MORI_MOE_MAX_INPUT_TOKENS_PREFILL" ]]; then + PREFILL_MORI_MOE_ENV="SGLANG_MORI_MOE_MAX_INPUT_TOKENS=${MORI_MOE_MAX_INPUT_TOKENS_PREFILL}" + fi + set +x + PREFILL_CMD="SGLANG_MORI_COMBINE_DTYPE=${MORI_COMBINE_DTYPE_PREFILL} ${PREFILL_SDMA_ENV} ${PREFILL_MORI_MOE_ENV} SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_PREFILL:-${MORI_MAX_DISPATCH_TOKENS_PREFILL}} MORI_IO_SQ_BACKOFF_TIMEOUT_US=${MORI_IO_SQ_BACKOFF_TIMEOUT_US} MORI_IO_QP_MAX_SEND_WR=${MORI_IO_QP_MAX_SEND_WR} ${LAUNCH_PREFIX:-} python3 -m sglang.launch_server \ + --model-path $MODEL_DIR/${MODEL_NAME} \ + --disaggregation-mode prefill \ + --disaggregation-ib-device ${IBDEVICES} \ + --host 0.0.0.0 \ + --port 8000 \ + --trust-remote-code \ + --log-level ${SGLANG_SERVER_LOG_LEVEL:-warning} \ + ${PREFILL_SERVER_CONFIG} " + + if [ "$PREFILL_NODES_PER_WORKER" -gt 1 ]; then + rank=$((NODE_RANK % PREFILL_NODES_PER_WORKER)) + prefill_idx=$((NODE_RANK / PREFILL_NODES_PER_WORKER)) + PREFILL_CMD="$PREFILL_CMD --dist-init-addr ${PREFILL_HEADNODE_URLS[$prefill_idx]} --nnodes ${PREFILL_NODES_PER_WORKER} --node-rank $rank" + fi + + dump_cmd "PREFILL (rank ${NODE_RANK})" "$PREFILL_CMD" + if [[ "$DRY_RUN" -eq 1 ]]; then + echo "DRY RUN: $PREFILL_CMD" + else + set -x + eval "$PREFILL_CMD" \ + 2>&1 | tee /run_logs/slurm_job-${SLURM_JOB_ID}/prefill_${host_name}.log & + set +x + prefill_pid=$! + fi + + echo "Waiting for proxy server to be up..." + BARRIER_CMD="python3 $SGLANG_WS_PATH/sync_agentic.py barrier \ + --node-ips ${NODE0_ADDR} \ + --node-ports 30000 \ + --wait-for-all-ports \ + --timeout 1800" + + if [[ "$DRY_RUN" -eq 1 ]]; then + echo "DRY RUN: $BARRIER_CMD" + else + eval "$BARRIER_CMD" + fi + + echo "Waiting until proxy server closes..." + WAIT_CMD="python3 $SGLANG_WS_PATH/sync_agentic.py wait \ + --remote-ip ${NODE0_ADDR} \ + --remote-port 30000" + + if [[ "$DRY_RUN" -eq 1 ]]; then + echo "DRY RUN: $WAIT_CMD" + else + eval "$WAIT_CMD" + fi + + echo "Killing the rank $NODE_RANK prefill server" + + if [[ "$DRY_RUN" -eq 0 ]]; then + kill $prefill_pid + fi + +else + RANK=$((NODE_RANK - xP * PREFILL_NODES_PER_WORKER)) + echo "${host_name}:${host_ip} is Decode Node (Model: ${MODEL_NAME:-'default'})" + echo "Using decode config: $DECODE_SERVER_CONFIG" + echo "Decode node rank: $RANK" + echo "Decode parallelism: TP=${DECODE_TP_SIZE}, EP enabled: ${DECODE_ENABLE_EP}, DP enabled: ${DECODE_ENABLE_DP}" + + CMD_DUMP="/run_logs/slurm_job-${SLURM_JOB_ID}/commands_${host_name}.txt" + dump_cmd() { echo -e "\n# ── $1 ──\n$2" >> "$CMD_DUMP"; } + echo "# Commands dump — $(date -u '+%Y-%m-%d %H:%M:%S UTC')" > "$CMD_DUMP" + echo "# Host: ${host_name} (${host_ip}) Node rank: ${NODE_RANK}" >> "$CMD_DUMP" + + DECODE_MORI_MOE_ENV="" + set -x + if [[ -n "$MORI_MOE_MAX_INPUT_TOKENS_DECODE" ]]; then + DECODE_MORI_MOE_ENV="SGLANG_MORI_MOE_MAX_INPUT_TOKENS=${MORI_MOE_MAX_INPUT_TOKENS_DECODE}" + fi + set +x + DECODE_CMD="SGLANG_MORI_COMBINE_DTYPE=${MORI_COMBINE_DTYPE_DECODE} ${DECODE_MORI_MOE_ENV} SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_DECODE:-${MORI_MAX_DISPATCH_TOKENS_DECODE}} MORI_IO_SQ_BACKOFF_TIMEOUT_US=${MORI_IO_SQ_BACKOFF_TIMEOUT_US} MORI_IO_QP_MAX_SEND_WR=${MORI_IO_QP_MAX_SEND_WR} ${LAUNCH_PREFIX:-} python3 -m sglang.launch_server \ + --model-path ${MODEL_DIR}/${MODEL_NAME} \ + --disaggregation-mode decode \ + --disaggregation-ib-device ${IBDEVICES} \ + --host 0.0.0.0 \ + --port 8000 \ + --trust-remote-code \ + --log-level ${SGLANG_SERVER_LOG_LEVEL:-warning} \ + ${DECODE_SERVER_CONFIG} " + + if [ "$DECODE_NODES_PER_WORKER" -gt 1 ]; then + rank=$((RANK % DECODE_NODES_PER_WORKER)) + decode_idx=$((RANK / DECODE_NODES_PER_WORKER)) + DECODE_CMD="$DECODE_CMD --dist-init-addr ${DECODE_HEADNODE_URLS[$decode_idx]} --nnodes ${DECODE_NODES_PER_WORKER} --node-rank $rank" + fi + + dump_cmd "DECODE (rank ${NODE_RANK})" "$DECODE_CMD" + if [[ "$DRY_RUN" -eq 1 ]]; then + echo "DRY RUN: $DECODE_CMD" + else + set -x + eval "$DECODE_CMD" \ + 2>&1 | tee /run_logs/slurm_job-${SLURM_JOB_ID}/decode_${host_name}.log & + + set +x + decode_pid=$! + fi + + + echo "Waiting for proxy server to be up..." + BARRIER_CMD="python3 $SGLANG_WS_PATH/sync_agentic.py barrier \ + --node-ips ${NODE0_ADDR} \ + --node-ports 30000 \ + --wait-for-all-ports \ + --timeout 1800" + + if [[ "$DRY_RUN" -eq 1 ]]; then + echo "DRY RUN: $BARRIER_CMD" + else + eval "$BARRIER_CMD" + fi + + + echo "Waiting until proxy server closes..." + WAIT_CMD="python3 $SGLANG_WS_PATH/sync_agentic.py wait \ + --remote-ip ${NODE0_ADDR} \ + --remote-port 30000" + + if [[ "$DRY_RUN" -eq 1 ]]; then + echo "DRY RUN: $WAIT_CMD" + else + eval "$WAIT_CMD" + fi + + echo "Killing the rank $RANK decode server" + if [[ "$DRY_RUN" -eq 0 ]]; then + kill $decode_pid + fi + +fi + +echo "Script completed successfully" +exit 0 diff --git a/benchmarks/multi_node/amd_utils/setup_deps_agentic.sh b/benchmarks/multi_node/amd_utils/setup_deps_agentic.sh new file mode 100755 index 0000000000..9f10958de1 --- /dev/null +++ b/benchmarks/multi_node/amd_utils/setup_deps_agentic.sh @@ -0,0 +1,174 @@ +#!/bin/bash +# ============================================================================= +# setup_deps.sh — Install missing disagg dependencies at container start. +# +# Dispatched by $ENGINE (set by server.sh dispatcher): +# vllm-disagg -> recipe deps + amd-quark + UCX/RIXL path exports +# (base image: vllm/vllm-openai-rocm:nightly) +# sglang-disagg -> SGLang aiter gluon patch + per-model installs +# (base image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-*) +# +# Sourced by server_vllm.sh and server_sglang.sh so PATH / LD_LIBRARY_PATH +# exports persist. Each patch is idempotent: skipped if already applied. +# +# Build steps run in subshells to avoid CWD pollution between installers. +# ============================================================================= + +ROCM_PATH="${ROCM_PATH:-/opt/rocm}" +UCX_HOME="${UCX_HOME:-/usr/local/ucx}" +RIXL_HOME="${RIXL_HOME:-/usr/local/rixl}" + +_SETUP_START=$(date +%s) +_SETUP_INSTALLED=() + +git_clone_retry() { + local url="$1" dest="$2" max_tries=3 try=1 + while (( try <= max_tries )); do + if git clone --quiet "$url" "$dest" 2>/dev/null; then return 0; fi + echo "[SETUP] git clone attempt $try/$max_tries failed for $url, retrying in 10s..." + rm -rf "$dest" + sleep 10 + (( try++ )) + done + echo "[SETUP] git clone failed after $max_tries attempts: $url" + return 1 +} + +# --------------------------------------------------------------------------- +# 5. Container RDMA/net tools +# - ibv_devinfo comes from ibverbs-utils +# - iproute2 provides the `ip` command +# Used for in-container NIC/RDMA validation and routing checks. +# --------------------------------------------------------------------------- +install_recipe_deps() { + if command -v ibv_devinfo >/dev/null 2>&1 && command -v ip >/dev/null 2>&1; then + echo "[SETUP] Container RDMA/net tools already present" + return 0 + fi + + echo "[SETUP] Installing ibv_devinfo + iproute2 in container..." + apt-get update -q -y && apt-get install -q -y \ + ibverbs-utils iproute2 \ + && rm -rf /var/lib/apt/lists/* + + if ! command -v ibv_devinfo >/dev/null 2>&1 || ! command -v ip >/dev/null 2>&1; then + echo "[SETUP] ERROR: Failed to install ibv_devinfo/iproute2"; exit 1 + fi + _SETUP_INSTALLED+=("ibverbs-utils+iproute2") +} + +# --------------------------------------------------------------------------- +# 6b. amd-quark (MXFP4 quantization support for Kimi-K2.5-MXFP4 and similar) +# Required due to ROCm vLLM missing the quark dependency: +# https://github.com/vllm-project/vllm/issues/35633 +# --------------------------------------------------------------------------- +install_amd_quark() { + if python3 -c "import quark" 2>/dev/null; then + echo "[SETUP] amd-quark already present" + return 0 + fi + + echo "[SETUP] Installing amd-quark for MXFP4 quantization support..." + pip install --quiet amd-quark + + if ! python3 -c "import quark" 2>/dev/null; then + echo "[SETUP] WARN: amd-quark install failed (non-fatal for non-MXFP4 models)" + return 0 + fi + _SETUP_INSTALLED+=("amd-quark") +} + +# --------------------------------------------------------------------------- +# SGLang: prevent TP-rank collective desync deadlock in disaggregation prefill. +# +# resolve_waiting_queue_bootstrap() runs poll_and_all_reduce_attn_cp_tp_group() +# over `candidates`. The upstream candidate set (all non-aborted waiting reqs) +# can differ across TP ranks, so some ranks enter the all_reduce while others +# skip it -> hang. Narrow candidates to optimistic (pending_bootstrap) requests, +# which is consistent across ranks and is the only set finalize_bootstrap acts on. +# --------------------------------------------------------------------------- +patch_disagg_prefill_bootstrap_desync() { + python3 -c ' +import os, sys + +target = "/sgl-workspace/sglang/python/sglang/srt/disaggregation/prefill.py" +if not os.path.isfile(target): + print("[SETUP] disaggregation/prefill.py not found, skipping") + sys.exit(0) + +src = open(target).read() + +old = " candidates = [req for req in self.waiting_queue if not is_aborted(req)]" +new = ( + " candidates = [\n" + " req\n" + " for req in self.waiting_queue\n" + " if req.pending_bootstrap and not is_aborted(req)\n" + " ]" +) + +if new in src: + print("[SETUP] prefill bootstrap-desync patch already applied") + sys.exit(0) + +if old not in src: + print("[SETUP] WARN: resolve_waiting_queue_bootstrap pattern not found — sglang version may have changed") + sys.exit(0) + +open(target, "w").write(src.replace(old, new)) +print("[SETUP] Patched: disaggregation/prefill.py resolve_waiting_queue_bootstrap candidates") +' + _SETUP_INSTALLED+=("prefill-bootstrap-desync-fix") +} + +# --------------------------------------------------------------------------- +# SGLang: Install latest transformers for GLM-5 model type support. +# +# GLM-5 (zai-org/GLM-5-FP8) requires a transformers build that includes +# the glm_moe_dsa model type. The mori images do not ship it. +# Only install if GLM-5 is the active model (avoid overhead otherwise). +# --------------------------------------------------------------------------- +install_transformers_glm5() { + if [[ "$MODEL_NAME" != "GLM-5-FP8" ]]; then + return 0 + fi + + if python3 -c "from transformers import AutoConfig; AutoConfig.from_pretrained('zai-org/GLM-5-FP8', trust_remote_code=True)" 2>/dev/null; then + echo "[SETUP] transformers already supports GLM-5 model type" + return 0 + fi + + echo "[SETUP] Installing transformers with GLM-5 (glm_moe_dsa) support..." + pip install --quiet -U --no-cache-dir \ + "git+https://github.com/huggingface/transformers.git@6ed9ee36f608fd145168377345bfc4a5de12e1e2" + _SETUP_INSTALLED+=("transformers-glm5") +} + +# ============================================================================= +# Run installers (engine-gated) +# ============================================================================= + +if [[ "$ENGINE" == "vllm-disagg" ]]; then + install_recipe_deps + install_amd_quark + + # ========================================================================= + # vLLM: Export UCX/RIXL paths (persists since this file is sourced) + # ========================================================================= + export ROCM_PATH="${ROCM_PATH}" + export UCX_HOME="${UCX_HOME}" + export RIXL_HOME="${RIXL_HOME}" + export PATH="${UCX_HOME}/bin:/usr/local/bin/etcd:/root/.cargo/bin:${PATH}" + export LD_LIBRARY_PATH="${UCX_HOME}/lib:${RIXL_HOME}/lib:${RIXL_HOME}/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH:-}" +else + patch_disagg_prefill_bootstrap_desync + + install_transformers_glm5 +fi + +_SETUP_END=$(date +%s) +if [[ ${#_SETUP_INSTALLED[@]} -eq 0 ]]; then + echo "[SETUP] All dependencies already present ($(( _SETUP_END - _SETUP_START ))s wallclock)" +else + echo "[SETUP] Installed: ${_SETUP_INSTALLED[*]} in $(( _SETUP_END - _SETUP_START ))s" +fi diff --git a/benchmarks/multi_node/amd_utils/submit_agentic.sh b/benchmarks/multi_node/amd_utils/submit_agentic.sh new file mode 100755 index 0000000000..39e6c3f4a1 --- /dev/null +++ b/benchmarks/multi_node/amd_utils/submit_agentic.sh @@ -0,0 +1,246 @@ +#!/bin/bash +# +# Cluster Configuration Template for Multi-Node Disaggregated Serving +# +# This script submits a multi-node disaggregated benchmark job to SLURM. +# It must be configured for your specific cluster before use. +# +# ENGINE=sglang (default): SGLang disaggregated serving +# ENGINE=vllm: vLLM disaggregated serving +# +# Router is co-located with the first prefill node (same for both engines), +# so NUM_NODES = PREFILL_NODES + DECODE_NODES. + +usage() { + cat << 'USAGE' +Usage: + bash submit.sh \ + \ + \ + \ + \ + [NODE_LIST] + +Arguments: + PREFILL_NODES Number of prefill nodes + PREFILL_WORKERS Number of prefill workers (usually 1) + DECODE_NODES Number of decode nodes + DECODE_WORKERS Number of decode workers (usually 1) + ISL Input sequence length + OSL Output sequence length + CONCURRENCIES Concurrency levels, delimited by 'x' (e.g., "8x16x32") + REQUEST_RATE Request rate ("inf" for max throughput) + PREFILL_ENABLE_EP true/false or 1/0 (expert parallelism on prefill) + PREFILL_ENABLE_DP true/false or 1/0 (data-parallel attention on prefill) + DECODE_ENABLE_EP true/false or 1/0 (expert parallelism on decode) + DECODE_ENABLE_DP true/false or 1/0 (data-parallel attention on decode) + PREFILL_TP Tensor parallel size per prefill node + DECODE_TP Tensor parallel size per decode node + RANDOM_RANGE_RATIO Random range ratio for benchmark client + NODE_LIST Optional: comma-separated hostnames (must match NUM_NODES) + +Required environment variables: + SLURM_ACCOUNT SLURM account name + SLURM_PARTITION SLURM partition + TIME_LIMIT Job time limit (e.g., "08:00:00") + MODEL_PATH Path to model directory (e.g., /nfsdata) + MODEL_NAME Model name directory + CONTAINER_IMAGE Docker image name (e.g., vllm_disagg_pd:latest) + RUNNER_NAME Runner identifier (for job name) + +Optional environment variables: + DRY_RUN 1 = echo composed server/router launch commands instead of + running them (preview a recipe against a real allocation). +USAGE +} + +check_env() { + local name="$1" + if [[ -z "${!name:-}" ]]; then + echo "Error: ${name} not specified" >&2 + usage >&2 + exit 1 + fi +} + +check_env SLURM_ACCOUNT +check_env SLURM_PARTITION +check_env TIME_LIMIT + +check_env MODEL_PATH +check_env MODEL_NAME +check_env CONTAINER_IMAGE +check_env RUNNER_NAME +check_env FRAMEWORK + +# GPUS_PER_NODE defaults to 8 (MI355X). Set to 4 for MI325X if needed. +GPUS_PER_NODE="${GPUS_PER_NODE:-8}" + +# COMMAND_LINE ARGS +PREFILL_NODES=$1 +PREFILL_WORKERS=${2:-1} +DECODE_NODES=$3 +DECODE_WORKERS=${4:-1} +ISL=$5 +OSL=$6 +CONCURRENCIES=$7 +REQUEST_RATE=$8 +PREFILL_ENABLE_EP=${9:-true} +PREFILL_ENABLE_DP=${10:-true} +DECODE_ENABLE_EP=${11:-true} +DECODE_ENABLE_DP=${12:-true} +PREFILL_TP=${13:-8} +DECODE_TP=${14:-8} +RANDOM_RANGE_RATIO=${15:-0.8} +NODE_LIST=${16} + +NUM_NODES=$((PREFILL_NODES + DECODE_NODES)) +profiler_args="${ISL} ${OSL} ${CONCURRENCIES} ${REQUEST_RATE}" + +# Export variables for the SLURM job +export ENGINE="${FRAMEWORK:-sglang}" +export MODEL_DIR=$MODEL_PATH +export DOCKER_IMAGE_NAME=$CONTAINER_IMAGE +export PROFILER_ARGS=$profiler_args + +# Engine-specific xP/yD semantics and TP exports +if [[ "$ENGINE" == "vllm-disagg" ]]; then + export PROXY_STREAM_IDLE_TIMEOUT=${PROXY_STREAM_IDLE_TIMEOUT:-300} +fi +# xP = prefill workers, yD = decode workers (may span multiple nodes) +export xP=$PREFILL_WORKERS +export yD=$DECODE_WORKERS +export PREFILL_TP_SIZE=$(( $PREFILL_NODES * $PREFILL_TP / $PREFILL_WORKERS )) +export PREFILL_ENABLE_EP=${PREFILL_ENABLE_EP} +export PREFILL_ENABLE_DP=${PREFILL_ENABLE_DP} +export DECODE_TP_SIZE=$(( $DECODE_NODES * $DECODE_TP / $DECODE_WORKERS )) +export DECODE_ENABLE_EP=${DECODE_ENABLE_EP} +export DECODE_ENABLE_DP=${DECODE_ENABLE_DP} +export DECODE_MTP_SIZE=${DECODE_MTP_SIZE} + +export NUM_NODES=$NUM_NODES +export GPUS_PER_NODE=$GPUS_PER_NODE +export MODEL_NAME=$MODEL_NAME +export BENCH_INPUT_LEN=${ISL} +export BENCH_OUTPUT_LEN=${OSL} +export BENCH_NUM_PROMPTS_MULTIPLIER=${BENCH_NUM_PROMPTS_MULTIPLIER:-10} +export BENCH_MAX_CONCURRENCY=${CONCURRENCIES} +export BENCH_REQUEST_RATE=${REQUEST_RATE} +export BENCH_RANDOM_RANGE_RATIO=${RANDOM_RANGE_RATIO:-0.8} + +# DRY_RUN=1 makes server_sglang.sh echo the composed prefill/decode/router launch +# commands instead of executing them (useful for previewing a recipe against a real +# allocation). Threaded here → job_agentic.slurm → Docker (-e DRY_RUN) → server_sglang.sh. +# sbatch defaults to --export=ALL, so exporting it is what carries it into the job. +export DRY_RUN="${DRY_RUN:-0}" + +# Eval-related env vars (threaded from workflow → runner → here → job_agentic.slurm → Docker) +export RUN_EVAL="${RUN_EVAL:-false}" +export EVAL_ONLY="${EVAL_ONLY:-false}" +export EVAL_CONC="${EVAL_CONC:-}" +export FRAMEWORK="${FRAMEWORK:-}" +export PRECISION="${PRECISION:-}" +export MODEL_PREFIX="${MODEL_PREFIX:-}" +export RUNNER_TYPE="${RUNNER_TYPE:-}" +export RESULT_FILENAME="${RESULT_FILENAME:-}" +export SPEC_DECODING="${SPEC_DECODING:-}" +export IS_MULTINODE="${IS_MULTINODE:-false}" + +# Log directory: must be on NFS (shared filesystem) so the submit host can read SLURM output. +export BENCHMARK_LOGS_DIR="${BENCHMARK_LOGS_DIR:-$(pwd)/benchmark_logs}" +mkdir -p "$BENCHMARK_LOGS_DIR" + +# Optional: pass an explicit node list to sbatch. +NODELIST_OPT=() +if [[ -n "${NODE_LIST//[[:space:]]/}" ]]; then + IFS=',' read -r -a NODE_ARR <<< "$NODE_LIST" + if [[ "${#NODE_ARR[@]}" -ne "$NUM_NODES" ]]; then + echo "Error: NODE_LIST has ${#NODE_ARR[@]} nodes but NUM_NODES=${NUM_NODES}" >&2 + echo "Error: NODE_LIST='${NODE_LIST}'" >&2 + exit 1 + fi + NODELIST_CSV="$(IFS=,; echo "${NODE_ARR[*]}")" + NODELIST_OPT=(--nodelist "$NODELIST_CSV") +fi + +# Optional: exclude specific nodes (e.g. nodes with broken Docker sockets). +# Set SLURM_EXCLUDE_NODES env var to a comma-separated list of hostnames. +EXCLUDE_OPT=() +SLURM_EXCLUDE_NODES="${SLURM_EXCLUDE_NODES:-mia1-p01-g11,mia1-p01-g12,mia1-p01-g15}" +if [[ -n "${SLURM_EXCLUDE_NODES:-}" ]]; then + EXCLUDE_OPT=(--exclude "$SLURM_EXCLUDE_NODES") +fi + +# ============================================================================= +# Reuse existing allocation (skip sbatch) +# ============================================================================= +# When SLURM_REUSE_JOBID is set, run job_agentic.slurm directly in the current shell, +# attaching to the existing allocation. Inner `srun` calls pick up the +# allocation via SLURM_JOB_ID; SLURM_OVERLAP=1 lets them share task slots with +# the interactive shell already holding the allocation. +if [[ -n "${SLURM_REUSE_JOBID:-}" ]]; then + REUSE_JID="$SLURM_REUSE_JOBID" + echo "Reusing existing Slurm allocation ${REUSE_JID} (skipping sbatch)" >&2 + + # Resolve allocation's nodelist if not already provided. + ALLOC_NODELIST="${SLURM_JOB_NODELIST:-$(squeue -h -j "$REUSE_JID" -o '%N' 2>/dev/null)}" + if [[ -z "$ALLOC_NODELIST" ]]; then + echo "Error: could not resolve nodelist for job ${REUSE_JID}" >&2 + exit 1 + fi + ALLOC_NNODES=$(scontrol show hostnames "$ALLOC_NODELIST" | wc -l) + if [[ "$ALLOC_NNODES" -lt "$NUM_NODES" ]]; then + echo "Error: allocation ${REUSE_JID} has ${ALLOC_NNODES} nodes, need ${NUM_NODES}" >&2 + exit 1 + fi + + export SLURM_JOB_ID="$REUSE_JID" + export SLURM_JOBID="$REUSE_JID" + export SLURM_JOB_NODELIST="$ALLOC_NODELIST" + export SLURM_NODELIST="$ALLOC_NODELIST" + export SLURM_NNODES="$ALLOC_NNODES" + export SLURM_JOB_NUM_NODES="$ALLOC_NNODES" + export SLURM_NTASKS="$ALLOC_NNODES" + export SLURM_NPROCS="$ALLOC_NNODES" + export SLURM_NTASKS_PER_NODE=1 + export SLURM_TASKS_PER_NODE="1(x${ALLOC_NNODES})" + export SLURM_OVERLAP=1 + export SLURM_SUBMIT_DIR="$(pwd)" + + STDOUT_LOG="${BENCHMARK_LOGS_DIR}/slurm_job-${REUSE_JID}.out" + STDERR_LOG="${BENCHMARK_LOGS_DIR}/slurm_job-${REUSE_JID}.err" + rm -f "$STDOUT_LOG" "$STDERR_LOG" + + nohup bash "$(dirname "$0")/job_agentic.slurm" >"$STDOUT_LOG" 2>"$STDERR_LOG" & + INLINE_PID=$! + echo "$INLINE_PID" > "${BENCHMARK_LOGS_DIR}/slurm_job-${REUSE_JID}.pid" + echo "Started job_agentic.slurm (pid=${INLINE_PID}); logs: ${STDOUT_LOG}" >&2 + + echo "$REUSE_JID" + exit 0 +fi + +# Construct the sbatch command +sbatch_cmd=( + sbatch + --parsable + --exclusive + -N "$NUM_NODES" + -n "$NUM_NODES" + "${NODELIST_OPT[@]}" + "${EXCLUDE_OPT[@]}" + --time "$TIME_LIMIT" + --partition "$SLURM_PARTITION" + --account "$SLURM_ACCOUNT" + --job-name "$RUNNER_NAME" + --output "${BENCHMARK_LOGS_DIR}/slurm_job-%j.out" + --error "${BENCHMARK_LOGS_DIR}/slurm_job-%j.err" + "$(dirname "$0")/job_agentic.slurm" +) + +JOB_ID=$("${sbatch_cmd[@]}") +if [[ $? -ne 0 ]]; then + echo "Error: Failed to submit job with sbatch" >&2 + exit 1 +fi +echo "$JOB_ID" diff --git a/benchmarks/multi_node/amd_utils/sync_agentic.py b/benchmarks/multi_node/amd_utils/sync_agentic.py new file mode 100755 index 0000000000..3678e76148 --- /dev/null +++ b/benchmarks/multi_node/amd_utils/sync_agentic.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +""" +Multi-node synchronization utilities for disaggregated inference. + +Subcommands: + barrier - Wait until all specified nodes have opened their ports (TCP barrier) + Optionally wait for HTTP health endpoints to return 200 + wait - Block until a remote port closes (shutdown coordination) +""" + +import socket +import time +import threading +import argparse +import sys +import urllib.request +import urllib.error + + +def is_port_open(ip, port, timeout=2): + """Check if a given IP and port are accessible.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.settimeout(timeout) + return s.connect_ex((ip, port)) == 0 + + +def check_health(ip, port, path="/health", timeout=2): + """Return True if http://ip:port/path returns HTTP 200.""" + try: + url = f"http://{ip}:{port}{path}" + req = urllib.request.Request(url) + with urllib.request.urlopen(req, timeout=timeout) as resp: + return getattr(resp, "status", 200) == 200 + except (urllib.error.URLError, urllib.error.HTTPError, OSError): + return False + + +# ============================================================================= +# barrier subcommand +# ============================================================================= + +def cmd_barrier(args): + """Wait until all nodes have opened the specified ports.""" + NODE_IPS = [ip.strip() for ip in args.node_ips.split(",") if ip.strip()] + NODE_PORTS = [int(p.strip()) for p in args.node_ports.split(",") if p.strip()] + + if not NODE_IPS: + print("Error: NODE_IPS argument is empty or not set.") + sys.exit(1) + + if len(NODE_PORTS) == 1: + NODE_PORTS *= len(NODE_IPS) + elif len(NODE_PORTS) != len(NODE_IPS): + print("Error: Number of ports must match number of node IPs or only one port should be given for all.") + sys.exit(1) + + server_socket = None + + def open_port(): + nonlocal server_socket + server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server_socket.bind((args.local_ip, args.local_port)) + server_socket.listen(5) + print(f"Port {args.local_port} is now open on {args.local_ip}.") + while True: + conn, addr = server_socket.accept() + conn.close() + + def close_port(): + nonlocal server_socket + if server_socket: + server_socket.close() + print(f"Port {args.local_port} has been closed on {args.local_ip}.") + + if args.enable_port: + threading.Thread(target=open_port, daemon=True).start() + + # Wait for all ports (TCP check) + if args.wait_for_all_ports: + start_time = time.time() + timeout = args.timeout + + while True: + if timeout > 0: + elapsed = time.time() - start_time + if elapsed >= timeout: + not_open = [(ip, port) for ip, port in zip(NODE_IPS, NODE_PORTS) + if not is_port_open(ip, port)] + print(f"ERROR: Timeout after {timeout} seconds waiting for ports to open.", flush=True) + print("The following nodes/ports are still not responding:", flush=True) + for ip, port in not_open: + print(f" - {ip}:{port}", flush=True) + sys.exit(1) + + all_open = all(is_port_open(ip, port) for ip, port in zip(NODE_IPS, NODE_PORTS)) + if all_open: + break + + if timeout > 0: + remaining = timeout - (time.time() - start_time) + print(f"Waiting for nodes.{NODE_PORTS},{NODE_IPS} . . ({remaining:.0f}s remaining)", flush=True) + else: + print(f"Waiting for nodes.{NODE_PORTS},{NODE_IPS} . .", flush=True) + time.sleep(5) + + # Wait for all health endpoints (HTTP check) + if args.wait_for_all_health: + health_path = args.health_endpoint + start_time = time.time() + timeout = args.timeout + + while True: + if timeout > 0: + elapsed = time.time() - start_time + if elapsed >= timeout: + not_ready = [ + (ip, port) + for ip, port in zip(NODE_IPS, NODE_PORTS) + if not check_health(ip, port, health_path) + ] + print(f"ERROR: Timeout after {timeout} seconds waiting for health endpoints.", flush=True) + print(f"The following (http://ip:port{health_path}) are still not responding:", flush=True) + for ip, port in not_ready: + print(f" - http://{ip}:{port}{health_path}", flush=True) + sys.exit(1) + + all_ready = all( + check_health(ip, port, health_path) + for ip, port in zip(NODE_IPS, NODE_PORTS) + ) + if all_ready: + break + + if timeout > 0: + remaining = timeout - (time.time() - start_time) + print( + f"Waiting for health on {list(zip(NODE_IPS, NODE_PORTS))} ({health_path}) .. ({remaining:.0f}s remaining)", + flush=True, + ) + else: + print(f"Waiting for health on {list(zip(NODE_IPS, NODE_PORTS))} ({health_path}) ..", flush=True) + time.sleep(30) + + if args.enable_port: + # Keep the port open long enough for slow nodes to pass their barrier. + # The previous 30s was too short when setup times vary by minutes. + grace = max(60, args.timeout // 2) if args.timeout > 0 else 300 + time.sleep(grace) + close_port() + + +# ============================================================================= +# wait subcommand +# ============================================================================= + +def cmd_wait(args): + """Wait while a remote port remains open, exit when it closes.""" + print(f"Waiting while port {args.remote_port} on {args.remote_ip} is open...") + while is_port_open(args.remote_ip, args.remote_port): + time.sleep(5) + print(f"Port {args.remote_port} on {args.remote_ip} is now closed.") + + +# ============================================================================= +# CLI +# ============================================================================= + +def main(): + parser = argparse.ArgumentParser(description="Multi-node synchronization utilities.") + subparsers = parser.add_subparsers(dest="command", required=True) + + # barrier subcommand + bp = subparsers.add_parser("barrier", help="Wait for all nodes to open specified ports.") + bp.add_argument("--local-ip", required=False, help="Local IP address to bind the server.") + bp.add_argument("--local-port", type=int, required=False, help="Port number to bind the server.") + bp.add_argument("--enable-port", action="store_true", help="Enable opening and closing of local port.") + bp.add_argument("--node-ips", required=True, help="Comma-separated list of node IPs.") + bp.add_argument("--node-ports", required=True, help="Comma-separated list of ports to check.") + bp.add_argument("--timeout", type=int, default=600, + help="Timeout in seconds (default: 600). Set to 0 for no timeout.") + bp.add_argument("--wait-for-all-ports", action="store_true", + help="Wait until all node ports are open (TCP).") + bp.add_argument("--wait-for-all-health", action="store_true", + help="Wait until http://ip:port/health returns 200 for all nodes.") + bp.add_argument("--health-endpoint", default="/health", + help="Path for health check (default: /health).") + bp.set_defaults(func=cmd_barrier) + + # wait subcommand + wp = subparsers.add_parser("wait", help="Wait while a remote port remains open.") + wp.add_argument("--remote-ip", required=True, help="Remote server IP address.") + wp.add_argument("--remote-port", type=int, required=True, help="Remote port number.") + wp.set_defaults(func=cmd_wait) + + args = parser.parse_args() + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/multi_node/amd_utils/trace_replay.sh b/benchmarks/multi_node/amd_utils/trace_replay.sh new file mode 100644 index 0000000000..a1d3428633 --- /dev/null +++ b/benchmarks/multi_node/amd_utils/trace_replay.sh @@ -0,0 +1,173 @@ +#!/bin/bash +# Dual-Engine Disaggregated Benchmark Runner +# +# ENGINE=sglang (default): SGLang benchmark +# ENGINE=vllm: vLLM benchmark +# +# Produces JSON result files via benchmark_serving.py so that the CI pipeline +# can collect and process results. +# +# Usage: bash bench.sh \ +# \ +# + +ENGINE="${ENGINE:-sglang-disagg}" + +model_path=$1 +model_name=$2 +concurrency_list=${3:-"1"} +MODEL_PATH="${MODEL_PATH:-${model_path}/${model_name}}" +# vllm-disagg uses --served-model-name MODEL_NAME; sglang defaults to MODEL_PATH +if [[ "$ENGINE" == "vllm-disagg" ]]; then + MODEL="${MODEL_NAME:-${MODEL_PATH}}" +else + MODEL="${MODEL_PATH}" +fi +log_path=${4:-/run_logs} + +# Split BENCH_MAX_CONCURRENCY (x-delimited, e.g. "8x16x32") into an array. +# Falls back to 1 if unset so the loop always runs at least once. +IFS='x' read -r -a chosen_concurrencies <<< "${concurrency_list}" + + +ROUTER_PORT="${ROUTER_PORT:-30000}" + +export TRANSFORMERS_VERBOSITY=error +export TOKENIZERS_PARALLELISM=false + +# echo "Config ${chosen_isl}; ${chosen_osl}; ${chosen_concurrencies[0]}; ${chosen_req_rate}" + +RESULT_DIR="${RESULT_DIR:-${log_path}/agentic}" +mkdir -p "$RESULT_DIR" + +source "$(dirname "$0")/../../benchmark_lib.sh" + +# Preserve the result schema from the original AgentX PR while fixed-sequence +# and other agentic paths use the current shared aggregation module. +write_agentic_result_json() { + local result_dir="$1" + ( + cd "$INFMAX_CONTAINER_WORKSPACE" + RESULT_DIR="$result_dir" AGENTIC_OUTPUT_DIR="${AGENTIC_OUTPUT_DIR:-$INFMAX_CONTAINER_WORKSPACE}" \ + "$AIPERF_PYTHON" -m utils.agentic.aggregation.process_agentic_result_agentic + ) + + "$AIPERF_PYTHON" "$INFMAX_CONTAINER_WORKSPACE/utils/generate_aiperf_plots.py" "$result_dir" 2>&1 || true +} + +# clear_kv_caches — wipe all KV cache tiers on every backend worker before a +# concurrency point, so each conc is measured cold (no prefix reuse bleeding in +# from the previous conc). Mirrors mori-scheduler/scripts/benchmark/lib/ +# clear_caches.sh, but the worker base URLs are already resolved by +# server_sglang.sh (SERVER_FLUSH_URLS_CSV) so no SSH/IP lookup is needed. +# +# Tiers (SGLang server APIs), hit on EACH worker directly (the router does not +# fan /flush_cache out): +# L1 (GPU radix) + L2 (host hicache): POST /flush_cache — NO-OP while any +# request is in flight, so we drain-retry until "Cache flushed" or +# FLUSH_DRAIN_TIMEOUT (default 120s) elapses. +# L3 (umbp / mooncake store): POST /hicache/storage-backend/clear +# — HTTP != 200 when L3 is off, tolerated. +# Best-effort: logs WARN, never hard-fails the sweep. +clear_kv_caches() { + local drain_tmo="${FLUSH_DRAIN_TIMEOUT:-120}" + local urls_csv="${SERVER_FLUSH_URLS_CSV:-}" + if [[ -z "$urls_csv" ]]; then + echo "[clear_caches] WARN: SERVER_FLUSH_URLS_CSV unset; skipping cache flush" >&2 + return 0 + fi + local -a urls + IFS=',' read -r -a urls <<< "$urls_csv" + local url start ok resp code + for url in "${urls[@]}"; do + [[ -n "$url" ]] || continue + # L1 + L2: drain-retry until flushed (no-op while requests in flight). + start=$(date +%s); ok=0; resp="" + while :; do + resp=$(curl -sf -m 10 -X POST "${url}/flush_cache" 2>/dev/null || true) + echo "$resp" | grep -qi "Cache flushed" && { ok=1; break; } + (( $(date +%s) - start >= drain_tmo )) && break + sleep 3 + done + if [[ "$ok" == 1 ]]; then + echo "[clear_caches] ${url}: L1+L2 flushed" + else + echo "[clear_caches] WARN ${url}: L1+L2 flush NOT confirmed after ${drain_tmo}s (resp='${resp:0:80}')" >&2 + fi + # L3: storage-backend clear (umbp / mooncake). 200 when a backend is attached. + code=$(curl -s -m 60 -o /dev/null -w '%{http_code}' -X POST "${url}/hicache/storage-backend/clear" 2>/dev/null || echo 000) + if [[ "$code" == 200 ]]; then + echo "[clear_caches] ${url}: L3 store cleared" + else + echo "[clear_caches] ${url}: L3 clear http=${code} (no storage backend / L3 off — ok)" + fi + done +} + +# REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" + +PORT="${ROUTER_PORT}" +MODEL="${MODEL:-${BENCH_MODEL}}" +DURATION="${DURATION:-1800}" +export MODEL DURATION MAX_MODEL_LEN +RESULT_DIR="${RESULT_DIR:-${profile_folder}}" +# Base name for the per-conc aggregate written by the existing +# utils.agentic.aggregation.process_agentic_result module. +# The workflow guard / upload steps expect a "${RESULT_FILENAME}_conc.json" +# file per concurrency, so each concurrency below is always suffixed with +# _conc (matching agentic_srt.sh on the gb200 path). +RESULT_FILENAME_BASE="${RESULT_FILENAME:-agentic_bench}" + +mkdir -p "$RESULT_DIR" + +resolve_trace_source +install_agentic_deps + +ANY_FAILED=0 +for max_concurrency in "${chosen_concurrencies[@]}"; do + + echo "==========================================" + echo "Agentic trace replay: conc=$max_concurrency" + echo "==========================================" + + # Clear all KV cache tiers on every backend before this conc point so it is + # measured cold (no prefix reuse from the previous conc). Default on; set + # CLEAR_CACHE_BETWEEN_CONC=0 to disable. Best-effort — never fails the run. + if [[ "${CLEAR_CACHE_BETWEEN_CONC:-1}" == "1" ]]; then + echo "conc=$max_concurrency: clearing L1/L2/L3 on all backends (no server restart)" + clear_kv_caches || echo "WARNING: cache clear had issues for conc=$max_concurrency" >&2 + fi + + # Mirror agentic_srt.sh (the srtctl/gb200 path): every concurrency writes + # its artifacts into a conc_/ subdir of RESULT_DIR. The CI matrix explodes + # agentic runs to one concurrency per job, but benchmark-multinode-tmpl.yml + # still expects the per-conc nesting (LOGS/agentic/conc_*/...) and the + # _conc result-file suffix, so we always nest to keep the layout identical + # across runners and avoid overwriting earlier runs in local multi-conc sweeps. + CONC_RESULT_DIR="$RESULT_DIR/conc_${max_concurrency}" + mkdir -p "$CONC_RESULT_DIR" + + CONC="$max_concurrency" + USERS="$max_concurrency" + export CONC USERS + build_replay_cmd "$CONC_RESULT_DIR" + + # Per-conc result name consumed by write_agentic_result_json. Always suffix + # with _conc so the file matches + # the workflow guard's "${RESULT_FILENAME}_conc*.json" glob (and the agg / + # checkpoint upload steps) for both single-conc CI runs and multi-conc sweeps. + export RESULT_FILENAME="${RESULT_FILENAME_BASE}_conc${max_concurrency}" + if ! run_agentic_replay_and_write_outputs "$CONC_RESULT_DIR"; then + echo "WARNING: agentic trace replay for conc=$max_concurrency failed (replay or validation) after writing available results" >&2 + ANY_FAILED=1 + fi + + echo "-----------------------------------------" + +done + +export RESULT_FILENAME="$RESULT_FILENAME_BASE" + +if [ "$ANY_FAILED" -ne 0 ]; then + echo "WARNING: at least one conc had a non-zero exit; per-conc result files were still written when possible." >&2 +fi diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 03c7a296ca..9ace1ac0a4 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -3122,3 +3122,36 @@ minimaxm3-fp8-mi325x-vllm-agentic: - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: native, conc-list: [3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] } - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: native, conc-list: [10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 32] } - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: native, conc-list: [24, 32, 36, 40, 44, 48, 52, 56, 60, 64, 72, 80, 96] } + +dsv4-fp4-mi355x-sglang-disagg-agentic-hicache: + image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:mi355x-amds + precision: fp4 + framework: sglang-disagg + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - spec-decoding: "none" + conc-list: [ 2,4,8,16,32 ] + kv-offloading: dram + kv-offload-backend: hicache + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MTP_SIZE=0" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index be28208ab4..028631183e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4716,3 +4716,10 @@ - "Clean the export envs" - "Enable two batch overlap" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2093 + +- config-keys: + - dsv4-fp4-mi355x-sglang-disagg-agentic-hicache + description: + - "Bump the SGLang ROCm image from 20260706 to 20260709 and retire the DeepSeek-V4 compress-state, DSA paged-MQA, AITER instruction-shape, and HiCache host-pool shims that are already included upstream." + - "Remove the SWA re-prefill and unified-KV HiCache overlays after their upstream PRs merged; retain only patch code without a merged upstream equivalent." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2127 diff --git a/runners/launch_mi355x-amds-agentic.sh b/runners/launch_mi355x-amds-agentic.sh new file mode 100755 index 0000000000..1ff0dffd59 --- /dev/null +++ b/runners/launch_mi355x-amds-agentic.sh @@ -0,0 +1,338 @@ +#!/usr/bin/env bash + +scancel_sync() { + local jobid=$1 + local timeout=${2:-600} + local interval=10 + local start + start=$(date +%s) + + echo "[scancel_sync] Requesting cancel of job $jobid" + scancel "$jobid" || true + + while [[ -n "$(squeue -j "$jobid" --noheader 2>/dev/null)" ]]; do + local now + now=$(date +%s) + if (( now - start >= timeout )); then + echo "[scancel_sync][WARN] job $jobid still present after ${timeout}s" + return 1 + fi + echo "[scancel_sync] waiting for job $jobid to exit. $((timeout-(now-start))) secs remaining..." + sleep "$interval" + done + echo "[scancel_sync] job $jobid exited" + return 0 +} + +if [[ "$IS_MULTINODE" == "true" ]]; then + # This sets up the environment and launches multi-node benchmarks + + set -x + + # Set up environment variables for SLURM + export SLURM_ACCOUNT="$USER" + export SLURM_PARTITION="compute" + export SLURM_JOB_NAME="benchmark-sglang-disagg.job" + + export MODEL_NAME=${MODEL##*/} + export MODEL_PATH="/it-share/data" + export IBDEVICES="rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7" + export MORI_RDMA_TC=104 + + # Set additional required env vars for multi_node scripts + export MODEL_DIR="$MODEL_PATH" # job.slurm uses MODEL_DIR + export GPUS_PER_NODE=8 # MI355X has 8 GPUs (set to 4 for MI325X) + + export ISL="$ISL" + export OSL="$OSL" + + # Logs go to BENCHMARK_LOGS_DIR (NFS-accessible, outside the repo tree) + export BENCHMARK_LOGS_DIR="${BENCHMARK_LOGS_DIR:-$GITHUB_WORKSPACE/benchmark_logs}" + mkdir -p "$BENCHMARK_LOGS_DIR" + sudo rm -rf "$BENCHMARK_LOGS_DIR/logs" 2>/dev/null || true + + # Ensure root-owned files are cleaned up even on early exit to prevent + # EACCES errors when the next GH Actions job checks out on this runner. + # Always preserve slurm logs as CI artifacts for debugging. + # KEEP_LOGS=1 disables the trap entirely (local-debug knob). + cleanup_and_save_logs() { + if [[ -n "${GITHUB_ACTIONS:-}" && -n "${JOB_ID:-}" ]]; then + local art_dir="$GITHUB_WORKSPACE/benchmark_artifacts" + mkdir -p "$art_dir" + cp -r "$BENCHMARK_LOGS_DIR"/slurm_job-${JOB_ID}.{out,err} "$art_dir/" 2>/dev/null || true + fi + # Print .err inline so failures are visible in CI output + local err_file="$BENCHMARK_LOGS_DIR/slurm_job-${JOB_ID:-unknown}.err" + if [[ -s "$err_file" ]]; then + echo "=== Slurm job stderr ===" + tail -100 "$err_file" + echo "========================" + fi + sudo rm -rf "$BENCHMARK_LOGS_DIR" 2>/dev/null || true + } + if [[ "${KEEP_LOGS:-0}" == "1" ]]; then + trap '' EXIT + else + trap cleanup_and_save_logs EXIT + fi + + SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_mi355x_${FRAMEWORK}.sh" + if [[ "$FRAMEWORK" == "sglang-disagg" ]] || [[ "$FRAMEWORK" == "vllm-disagg" ]] || [[ "$FRAMEWORK" == "atom-disagg" ]]; then + # Agentic recipes live under multi_node/agentic/ and export the + # HiCache tunables (page-size, io-backend, ...); fixed-seq-len recipes + # live at the multi_node/ root. Honor SCENARIO_SUBDIR so agentic-coding + # configs pick the agentic recipe instead of the root one. + if [[ "${SCENARIO_SUBDIR}" == "agentic/" ]]; then + BENCHMARK_SUBDIR="multi_node/agentic" + else + BENCHMARK_SUBDIR="multi_node" + fi + else + BENCHMARK_SUBDIR="single_node/fixed_seq_len" + fi + JOB_ID=$(bash "benchmarks/${BENCHMARK_SUBDIR}/${SCRIPT_NAME}") + + # Wait for job to complete + LOG_FILE="$BENCHMARK_LOGS_DIR/slurm_job-${JOB_ID}.out" + + # Give slurm time to start the job and create log file + sleep 10 + + # Wait for log file to appear (also check job is still alive) + while ! ls "$LOG_FILE" &>/dev/null; do + if ! squeue -u "$USER" --noheader --format='%i' | grep -q "$JOB_ID"; then + echo "ERROR: Job $JOB_ID failed before creating log file" + scontrol show job "$JOB_ID" + exit 1 + fi + sleep 5 + done + + set +x + + # Poll for job completion in background + ( + while squeue -u $USER --noheader --format='%i' | grep -q "$JOB_ID"; do + sleep 10 + done + ) & + POLL_PID=$! + + # Tail the log file until job completes (-F follows by name, polls instead of inotify for NFS) + tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null + + wait $POLL_PID + + set -x + + # FIXME: The below is bad and is a result of the indirection of the ways in which + # Dynamo jobs are launched. In a follow-up PR, the location of the result file should not + # depend on the runner, it should always be in the same spot in the GH workspace. + + # Process results from all configurations + + # search for "FRAMEWORK_DIFF_IF_STATEMENT #3" for this if-statement + # Find the latest log directory that contains the data + + if [[ "${EVAL_ONLY:-false}" != "true" && "${IS_AGENTIC:-0}" != "1" ]]; then + cat > collect_latest_results.py <<'PY' +import os, sys +job_dir, isl, osl, nexp, framework = sys.argv[1], int(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4]), sys.argv[5] +logs_root = f"{job_dir}/logs/" +candidates = [] +if os.path.isdir(logs_root): + for name in os.listdir(logs_root): + subdir = f"{logs_root}{name}/{framework}_isl_{isl}_osl_{osl}" + if os.path.isdir(subdir): + candidates.append(subdir) +for path in sorted(candidates, key=os.path.getmtime, reverse=True)[:nexp]: + print(path) +PY + + LOGS_DIR=$(python3 collect_latest_results.py "$BENCHMARK_LOGS_DIR" "$ISL" "$OSL" 1 "$FRAMEWORK") + if [ -z "$LOGS_DIR" ]; then + echo "No logs directory found for ISL=${ISL}, OSL=${OSL}" + exit 1 + fi + + echo "Found logs directory: $LOGS_DIR" + ls -la "$LOGS_DIR" + + # Result JSON are contained within the result directory + for result_file in $(find $LOGS_DIR -type f); do + # result_file should directly be isl_ISL_osl_OSL_concurrency_CONC_req_rate_R_gpus_N_ctx_M_gen_N.json + file_name=$(basename $result_file) + if [ -f $result_file ]; then + # Copy the result file to workspace with a unique name + WORKSPACE_RESULT_FILE="$GITHUB_WORKSPACE/${RESULT_FILENAME}_${file_name}" + echo "Found result file ${result_file}. Copying it to ${WORKSPACE_RESULT_FILE}" + cp $result_file $WORKSPACE_RESULT_FILE + fi + done + fi + + # Extract eval results if eval was requested + if [[ "${RUN_EVAL:-false}" == "true" ]]; then + # Find eval_results in the slurm job logs directory + EVAL_DIR=$(find "$BENCHMARK_LOGS_DIR/logs" -type d -name eval_results 2>/dev/null | head -1) + if [ -n "$EVAL_DIR" ] && [ -d "$EVAL_DIR" ]; then + echo "Extracting eval results from $EVAL_DIR" + shopt -s nullglob + for eval_file in "$EVAL_DIR"/*; do + [ -f "$eval_file" ] || continue + cp "$eval_file" "$GITHUB_WORKSPACE/" + echo "Copied eval artifact: $(basename "$eval_file")" + done + shopt -u nullglob + else + echo "WARNING: RUN_EVAL=true but no eval results found under $BENCHMARK_LOGS_DIR/logs" + fi + fi + + # Stage agentic raw artifacts + server logs for the CI upload steps. + # server_sglang.sh copies /run_logs/slurm_job- to + # $BENCHMARK_LOGS_DIR/logs/slurm_job- on shared storage, and + # trace_replay.sh writes each concurrency's aiperf artifacts under + # agentic/conc_/ (mirroring agentic_srt.sh). benchmark-multinode-tmpl.yml + # uploads them from $GITHUB_WORKSPACE/LOGS/agentic/conc_*/... plus a + # multinode_server_logs.tar.gz, so preserve the conc_/ nesting here + # before the logs dir is removed below. The agg result JSON is already + # written straight to the mounted workspace by the existing agentic + # aggregation module. + if [[ "${IS_AGENTIC:-0}" == "1" ]]; then + JOB_LOGS_DIR="$BENCHMARK_LOGS_DIR/logs/slurm_job-${JOB_ID}" + if [ -d "$JOB_LOGS_DIR" ]; then + # trace_replay.sh always nests artifacts under agentic/conc_/. + # Copy the whole agentic/ tree so the conc_/ subdirs are + # preserved for the LOGS/agentic/conc_*/... upload globs. + AGENTIC_SRC="$JOB_LOGS_DIR/agentic" + if [ -d "$AGENTIC_SRC" ] && find "$AGENTIC_SRC" -mindepth 1 -maxdepth 1 -type d -name 'conc_*' -print -quit 2>/dev/null | grep -q .; then + echo "Staging agentic raw artifacts from $AGENTIC_SRC" + mkdir -p "$GITHUB_WORKSPACE/LOGS/agentic" + cp -r "$AGENTIC_SRC"/. "$GITHUB_WORKSPACE/LOGS/agentic/" + # The source artifacts are created inside the container as root + # (--container-remap-root), so depending on how the runner + # invokes this script the copies can end up root-owned and/or + # read-only (aiperf/server_sglang make some dirs mode 0555). If + # the staged tree isn't owned+writable by the runner user, the + # next checkout's `git clean` fails with + # EACCES: permission denied, rmdir '.../LOGS/agentic'. + # chown to the invoking user (the same one that runs git clean) + # via sudo (already passwordless here for rm -rf). The follow-up + # chmod uses a+rwX (not just u+rwX): the *next* job against this + # same $GITHUB_WORKSPACE may be picked up by a different runner + # process running as a different OS user, which would otherwise + # fall outside the owner bits and still fail the same + # `git clean` with EACCES despite the chown above. + sudo chown -R "$(id -u):$(id -g)" "$GITHUB_WORKSPACE/LOGS" 2>/dev/null || true + chmod -R a+rwX "$GITHUB_WORKSPACE/LOGS" 2>/dev/null || true + ls -laR "$GITHUB_WORKSPACE/LOGS/agentic" + else + echo "WARNING: no agentic conc_*/ artifacts found under $JOB_LOGS_DIR/agentic" + fi + # Server/router/prefill/decode logs for the multinode_server_logs_* artifact. + if tar czf "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz" -C "$JOB_LOGS_DIR" . 2>/dev/null; then + echo "Created multinode_server_logs.tar.gz" + else + echo "WARNING: failed to create multinode_server_logs.tar.gz" + fi + else + echo "WARNING: agentic staging skipped; $JOB_LOGS_DIR not found" + fi + fi + + echo "All result files processed" + # Use sync scancel to ensure nfs file handle is released in time + set +x + scancel_sync $JOB_ID + set -x + echo "Canceled the slurm job $JOB_ID" + + sudo rm -rf "$BENCHMARK_LOGS_DIR/logs" 2>/dev/null || true + + # Log preservation and cleanup handled by EXIT trap (cleanup_and_save_logs) + +else + + export HF_HUB_CACHE_MOUNT="/var/lib/hf-hub-cache/" + export AIPERF_MMAP_CACHE_HOST_PATH="/it-share/aiperf-cache/" + export PORT_OFFSET=${RUNNER_NAME: -1} + export PORT=$(( 8888 + ${PORT_OFFSET} )) + FRAMEWORK_SUFFIX=$([[ "$FRAMEWORK" == "atom" ]] && printf '_atom' || printf '') + SPEC_SUFFIX=$([[ "$SPEC_DECODING" == "mtp" ]] && printf '_mtp' || printf '') + + PARTITION="compute" + SQUASH_FILE="/var/lib/squash/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" + LOCK_FILE="${SQUASH_FILE}.lock" + + export GPU_COUNT="${GPU_COUNT:-${TP:?TP must be set}}" + + set -x + # Exclude known-bad mi355x compute nodes (KLAUD_DEBUG §5.1 / §5.2): + # mia1-p01-g09: pyxis broken (persistently fails to create container filesystem) + # mia1-p01-g11: docker.sock permissions denied (cluster-cleanup step fails) + # Both have been root-caused via #1431/#1432/#1440/#1441/#1443 sweep failures. + salloc --partition=$PARTITION --exclude=mia1-p01-g09,mia1-p01-g11 --gres=gpu:$GPU_COUNT --exclusive --cpus-per-task=128 --time=500 --no-shell --job-name="$RUNNER_NAME" + JOB_ID=$(squeue --name="$RUNNER_NAME" -h -o %A | head -n1) + + srun --jobid=$JOB_ID bash -c "docker stop \$(docker ps -a -q)" + + # Use flock to serialize concurrent imports to the same squash file + srun --jobid=$JOB_ID bash -c " + exec 9>\"$LOCK_FILE\" + flock -w 600 9 || { echo 'Failed to acquire lock for $SQUASH_FILE'; exit 1; } + if unsquashfs -l \"$SQUASH_FILE\" > /dev/null 2>&1; then + echo 'Squash file already exists and is valid, skipping import' + else + rm -f \"$SQUASH_FILE\" + enroot import -o \"$SQUASH_FILE\" docker://$IMAGE + fi + " + + export VLLM_CACHE_ROOT="/it-share/gharunners/.cache/vllm" + #--container-mount-home \ + + if [[ "$FRAMEWORK" == "atom" ]] || [[ "$FRAMEWORK" == "sglang" ]]; then + SLRUM_HOME_MOUNT="" + else + SLRUM_HOME_MOUNT=" --container-mount-home " + fi + + # to prevent reading outdated saved model. use a fresh model from hf repo + if [[ ("$FRAMEWORK" == "vllm" || "$FRAMEWORK" == "atom") ]] && [[ "$MODEL" == "deepseek-ai/DeepSeek-V4-Pro" ]]; then + export HF_HUB_CACHE_MOUNT="/it-share/hf-hub-cache/" + fi + + # MiniMax-M3 weights are not staged on the node-local /var/lib NVMe cache; + # they are pre-downloaded once to the NFS share instead. Covers both the + # MiniMaxAI MXFP8 checkpoint and the amd MXFP4 atom checkpoint. + if [[ "$MODEL" == MiniMaxAI/MiniMax-M3* || "$MODEL" == amd/MiniMax-M3* ]]; then + export HF_HUB_CACHE_MOUNT="/it-share/hf-hub-cache/" + fi + + SCRIPT_BASE="${EXP_NAME%%_*}_${PRECISION}_mi355x" + SCRIPT_FW="benchmarks/single_node/${SCENARIO_SUBDIR:-fixed_seq_len/}${SCRIPT_BASE}_${FRAMEWORK}${SPEC_SUFFIX}.sh" + SCRIPT_FALLBACK="benchmarks/single_node/${SCENARIO_SUBDIR:-fixed_seq_len/}${SCRIPT_BASE}${FRAMEWORK_SUFFIX}${SPEC_SUFFIX}.sh" + if [[ -f "$SCRIPT_FW" ]]; then + BENCHMARK_SCRIPT="$SCRIPT_FW" + else + BENCHMARK_SCRIPT="$SCRIPT_FALLBACK" + fi + + srun --jobid=$JOB_ID \ + --container-image=$SQUASH_FILE \ + --container-mounts=$GITHUB_WORKSPACE:/workspace/,$HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE,$AIPERF_MMAP_CACHE_HOST_PATH:/aiperf_mmap_cache \ + $SLRUM_HOME_MOUNT \ + --container-writable \ + --container-workdir=/workspace/ \ + --container-remap-root \ + --no-container-entrypoint --export=ALL,AIPERF_DATASET_MMAP_CACHE_DIR=/aiperf_mmap_cache \ + bash "$BENCHMARK_SCRIPT" + + scancel $JOB_ID + + if ls gpucore.* 1> /dev/null 2>&1; then + echo "gpucore files exist. not good" + rm -f gpucore.* + fi +fi diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index 8cb92b7a16..ce8d5c1022 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -1,5 +1,9 @@ #!/usr/bin/env bash +if [[ "${SCENARIO_SUBDIR:-}" == "agentic/" ]]; then + exec bash "$(dirname "$0")/launch_mi355x-amds-agentic.sh" +fi + scancel_sync() { local jobid=$1 local timeout=${2:-600} diff --git a/utils/agentic/aggregation/backends/sglang.py b/utils/agentic/aggregation/backends/sglang.py index 2f560f3bf7..acbbc12b02 100644 --- a/utils/agentic/aggregation/backends/sglang.py +++ b/utils/agentic/aggregation/backends/sglang.py @@ -61,6 +61,8 @@ def populate( flat["server_gpu_cache_hit_rate"] = rate(device_hits, prompt_total) flat["server_cpu_cache_hit_rate"] = rate(host_hits, prompt_total) + # HiCache host hits are external to the GPU cache. + flat["server_external_cache_hit_rate"] = flat["server_cpu_cache_hit_rate"] flat["server_overall_cache_hit_rate"] = rate(total_cached, prompt_total) if flat["server_overall_cache_hit_rate"] is None: diff --git a/utils/agentic/aggregation/process_agentic_result_agentic.py b/utils/agentic/aggregation/process_agentic_result_agentic.py new file mode 100644 index 0000000000..ff685e2c75 --- /dev/null +++ b/utils/agentic/aggregation/process_agentic_result_agentic.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 +"""Process aiperf agentic-replay output into InferenceX aggregate JSON.""" + +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path +from typing import Any + +from .aggregation_common import round_floats +from .request_metrics import compute_request_metrics, load_aggregate, load_records_with_accounting +from .server_log_metrics import find_server_log_paths +from .server_metrics import compute_server_metrics, load_server_metrics + + +def env_int(name: str, default: int = 0) -> int: + value = os.environ.get(name) + if value in (None, ""): + return default + return int(value) + + +def env_bool(name: str, default: bool = False) -> bool: + value = os.environ.get(name) + if value in (None, ""): + return default + return value.lower() in ("1", "true", "yes", "on") + + +def required_env(name: str) -> str: + value = os.environ.get(name) + if value in (None, ""): + raise SystemExit(f"Missing required environment variable: {name}") + return value + + +def _validate_kv_offload_env() -> tuple[str, str]: + kv_offloading = required_env("KV_OFFLOADING") + kv_offload_backend = os.environ.get("KV_OFFLOAD_BACKEND", "") + if kv_offloading == "none": + if kv_offload_backend: + raise SystemExit("KV_OFFLOAD_BACKEND must be empty when KV_OFFLOADING=none") + else: + if not kv_offload_backend or kv_offload_backend == "none": + raise SystemExit("KV_OFFLOAD_BACKEND is required when KV_OFFLOADING is enabled") + return kv_offloading, kv_offload_backend + + +def _gpu_shape() -> tuple[dict[str, Any], int, int, int, str]: + is_multinode = env_bool("IS_MULTINODE") + tp = env_int("TP", 1) + ep = env_int("EP_SIZE", 1) + dp_attention = os.environ.get("DP_ATTENTION", "false") + fields: dict[str, Any] = {} + + if not is_multinode: + dcp_size = env_int("DCP_SIZE", 1) + pcp_size = env_int("PCP_SIZE", 1) + if dcp_size <= 0 or pcp_size <= 0: + raise SystemExit("DCP_SIZE and PCP_SIZE must be positive integers.") + fields.update({"dcp_size": dcp_size, "pcp_size": pcp_size}) + return fields, tp * pcp_size, tp, ep, dp_attention + + prefill_num_workers = env_int("PREFILL_NUM_WORKERS") + prefill_tp = env_int("PREFILL_TP") + prefill_ep = env_int("PREFILL_EP", 1) + prefill_dp_attention = os.environ.get("PREFILL_DP_ATTN", "false") + decode_num_workers = env_int("DECODE_NUM_WORKERS") + decode_tp = env_int("DECODE_TP") + decode_ep = env_int("DECODE_EP", 1) + decode_dp_attention = os.environ.get("DECODE_DP_ATTN", "false") + prefill_hardware = os.environ.get("PREFILL_HARDWARE", "") + decode_hardware = os.environ.get("DECODE_HARDWARE", "") + if bool(prefill_hardware) != bool(decode_hardware): + raise SystemExit( + "PREFILL_HARDWARE and DECODE_HARDWARE must be specified together." + ) + num_prefill_gpu = prefill_num_workers * prefill_tp + num_decode_gpu = decode_num_workers * decode_tp + num_gpus = num_prefill_gpu + num_decode_gpu + tp = prefill_tp + decode_tp + ep = max(prefill_ep, decode_ep) + dp_attention = ( + "true" + if env_bool("PREFILL_DP_ATTN") or env_bool("DECODE_DP_ATTN") + else "false" + ) + fields.update( + { + "prefill_num_workers": prefill_num_workers, + "prefill_tp": prefill_tp, + "prefill_ep": prefill_ep, + "prefill_dp_attention": prefill_dp_attention, + "num_prefill_gpu": num_prefill_gpu, + "decode_num_workers": decode_num_workers, + "decode_tp": decode_tp, + "decode_ep": decode_ep, + "decode_dp_attention": decode_dp_attention, + "num_decode_gpu": num_decode_gpu, + } + ) + if prefill_hardware: + fields["prefill_hw"] = prefill_hardware + fields["decode_hw"] = decode_hardware + return fields, num_gpus, tp, ep, dp_attention + + +def build_agg( + records: list[dict[str, Any]], + aggregate: dict[str, Any], + server_metrics: dict[str, Any], + *, + request_accounting: dict[str, Any] | None = None, + server_log_paths: list[Path] | None = None, +) -> dict[str, Any]: + """Compose the agg_*.json body from the three aiperf inputs.""" + kv_offloading, kv_offload_backend = _validate_kv_offload_env() + multinode_fields, num_gpus, tp, ep, dp_attention = _gpu_shape() + framework = os.environ.get("FRAMEWORK", "") + request_accounting = request_accounting or { + "records_total": len(records), + "records_profiled": len(records), + "records_dropped_total": 0, + "records_warmup_dropped": 0, + "records_error_dropped": 0, + "error_categories": {}, + } + + agg: dict[str, Any] = { + "hw": os.environ.get("RUNNER_TYPE", ""), + "conc": int(os.environ.get("CONC", "0")), + "image": os.environ.get("IMAGE", ""), + "model": os.environ.get("MODEL", ""), + "infmax_model_prefix": os.environ.get("MODEL_PREFIX", ""), + "framework": framework, + "precision": os.environ.get("PRECISION", ""), + "spec_decoding": os.environ.get("SPEC_DECODING", "none"), + "disagg": env_bool("DISAGG"), + "scenario_type": "agentic-coding", + "is_multinode": env_bool("IS_MULTINODE"), + "tp": tp, + "ep": ep, + "dp_attention": dp_attention, + "kv_offloading": kv_offloading, + "kv_offload_backend": kv_offload_backend, + "allocated_cpu_dram_gb": env_int("TOTAL_CPU_DRAM_GB"), + "num_requests_total": request_accounting["records_total"], + "num_requests_successful": len(records), + "request_accounting": request_accounting, + } + agg.update(multinode_fields) + + metadata = aggregate.get("metadata") + if isinstance(metadata, dict): + dataset = metadata.get("dataset") + if isinstance(dataset, dict): + agg["dataset"] = dataset + + request_flat, request_nested = compute_request_metrics(records, aggregate) + _, server_nested, warnings = compute_server_metrics( + server_metrics, + framework=framework, + records=records, + server_log_paths=server_log_paths, + ) + + if "total_tput_tps" in request_flat and num_gpus > 0: + request_nested["throughput"]["per_gpu"] = { + "total_tput_tps": request_flat["total_tput_tps"] / num_gpus, + "output_tput_tps": request_flat.get("output_tput_tps", 0) / num_gpus, + "input_tput_tps": request_flat.get("input_tput_tps", 0) / num_gpus, + } + + agg["request_metrics"] = request_nested + agg["server_metrics"] = server_nested + if warnings: + agg["warnings"] = warnings + return agg + + +def _resolve_artifact_dir(result_dir: Path) -> Path: + """Find the dir containing aiperf's profile_export* files.""" + base = result_dir / "aiperf_artifacts" + if (base / "profile_export.jsonl").is_file(): + return base + if base.is_dir(): + for child in sorted(base.iterdir()): + if child.is_dir() and (child / "profile_export.jsonl").is_file(): + return child + return base + + +def main() -> int: + result_filename = os.environ.get("RESULT_FILENAME", "") + if not result_filename: + print("ERROR: RESULT_FILENAME env var not set", file=sys.stderr) + return 1 + + result_dir = Path(os.environ.get("RESULT_DIR", "results")) + output_dir = Path(os.environ.get("AGENTIC_OUTPUT_DIR", ".")) + + artifact_dir = _resolve_artifact_dir(result_dir) + aggregate_path = artifact_dir / "profile_export_aiperf.json" + jsonl_path = artifact_dir / "profile_export.jsonl" + server_metrics_path = artifact_dir / "server_metrics_export.json" + + if not jsonl_path.exists(): + print(f"ERROR: {jsonl_path} not found", file=sys.stderr) + return 1 + + records, request_accounting = load_records_with_accounting(jsonl_path) + aggregate = load_aggregate(aggregate_path) if aggregate_path.exists() else {} + server_metrics = load_server_metrics(server_metrics_path) + server_log_paths = find_server_log_paths(result_dir) + agg = round_floats( + build_agg( + records, + aggregate, + server_metrics, + request_accounting=request_accounting, + server_log_paths=server_log_paths, + ) + ) + + output_path = output_dir / f"{result_filename}.json" + output_dir.mkdir(parents=True, exist_ok=True) + with open(output_path, "w") as f: + json.dump(agg, f, indent=2) + + print(f"Saved aggregated agentic result to {output_path}") + print( + f" Requests: {len(records)} successful / " + f"{request_accounting['records_total']} total " + f"({request_accounting['records_warmup_dropped']} warmup, " + f"{request_accounting['records_error_dropped']} error dropped)" + ) + request_metrics = agg.get("request_metrics", {}) + qps_metrics = request_metrics.get("qps", {}) + if "mean" in qps_metrics: + print( + f" QPS: mean={qps_metrics['mean']:.2f} " + f"p75={qps_metrics.get('p75', 0):.2f} " + f"p95={qps_metrics.get('p95', 0):.2f}" + ) + server_metrics = agg.get("server_metrics", {}) + server_cache = server_metrics.get("cache", {}) + server_kv_cache = server_metrics.get("kv_cache", {}) + if server_cache.get("gpu_cache_hit_rate") is not None: + print(f" GPU cache hit rate: {server_cache['gpu_cache_hit_rate']:.1%}") + if server_cache.get("cpu_cache_hit_rate") is not None: + print(f" CPU/offload cache hit rate: {server_cache['cpu_cache_hit_rate']:.1%}") + if server_cache.get("external_cache_hit_rate") is not None: + print(f" External cache hit rate: {server_cache['external_cache_hit_rate']:.1%}") + if server_kv_cache.get("gpu_usage_pct") is not None: + print(f" GPU KV cache usage: {server_kv_cache['gpu_usage_pct']:.1%}") + if server_kv_cache.get("gpu_total_tokens") is not None: + print(f" GPU KV cache capacity: {server_kv_cache['gpu_total_tokens']} tokens") + request_cache = request_metrics.get("cache", {}) + if request_cache.get("theoretical_cache_hit_rate") is not None: + print(f" Theoretical cache hit rate: {request_cache['theoretical_cache_hit_rate']:.1%}") + throughput_per_gpu = request_metrics.get("throughput", {}).get("per_gpu", {}) + if throughput_per_gpu.get("total_tput_tps") is not None: + print(f" Throughput per GPU: {throughput_per_gpu['total_tput_tps']:.0f} tok/s") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/utils/agentic/aggregation/test_process_agentic_result.py b/utils/agentic/aggregation/test_process_agentic_result.py index 45fd9d563e..0b9b6ab49b 100644 --- a/utils/agentic/aggregation/test_process_agentic_result.py +++ b/utils/agentic/aggregation/test_process_agentic_result.py @@ -1050,6 +1050,7 @@ def test_processor_normalizes_sglang_server_metrics(tmp_path: Path): _assert_stable_server_metrics_schema(agg) assert agg["server_metrics"]["cache"]["gpu_cache_hit_rate"] == pytest.approx(0.4) assert agg["server_metrics"]["cache"]["cpu_cache_hit_rate"] == pytest.approx(0.1) + assert agg["server_metrics"]["cache"]["external_cache_hit_rate"] == pytest.approx(0.1) assert agg["server_metrics"]["cache"]["overall_cache_hit_rate"] == pytest.approx(0.5) assert agg["server_metrics"]["kv_cache"]["gpu_usage_pct"] == pytest.approx(0.75) assert agg["server_metrics"]["kv_cache"]["cpu_usage_pct"] == pytest.approx(0.3)