From 6927574295d37fcf302e04a7f6ad68525fd8656e Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 10 Jul 2026 16:46:49 -0500 Subject: [PATCH] Revert #2127: contains inference-engine patches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reverts SemiAnalysisAI/InferenceX#2127. It live-patches SGLang (disaggregation/prefill.py resolve_waiting_queue_bootstrap) during setup, which #2155 forbids without a docs/waiver/ entry. 中文：回滚 #2127，因其在 setup 阶段对 SGLang（disaggregation/prefill.py 的 resolve_waiting_queue_bootstrap）进行运行时打补丁，违反 #2155（未经 docs/waiver/ 豁免不得修改推理引擎）。 --- .github/workflows/run-sweep.yml | 3 +- .../agentic/dsv4_fp4_mi355x_sglang-disagg.sh | 172 --- .../multi_node/amd_utils/env_agentic.sh | 354 ------ .../multi_node/amd_utils/job_agentic.slurm | 708 ----------- .../multi_node/amd_utils/models_agentic.yaml | 394 ------- .../multi_node/amd_utils/server_agentic.sh | 23 - .../amd_utils/server_sglang_agentic.sh | 1039 ----------------- .../amd_utils/setup_deps_agentic.sh | 174 --- .../multi_node/amd_utils/submit_agentic.sh | 246 ---- .../multi_node/amd_utils/sync_agentic.py | 201 ---- .../multi_node/amd_utils/trace_replay.sh | 173 --- configs/amd-master.yaml | 33 - perf-changelog.yaml | 7 - runners/launch_mi355x-amds-agentic.sh | 338 ------ runners/launch_mi355x-amds.sh | 4 - utils/agentic/aggregation/backends/sglang.py | 2 - .../process_agentic_result_agentic.py | 270 ----- .../test_process_agentic_result.py | 1 - 18 files changed, 2 insertions(+), 4140 deletions(-) delete mode 100755 benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh delete mode 100755 benchmarks/multi_node/amd_utils/env_agentic.sh delete mode 100755 benchmarks/multi_node/amd_utils/job_agentic.slurm delete mode 100644 benchmarks/multi_node/amd_utils/models_agentic.yaml delete mode 100755 benchmarks/multi_node/amd_utils/server_agentic.sh delete mode 100755 benchmarks/multi_node/amd_utils/server_sglang_agentic.sh delete mode 100755 benchmarks/multi_node/amd_utils/setup_deps_agentic.sh delete mode 100755 benchmarks/multi_node/amd_utils/submit_agentic.sh delete mode 100755 benchmarks/multi_node/amd_utils/sync_agentic.py delete mode 100644 benchmarks/multi_node/amd_utils/trace_replay.sh delete mode 100755 runners/launch_mi355x-amds-agentic.sh delete mode 100644 utils/agentic/aggregation/process_agentic_result_agentic.py diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index 05cdf107a9..5fc64b0af3 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -536,7 +536,7 @@ jobs: model-prefix: ${{ matrix.config.model-prefix }} framework: ${{ matrix.config.framework }} precision: ${{ matrix.config.precision }} - conc-list: ${{ toJson(matrix.config.conc) }} + conc-list: '[${{ matrix.config.conc }}]' spec-decoding: ${{ matrix.config.spec-decoding }} disagg: ${{ matrix.config.disagg }} prefill-hardware: ${{ matrix.config.prefill.hardware }} @@ -551,6 +551,7 @@ jobs: decode-ep: ${{ matrix.config.decode.ep }} decode-dp-attn: ${{ matrix.config.decode.dp-attn }} decode-additional-settings: ${{ toJson(matrix.config.decode.additional-settings) }} + conc: ${{ matrix.config.conc }} kv-offloading: ${{ matrix.config.kv-offloading }} kv-offload-backend: ${{ matrix.config.kv-offload-backend }} duration: ${{ matrix.config.duration }} diff --git a/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh b/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh deleted file mode 100755 index a45daba41a..0000000000 --- a/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh +++ /dev/null @@ -1,172 +0,0 @@ -#!/usr/bin/env bash - -# Agentic trace-replay recipe for a disaggregated SGLang server on MI355X -# (DeepSeek-V4-Pro FP4, 1P1D TP8). -# -# CI-style sibling of dsr1_fp4_mi355x_sglang-disagg.sh: driven entirely by -# environment variables and submits a SLURM job via submit.sh. The agentic / -# HiCache-offload configuration mirrors the DSR1 recipe but uses DSV4-Pro -# specific flags (dsv4 attention backend, page-size 256, SWA settings). - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - DURATION \ - KV_OFFLOADING \ - IS_AGENTIC \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="${TIME_LIMIT:-08:00:00}" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -# ── Identity / result naming ── -export MODEL_PREFIX="${MODEL_PREFIX:-dsv4}" -export PRECISION="${PRECISION:-fp4}" -export RESULT_FILENAME="${RESULT_FILENAME:-${RUNNER_NAME:-dsv4-fp4-agentic}}" - -# ── Agentic benchmark params ── -export DURATION="${DURATION:-1800}" -# DSV4-Pro max model len for agentic traces (matches single-node recipe). -export MAX_MODEL_LEN="${MAX_MODEL_LEN:-1000000}" - -# ── In-tree sglang patches ── -# mori_conn.py targets hybrid-state bugs (GLM-5, Qwen3.5). DSV4-Pro uses a -# pure MoE/DSA architecture without hybrid state; skip to avoid interference. -export MORI_CONN_PATCH="${MORI_CONN_PATCH:-skip}" - -# ── Aiter fault mitigation ── -# --disable-custom-all-reduce avoids a known aiter fault on MI355X. -export DISABLE_CUSTOM_ALL_REDUCE="${DISABLE_CUSTOM_ALL_REDUCE:-0}" - -# ── KV cache offloading (HiCache) ── -# KV_OFFLOADING=none | dram (passed from YAML; default none for disagg). -# KV_OFFLOAD_BACKEND selects the backend when offloading is on; this recipe -# only implements HiCache, so "hicache" is the only supported value. -# HICACHE_TIER: L2 -> GPU + CPU-DRAM host pool. L3 -> + Mooncake store. -export KV_OFFLOADING="${KV_OFFLOADING:-none}" -if [[ "$KV_OFFLOADING" != "none" ]]; then - export KV_OFFLOAD_BACKEND="${KV_OFFLOAD_BACKEND:-hicache}" -fi -# HiCache/Mooncake tunables only matter when KV offloading is enabled. -if [[ "$KV_OFFLOADING" != "none" && "${KV_OFFLOAD_BACKEND:-}" == "hicache" ]]; then - export HICACHE_TIER="${HICACHE_TIER:-L2}" - export HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-1}" - # DSV4 uses page-size 256 (set in models.yaml); HiCache must match. - export HICACHE_PAGE_SIZE="${HICACHE_PAGE_SIZE:-256}" - # HiCache ratio (host pool = ratio * GPU KV pool). Default derived in server_sglang.sh. - export HICACHE_RATIO="${HICACHE_RATIO:-}" - - # ── HiCache layout/backend by tier ── - # L3 (Mooncake): page_first + direct + write_through + storage=mooncake - # L2 (CPU DRAM): layer_first + direct + write_through_selective + storage=none - # NOTE: write_through_selective evicts only under GPU memory pressure, avoiding - # the mori RDMA race that causes GPU memory access faults with write_through. - if [[ "${HICACHE_TIER^^}" == "L3" ]]; then - export HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first}" - export HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" - export HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" - export HICACHE_STORAGE_BACKEND="${HICACHE_STORAGE_BACKEND:-mooncake}" - else - export HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first}" - export HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" - export HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" - export HICACHE_STORAGE_BACKEND="${HICACHE_STORAGE_BACKEND:-}" - fi - export HICACHE_PREFETCH_POLICY="${HICACHE_PREFETCH_POLICY:-best_effort}" - # Shared nodes: use non-default Mooncake ports to avoid collisions. - export MC_MASTER_PORT="${MC_MASTER_PORT:-58137}" - export MC_METADATA_PORT="${MC_METADATA_PORT:-8080}" - export MC_METRICS_PORT="${MC_METRICS_PORT:-19003}" - export MC_MASTER_THREADS="${MC_MASTER_THREADS:-64}" - export MC_EVICTION_HIGH_WATERMARK="${MC_EVICTION_HIGH_WATERMARK:-0.95}" - export MC_PATCH_HOSTPOOL="${MC_PATCH_HOSTPOOL:-1}" - export MC_PROTOCOL="${MC_PROTOCOL:-tcp}" - export MC_GLOBAL_SEG="${MC_GLOBAL_SEG:-64gb}" - export MC_DEVICE="${MC_DEVICE:-}" - export MC_MASTER_ADDR="${MC_MASTER_ADDR:-}" - export MC_METADATA_SERVER="${MC_METADATA_SERVER:-}" -fi - -# ── MoRIIO RDMA Send Queue tuning ── -export MORI_IO_SQ_BACKOFF_TIMEOUT_US="${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-500000}" -export MORI_IO_QP_MAX_SEND_WR="${MORI_IO_QP_MAX_SEND_WR:-32768}" - -# ── SGLang PD router policy + server metrics ── -export PREFILL_ROUTER_POLICY="${PREFILL_ROUTER_POLICY:-cache_aware}" -export ENABLE_METRICS="${ENABLE_METRICS:-1}" - -# ── MTP ── -export DECODE_MTP_SIZE="${DECODE_MTP_SIZE:-0}" - -# Derive EP/DP enable flags from the topology inputs. -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# Launch the job. CONC_LIST is space-delimited in YAML; submit.sh wants 'x'. -JOB_ID=$(bash ./submit_agentic.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/amd_utils/env_agentic.sh b/benchmarks/multi_node/amd_utils/env_agentic.sh deleted file mode 100755 index 2a3005c644..0000000000 --- a/benchmarks/multi_node/amd_utils/env_agentic.sh +++ /dev/null @@ -1,354 +0,0 @@ -#!/bin/bash -# Dual-engine environment setup for multi-node disaggregated serving. -# -# ENGINE=sglang (default): SGLang/MoRI environment -# ENGINE=vllm: vLLM/Nixl environment -# -# REQUIRED ENVIRONMENT VARIABLES: -# IBDEVICES - RDMA/InfiniBand device names (e.g., ionic_0,ionic_1,... or mlx5_0,mlx5_1,...) -# Set by runner or auto-detected from hostname. -set -x - -ENGINE="${ENGINE:-sglang-disagg}" -export PYTHONDONTWRITEBYTECODE=1 - -# ============================================================================= -# HiCache / Mooncake settings from job.slurm -# ============================================================================= -# job.slurm writes the recipe-provided HiCache/Mooncake tunables to -# hicache_mc_.env and mounts it read-only at /config/hicache_mc.env. Source -# it here (auto-export) so values like HICACHE_PAGE_SIZE=256 reach the container -# before server_sglang.sh applies its "${VAR:-default}" fallbacks. Without this -# the vars arrive unset and server_sglang.sh defaults HICACHE_PAGE_SIZE to 1, -# overriding the recipe's --page-size. Empty values in the file are harmless: -# the "${VAR:-default}" fallbacks still treat "" as unset. -if [[ -f /config/hicache_mc.env ]]; then - set -a - source /config/hicache_mc.env - set +a - echo "[env.sh] sourced HiCache config from /config/hicache_mc.env (HICACHE_PAGE_SIZE=${HICACHE_PAGE_SIZE:-unset})" -fi - -# ============================================================================= -# Shared: IBDEVICES detection -# ============================================================================= - -# Prefer IBDEVICES set by runner (runners/launch_mi355x-amds.sh) -# Fall back to hostname detection if not set (for direct script execution) -if [[ -z "$IBDEVICES" ]]; then - DETECTED=$(ibv_devinfo 2>/dev/null | grep "hca_id:" | awk '{print $2}' | paste -sd',') - if [[ -n "$DETECTED" ]]; then - export IBDEVICES="$DETECTED" - echo "[INFO] Auto-detected IBDEVICES=$IBDEVICES via ibv_devinfo on $(hostname -s)" - else - echo "ERROR: Unable to detect RDMA devices. Set IBDEVICES explicitly." >&2 - exit 1 - fi -else - echo "[INFO] Using IBDEVICES=$IBDEVICES (set by runner or environment)" -fi -export IBDEVICES - -# Shared: Auto-detect default network interface (portable across clusters) -# Only auto-detect if not already set by the runner/environment -if [[ -z "$GLOO_SOCKET_IFNAME" ]]; then - export GLOO_SOCKET_IFNAME=$(ip route 2>/dev/null | grep '^default' | awk '{print $5}' | head -n 1) -fi -if [[ -z "$NCCL_SOCKET_IFNAME" ]]; then - export NCCL_SOCKET_IFNAME=$(ip route 2>/dev/null | grep '^default' | awk '{print $5}' | head -n 1) -fi - -set +x - -export NCCL_IB_HCA=${NCCL_IB_HCA:-$IBDEVICES} - -# ============================================================================= -# MoRI-specific environment -# ============================================================================= -# Shared by the vLLM MoRIIOConnector and the SGLang/MoRI KV-transfer path. - -export MORI_IO_SQ_BACKOFF_TIMEOUT_US="${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-50000}" -export MORI_IO_QP_MAX_SEND_WR="${MORI_IO_QP_MAX_SEND_WR:-16384}" -export MORI_IO_QP_MAX_CQE="${MORI_IO_QP_MAX_CQE:-32768}" -export MORI_IO_QP_MAX_SGE="${MORI_IO_QP_MAX_SGE:-2}" -export MORI_IO_TC_DISABLE="${MORI_IO_TC_DISABLE:-0}" - -# QoS/DSCP configuration -# Priority order: 1) Set by runner, 2) Detect via nicctl, 3) Detect from hostname -if [[ -n "$MORI_RDMA_TC" ]]; then - echo "[INFO] Using MORI_RDMA_TC=$MORI_RDMA_TC (set by runner or environment)" -elif command -v nicctl &> /dev/null; then - ND_PRIO=$(nicctl show qos 2>/dev/null | awk '/PFC no-drop priorities/ {print $NF; exit}') - ND_DSCP=$(nicctl show qos 2>/dev/null| awk -v p="$ND_PRIO" ' -$1 == "DSCP" && $2 == ":" && $NF == p { - print $3; exit -}') - # nicctl may emit trailing commas (e.g. "24,"); keep the leading integer so the - # arithmetic can't choke and unparseable output falls back to hostname detection. - ND_PRIO="${ND_PRIO%%,*}"; ND_PRIO="${ND_PRIO//[!0-9]/}" - ND_DSCP="${ND_DSCP%%,*}"; ND_DSCP="${ND_DSCP//[!0-9]/}" - - if [[ "$ND_DSCP" =~ ^[0-9]+$ ]] && [[ "$ND_PRIO" =~ ^[0-9]+$ ]]; then - TC=$(( 4 * ND_DSCP )) - export MORI_RDMA_SL=$ND_PRIO - export MORI_IO_SL=$ND_PRIO - export MORI_RDMA_TC=$TC - export MORI_IO_TC=$TC - echo "[INFO] Detected QoS config from nicctl: MORI_RDMA_TC=$MORI_RDMA_TC, MORI_RDMA_SL=$MORI_RDMA_SL, MORI_IO_TC=$MORI_IO_TC, MORI_IO_SL=$MORI_IO_SL" - else - echo "[WARN] nicctl available but QoS data unavailable; trying hostname detection." - # Fall back to hostname-based detection - NODENAME=$(hostname -s) - if [[ $NODENAME == GPU* ]] || [[ $NODENAME == smci355-ccs-aus* ]]; then - export MORI_RDMA_TC=96 - export MORI_IO_TC=96 - echo "[INFO] Auto-detected MORI_RDMA_TC=$MORI_RDMA_TC from hostname $NODENAME" - elif [[ $NODENAME == mia1* ]]; then - export MORI_RDMA_TC=104 - export MORI_IO_TC=104 - echo "[INFO] Auto-detected MORI_RDMA_TC=$MORI_RDMA_TC from hostname $NODENAME" - else - echo "[INFO] Unable to detect MORI_RDMA_TC from hostname. Skipping RDMA QoS configuration." - fi - fi -else - # nicctl not available, try hostname-based detection - NODENAME=$(hostname -s) - if [[ $NODENAME == GPU* ]] || [[ $NODENAME == smci355-ccs-aus* ]]; then - export MORI_RDMA_TC=96 - export MORI_IO_TC=96 - echo "[INFO] Auto-detected MORI_RDMA_TC=$MORI_RDMA_TC from hostname $NODENAME" - elif [[ $NODENAME == mia1* ]]; then - export MORI_RDMA_TC=104 - export MORI_IO_TC=104 - echo "[INFO] Auto-detected MORI_RDMA_TC=$MORI_RDMA_TC from hostname $NODENAME" - else - echo "[INFO] nicctl not found and unable to detect from hostname. Skipping RDMA QoS configuration." - echo " This is normal for clusters without QoS or outside Docker containers." - fi -fi - -# ============================================================================= -# Engine-specific environment -# ============================================================================= - -if [[ "$ENGINE" == "vllm-disagg" ]]; then - # ========================================================================= - # vLLM/Nixl-specific environment - # ========================================================================= - export VLLM_USE_V1=1 - export VLLM_SERVER_DEV_MODE=0 - export VLLM_DISABLE_REQUEST_ID_RANDOMIZATION=1 - - set -x - - # UCX_NET_DEVICES: Use the first tw-eth interface for UCX TCP transport - if [[ -z "$UCX_NET_DEVICES" ]]; then - UCX_NET_DEV=$(ip -o link show 2>/dev/null | awk -F': ' '/tw-eth/{print $2}' | head -1) - if [[ -n "$UCX_NET_DEV" ]]; then - export UCX_NET_DEVICES="$UCX_NET_DEV" - else - FIRST_IB=$(echo "$IBDEVICES" | cut -d',' -f1) - if [[ -n "$FIRST_IB" ]]; then - export UCX_NET_DEVICES="${FIRST_IB}:1" - fi - fi - echo "[INFO] Auto-set UCX_NET_DEVICES=$UCX_NET_DEVICES" - else - echo "[INFO] Using UCX_NET_DEVICES=$UCX_NET_DEVICES (set by environment)" - fi - - # RoCEv2: use IPv4-mapped GID (index 1) for inter-node RDMA routing - export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-1} - - # QoS/DSCP configuration for lossless RoCEv2 fabric. - if [[ -n "$UCX_IB_TRAFFIC_CLASS" ]]; then - echo "[INFO] Using UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS (set by environment)" - elif command -v nicctl &> /dev/null; then - ND_PRIO=$(nicctl show qos 2>/dev/null | awk '/PFC no-drop priorities/ {print $NF; exit}') - ND_DSCP=$(nicctl show qos 2>/dev/null | awk -v p="$ND_PRIO" ' -$1 == "DSCP" && $2 == ":" && $NF == p { - print $3; exit -}') - # nicctl may emit trailing commas (e.g. "24,"); keep the leading integer so the - # arithmetic can't choke and unparseable output falls back to hostname detection. - ND_PRIO="${ND_PRIO%%,*}"; ND_PRIO="${ND_PRIO//[!0-9]/}" - ND_DSCP="${ND_DSCP%%,*}"; ND_DSCP="${ND_DSCP//[!0-9]/}" - if [[ "$ND_DSCP" =~ ^[0-9]+$ ]] && [[ "$ND_PRIO" =~ ^[0-9]+$ ]]; then - export UCX_IB_TRAFFIC_CLASS=$(( 4 * ND_DSCP )) - export UCX_IB_SL=$ND_PRIO - echo "[INFO] Detected QoS from nicctl: UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS, UCX_IB_SL=$UCX_IB_SL" - else - echo "[WARN] nicctl available but QoS data unavailable; trying hostname detection." - NODENAME=$(hostname -s) - if [[ $NODENAME == GPU* ]] || [[ $NODENAME == smci355-ccs-aus* ]]; then - export UCX_IB_TRAFFIC_CLASS=96 - echo "[INFO] Auto-detected UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS from hostname $NODENAME" - elif [[ $NODENAME == mia1* ]]; then - export UCX_IB_TRAFFIC_CLASS=104 - echo "[INFO] Auto-detected UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS from hostname $NODENAME" - fi - fi - else - NODENAME=$(hostname -s) - if [[ $NODENAME == GPU* ]] || [[ $NODENAME == smci355-ccs-aus* ]]; then - export UCX_IB_TRAFFIC_CLASS=96 - echo "[INFO] Auto-detected UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS from hostname $NODENAME" - elif [[ $NODENAME == mia1* ]]; then - export UCX_IB_TRAFFIC_CLASS=104 - echo "[INFO] Auto-detected UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS from hostname $NODENAME" - else - echo "[INFO] No nicctl and unable to detect from hostname. Skipping QoS configuration." - fi - fi - - set +x - echo "[INFO] IBDEVICES=$IBDEVICES UCX_NET_DEVICES=$UCX_NET_DEVICES NCCL_SOCKET_IFNAME=$NCCL_SOCKET_IFNAME UCX_IB_GID_INDEX=$UCX_IB_GID_INDEX UCX_IB_TRAFFIC_CLASS=${UCX_IB_TRAFFIC_CLASS:-unset}" - -else - # ========================================================================= - # SGLang-specific environment - # ========================================================================= - - export SGLANG_USE_AITER=1 - export AITER_LOG_LEVEL=ERROR - - export SGLANG_MORI_DISPATCH_DTYPE=auto - # export MORI_COMBINE_DTYPE_PREFILL=fp8_direct_cast - # export MORI_COMBINE_DTYPE_DECODE=fp8 - export MORI_COMBINE_DTYPE_PREFILL="" - export MORI_COMBINE_DTYPE_DECODE="" - export SGLANG_MORI_QP_PER_TRANSFER=4 - export SGLANG_MORI_NUM_WORKERS=4 - # Keep these as overridable defaults (not hard assignments), otherwise - # later tuning blocks cannot raise them for high-concurrency runs. - # export MORI_IO_SQ_BACKOFF_TIMEOUT_US="${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-500000}" - - # export MORI_IO_QP_MAX_SEND_WR="${MORI_IO_QP_MAX_SEND_WR:-16384}" - # export MORI_IO_QP_MAX_CQE=32768 - # export MORI_IO_QP_MAX_SGE=1 - - # export MORI_IO_TC_DISABLE=0 - - export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600 - export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600 - - export SGLANG_HEALTH_CHECK_TIMEOUT=600 - - # GLM-5: uses NSA (not MLA), needs fused-decode-MLA disabled + fast loading - if [[ "$MODEL_NAME" == "GLM-5-FP8" ]]; then - export SGLANG_ROCM_FUSED_DECODE_MLA=0 - export ROCM_QUICK_REDUCE_QUANTIZATION=INT4 - export SAFETENSORS_FAST_GPU=1 - fi - - # Disable allocating memory in one pass - export MORI_SHMEM_MODE=ISOLATION - - # Enable spec v2 - export SGLANG_ENABLE_SPEC_V2=1 - export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 - - export SGLANG_LOG_MS=true - export SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS=32 - - export MORI_MAX_DISPATCH_TOKENS_PREFILL=8192 - export MORI_MAX_DISPATCH_TOKENS_DECODE=512 - - export MORI_MOE_MAX_INPUT_TOKENS_PREFILL=32768 - export MORI_MOE_MAX_INPUT_TOKENS_DECODE=2703 - - # set MTP size=1 when EP16 - export SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD=$((MORI_MAX_DISPATCH_TOKENS_DECODE * 2)) - - export MORI_EP_LAUNCH_CONFIG_MODE=AUTO - - # Default to WARNING to cut per-op MoRI log spam on long multinode/eval - # runs; override with MORI_APP_LOG_LEVEL=INFO when debugging. - export MORI_APP_LOG_LEVEL="${MORI_APP_LOG_LEVEL:-WARNING}" - - # Router logging control: - # 0 (default) keeps noisy per-request access logs out of stdout while still logging to file. - # 1 mirrors router logs to stdout via tee (useful for live debugging). - export SGLANG_ROUTER_STDOUT_LOGS="${SGLANG_ROUTER_STDOUT_LOGS:-0}" - - # FIXME: WA for latest upstream 0305 image - export PYTHONPATH=/sgl-workspace/aiter:${PYTHONPATH} - - # ========================================================================= - # DeepSeek-V4-Pro PD recipe overrides - # Placed at the end of the SGLang env block so it wins over the global - # MoRI/SGLang defaults set above. Mirrors the validated DSv4 manual PD - # commands (ported from InferenceX amd/dsv4_sgl_di). These SGLANG_OPT_* / - # AITER_* kernel-routing knobs steer DSv4 away from the default aiter CK - # fused-MoE path, which raises "Unsupported kernel config for moe heuristic - # dispatch" at decode time on this fp4 model (job 19034 crash). Only the - # SGLang/MoRI env knobs are pinned here; CLI flags live in models.yaml and - # the cluster NIC/socket vars stay runner-derived. - # ========================================================================= - if [[ "$MODEL_NAME" == "DeepSeek-V4-Pro" ]]; then - # MoRI RDMA send-queue depth for DSv4 (overrides the global default above). - export MORI_IO_QP_MAX_SEND_WR=32767 - # Unified radix tree: cache impl with per-component (full-attn / SWA) - # management for hybrid-attention models. Set unconditionally (not gated on - # hicache) so all SGLang runs use it. - export SGLANG_ENABLE_UNIFIED_RADIX_TREE=1 - # Proactively free out-of-window SWA KV slots during chunked prefill. - # Without it, in-flight requests pin SWA KV for their whole context, keeping - # the SWA pool under constant eviction pressure; under LRU the trailing - # window of cached sessions gets flushed, making prefix-cache hits bimodal - # and collapsing the effective hit rate on multi-turn agentic workloads. - export SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS=1 - - # MoRI dispatch/combine dtypes: auto for both roles (not the fp8 split default) - export SGLANG_MORI_DISPATCH_DTYPE=auto - export MORI_COMBINE_DTYPE_PREFILL=auto - export MORI_COMBINE_DTYPE_DECODE=auto - - # Per-role MoRI dispatch sizing (used by the harness chunked/MoE math) - export MORI_MAX_DISPATCH_TOKENS_PREFILL=8192 - export MORI_MAX_DISPATCH_TOKENS_DECODE=64 - unset MORI_MOE_MAX_INPUT_TOKENS_PREFILL - unset MORI_MOE_MAX_INPUT_TOKENS_DECODE - - # PER_RANK dispatch tokens pinned independently (16384 prefill / 128 - # decode); server_sglang.sh prefers these over the MORI_MAX_DISPATCH_* - # coupling when set. - export MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_PREFILL=16384 - export MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_DECODE=128 - - # Fixed inter-kernel switch threshold (not derived). - export SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD=4096 - - # Overlap plan stream on for DSv4 (global default is 0) - # export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 - - # DSv4 model kernel routing (mirrors the single-node / manual PD recipe) - export SGLANG_DEFAULT_THINKING=1 - export SGLANG_DSV4_REASONING_EFFORT=max - export SGLANG_OPT_DEEPGEMM_HC_PRENORM=false - export SGLANG_USE_AITER=1 - export SGLANG_USE_ROCM700A=0 - export SGLANG_OPT_USE_FUSED_COMPRESS=true - export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton - export SGLANG_OPT_FP8_WO_A_GEMM=false - export SGLANG_OPT_USE_JIT_INDEXER_METADATA=false - export SGLANG_OPT_USE_TOPK_V2=false - export SGLANG_OPT_USE_AITER_INDEXER=${SGLANG_OPT_USE_AITER_INDEXER:-true} - export SGLANG_OPT_USE_TILELANG_INDEXER=false - export SGLANG_OPT_USE_TILELANG_MHC_PRE=false - export SGLANG_OPT_USE_TILELANG_MHC_POST=false - export SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1 - export SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true - export SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false - export SGLANG_ROCM_USE_MULTI_STREAM=false - export AITER_BF16_FP8_MOE_BOUND=0 - export SGLANG_EAGER_INPUT_NO_COPY=true - export SGLANG_SHARED_EXPERT_TP1=1 - export SGLANG_DP_SHARED_EXPERT_LOCAL=1 - export SGLANG_DP_USE_GATHERV=1 - export SGLANG_DP_USE_REDUCE_SCATTER=1 - export GPU_MAX_HW_QUEUES=5 - fi - -fi \ No newline at end of file diff --git a/benchmarks/multi_node/amd_utils/job_agentic.slurm b/benchmarks/multi_node/amd_utils/job_agentic.slurm deleted file mode 100755 index f859b86365..0000000000 --- a/benchmarks/multi_node/amd_utils/job_agentic.slurm +++ /dev/null @@ -1,708 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=disagg-bench -#SBATCH -N 3 # Overridden by submit.sh -N flag -#SBATCH -n 3 # Overridden by submit.sh -n flag -#SBATCH --ntasks-per-node=1 -#SBATCH --spread-job -#SBATCH --gres=gpu:8 -#SBATCH --time=24:00:00 -# --output and --error are set by submit.sh via BENCHMARK_LOGS_DIR - -ENGINE="${ENGINE:-sglang-disagg}" - -echo "=== Job Start Time ===" -echo "UTC Time: $(TZ=UTC date '+%Y-%m-%d %H:%M:%S %Z')" -echo "PST Time: $(TZ=America/Los_Angeles date '+%Y-%m-%d %H:%M:%S %Z')" -echo "ENGINE: $ENGINE" -echo "=======================" -echo "" - -# ============================================================================= -# Model Validation -# ============================================================================= - -# Use $(pwd) not BASH_SOURCE — sbatch copies the script to /var/spool/slurmd/ -# at runtime, but the CWD remains the submit-time directory (amd_utils/). -if [[ "$ENGINE" == "vllm-disagg" ]]; then - MODELS_YAML="$(pwd)/models_vllm.yaml" -elif [[ "$ENGINE" == "atom-disagg" ]]; then - MODELS_YAML="$(pwd)/models_atom.yaml" -else - MODELS_YAML="$(pwd)/models_agentic.yaml" -fi - -if [[ ! -f "$MODELS_YAML" ]]; then - echo "Error: models YAML not found at $MODELS_YAML" - exit 1 -fi - -if [[ -z "${DOCKER_IMAGE_NAME:-}" ]]; then - echo "Error: DOCKER_IMAGE_NAME is not set." - exit 1 -fi - -MODEL_NAME="${MODEL_NAME:-None}" -if ! grep -q "^${MODEL_NAME}:" "$MODELS_YAML"; then - echo "Error: Model '$MODEL_NAME' not found in $MODELS_YAML" - echo "Available models:" - grep -E '^[A-Za-z]' "$MODELS_YAML" | sed 's/:.*$//' | sed 's/^/ - /' - exit 1 -fi -echo "Model found: $MODEL_NAME" - -RUN_FILE="server_agentic.sh" -echo "Runfile set: $RUN_FILE" - -# DI_REPO_DIR points to the repo root. -# $(pwd) is amd_utils/ (the sbatch submit dir); go up 3 levels to reach the repo root. -export DI_REPO_DIR=$(cd "$(pwd)/../../.." && pwd) - -# ── In-tree sglang patches: auto-apply for known-affected images ────── -# sglang v0.5.12.post1 ships a known-broken MoRI PD-disaggregation -# backend that crashes hybrid-attention models (GLM-5, Qwen3.5-MoE, -# anything with state_types: List[StateType]) at startup. We carry an -# in-tree overlay of mori/conn.py that fixes the wire format + the -# legacy state_type fallback (see patches/README.md for the bug -# analysis and patch detail). -# -# Auto-applied when the image tag contains "v0.5.12.post1", unless the -# caller sets MORI_CONN_PATCH=skip. The overlay is appended to -# ${EXTRA_DOCKER_MOUNTS:-} so callers can still inject other mounts. -# Dedup guard avoids double-mounting if EXTRA_DOCKER_MOUNTS already -# contains the target path (docker rejects duplicate destinations). -_MORI_PATCH_FILE="$DI_REPO_DIR/benchmarks/multi_node/amd_utils/patches/mori_conn.py" -_MORI_PATCH_TARGET="/sgl-workspace/sglang/python/sglang/srt/disaggregation/mori/conn.py" -if [[ "${MORI_CONN_PATCH:-auto}" != "skip" ]] \ - && [[ -f "$_MORI_PATCH_FILE" ]] \ - && [[ "${DOCKER_IMAGE_NAME:-}" == *"v0.5.12.post1"* ]] \ - && [[ "${EXTRA_DOCKER_MOUNTS:-}" != *"$_MORI_PATCH_TARGET"* ]]; then - EXTRA_DOCKER_MOUNTS="${EXTRA_DOCKER_MOUNTS:-} -v ${_MORI_PATCH_FILE}:${_MORI_PATCH_TARGET}:ro" - export EXTRA_DOCKER_MOUNTS - echo "[job.slurm] auto-applied MoRI conn.py overlay: ${_MORI_PATCH_FILE}" -fi -xP="${xP:-1}" -yD="${yD:-1}" - -# Benchmark configuration -BENCH_INPUT_LEN="${BENCH_INPUT_LEN:-1024}" -BENCH_OUTPUT_LEN="${BENCH_OUTPUT_LEN:-1024}" -BENCH_RANDOM_RANGE_RATIO="${BENCH_RANDOM_RANGE_RATIO:-1}" -BENCH_NUM_PROMPTS_MULTIPLIER="${BENCH_NUM_PROMPTS_MULTIPLIER:-10}" -BENCH_MAX_CONCURRENCY="${BENCH_MAX_CONCURRENCY:-512}" -BENCH_REQUEST_RATE="${BENCH_REQUEST_RATE:-inf}" - -GPUS_PER_NODE="${GPUS_PER_NODE:-8}" - -# Engine-specific defaults -PREFILL_ENABLE_EP="${PREFILL_ENABLE_EP:-false}" -PREFILL_ENABLE_DP="${PREFILL_ENABLE_DP:-false}" -DECODE_ENABLE_EP="${DECODE_ENABLE_EP:-false}" -DECODE_ENABLE_DP="${DECODE_ENABLE_DP:-false}" -PREFILL_TP_SIZE="${PREFILL_TP_SIZE:-8}" -DECODE_TP_SIZE="${DECODE_TP_SIZE:-8}" -DECODE_MTP_SIZE=${DECODE_MTP_SIZE:-0} - -# Router selection: "vllm-router" (external container) or "moriio" (in-container proxy) -ROUTER_TYPE="${ROUTER_TYPE:-vllm-router}" -ROUTER_PORT="${ROUTER_PORT:-30000}" -PROXY_PING_PORT="${PROXY_PING_PORT:-36367}" - -# ============================================================================= -# Model Path Resolution -# ============================================================================= - -# MODEL_DIR detection: prefer env var, fall back to hostname detection -if [[ -z "$MODEL_DIR" ]]; then - NODENAME=$(hostname -s) - if [[ $NODENAME == GPU* ]] || [[ $NODENAME == smci355-ccs-aus* ]]; then - MODEL_DIR="/nfsdata" - elif [[ $NODENAME == mia1* ]]; then - MODEL_DIR="/it-share/data" - else - MODEL_DIR="/nfsdata" - fi - echo "[INFO] Auto-detected MODEL_DIR=$MODEL_DIR from hostname $(hostname -s)" -fi -export MODEL_DIR - -if [[ "$ENGINE" == "vllm-disagg" ]]; then - # vLLM: Extract hf_dir from models.yaml, search multiple paths, resolve HF cache snapshots - DISK_DIR_NAME=$(awk '/^'"$MODEL_NAME"':/{found=1; next} - found && /^[^ ]/{exit} - found && /hf_dir:/{gsub(/[" ]/, "", $2); print $2; exit}' "$MODELS_YAML") - DISK_DIR_NAME="${DISK_DIR_NAME:-$MODEL_NAME}" - echo "Looking for model: $MODEL_NAME (disk dir: $DISK_DIR_NAME)" - - resolve_hf_cache_path() { - local base_path=$1 - if [[ -d "${base_path}/snapshots" ]]; then - local snapshot=$(ls -1 "${base_path}/snapshots" 2>/dev/null | head -1) - if [[ -n "$snapshot" ]]; then - echo "${base_path}/snapshots/${snapshot}" - return 0 - fi - fi - echo "$base_path" - return 1 - } - - MODEL_PATH="" - SEARCH_PATHS=( - "${MODEL_DIR}/${DISK_DIR_NAME}" - "${MODEL_DIR}/${MODEL_NAME}" - "/nfsdata/hf_hub_cache-0/${DISK_DIR_NAME}" - "/nfsdata/hf_hub_cache-0/${MODEL_NAME}" - ) - - for search_path in "${SEARCH_PATHS[@]}"; do - if [[ -d "$search_path" ]]; then - RESOLVED=$(resolve_hf_cache_path "$search_path") - MODEL_PATH="$RESOLVED" - echo "Found MODEL_PATH: $MODEL_PATH" - break - fi - done - - if [[ -z "$MODEL_PATH" ]]; then - echo "FATAL: Model '$MODEL_NAME' not found. Searched:" - for p in "${SEARCH_PATHS[@]}"; do echo " - $p"; done - exit 1 - fi - echo "Final MODEL_PATH: $MODEL_PATH" -else - # SGLang: Validate model path across all allocated nodes - echo "Looking for model: $MODEL_NAME" - echo "Checking model availability across all allocated nodes..." - - ALL_NODES=$(scontrol show hostnames "$SLURM_JOB_NODELIST") - TOTAL_NODES=$(echo "$ALL_NODES" | wc -l) - echo "Total allocated nodes: $TOTAL_NODES" - echo "Nodes: $(echo "$ALL_NODES" | tr '\n' ' ')" - - check_model_path() { - local path=$1 - local check_name=$2 - echo "Checking $check_name: $path" - srun --nodes=$SLURM_NNODES --ntasks=$SLURM_NNODES /bin/bash -c " - if [ -d '$path' ]; then - echo \"\$(hostname): Found $path\" - exit 0 - else - echo \"\$(hostname): Missing $path\" - exit 1 - fi - " - local exit_code=$? - if [ $exit_code -eq 0 ]; then - echo "$check_name available on ALL nodes" - return 0 - else - echo "$check_name NOT available on all nodes" - return 1 - fi - } - - # Extract hf_dir from models.yaml (same as vllm-disagg path above) - SGL_DISK_DIR_NAME=$(awk '/^'"$MODEL_NAME"':/{found=1; next} - found && /^[^ ]/{exit} - found && /hf_dir:/{gsub(/[" ]/, "", $2); print $2; exit}' "$MODELS_YAML") - SGL_DISK_DIR_NAME="${SGL_DISK_DIR_NAME:-$MODEL_NAME}" - - # Prefer the caller-supplied MODEL_PATH (recipe scripts set this explicitly); - # fall back to MODEL_DIR/hf_dir then MODEL_DIR/MODEL_NAME. - if [[ -n "${MODEL_PATH:-}" && "$MODEL_PATH" != "$MODEL_DIR" ]]; then - # Caller already resolved the path (e.g. MODEL_PATH=/it-share/hf_cache/models--...) - # Use it directly if it exists on all nodes, otherwise try subdirectory combos. - if check_model_path "$MODEL_PATH" "MODEL_PATH (caller-supplied)"; then - echo "Selected MODEL_PATH: $MODEL_PATH (caller-supplied, available on all nodes)" - elif check_model_path "$MODEL_PATH/$SGL_DISK_DIR_NAME" "$MODEL_PATH/$SGL_DISK_DIR_NAME"; then - MODEL_PATH="$MODEL_PATH/$SGL_DISK_DIR_NAME" - echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" - elif check_model_path "$MODEL_PATH/$MODEL_NAME" "$MODEL_PATH/$MODEL_NAME"; then - MODEL_PATH="$MODEL_PATH/$MODEL_NAME" - echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" - else - echo "FATAL ERROR: Model '$MODEL_NAME' not found on ALL allocated nodes in:" - echo " - $MODEL_PATH" - echo " - $MODEL_PATH/$SGL_DISK_DIR_NAME" - echo " - $MODEL_PATH/$MODEL_NAME" - exit 1 - fi - elif check_model_path "$MODEL_DIR/$SGL_DISK_DIR_NAME" "$MODEL_DIR/$SGL_DISK_DIR_NAME"; then - MODEL_PATH="$MODEL_DIR/$SGL_DISK_DIR_NAME" - echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" - elif check_model_path "$MODEL_DIR/$MODEL_NAME" "$MODEL_DIR"; then - MODEL_PATH="$MODEL_DIR/$MODEL_NAME" - echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" - else - echo "FATAL ERROR: Model '$MODEL_NAME' not found on ALL allocated nodes in:" - echo " - $MODEL_DIR/$SGL_DISK_DIR_NAME" - echo " - $MODEL_DIR/$MODEL_NAME" - exit 1 - fi - echo "Final MODEL_PATH: $MODEL_PATH" -fi - -# ============================================================================= -# Node Selection -# ============================================================================= - -NUM_NODES=$((xP + yD)) -echo "NUM_NODES: $NUM_NODES (xP=$xP + yD=$yD)" - -FULL_NODELIST=$(scontrol show hostnames "$SLURM_JOB_NODELIST") -SELECTED_NODES=$(echo "$FULL_NODELIST" | head -n $NUM_NODES) -SELECTED_NODELIST_STR=$(echo "$SELECTED_NODES" | tr '\n' ',' | sed 's/,$//') - -# Docker privilege detection — evaluated per-node since group membership varies. -# Exported as a snippet so every srun participant resolves it locally. -export DOCKER_CMD_DETECT='if docker ps &>/dev/null 2>&1; then DOCKER_CMD=docker; else DOCKER_CMD="sudo docker"; fi' - -# Update SLURM environment variables -export SLURM_NNODES=$NUM_NODES -export SLURM_NTASKS=$NUM_NODES -export SLURM_JOB_NUM_NODES=$NUM_NODES -export SLURM_NPROCS=$NUM_NODES -export SLURM_JOB_NODELIST="$SELECTED_NODELIST_STR" -export SLURM_NODELIST="$SELECTED_NODELIST_STR" -export SLURM_TASKS_PER_NODE="1(x$NUM_NODES)" -export SLURM_NTASKS_PER_NODE=1 - -echo "" -echo "Selected nodes: $SELECTED_NODELIST_STR" - -# ============================================================================= -# IP Resolution -# ============================================================================= - -USER_NAME=$(whoami) -MASTER_NODE=$(echo "$SELECTED_NODES" | head -n 1) -NODE0_ADDR=$(srun --nodes=1 --ntasks=1 --time=00:20:00 --nodelist="$MASTER_NODE" bash -c 'ip route get 1.1.1.1') -NODE0_ADDR=$(echo "$NODE0_ADDR" | awk '/src/ {print $7}') - -IPS=() -for NODE in $SELECTED_NODES; do - IP=$(srun --nodes=1 --ntasks=1 --time=00:20:00 --nodelist="$NODE" bash -c 'ip route get 1.1.1.1') - IP=$(echo "$IP" | awk '/src/ {print $7}') - IPS+=("$IP") -done - -echo "Node IPs: ${IPS[*]}" - -DOCKER_MOUNT_PATH="/workspace" -WS_PATH="${DOCKER_MOUNT_PATH}/benchmarks/multi_node/amd_utils" - -NNODES=$NUM_NODES - -echo "MASTER_NODE: ${MASTER_NODE}" -echo "NODE0_ADDR: ${NODE0_ADDR}" -echo "NNODES: ${NNODES}" -echo "REPO DIR: ${DI_REPO_DIR}" -echo "USER: ${USER_NAME}" - -# Reduce log spam -export TQDM_MININTERVAL=20 - -# Translate the host-resolved MODEL_PATH to the Docker mount namespace -DOCKER_MODEL_PATH="${MODEL_PATH/#$MODEL_DIR//models}" - -export DI_REPO_DIR=$DI_REPO_DIR -export WS_PATH=$WS_PATH -export NNODES=$NNODES -export NODE0_ADDR=$NODE0_ADDR -export MODEL_PATH=$MODEL_PATH -export MODEL_DIR=$MODEL_DIR -export xP=$xP -export yD=$yD -export MODEL_NAME=$MODEL_NAME -export USER_NAME=$USER_NAME -export IPADDRS="$(echo "${IPS[*]}" | sed 's/ /,/g')" -export GPUS_PER_NODE=$GPUS_PER_NODE -export BENCH_INPUT_LEN=$BENCH_INPUT_LEN -export BENCH_OUTPUT_LEN=$BENCH_OUTPUT_LEN -export BENCH_RANDOM_RANGE_RATIO=$BENCH_RANDOM_RANGE_RATIO -export BENCH_NUM_PROMPTS_MULTIPLIER=$BENCH_NUM_PROMPTS_MULTIPLIER -export BENCH_MAX_CONCURRENCY=$BENCH_MAX_CONCURRENCY -export BENCH_REQUEST_RATE=$BENCH_REQUEST_RATE -export DRY_RUN="${DRY_RUN:-0}" -export BENCHMARK_LOGS_DIR="${BENCHMARK_LOGS_DIR:-$(pwd)/benchmark_logs}" -export KEEP_CONTAINERS="${KEEP_CONTAINERS:-0}" -export ENGINE=$ENGINE - -# Eval-related env vars (threaded from submit.sh) -export RUN_EVAL="${RUN_EVAL:-false}" -export EVAL_ONLY="${EVAL_ONLY:-false}" -export EVAL_CONC="${EVAL_CONC:-}" -export FRAMEWORK="${FRAMEWORK:-}" -export PRECISION="${PRECISION:-}" -export MODEL_PREFIX="${MODEL_PREFIX:-}" -export RUNNER_TYPE="${RUNNER_TYPE:-}" -export RESULT_FILENAME="${RESULT_FILENAME:-}" -export SPEC_DECODING="${SPEC_DECODING:-}" -export IS_MULTINODE="${IS_MULTINODE:-false}" - -SANITIZED_USER=$(echo "$USER_NAME" | tr -c 'a-zA-Z0-9_.-' '_') -export DOCKER_CONT_NAME="container_${ENGINE}_${SANITIZED_USER}_${MODEL_NAME}_${SLURM_JOB_ID}" - -# vLLM external router container. -# NOTE: vllm/vllm-router only retains ~16 recent nightlies on Docker Hub; older -# dated tags are garbage-collected (manifest unknown) -VLLM_ROUTER_IMAGE="${VLLM_ROUTER_IMAGE:-vllm/vllm-router:nightly-20260629-e667ebb}" -ROUTER_CONT_NAME="router_vllm_${SANITIZED_USER}_${SLURM_JOB_ID}" -export RUN_FILE_FULL="$WS_PATH/${RUN_FILE}" - -SELECTED_NODELIST_SRUN=$(echo "$SELECTED_NODES" | paste -sd,) - -cleanup() { - echo "[${SLURM_JOB_ID}] termination received on $(hostname); cleaning up..." - rm -rf ${SLURM_SUBMIT_DIR}/logs 2>/dev/null || true - echo "[${SLURM_JOB_ID}] cleanup done." -} - -trap cleanup INT TERM HUP - -# Force NFS cache refresh on all nodes -echo "Refreshing NFS caches on all nodes..." -srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c ' - sync - ls -la '"$DI_REPO_DIR"'/benchmarks/multi_node/amd_utils > /dev/null 2>&1 - stat '"$DI_REPO_DIR"'/benchmarks/multi_node/amd_utils/server_agentic.sh > /dev/null 2>&1 - cat '"$DI_REPO_DIR"'/benchmarks/multi_node/amd_utils/server_agentic.sh > /dev/null 2>&1 - echo 3 | sudo tee /proc/sys/vm/drop_caches > /dev/null 2>&1 || true - echo "NFS cache refreshed on $(hostname)" -' - -# ============================================================================= -# Build engine-specific Docker environment variables -# ============================================================================= - -# Common env vars (always passed) -DOCKER_ENV_COMMON=( - -e SLURM_JOB_ID=\$SLURM_JOB_ID - -e SLURM_JOB_NODELIST=\$SLURM_JOB_NODELIST - -e NNODES=\$NNODES - -e NODE_RANK=\$SLURM_PROCID - -e NODE0_ADDR=\$NODE0_ADDR - -e MODEL_DIR=/models - -e MODEL_NAME=\$MODEL_NAME - -e GPUS_PER_NODE=\$GPUS_PER_NODE - -e xP=\$xP - -e yD=\$yD - -e IPADDRS=\$IPADDRS - -e BENCH_INPUT_LEN=\$BENCH_INPUT_LEN - -e BENCH_OUTPUT_LEN=\$BENCH_OUTPUT_LEN - -e BENCH_RANDOM_RANGE_RATIO=\$BENCH_RANDOM_RANGE_RATIO - -e BENCH_NUM_PROMPTS_MULTIPLIER=\$BENCH_NUM_PROMPTS_MULTIPLIER - -e BENCH_MAX_CONCURRENCY=\$BENCH_MAX_CONCURRENCY - -e BENCH_REQUEST_RATE=\$BENCH_REQUEST_RATE - -e TQDM_MININTERVAL=\$TQDM_MININTERVAL - -e BENCHMARK_LOGS_DIR=/benchmark_logs - -e ENGINE=\$ENGINE - -e WS_PATH=${WS_PATH} - -e RUN_EVAL=\$RUN_EVAL - -e EVAL_ONLY=\$EVAL_ONLY - -e \"EVAL_CONC=\$EVAL_CONC\" - -e FRAMEWORK=\$FRAMEWORK - -e PRECISION=\$PRECISION - -e MODEL_PREFIX=\$MODEL_PREFIX - -e RUNNER_TYPE=\$RUNNER_TYPE - -e RESULT_FILENAME=\$RESULT_FILENAME - -e SPEC_DECODING=\$SPEC_DECODING - # DISAGG was never forwarded into the container at all (not even under a - # different name), so process_agentic_result.py's env_bool("DISAGG") always - # defaulted to false in the result JSON regardless of the actual topology. - -e DISAGG=\${DISAGG:-false} - -e PREFILL_TP_SIZE=\$PREFILL_TP_SIZE - # PREFILL_TP/DECODE_TP/*_NUM_WORKERS (below, undecorated -- distinct from the - # *_SIZE vars server_sglang.sh uses for launch args) are what - # process_agentic_result.py's _gpu_shape() reads for multinode runs. Without - # these the container never sees them (only *_SIZE was passed), so agentic - # result JSONs silently recorded tp=0 / prefill_tp=0 / prefill_num_workers=0 - # for every multinode run. - -e PREFILL_TP=\$PREFILL_TP - -e PREFILL_NUM_WORKERS=\$PREFILL_NUM_WORKERS - -e PREFILL_ENABLE_EP=\$PREFILL_ENABLE_EP - -e PREFILL_ENABLE_DP=\$PREFILL_ENABLE_DP - -e PREFILL_CONTEXT_LENGTH=\${PREFILL_CONTEXT_LENGTH:-} - -e PREFILL_CHUNKED_PREFILL_SIZE=\${PREFILL_CHUNKED_PREFILL_SIZE:-} - -e SGLANG_AITER_MLA_PERSIST=\${SGLANG_AITER_MLA_PERSIST:-0} - -e DISABLE_CUSTOM_ALL_REDUCE=\${DISABLE_CUSTOM_ALL_REDUCE:-0} - -e MAX_MODEL_LEN=\${MAX_MODEL_LEN:-} - -e DURATION=\${DURATION:-1800} - -e IS_AGENTIC=\${IS_AGENTIC:-0} - -e KV_OFFLOADING=\${KV_OFFLOADING:-none} - -e KV_OFFLOAD_BACKEND=\${KV_OFFLOAD_BACKEND:-} - -e TOTAL_CPU_DRAM_GB=\${TOTAL_CPU_DRAM_GB:-} - -e ENABLE_METRICS=\${ENABLE_METRICS:-0} - -e PREFILL_ROUTER_POLICY=\${PREFILL_ROUTER_POLICY:-random} - -e DECODE_ROUTER_POLICY=\${DECODE_ROUTER_POLICY:-random} - -e MORI_IO_SQ_BACKOFF_TIMEOUT_US=\${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-} - -e MORI_IO_QP_MAX_SEND_WR=\${MORI_IO_QP_MAX_SEND_WR:-} - -e DECODE_TP_SIZE=\$DECODE_TP_SIZE - -e DECODE_TP=\$DECODE_TP - -e DECODE_NUM_WORKERS=\$DECODE_NUM_WORKERS - -e DECODE_ENABLE_EP=\$DECODE_ENABLE_EP - -e DECODE_ENABLE_DP=\$DECODE_ENABLE_DP - -e DECODE_MTP_SIZE=\$DECODE_MTP_SIZE - -e IS_MULTINODE=\$IS_MULTINODE - -e DRY_RUN=\${DRY_RUN:-0} -) - -# Engine-specific env vars -if [[ "$ENGINE" == "vllm-disagg" ]]; then - DOCKER_ENV_ENGINE=( - -e VLLM_WS_PATH=${WS_PATH} - -e MODEL_PATH=$DOCKER_MODEL_PATH - -e UCX_TLS=tcp,self,shm,rocm_ipc,rocm_copy,cma - -e UCX_SOCKADDR_TLS_PRIORITY=tcp - -e UCX_MEMTYPE_CACHE=y - -e UCX_RNDV_SCHEME=get_zcopy - -e UCX_RNDV_THRESH=4k - -e UCX_ROCM_IPC_MIN_ZCOPY=0 - -e UCX_LOG_LEVEL=warn - -e HSA_ENABLE_SDMA=1 - -e PROXY_STREAM_IDLE_TIMEOUT=\${PROXY_STREAM_IDLE_TIMEOUT:-300} - -e PYTHONPYCACHEPREFIX=/tmp/pycache - ) -elif [[ "$ENGINE" == "atom-disagg" ]]; then - DOCKER_ENV_ENGINE=( - -e ATOM_WS_PATH=${WS_PATH} - -e PREFILL_PORT=${PREFILL_PORT:-8010} - -e DECODE_PORT=${DECODE_PORT:-8020} - -e ROUTER_PORT=${ROUTER_PORT:-30000} - -e HANDSHAKE_PORT=${HANDSHAKE_PORT:-6301} - -e MEM_FRAC_STATIC=${MEM_FRAC_STATIC:-0.85} - -e KV_CACHE_DTYPE=${KV_CACHE_DTYPE:-fp8} - -e BLOCK_SIZE=${BLOCK_SIZE:-16} - -e MAX_NUM_SEQS=${MAX_NUM_SEQS:-256} - -e MAX_MODEL_LEN=${MAX_MODEL_LEN:-} - -e MAX_NUM_BATCHED_TOKENS=${MAX_NUM_BATCHED_TOKENS:-} - -e EXTRA_SERVER_ARGS=\${EXTRA_SERVER_ARGS:-} - -e IBDEVICES=${IBDEVICES:-} - ) -else - DOCKER_ENV_ENGINE=( - -e SGLANG_WS_PATH=${WS_PATH} - ) -fi - -# HiCache / Mooncake settings are delivered via a bind-mounted config file rather -# than a long list of docker -e flags. Write it once to the shared benchmark-logs -# dir (already a host path, visible on every node) and mount it read-only at -# /config/hicache_mc.env, where env.sh sources it before applying its defaults. -# Empty values are preserved so env.sh's "${VAR:-default}" fallbacks still apply. -HICACHE_MC_CONFIG="${BENCHMARK_LOGS_DIR}/hicache_mc_${SLURM_JOB_ID}.env" -cat > "$HICACHE_MC_CONFIG" < $HICACHE_MC_CONFIG" - -# Engine-specific container filter for pre-clean -CONT_FILTER="name=^container_${ENGINE}_" - -srun \ - --nodelist="$SELECTED_NODELIST_SRUN" \ - --kill-on-bad-exit=1 \ - --signal=TERM@30 \ - --unbuffered \ - bash -lc " -set -euo pipefail - -echo \"Rank \$SLURM_PROCID on \$(hostname)\" - -# Per-node docker privilege detection -eval \"\$DOCKER_CMD_DETECT\" -echo \"[docker-detect] rank \$SLURM_PROCID: DOCKER_CMD=\$DOCKER_CMD\" - -# Enable out-of-tree RDMA library mounts for atom-disagg (mooncake requires host RDMA stack) -RDMA_MOUNTS=() -if [[ "$ENGINE" == "atom-disagg" ]]; then - -# When the container base OS differs from the host (e.g. Ubuntu 24.04 image -# on a 22.04 host), the container's bundled libibverbs/libionic may be -# ABI-incompatible with the host kernel drivers. Detect the NIC type and -# bind-mount the host's out-of-tree RDMA userspace libraries into the -# container so the RDMA stack always matches the running kernel. -_detect_nic_type() { - if [[ -n \"\${MORI_NIC_TYPE:-}\" ]]; then echo \"\$MORI_NIC_TYPE\"; return; fi - local bnxt=0 mlx5=0 ionic=0 - if [[ -d /sys/class/infiniband ]]; then - for dev in /sys/class/infiniband/*; do - local name; name=\$(basename \"\$dev\") - case \"\$name\" in - bnxt_re*) ((bnxt++)) ;; mlx5*) ((mlx5++)) ;; ionic*) ((ionic++)) ;; - *) - local drv; drv=\$(basename \"\$(readlink -f \"\$dev/device/driver\" 2>/dev/null)\" 2>/dev/null || true) - case \"\$drv\" in bnxt*) ((bnxt++)) ;; mlx5*) ((mlx5++)) ;; ionic*) ((ionic++)) ;; esac ;; - esac - done - fi - if (( bnxt >= mlx5 && bnxt >= ionic && bnxt > 0 )); then echo bnxt - elif (( ionic >= mlx5 && ionic > 0 )); then echo ionic - else echo mlx5; fi -} - -_find_host_ibverbs() { - for c in /usr/lib64/libibverbs.so.1 /lib/x86_64-linux-gnu/libibverbs.so.1 /usr/lib/x86_64-linux-gnu/libibverbs.so.1.14.39.0 /usr/lib/x86_64-linux-gnu/libibverbs.so.1; do - local r; r=\$(readlink -f \"\$c\" 2>/dev/null || true) - [[ \"\$r\" == *libibverbs.so.1.14.57.0 ]] && continue - if [[ -f \"\$r\" ]]; then echo \"\$r\"; return; fi - done -} - -_NIC_TYPE=\$(_detect_nic_type) -echo \"[rdma] NIC type: \${_NIC_TYPE} on \$(hostname)\" - -if [[ \"\$_NIC_TYPE\" == \"ionic\" || \"\$_NIC_TYPE\" == \"bnxt\" ]]; then - _host_ibv=\$(_find_host_ibverbs) - if [[ -n \"\$_host_ibv\" ]]; then - RDMA_MOUNTS+=(-v \"\$_host_ibv:/lib/x86_64-linux-gnu/libibverbs.so.1\") - fi -fi - -if [[ \"\$_NIC_TYPE\" == \"ionic\" ]]; then - for _dir in /usr/local/lib /usr/lib/x86_64-linux-gnu; do - for _lib in \"\$_dir\"/libionic*.so; do - [[ -f \"\$_lib\" ]] || continue - _real=\$(readlink -f \"\$_lib\") - [[ -f \"\$_real\" ]] && RDMA_MOUNTS+=(-v \"\$_real:\$_real\") - RDMA_MOUNTS+=(-v \"\$_lib:/usr/lib/x86_64-linux-gnu/\$(basename \"\$_lib\")\") - done - done - if [[ -d /usr/lib/x86_64-linux-gnu/libibverbs ]]; then - for _lib in /usr/lib/x86_64-linux-gnu/libibverbs/libionic-rdmav*.so; do - [[ -f \"\$_lib\" ]] && RDMA_MOUNTS+=(-v \"\$_lib:\$_lib\") - done - fi - [[ -d /etc/libibverbs.d ]] && RDMA_MOUNTS+=(-v /etc/libibverbs.d:/etc/libibverbs.d:ro) -elif [[ \"\$_NIC_TYPE\" == \"bnxt\" ]]; then - for _lib in /usr/local/lib/libbnxt_re-rdmav*.so; do - [[ -f \"\$_lib\" ]] && RDMA_MOUNTS+=(-v \"\$_lib:/usr/lib/x86_64-linux-gnu/libibverbs/\$(basename \"\$_lib\")\") - done - for _lib in /usr/local/lib/libbnxt_re.so; do - [[ -f \"\$_lib\" ]] && RDMA_MOUNTS+=(-v \"\$_lib:/usr/lib/x86_64-linux-gnu/\$(basename \"\$_lib\")\") - done - [[ -d /etc/libibverbs.d ]] && RDMA_MOUNTS+=(-v /etc/libibverbs.d:/etc/libibverbs.d:ro) -fi - -if [[ \${#RDMA_MOUNTS[@]} -gt 0 ]]; then - echo \"[rdma] bind-mounts: \${RDMA_MOUNTS[*]}\" -else - echo \"[rdma] no out-of-tree RDMA mounts needed\" -fi -fi # end: if ENGINE == atom-disagg - -# Pre-clean (idempotent) -\$DOCKER_CMD ps -aq --filter \"$CONT_FILTER\" | xargs -r \$DOCKER_CMD rm -f || true -\$DOCKER_CMD ps -aq | xargs -r \$DOCKER_CMD stop || true - -# Start vLLM external router container on node 0 -if [[ \"$ENGINE\" == \"vllm-disagg\" && \"$ROUTER_TYPE\" == \"vllm-router\" && \"\$SLURM_PROCID\" == \"0\" ]]; then - \$DOCKER_CMD rm -f \"$ROUTER_CONT_NAME\" 2>/dev/null || true - \$DOCKER_CMD run -d \ - --name \"$ROUTER_CONT_NAME\" \ - --network host \ - --ulimit nofile=1048576:1048576 \ - -v /tmp:/run_logs \ - \"$VLLM_ROUTER_IMAGE\" \ - bash -lc \"mkdir -p /run_logs/slurm_job-${SLURM_JOB_ID} && exec vllm-router \ - --vllm-pd-disaggregation \ - --kv-connector moriio \ - --vllm-discovery-address 0.0.0.0:${PROXY_PING_PORT} \ - --port ${ROUTER_PORT} \ - --host 0.0.0.0 \ - --policy consistent_hash \ - --prefill-policy consistent_hash \ - --decode-policy consistent_hash \ - --log-level info 2>&1 | tee /run_logs/slurm_job-${SLURM_JOB_ID}/vllm_router_\$(hostname).log \" -fi - -# Skip exec on vllm-disagg rank 0 so we can stop the router after the main -# container exits. Without this, decode nodes block forever waiting for the -# router port to close (the router is a separate container). -MAYBE_EXEC=exec -if [[ \"$ENGINE\" == \"vllm-disagg\" && \"$ROUTER_TYPE\" == \"vllm-router\" && \"\$SLURM_PROCID\" == \"0\" ]]; then - MAYBE_EXEC= - set +e -fi - -\$MAYBE_EXEC \$DOCKER_CMD run \ - --init \ - --stop-timeout 10 \ - --device /dev/dri \ - --device /dev/kfd \ - --device /dev/infiniband \ - --device=/dev/infiniband/rdma_cm \ - --device=/dev/infiniband/uverbs0 \ - --device=/dev/infiniband/uverbs1 \ - --device=/dev/infiniband/uverbs2 \ - --device=/dev/infiniband/uverbs3 \ - --device=/dev/infiniband/uverbs4 \ - --device=/dev/infiniband/uverbs5 \ - --device=/dev/infiniband/uverbs6 \ - --device=/dev/infiniband/uverbs7 \ - --ulimit memlock=-1 \ - --ulimit stack=67108864 \ - --ulimit nofile=1048576:1048576 \ - --network host \ - --ipc host \ - --group-add video \ - --cap-add SYS_PTRACE \ - --security-opt seccomp=unconfined \ - --privileged \ - -v /sys:/sys \ - $(command -v nicctl >/dev/null 2>&1 && echo "-v $(which nicctl):/usr/sbin/nicctl") \ - -v ${MODEL_DIR}:/models \ - -v \$HOME/.ssh:/root/.ssh \ - --shm-size 128G \ - -v /tmp:/run_logs \ - -v ${BENCHMARK_LOGS_DIR}:/benchmark_logs \ - -v ${DI_REPO_DIR}:${DOCKER_MOUNT_PATH} \ - -v ${HICACHE_MC_CONFIG}:/config/hicache_mc.env:ro \ - ${EXTRA_DOCKER_MOUNTS:-} \ - \${RDMA_MOUNTS[@]+"\${RDMA_MOUNTS[@]}"} \ - ${DOCKER_ENV_COMMON[*]} \ - ${DOCKER_ENV_ENGINE[*]} \ - --name \"$DOCKER_CONT_NAME\" \ - --entrypoint \"\" \ - \"$DOCKER_IMAGE_NAME\" bash -lc ' - set -o pipefail - mkdir -p /run_logs/slurm_job-'\"\$SLURM_JOB_ID\"' - '"$RUN_FILE_FULL"' 2>&1 | tee /run_logs/slurm_job-'\"\$SLURM_JOB_ID\"'/server_\$(hostname).log - ' - -# Only reached when exec was skipped (vllm-disagg rank 0) -DOCKER_EXIT_CODE=\$? -echo \"[rank 0] Main container exited (rc=\$DOCKER_EXIT_CODE). Stopping vllm-router...\" -\$DOCKER_CMD rm -f \"$ROUTER_CONT_NAME\" 2>/dev/null || true -exit \$DOCKER_EXIT_CODE -" - -if [[ "${KEEP_CONTAINERS}" != "1" ]]; then - srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c 'eval "$DOCKER_CMD_DETECT"; $DOCKER_CMD rm -f '"$DOCKER_CONT_NAME"' 2>/dev/null || true' - - # Clean up vLLM external router container on node 0 - if [[ "$ENGINE" == "vllm-disagg" && "$ROUTER_TYPE" == "vllm-router" ]]; then - srun --nodes=1 --ntasks=1 --nodelist="$MASTER_NODE" bash -c ' - eval "$DOCKER_CMD_DETECT"; $DOCKER_CMD rm -f '"$ROUTER_CONT_NAME"' 2>/dev/null || true - ' - fi -fi diff --git a/benchmarks/multi_node/amd_utils/models_agentic.yaml b/benchmarks/multi_node/amd_utils/models_agentic.yaml deleted file mode 100644 index 698fcbc5eb..0000000000 --- a/benchmarks/multi_node/amd_utils/models_agentic.yaml +++ /dev/null @@ -1,394 +0,0 @@ -# Model-specific SGLang server configurations for disaggregated inference. -# -# Each top-level key is a MODEL_NAME value (must match the directory name under MODEL_DIR). -# -# To add a new model: add a new top-level entry following the same schema. -# No script changes are required. -# -# Schema: -# : -# base_flags: str # Common flags for both prefill and decode -# mtp_flags: str # Appended to decode when DECODE_MTP_SIZE > 0 -# dp_flags: str # Appended when DP is enabled (prefill or decode) -# prefill: -# mem_fraction_static: float -# disable_radix_cache: bool -# dp: # Config when data-parallel attention is enabled -# max_running_requests: int -# chunked_prefill_size: str # Can be integer or bash arithmetic expression -# cuda_graph_bs: str # Space-separated values -# no_dp: # Config when data-parallel attention is disabled -# max_running_requests: int -# chunked_prefill_size: int -# cuda_graph_bs_range: str # "start-end" expanded via seq -# decode: -# mem_fraction_static: float -# prefill_round_robin_balance: bool -# dp: -# max_running_requests: int -# chunked_prefill_size: str -# cuda_graph_bs_range: str -# ep_only: # Config when EP is enabled but DP is disabled -# max_running_requests: int -# chunked_prefill_size: int -# cuda_graph_bs_range: str -# no_dp: -# max_running_requests: int -# chunked_prefill_size: int -# cuda_graph_bs_range: str - -DeepSeek-V3: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --ep-dispatch-algorithm fake --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" - mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1" - dp_flags: "--moe-a2a-backend mori --deepep-mode normal --enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - prefill_round_robin_balance: true - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -DeepSeek-V3-0324: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --ep-dispatch-algorithm fake --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" - mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1" - dp_flags: "--moe-a2a-backend mori --deepep-mode normal --enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - prefill_round_robin_balance: true - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -DeepSeek-R1: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --ep-dispatch-algorithm fake --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" - mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1" - dp_flags: "--moe-a2a-backend mori --deepep-mode normal --enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - prefill_round_robin_balance: true - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -DeepSeek-R1-0528: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --ep-dispatch-algorithm fake --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" - mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1" - dp_flags: "--moe-a2a-backend mori --deepep-mode normal --enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - prefill_round_robin_balance: true - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -Qwen3.5-397B-A17B-MXFP4: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori --moe-dense-tp-size 1" - mtp_flags: "" - dp_flags: "--moe-a2a-backend mori --enable-dp-attention --enable-dp-lm-head" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - prefill_round_robin_balance: true - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -Qwen3.5-397B-A17B-FP8: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori --moe-dense-tp-size 1" - mtp_flags: "" - dp_flags: "--moe-a2a-backend mori --enable-dp-attention --enable-dp-lm-head" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - prefill_round_robin_balance: true - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -GLM-5-FP8: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --disaggregation-transfer-backend mori --tool-call-parser glm47 --reasoning-parser glm45 --model-loader-extra-config '{\\\"enable_multithread_load\\\": true, \\\"num_threads\\\": 8}'" - mtp_flags: "" - dp_flags: "--moe-a2a-backend mori --enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - prefill_round_robin_balance: true - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -DeepSeek-R1-0528-MXFP4-Preview: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --ep-dispatch-algorithm fake --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" - mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1" - dp_flags: "--moe-a2a-backend mori --deepep-mode normal --enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: 16384 - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 16384 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - prefill_round_robin_balance: true - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -DeepSeek-R1-0528-MXFP4: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --ep-dispatch-algorithm fake --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" - mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1" - dp_flags: "--moe-a2a-backend mori --deepep-mode normal --enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 16384 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - prefill_round_robin_balance: true - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -DeepSeek-R1-0528-MXFP4-v2: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --ep-dispatch-algorithm fake --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" - mtp_flags: "--speculative-draft-model-path SGLang/DeepSeek-R1-NextN --speculative-algorithm NEXTN --speculative-eagle-topk 1 --speculative-attention-mode decode " - dp_flags: "--moe-a2a-backend mori --deepep-mode normal --enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head --stream-interval 100 --tokenizer-worker-num 32 " - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - context_length: 9217 - max_total_tokens: 131072 - enable_two_batch_overlap: true - no_dp: - max_running_requests: 128 - chunked_prefill_size: 16384 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - prefill_round_robin_balance: true - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-512" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -# DeepSeek-V4-Pro PD-disaggregation recipe (MI355X, SGLang + MoRI). -# KV transfer = mori for both topologies (pure-TP and DEP); the DP path additionally -# routes the MoE all-to-all through mori (--moe-a2a-backend mori) with dp-attention. -# DSv4-specific kernel routing (unified_kv_triton, AITER indexer, fp8 wo_a fallback, -# thinking/reasoning-effort, dispatch dtypes, per-role PER_RANK dispatch tokens) is set -# in env.sh's DeepSeek-V4-Pro block. The bench client uses --dsv4 framing (bench.sh). -# prefill.disable_cuda_graph routes prefill to --disable-cuda-graph; decode keeps -# --cuda-graph-bs. See dsv4_mi355x_sglang_disagg_plan.md. hf_dir: "models--deepseek-ai--DeepSeek-V4-Pro" -DeepSeek-V4-Pro: - base_flags: "--watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend dsv4 --page-size 256 --swa-full-tokens-ratio 0.1 --disable-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --disaggregation-transfer-backend mori --log-level error --log-level-http error" - dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - disable_cuda_graph: true - dp: - max_running_requests: 1024 - chunked_prefill_size: 65280 # dsv4 compressor kernel uint16 token cap (255*256) - context_length: 1048576 - max_total_tokens: 1048576 - no_dp: - max_running_requests: 128 - # Small prefill chunks interleave long-context agentic prefills across - # requests instead of letting one ~100K-token prefill monopolize the - # engine (the conc>=16 queue-saturation / decode-stall failure mode). - # Mirrors the single-node DSv4 agentic recipe (dsv4_fp4_mi355x.sh=8192). - # Was 65280 (255*256, the dsv4 compressor kernel uint16 token cap); 8192 - # (32*256) stays a page-size multiple well under that cap. - chunked_prefill_size: 8192 - context_length: 1048576 - max_total_tokens: 1048576 - decode: - mem_fraction_static: 0.85 - prefill_round_robin_balance: true - disagg_decode_enable_radix_cache: false - dp: - max_running_requests: 1024 - cuda_graph_bs_range: "1-128" - no_dp: - max_running_requests: 128 - cuda_graph_bs_range: "1-128" diff --git a/benchmarks/multi_node/amd_utils/server_agentic.sh b/benchmarks/multi_node/amd_utils/server_agentic.sh deleted file mode 100755 index c2ba9c3553..0000000000 --- a/benchmarks/multi_node/amd_utils/server_agentic.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# Multi-Engine Disaggregated Server Dispatcher -# ============================================================================= -# Dispatches to the engine-specific server launcher based on ENGINE env var. -# ENGINE=sglang-disagg (default) -> server_sglang_agentic.sh (SGLang + MoRI) -# ENGINE=vllm-disagg -> server_vllm.sh (vLLM + Nixl/MoRI-IO) -# ENGINE=atom-disagg -> server_atom.sh (ATOM + mooncake) -# ============================================================================= - -ENGINE="${ENGINE:-sglang-disagg}" -WS_PATH="${WS_PATH:-${SGLANG_WS_PATH:-${VLLM_WS_PATH:-${ATOM_WS_PATH:-$(dirname "${BASH_SOURCE[0]}")}}}}" -export WS_PATH ENGINE - -echo "[DISPATCHER] ENGINE=$ENGINE WS_PATH=$WS_PATH" - -if [[ "$ENGINE" == "vllm-disagg" ]]; then - source "$WS_PATH/server_vllm.sh" -elif [[ "$ENGINE" == "atom-disagg" ]]; then - export ATOM_WS_PATH="$WS_PATH" - source "$WS_PATH/server_atom.sh" -else - source "$WS_PATH/server_sglang_agentic.sh" -fi diff --git a/benchmarks/multi_node/amd_utils/server_sglang_agentic.sh b/benchmarks/multi_node/amd_utils/server_sglang_agentic.sh deleted file mode 100755 index d19691f428..0000000000 --- a/benchmarks/multi_node/amd_utils/server_sglang_agentic.sh +++ /dev/null @@ -1,1039 +0,0 @@ -#!/bin/bash -# SGLang Disaggregated Server Launcher with Model-Specific Configurations -# ============================================================================= - -# ============================================================================= -# Environment Configuration -# ============================================================================= - -NODE0_ADDR="${NODE0_ADDR:-localhost}" -NODE_RANK="${NODE_RANK:-0}" -MODEL_DIR="${MODEL_DIR:-}" -MODEL_NAME="${MODEL_NAME:-}" - -xP="${xP:-1}" #-> Number of Prefill Workers -yD="${yD:-1}" #-> Number of Decode Workers - -IPADDRS="${IPADDRS:-localhost}" -HEADNODE_PORT="${HEADNODE_PORT:-20000}" -# Parallelism Configuration -PREFILL_TP_SIZE="${PREFILL_TP_SIZE:-8}" -PREFILL_ENABLE_EP="${PREFILL_ENABLE_EP:-true}" -PREFILL_ENABLE_DP="${PREFILL_ENABLE_DP:-true}" -DECODE_TP_SIZE="${DECODE_TP_SIZE:-8}" -DECODE_ENABLE_EP="${DECODE_ENABLE_EP:-true}" -DECODE_ENABLE_DP="${DECODE_ENABLE_DP:-true}" -DECODE_MTP_SIZE="${DECODE_MTP_SIZE:-0}" - -# Benchmark Configuration -BENCH_INPUT_LEN="${BENCH_INPUT_LEN:-1024}" -BENCH_OUTPUT_LEN="${BENCH_OUTPUT_LEN:-1024}" -BENCH_RANDOM_RANGE_RATIO="${BENCH_RANDOM_RANGE_RATIO:-1}" -BENCH_REQUEST_RATE="${BENCH_REQUEST_RATE:-inf}" -BENCH_NUM_PROMPTS_MULTIPLIER="${BENCH_NUM_PROMPTS_MULTIPLIER:-10}" -BENCH_MAX_CONCURRENCY="${BENCH_MAX_CONCURRENCY:-512}" - -# Extract the maximum concurrency from the x-delimited list -BENCH_MAX_CONC_VALUE=$(echo "$BENCH_MAX_CONCURRENCY" | tr 'x' '\n' | sort -n | tail -1) - -# Dry Run for debugging purpose -DRY_RUN="${DRY_RUN:-0}" - -# GPU count (expandable for different hardware) -GPUS_PER_NODE="${GPUS_PER_NODE:-8}" - - -# ============================================================================= -# Dependencies and Environment Setup -# ============================================================================= -source $SGLANG_WS_PATH/setup_deps_agentic.sh -source $SGLANG_WS_PATH/env_agentic.sh - -host_ip=$(ip route get 1.1.1.1 | awk '/src/ {print $7}') -host_name=$(hostname) - -# MORI_RDMA_TC configuration (optional) -# If set by runner, use it for RDMA traffic class configuration -# If not set, RDMA operations will proceed without QoS/traffic class settings -if [[ -n "${MORI_RDMA_TC}" ]]; then - echo "[INFO] Using MORI_RDMA_TC=$MORI_RDMA_TC for RDMA traffic class configuration" - echo "[INFO] Host '$host_name' configured with MORI_RDMA_TC=$MORI_RDMA_TC" -else - echo "[INFO] MORI_RDMA_TC not set. Skipping RDMA traffic class configuration." - echo "[INFO] This is normal for clusters without QoS requirements." -fi - -# ============================================================================= -# Model-Specific Configuration from YAML -# ============================================================================= -MODELS_YAML="${SGLANG_WS_PATH}/models_agentic.yaml" - -if [[ ! -f "$MODELS_YAML" ]]; then - echo "ERROR: models.yaml not found at $MODELS_YAML" - exit 1 -fi - -# Load model config via inline Python (PyYAML is available in SGLang containers) -# Formula evaluation (e.g. "SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK * TP * xP") -# is done here in Python to avoid bash glob-expanding the * characters. -eval "$(python3 -c " -import yaml, sys, os - -config_path = '${MODELS_YAML}' -model_name = '${MODEL_NAME}' - -with open(config_path) as f: - models = yaml.safe_load(f) - -if model_name not in models: - print(f'echo \"ERROR: Model {model_name} not in models.yaml\"; exit 1') - sys.exit(0) - -m = models[model_name] - -def eval_formula(val): - \"\"\"Evaluate chunked_prefill_size: if string, resolve variable names from env and compute.\"\"\" - if isinstance(val, (int, float)): - return int(val) - s = str(val) - # Build a namespace from env vars (convert numeric values to int) - ns = {} - for k, v in os.environ.items(): - try: - ns[k] = int(v) - except (ValueError, TypeError): - pass - try: - return int(eval(s, {'__builtins__': {}}, ns)) - except Exception as e: - print(f'echo \"WARNING: Cannot evaluate formula: {s} ({e})\"', file=sys.stderr) - return val - -def parse_range(cuda_range, default_start, default_end): - if '-' in str(cuda_range): - s, e = str(cuda_range).split('-') - return s, e - return str(default_start), str(default_end) - -# Output shell variables -print(f'MODEL_BASE_FLAGS=\"{m.get(\"base_flags\", \"\")}\"') -print(f'MODEL_MTP_FLAGS=\"{m.get(\"mtp_flags\", \"\")}\"') -print(f'MODEL_DP_FLAGS=\"{m.get(\"dp_flags\", \"\")}\"') -print(f'MODEL_EP_FLAGS=\"{m.get(\"ep_flags\", \"\")}\"') - -prefill = m.get('prefill', {}) -decode = m.get('decode', {}) - -print(f'PREFILL_MEM_FRACTION_STATIC=\"{prefill.get(\"mem_fraction_static\", 0.8)}\"') -print(f'PREFILL_DISABLE_RADIX_CACHE=\"{prefill.get(\"disable_radix_cache\", True)}\"') -print(f'PREFILL_DISABLE_CUDA_GRAPH=\"{prefill.get(\"disable_cuda_graph\", False)}\"') - -dp = prefill.get('dp', {}) -no_dp = prefill.get('no_dp', {}) -print(f'PREFILL_MAX_RUNNING_REQUESTS_DP=\"{dp.get(\"max_running_requests\", 24)}\"') -print(f'PREFILL_CHUNKED_PREFILL_SIZE_DP=\"{eval_formula(dp.get(\"chunked_prefill_size\", 262144))}\"') -print(f'PREFILL_CUDA_GRAPH_BS_DP=\"{dp.get(\"cuda_graph_bs\", \"1 2 3\")}\"') -print(f'PREFILL_CONTEXT_LENGTH_DP=\"{dp.get(\"context_length\", \"\")}\"') -print(f'PREFILL_MAX_TOTAL_TOKENS_DP=\"{dp.get(\"max_total_tokens\", \"\")}\"') -print(f'PREFILL_ENABLE_TWO_BATCH_OVERLAP_DP=\"{dp.get(\"enable_two_batch_overlap\", False)}\"') -print(f'PREFILL_MAX_RUNNING_REQUESTS_NO_DP=\"{no_dp.get(\"max_running_requests\", 128)}\"') -print(f'PREFILL_CHUNKED_PREFILL_SIZE_NO_DP=\"{eval_formula(no_dp.get(\"chunked_prefill_size\", 262144))}\"') -print(f'PREFILL_CONTEXT_LENGTH_NO_DP=\"{no_dp.get(\"context_length\", \"\")}\"') -print(f'PREFILL_MAX_TOTAL_TOKENS_NO_DP=\"{no_dp.get(\"max_total_tokens\", \"\")}\"') -s, e = parse_range(no_dp.get('cuda_graph_bs_range', '1-128'), 1, 128) -print(f'PREFILL_CUDA_GRAPH_BS_NO_DP_START=\"{s}\"') -print(f'PREFILL_CUDA_GRAPH_BS_NO_DP_END=\"{e}\"') - -print(f'DECODE_MEM_FRACTION_STATIC=\"{decode.get(\"mem_fraction_static\", 0.85)}\"') -print(f'DECODE_PREFILL_ROUND_ROBIN_BALANCE=\"{decode.get(\"prefill_round_robin_balance\", True)}\"') -print(f'DECODE_DISAGG_ENABLE_RADIX_CACHE=\"{decode.get(\"disagg_decode_enable_radix_cache\", False)}\"') - -dp = decode.get('dp', {}) -ep_only = decode.get('ep_only', {}) -no_dp = decode.get('no_dp', {}) - -# Decode DP config -print(f'DECODE_MAX_RUNNING_REQUESTS_DP=\"{dp.get(\"max_running_requests\", 4096)}\"') -print(f'DECODE_CHUNKED_PREFILL_SIZE_DP=\"{eval_formula(dp.get(\"chunked_prefill_size\", 262144))}\"') -s, e = parse_range(dp.get('cuda_graph_bs_range', '1-160'), 1, 160) -print(f'DECODE_CUDA_GRAPH_BS_DP_START=\"{s}\"') -print(f'DECODE_CUDA_GRAPH_BS_DP_END=\"{e}\"') - -# Decode EP-only config (EP enabled but DP disabled) -print(f'DECODE_MAX_RUNNING_REQUESTS_EP_ONLY=\"{ep_only.get(\"max_running_requests\", 256)}\"') -print(f'DECODE_CHUNKED_PREFILL_SIZE_EP_ONLY=\"{eval_formula(ep_only.get(\"chunked_prefill_size\", 262144))}\"') -s, e = parse_range(ep_only.get('cuda_graph_bs_range', '1-256'), 1, 256) -print(f'DECODE_CUDA_GRAPH_BS_EP_ONLY_START=\"{s}\"') -print(f'DECODE_CUDA_GRAPH_BS_EP_ONLY_END=\"{e}\"') - -# Decode no-DP config -print(f'DECODE_MAX_RUNNING_REQUESTS_NO_DP=\"{no_dp.get(\"max_running_requests\", 128)}\"') -print(f'DECODE_CHUNKED_PREFILL_SIZE_NO_DP=\"{eval_formula(no_dp.get(\"chunked_prefill_size\", 262144))}\"') -s, e = parse_range(no_dp.get('cuda_graph_bs_range', '1-128'), 1, 128) -print(f'DECODE_CUDA_GRAPH_BS_NO_DP_START=\"{s}\"') -print(f'DECODE_CUDA_GRAPH_BS_NO_DP_END=\"{e}\"') -")" - -echo "Loaded model configuration for: $MODEL_NAME" - -# Compute DP-dependent prefill parameters -if [[ "$PREFILL_ENABLE_DP" == "true" ]]; then - prefill_cuda_graph_bs=($PREFILL_CUDA_GRAPH_BS_DP) - prefill_max_running_requests=$PREFILL_MAX_RUNNING_REQUESTS_DP - prefill_chunked_prefill_size=$PREFILL_CHUNKED_PREFILL_SIZE_DP - prefill_context_length=$PREFILL_CONTEXT_LENGTH_DP - prefill_max_total_tokens=$PREFILL_MAX_TOTAL_TOKENS_DP - prefill_enable_two_batch_overlap=$PREFILL_ENABLE_TWO_BATCH_OVERLAP_DP -else - prefill_cuda_graph_bs=($(seq $PREFILL_CUDA_GRAPH_BS_NO_DP_START $PREFILL_CUDA_GRAPH_BS_NO_DP_END)) - prefill_max_running_requests=$PREFILL_MAX_RUNNING_REQUESTS_NO_DP - prefill_chunked_prefill_size=$PREFILL_CHUNKED_PREFILL_SIZE_NO_DP - prefill_context_length=$PREFILL_CONTEXT_LENGTH_NO_DP - prefill_max_total_tokens=$PREFILL_MAX_TOTAL_TOKENS_NO_DP - prefill_enable_two_batch_overlap="false" -fi - -# When both DP and EP are enabled, override max-running-requests with max bench concurrency -if [[ "$PREFILL_ENABLE_DP" == "true" ]] && [[ "$PREFILL_ENABLE_EP" == "true" ]]; then - prefill_max_running_requests=$BENCH_MAX_CONC_VALUE - prefill_dp_ranks=$PREFILL_TP_SIZE - # MORI_MAX_DISPATCH_TOKENS_PREFILL stays at 8192 (no change) - # MORI_MOE_MAX_INPUT_TOKENS_PREFILL=$((MORI_MAX_DISPATCH_TOKENS_PREFILL * prefill_dp_ranks / 2)) - echo "[DP+EP override] Prefill: max-running-requests=$prefill_max_running_requests, MOE_MAX_INPUT=$MORI_MOE_MAX_INPUT_TOKENS_PREFILL" -fi - -# Compute DP-dependent decode parameters (3-way: DP > EP-only > no_dp) -if [[ "$DECODE_ENABLE_DP" == "true" ]]; then - decode_cuda_graph_bs=($(seq $DECODE_CUDA_GRAPH_BS_DP_START $DECODE_CUDA_GRAPH_BS_DP_END)) - decode_max_running_requests=$((DECODE_CUDA_GRAPH_BS_DP_END * DECODE_TP_SIZE)) -elif [[ "$DECODE_ENABLE_EP" == "true" ]]; then - decode_cuda_graph_bs=($(seq $DECODE_CUDA_GRAPH_BS_EP_ONLY_START $DECODE_CUDA_GRAPH_BS_EP_ONLY_END)) - decode_max_running_requests=$DECODE_MAX_RUNNING_REQUESTS_EP_ONLY -else - decode_cuda_graph_bs=($(seq $DECODE_CUDA_GRAPH_BS_NO_DP_START $DECODE_CUDA_GRAPH_BS_NO_DP_END)) - decode_max_running_requests=$DECODE_MAX_RUNNING_REQUESTS_NO_DP -fi - -# When both DP and EP are enabled, override max-running-requests and dispatch tokens -if [[ "$DECODE_ENABLE_DP" == "true" ]] && [[ "$DECODE_ENABLE_EP" == "true" ]]; then - decode_max_running_requests=$BENCH_MAX_CONC_VALUE - decode_dp_ranks=$DECODE_TP_SIZE - MORI_MAX_DISPATCH_TOKENS_DECODE=$((BENCH_MAX_CONC_VALUE / decode_dp_ranks)) - # MORI_MOE_MAX_INPUT_TOKENS_DECODE=$((MORI_MAX_DISPATCH_TOKENS_DECODE * decode_dp_ranks * 7 / 10)) - # Update derived variable - SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD=$((MORI_MAX_DISPATCH_TOKENS_DECODE * 2)) - export SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD - echo "[DP+EP override] Decode: max-running-requests=$decode_max_running_requests, DISPATCH_TOKENS=$MORI_MAX_DISPATCH_TOKENS_DECODE, MOE_MAX_INPUT=$MORI_MOE_MAX_INPUT_TOKENS_DECODE, INTER_KERNEL_SWITCH=$SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD" -fi - -# Build the composed config strings (equivalent to the old MODEL_PREFILL_CONFIGS / MODEL_DECODE_CONFIGS) -# disable_cuda_graph (model-level) routes prefill to --disable-cuda-graph instead of --cuda-graph-bs. -if [[ "$PREFILL_DISABLE_CUDA_GRAPH" == "True" ]] || [[ "$PREFILL_DISABLE_CUDA_GRAPH" == "true" ]]; then - PREFILL_MODE_FLAGS="--mem-fraction-static ${PREFILL_MEM_FRACTION_STATIC} --max-running-requests ${prefill_max_running_requests} --chunked-prefill-size ${prefill_chunked_prefill_size} --disable-cuda-graph " -else - PREFILL_MODE_FLAGS="--mem-fraction-static ${PREFILL_MEM_FRACTION_STATIC} --max-running-requests ${prefill_max_running_requests} --chunked-prefill-size ${prefill_chunked_prefill_size} --cuda-graph-bs ${prefill_cuda_graph_bs[*]} " -fi - -if [[ "$PREFILL_DISABLE_RADIX_CACHE" == "True" ]] || [[ "$PREFILL_DISABLE_RADIX_CACHE" == "true" ]]; then - PREFILL_MODE_FLAGS="$PREFILL_MODE_FLAGS --disable-radix-cache" -fi -# Agentic runs: keep radix/prefix cache enabled by replacing --disable-radix-cache with empty. -if [[ "${IS_AGENTIC:-0}" == "1" || "${IS_AGENTIC:-}" == "true" ]]; then - PREFILL_MODE_FLAGS="${PREFILL_MODE_FLAGS//--disable-radix-cache/}" -fi -if [[ -n "$prefill_context_length" ]]; then - PREFILL_MODE_FLAGS="$PREFILL_MODE_FLAGS --context-length ${prefill_context_length}" -fi -if [[ -n "$prefill_max_total_tokens" ]]; then - PREFILL_MODE_FLAGS="$PREFILL_MODE_FLAGS --max-total-tokens ${prefill_max_total_tokens}" -fi -if [[ "$prefill_enable_two_batch_overlap" == "True" ]] || [[ "$prefill_enable_two_batch_overlap" == "true" ]]; then - PREFILL_MODE_FLAGS="$PREFILL_MODE_FLAGS --enable-two-batch-overlap" - PREFILL_SDMA_ENV="MORI_ENABLE_SDMA=true" -fi - -DECODE_MODE_FLAGS="--mem-fraction-static ${DECODE_MEM_FRACTION_STATIC} --max-running-requests ${decode_max_running_requests} --cuda-graph-bs ${decode_cuda_graph_bs[*]} " - -if [[ "$DECODE_PREFILL_ROUND_ROBIN_BALANCE" == "True" ]] || [[ "$DECODE_PREFILL_ROUND_ROBIN_BALANCE" == "true" ]]; then - DECODE_MODE_FLAGS="$DECODE_MODE_FLAGS --prefill-round-robin-balance" -fi -if [[ "$DECODE_DISAGG_ENABLE_RADIX_CACHE" == "True" ]] || [[ "$DECODE_DISAGG_ENABLE_RADIX_CACHE" == "true" ]]; then - DECODE_MODE_FLAGS="$DECODE_MODE_FLAGS --disaggregation-decode-enable-radix-cache" -fi - -if [[ "$DECODE_MTP_SIZE" -gt 0 ]]; then - MORI_MAX_DISPATCH_TOKENS_DECODE=$((MORI_MAX_DISPATCH_TOKENS_DECODE * (DECODE_MTP_SIZE + 1))) - # MORI_MOE_MAX_INPUT_TOKENS_DECODE=$((MORI_MOE_MAX_INPUT_TOKENS_DECODE * (DECODE_MTP_SIZE + 1))) -fi - -# ============================================================================= -# Cluster Topology Configuration -# ============================================================================= -IFS=',' read -ra IP_ARRAY <<< "$IPADDRS" - -# Ceiling division by GPUS_PER_NODE for nodes-per-worker -PREFILL_NODES_PER_WORKER=$(((PREFILL_TP_SIZE + 7) / GPUS_PER_NODE)) -DECODE_NODES_PER_WORKER=$(((DECODE_TP_SIZE + 7) / GPUS_PER_NODE)) -NODE_OFFSET=$((PREFILL_NODES_PER_WORKER * xP)) - -# Build prefill arguments dynamically based on xP -PREFILL_HEADNODE_URLS=() -PREFILL_ARGS="" -# Per-worker Prometheus /metrics endpoints (port 8000) for aiperf's -# --server-metrics scrape. The router on :30000 does not serve Prometheus, so -# aiperf must scrape each prefill/decode worker directly (see ENABLE_METRICS). -SERVER_METRICS_URLS=() -# Per-worker base URLs (port 8000) for direct cache flushing between -# concurrency points. The router (:30000) does not fan /flush_cache out, so -# trace_replay.sh must POST to each prefill/decode worker directly. -SERVER_FLUSH_URLS=() -for i in $(seq 0 $((xP - 1))); do - prefill_idx=$((i * PREFILL_NODES_PER_WORKER)) - PREFILL_HEADNODE_URLS[$i]="${IP_ARRAY[$prefill_idx]}:${HEADNODE_PORT}" - PREFILL_ARGS="$PREFILL_ARGS --prefill http://${IP_ARRAY[$prefill_idx]}:8000" - SERVER_METRICS_URLS+=("http://${IP_ARRAY[$prefill_idx]}:8000/metrics") - SERVER_FLUSH_URLS+=("http://${IP_ARRAY[$prefill_idx]}:8000") -done - -# Build decode arguments dynamically based on yD -DECODE_HEADNODE_URLS=() -DECODE_ARGS="" -for i in $(seq 0 $((yD - 1))); do - decode_idx=$((i * DECODE_NODES_PER_WORKER + NODE_OFFSET)) - DECODE_HEADNODE_URLS[$i]="${IP_ARRAY[$decode_idx]}:${HEADNODE_PORT}" - DECODE_ARGS="$DECODE_ARGS --decode http://${IP_ARRAY[$decode_idx]}:8000" - SERVER_METRICS_URLS+=("http://${IP_ARRAY[$decode_idx]}:8000/metrics") - SERVER_FLUSH_URLS+=("http://${IP_ARRAY[$decode_idx]}:8000") -done - -echo "Prefill worker headnode list: ${PREFILL_HEADNODE_URLS[@]}" -echo "Decode worker headnode list: ${DECODE_HEADNODE_URLS[@]}" -echo "Server metrics endpoints: ${SERVER_METRICS_URLS[@]}" -echo "Server flush endpoints: ${SERVER_FLUSH_URLS[@]}" - -# ============================================================================= -# Configuration Builder Functions -# ============================================================================= - -build_server_config() { - local mode="$1" - local model_name="$2" - local tp_size="$3" - local enable_ep="$4" - local enable_dp="$5" - local decode_mtp_size="$6" - - # Calculate EP and DP sizes based on enable flags - local ep_size=1 - local dp_size=1 - - if [[ "$enable_ep" == "true" ]]; then - ep_size=$tp_size - fi - - if [[ "$enable_dp" == "true" ]]; then - dp_size=$tp_size - fi - - # Build parallelism arguments - local parallel_args="--tp-size ${tp_size}" - - if [[ "$enable_ep" == "true" ]]; then - parallel_args="$parallel_args --ep-size ${ep_size}" - fi - - if [[ "$enable_dp" == "true" ]]; then - parallel_args="$parallel_args --dp-size ${dp_size}" - fi - - # Get model-specific configuration from YAML-loaded variables - local base_config="$MODEL_BASE_FLAGS" - local mtp_config="" - local dp_config="" - local ep_config="" - local specific_config="" - - # MTP config (only if MTP is enabled and mode is decode) - if [ "$decode_mtp_size" -gt 0 ]; then - mtp_config="${MODEL_MTP_FLAGS} --speculative-num-steps ${decode_mtp_size} --speculative-num-draft-tokens $((decode_mtp_size + 1))" - fi - - # DP config (only if DP is enabled) - if [[ "$enable_dp" == "true" ]]; then - dp_config="$MODEL_DP_FLAGS" - fi - - # EP config (only if EP is enabled): a2a backend, deepep mode, ep-dispatch algo. - # With ep=1 (EP disabled) these are dropped, so the MoE runs tensor-parallel (TP) - # instead of expert-parallel — even when dp-attention is on. - if [[ "$enable_ep" == "true" ]]; then - ep_config="$MODEL_EP_FLAGS" - fi - - # Mode-specific config - if [[ "$mode" == "prefill" ]]; then - specific_config="$PREFILL_MODE_FLAGS" - elif [[ "$mode" == "decode" ]]; then - specific_config="$DECODE_MODE_FLAGS" - fi - - # Combine: parallel args + base config + ep config + mtp config (decode only) + dp config + specific config - local full_config="$parallel_args" - if [[ -n "$base_config" ]]; then - full_config="$full_config $base_config" - fi - if [[ -n "$ep_config" ]]; then - full_config="$full_config $ep_config" - fi - if [[ -n "$mtp_config" ]] && [[ "$mode" == "decode" ]]; then - full_config="$full_config $mtp_config" - fi - if [[ -n "$dp_config" ]]; then - full_config="$full_config $dp_config" - fi - if [[ -n "$specific_config" ]]; then - full_config="$full_config $specific_config" - fi - - echo "$full_config" -} - -# Build complete server configurations -PREFILL_SERVER_CONFIG=$(build_server_config "prefill" "$MODEL_NAME" "$PREFILL_TP_SIZE" "$PREFILL_ENABLE_EP" "$PREFILL_ENABLE_DP" "$DECODE_MTP_SIZE") -DECODE_SERVER_CONFIG=$(build_server_config "decode" "$MODEL_NAME" "$DECODE_TP_SIZE" "$DECODE_ENABLE_EP" "$DECODE_ENABLE_DP" "$DECODE_MTP_SIZE") - -# Expose Prometheus /metrics on the servers when requested (ENABLE_METRICS=1). -if [[ "${ENABLE_METRICS:-0}" == "1" ]]; then - [[ "$PREFILL_SERVER_CONFIG" != *"--enable-metrics"* ]] && PREFILL_SERVER_CONFIG="$PREFILL_SERVER_CONFIG --enable-metrics" - [[ "$DECODE_SERVER_CONFIG" != *"--enable-metrics"* ]] && DECODE_SERVER_CONFIG="$DECODE_SERVER_CONFIG --enable-metrics" -fi - -if [[ -n "$MODEL_NAME" ]]; then - echo "Using model-specific configuration for: $MODEL_NAME" -fi - -# ============================================================================= -# Optional KV cache offloading (HiCache) — enabled when -# KV_OFFLOADING != none AND KV_OFFLOAD_BACKEND == hicache. -# HiCache extends RadixAttention, so radix cache MUST stay on (drop -# --disable-radix-cache). The --hicache-* flags are appended to BOTH the -# prefill and decode server configs. -# ============================================================================= -KV_OFFLOADING="${KV_OFFLOADING:-none}" -KV_OFFLOAD_BACKEND="${KV_OFFLOAD_BACKEND:-}" -if [[ "$KV_OFFLOADING" != "none" && "$KV_OFFLOAD_BACKEND" == "hicache" ]]; then - HICACHE_TOTAL_CPU_DRAM_GB="${HICACHE_TOTAL_CPU_DRAM_GB:-2000}" - HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-1}" - HICACHE_PAGE_SIZE="${HICACHE_PAGE_SIZE:-1}" - HICACHE_PREFETCH_POLICY="${HICACHE_PREFETCH_POLICY:-wait_complete}" - - # Optional L3 storage tier behind the CPU-DRAM (L2) cache. - # "" -> CPU DRAM only (default) - # "mooncake"-> Mooncake distributed KV store (needs a mooncake_master) - HICACHE_STORAGE_BACKEND="${HICACHE_STORAGE_BACKEND:-}" - - # Layout / IO backend / write policy are backend-specific: - # mooncake L3: page_first_direct + the "direct" IO backend (the Mooncake - # store maps a page-contiguous segment for RDMA/zero-copy). This layout - # asserts host_pool > device_pool, so it needs a large CPU-DRAM budget. - # L2-only (CPU DRAM): layer_first + the "kernel" IO backend. layer_first - # has no host>device constraint (the "direct" IO backend REQUIRES a - # page_first layout, so it cannot be paired with layer_first). - if [[ "$HICACHE_STORAGE_BACKEND" == "mooncake" ]]; then - HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first}" - HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" - HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" - else - HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first_direct}" - HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" - HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" - fi - - # Mooncake master/connection settings (used only when storage=mooncake). - # The master runs once on node 0; every prefill/decode server connects to - # it via NODE0_ADDR so it is reachable across nodes. - MC_MASTER_PORT="${MC_MASTER_PORT:-50061}" - MC_METADATA_PORT="${MC_METADATA_PORT:-8080}" - MC_METRICS_PORT="${MC_METRICS_PORT:-9003}" - MC_MASTER_THREADS="${MC_MASTER_THREADS:-64}" - MC_EVICTION_HIGH_WATERMARK="${MC_EVICTION_HIGH_WATERMARK:-0.95}" - MC_PROTOCOL="${MC_PROTOCOL:-tcp}" - MC_GLOBAL_SEG="${MC_GLOBAL_SEG:-64gb}" - MC_DEVICE="${MC_DEVICE:-$IBDEVICES}" - MC_MASTER_ADDR="${MC_MASTER_ADDR:-${NODE0_ADDR}:${MC_MASTER_PORT}}" - MC_METADATA_SERVER="${MC_METADATA_SERVER:-http://${NODE0_ADDR}:${MC_METADATA_PORT}/metadata}" - - # Emit the --hicache-storage-backend flags (empty unless mooncake). The - # extra-config JSON is single-quoted so it survives the later `eval` of the - # launch command as a single argument. - build_storage_flags() { - [[ "$HICACHE_STORAGE_BACKEND" != "mooncake" ]] && return 0 - local extra="{\"master_server_address\": \"${MC_MASTER_ADDR}\", \"protocol\": \"${MC_PROTOCOL}\", \"device_name\": \"${MC_DEVICE}\", \"local_hostname\": \"${host_ip}\", \"global_segment_size\": \"${MC_GLOBAL_SEG}\", \"metadata_server\": \"${MC_METADATA_SERVER}\", \"check_server\": false}" - echo "--hicache-storage-backend mooncake --hicache-storage-backend-extra-config '${extra}' --enable-metrics --enable-cache-report" - } - - # HiCache capacity via --hicache-ratio (scales with GPU KV pool). - HICACHE_RATIO="${HICACHE_RATIO:-16}" - - build_hicache_flags() { - echo "--page-size ${HICACHE_PAGE_SIZE} --enable-hierarchical-cache --hicache-ratio ${HICACHE_RATIO} --hicache-io-backend ${HICACHE_IO_BACKEND} --hicache-mem-layout ${HICACHE_MEM_LAYOUT} --hicache-write-policy ${HICACHE_WRITE_POLICY} --hicache-storage-prefetch-policy ${HICACHE_PREFETCH_POLICY} $(build_storage_flags)" - } - - # HiCache requires RadixAttention; strip any --disable-radix-cache. - PREFILL_SERVER_CONFIG="${PREFILL_SERVER_CONFIG//--disable-radix-cache/}" - DECODE_SERVER_CONFIG="${DECODE_SERVER_CONFIG//--disable-radix-cache/}" - - # Prefill always gets HiCache. - PREFILL_SERVER_CONFIG="$PREFILL_SERVER_CONFIG $(build_hicache_flags "$PREFILL_TP_SIZE")" - - - DECODE_SERVER_CONFIG="$DECODE_SERVER_CONFIG --page-size ${HICACHE_PAGE_SIZE}" - echo "[HiCache] KV_OFFLOADING=${KV_OFFLOADING} backend=${KV_OFFLOAD_BACKEND} applied to prefill only; decode mirrors --page-size ${HICACHE_PAGE_SIZE} for transfer compatibility (chunk cache under the mori transfer backend)" - echo "[HiCache] params: io_backend=${HICACHE_IO_BACKEND}, mem_layout=${HICACHE_MEM_LAYOUT}, page_size=${HICACHE_PAGE_SIZE}, write_policy=${HICACHE_WRITE_POLICY}, prefetch_policy=${HICACHE_PREFETCH_POLICY}, storage_backend=${HICACHE_STORAGE_BACKEND:-none}" - if [[ "$HICACHE_STORAGE_BACKEND" == "mooncake" ]]; then - echo "[HiCache] Mooncake store: master=${MC_MASTER_ADDR} metadata=${MC_METADATA_SERVER} protocol=${MC_PROTOCOL} device=${MC_DEVICE} segment=${MC_GLOBAL_SEG} threads=${MC_MASTER_THREADS} eviction_watermark=${MC_EVICTION_HIGH_WATERMARK}" - fi -else - echo "[HiCache] KV_OFFLOADING=${KV_OFFLOADING} backend=${KV_OFFLOAD_BACKEND:-none} (HiCache disabled)" -fi - -if [[ "${EVAL_ONLY:-false}" == "true" ]] || [[ "${RUN_EVAL:-false}" == "true" ]]; then - PREFILL_SERVER_CONFIG=$(echo "$PREFILL_SERVER_CONFIG" | sed 's/--ep-dispatch-algorithm fake//g') - DECODE_SERVER_CONFIG=$(echo "$DECODE_SERVER_CONFIG" | sed 's/--ep-dispatch-algorithm fake//g') - unset MORI_MOE_MAX_INPUT_TOKENS_PREFILL - unset MORI_MOE_MAX_INPUT_TOKENS_DECODE -fi - -# ============================================================================= -# Container Synchronization -# ============================================================================= - -echo "Waiting at the container creation barrier on $host_name" -python3 $SGLANG_WS_PATH/sync_agentic.py barrier \ - --local-ip ${host_ip} \ - --local-port 5000 \ - --enable-port \ - --node-ips ${IPADDRS} \ - --node-ports 5000 \ - --wait-for-all-ports \ - --timeout 300 - - -# ============================================================================= -# Node Role Assignment and Server Launch -# ============================================================================= - -if [ "$NODE_RANK" -eq 0 ]; then - echo "NODE INFO =======================================" - echo "================================================" - echo "Node List : ${SLURM_JOB_NODELIST}" - echo "Node IPs : ${IPADDRS}" - echo "Model Name : ${MODEL_NAME:-'Not specified'}" - echo "================================================" - - echo "CLUSTER INFO ====================================" - echo "================================================" - echo "${host_name}:${host_ip} is Proxy Node and Prefill Node" - echo "Using prefill config: $PREFILL_SERVER_CONFIG" - echo "Prefill parallelism: TP=${PREFILL_TP_SIZE}, EP enabled: ${PREFILL_ENABLE_EP}, DP enabled: ${PREFILL_ENABLE_DP}, MTP size=${DECODE_MTP_SIZE}" - echo "Decode parallelism: TP=${DECODE_TP_SIZE}, EP enabled: ${DECODE_ENABLE_EP}, DP enabled: ${DECODE_ENABLE_DP}, MTP size=${DECODE_MTP_SIZE}" - echo "Prefill servers ($((PREFILL_TP_SIZE/GPUS_PER_NODE)) nodes): ${PREFILL_ARGS}" - echo "Decode servers ($((DECODE_TP_SIZE/GPUS_PER_NODE)) nodes): ${DECODE_ARGS}" - echo "Prefill env: SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_MAX_DISPATCH_TOKENS_PREFILL}" - echo "Decode env: SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_MAX_DISPATCH_TOKENS_DECODE} " - echo "Decode env: SGLANG_MORI_MOE_MAX_INPUT_TOKENS=${MORI_MOE_MAX_INPUT_TOKENS_DECODE} " - - echo "================================================" - - # Dump all resolved commands to a text file for debugging / reproducibility. - CMD_DUMP="/run_logs/slurm_job-${SLURM_JOB_ID}/commands_${host_name}.txt" - dump_cmd() { echo -e "\n# ── $1 ──\n$2" >> "$CMD_DUMP"; } - echo "# Commands dump — $(date -u '+%Y-%m-%d %H:%M:%S UTC')" > "$CMD_DUMP" - echo "# Host: ${host_name} (${host_ip}) Node rank: ${NODE_RANK}" >> "$CMD_DUMP" - echo "# Model: ${MODEL_NAME} Image: ${DOCKER_IMAGE_NAME:-unknown}" >> "$CMD_DUMP" - - # Start the Mooncake store master (L3 HiCache backend) on node 0 only. - # All prefill/decode servers connect to it via NODE0_ADDR:MC_MASTER_PORT. - if [[ "${KV_OFFLOADING:-none}" != "none" && "${KV_OFFLOAD_BACKEND:-}" == "hicache" && "${HICACHE_STORAGE_BACKEND:-}" == "mooncake" ]]; then - echo "Starting Mooncake master on ${host_ip}:${MC_MASTER_PORT} (metadata :${MC_METADATA_PORT}, metrics :${MC_METRICS_PORT})" - MC_MASTER_CMD="mooncake_master \ - --enable_http_metadata_server=true \ - --http_metadata_server_host=0.0.0.0 \ - --http_metadata_server_port=${MC_METADATA_PORT} \ - --rpc_port=${MC_MASTER_PORT} \ - --rpc_thread_num=${MC_MASTER_THREADS} \ - --metrics_port=${MC_METRICS_PORT} \ - --enable_metric_reporting=true \ - --eviction_high_watermark_ratio=${MC_EVICTION_HIGH_WATERMARK}" - dump_cmd "MOONCAKE MASTER" "$MC_MASTER_CMD" - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $MC_MASTER_CMD" - else - MC_MASTER_LOG="/run_logs/slurm_job-${SLURM_JOB_ID}/mooncake_master_${host_name}.log" - mooncake_master \ - --enable_http_metadata_server=true \ - --http_metadata_server_host=0.0.0.0 \ - --http_metadata_server_port="${MC_METADATA_PORT}" \ - --rpc_port="${MC_MASTER_PORT}" \ - --rpc_thread_num="${MC_MASTER_THREADS}" \ - --metrics_port="${MC_METRICS_PORT}" \ - --enable_metric_reporting=true \ - --eviction_high_watermark_ratio="${MC_EVICTION_HIGH_WATERMARK}" \ - > "${MC_MASTER_LOG}" 2>&1 & - mc_master_pid=$! - sleep 3 - # Fail loudly on a port collision. On shared nodes the Mooncake RPC - # port may already be taken by another user's master; in that case the - # metrics-port health check below can still pass against the foreign - # master while our RPC port is dead, and the prefill then hangs. - if grep -qiE "Address already in use|bind .*error" "${MC_MASTER_LOG}" 2>/dev/null; then - echo "ERROR: mooncake_master failed to bind port ${MC_MASTER_PORT} (already in use)." - echo " Set MC_MASTER_PORT/MC_METRICS_PORT to free ports and resubmit." - grep -iE "Address already in use|bind .*error" "${MC_MASTER_LOG}" | tail -3 - exit 1 - fi - for ((i=3; i<=60; i+=3)); do - if curl -sf "http://127.0.0.1:${MC_METRICS_PORT}/get_all_segments" >/dev/null 2>&1; then - echo " mooncake master OK at ${i}s" - break - fi - sleep 3 - done - fi - fi - - # start the head prefill server - PREFILL_MORI_MOE_ENV="" - set -x - if [[ -n "$MORI_MOE_MAX_INPUT_TOKENS_PREFILL" ]]; then - PREFILL_MORI_MOE_ENV="SGLANG_MORI_MOE_MAX_INPUT_TOKENS=${MORI_MOE_MAX_INPUT_TOKENS_PREFILL}" - fi - set +x - PREFILL_CMD="SGLANG_MORI_COMBINE_DTYPE=${MORI_COMBINE_DTYPE_PREFILL} ${PREFILL_SDMA_ENV} ${PREFILL_MORI_MOE_ENV} SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_PREFILL:-${MORI_MAX_DISPATCH_TOKENS_PREFILL}} MORI_IO_SQ_BACKOFF_TIMEOUT_US=${MORI_IO_SQ_BACKOFF_TIMEOUT_US} MORI_IO_QP_MAX_SEND_WR=${MORI_IO_QP_MAX_SEND_WR} ${LAUNCH_PREFIX:-} python3 -m sglang.launch_server \ - --model-path $MODEL_DIR/$MODEL_NAME \ - --disaggregation-mode prefill \ - --disaggregation-ib-device ${IBDEVICES} \ - --host 0.0.0.0 \ - --port 8000 \ - --trust-remote-code \ - --log-level ${SGLANG_SERVER_LOG_LEVEL:-warning} \ - ${PREFILL_SERVER_CONFIG} " - - if [ "$PREFILL_NODES_PER_WORKER" -gt 1 ]; then - PREFILL_CMD="$PREFILL_CMD --dist-init-addr ${PREFILL_HEADNODE_URLS[0]} --nnodes ${PREFILL_NODES_PER_WORKER} --node-rank 0" - fi - - - dump_cmd "PREFILL (node 0)" "$PREFILL_CMD" - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $PREFILL_CMD" - else - set -x - eval "$PREFILL_CMD" \ - 2>&1 | tee /run_logs/slurm_job-${SLURM_JOB_ID}/prefill_${host_name}.log & - set +x - prefill0_pid=$! - fi - - - echo "Waiting for all prefill and decode servers to be up . . ." - - - BARRIER_CMD="python3 $SGLANG_WS_PATH/sync_agentic.py barrier \ - --node-ips ${IPADDRS} \ - --node-ports 8000 \ - --wait-for-all-ports \ - --timeout 1800" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $BARRIER_CMD" - else - eval "$BARRIER_CMD" - fi - echo "Congratulations!!! All prefill and decode servers are up . . ." - - # Router resilience: a single prefill worker doing huge long-context prefills - # (256K+ token agentic prompts in 65280-token chunks) can be slow to drain a - # concurrent burst. With defaults the circuit breaker opens after 10 failures - # and short-circuits the whole worker, so the aiperf profiling burst sees - # "No available prefill workers (all circuits open or unhealthy)" and aborts - # with 100% errors. Disable the breaker and relax health-check sensitivity so - # a busy-but-alive worker is not ejected. Override via ROUTER_RESILIENCE_FLAGS. - ROUTER_RESILIENCE_FLAGS="${ROUTER_RESILIENCE_FLAGS:---disable-circuit-breaker --health-failure-threshold 100 --health-check-timeout-secs 600 --health-check-interval-secs 30}" - - # Router scheduling policy. cache_aware prefill routing exploits HiCache/radix - # prefix reuse across the agentic trace; round_robin decode keeps the single - # decode worker fed evenly. cache_threshold / balance_*_threshold tune the - # cache_aware load-balancing (router defaults are 0.5 / 64 / 1.5). Override any - # of these via env. - ROUTER_PREFILL_POLICY="${ROUTER_PREFILL_POLICY:-cache_aware}" - ROUTER_DECODE_POLICY="${ROUTER_DECODE_POLICY:-round_robin}" - ROUTER_CACHE_THRESHOLD="${ROUTER_CACHE_THRESHOLD:-0.3}" - ROUTER_BALANCE_ABS_THRESHOLD="${ROUTER_BALANCE_ABS_THRESHOLD:-2}" - ROUTER_BALANCE_REL_THRESHOLD="${ROUTER_BALANCE_REL_THRESHOLD:-1.1}" - ROUTER_POLICY_FLAGS="${ROUTER_POLICY_FLAGS:---policy ${ROUTER_PREFILL_POLICY} --prefill-policy ${ROUTER_PREFILL_POLICY} --decode-policy ${ROUTER_DECODE_POLICY} --cache-threshold ${ROUTER_CACHE_THRESHOLD} --balance-abs-threshold ${ROUTER_BALANCE_ABS_THRESHOLD} --balance-rel-threshold ${ROUTER_BALANCE_REL_THRESHOLD}}" - - ROUTER_CMD="python -m sglang_router.launch_router \ - --pd-disaggregation \ - --port 30000 \ - ${ROUTER_POLICY_FLAGS} \ - ${ROUTER_RESILIENCE_FLAGS} \ - ${PREFILL_ARGS} \ - ${DECODE_ARGS}" - - - dump_cmd "ROUTER" "$ROUTER_CMD" - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $ROUTER_CMD" - else - ROUTER_LOG_FILE="/run_logs/slurm_job-${SLURM_JOB_ID}/router_${host_name}.log" - set -x - if [[ "${SGLANG_ROUTER_STDOUT_LOGS:-0}" == "1" ]]; then - eval "$ROUTER_CMD" 2>&1 | tee "$ROUTER_LOG_FILE" & - else - eval "$ROUTER_CMD" >"$ROUTER_LOG_FILE" 2>&1 & - fi - set +x - proxy_pid=$! - - # Wait for router to be ready via health endpoint - HEALTH_BARRIER_CMD="python3 $SGLANG_WS_PATH/sync_agentic.py barrier \ - --node-ips ${NODE0_ADDR} \ - --node-ports 30000 \ - --wait-for-all-health \ - --health-endpoint /readiness \ - --timeout 1800" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $HEALTH_BARRIER_CMD" - else - eval "$HEALTH_BARRIER_CMD" - fi - - echo "Router is ready for benchmarking" - fi - - - echo "Ready for benchmarking on ${host_name}:${host_ip}" - - echo "Benchmarking on ${host_name}:${host_ip}" - cd $SGLANG_WS_PATH - - # Export IS_MTP based on whether MTP is enabled - if [ "$DECODE_MTP_SIZE" -gt 0 ]; then - export IS_MTP=true - else - export IS_MTP=false - fi - - # Select the benchmark runner. - # IS_AGENTIC=1/true → agentic trace replay (trace_replay.sh) - # IS_AGENTIC unset/0 → fixed-seq-len throughput benchmark (bench.sh) - if [[ "${IS_AGENTIC:-0}" == "1" || "${IS_AGENTIC:-}" == "true" ]]; then - # Point aiperf's server-metrics scrape at the per-worker Prometheus - # /metrics endpoints. The router (:30000) that aiperf auto-detects from - # --url does not expose Prometheus, so without this the scrape finds no - # reachable endpoint and all server-side cache/KV fields come out null. - # Only set it when the workers were actually started with --enable-metrics. - if [[ "${ENABLE_METRICS:-0}" == "1" && "${#SERVER_METRICS_URLS[@]}" -gt 0 ]]; then - AIPERF_SERVER_METRICS_URLS=$(IFS=,; echo "${SERVER_METRICS_URLS[*]}") - export AIPERF_SERVER_METRICS_URLS - echo "AIPERF_SERVER_METRICS_URLS=${AIPERF_SERVER_METRICS_URLS}" - fi - # Per-worker base URLs for cache flushing between concurrency points. - # trace_replay.sh consults these when CLEAR_CACHE_BETWEEN_CONC=1. - if [[ "${#SERVER_FLUSH_URLS[@]}" -gt 0 ]]; then - SERVER_FLUSH_URLS_CSV=$(IFS=,; echo "${SERVER_FLUSH_URLS[*]}") - export SERVER_FLUSH_URLS_CSV - echo "SERVER_FLUSH_URLS_CSV=${SERVER_FLUSH_URLS_CSV}" - fi - # trace_replay.sh signature: model_path model_name concurrency_list log_path - BENCH_CMD="bash $SGLANG_WS_PATH/trace_replay.sh \ - $MODEL_DIR $MODEL_NAME $BENCH_MAX_CONCURRENCY /run_logs/slurm_job-${SLURM_JOB_ID}" - echo "Benchmark runner: trace_replay.sh (agentic, KV_OFFLOADING=${KV_OFFLOADING:-none}, backend=${KV_OFFLOAD_BACKEND:-none}, CONC=${BENCH_MAX_CONCURRENCY})" - else - # bench.sh signature: - # n_prefill n_decode prefill_gpus decode_gpus model_dir model_name log_path - # isl osl concurrency_list req_rate random_range_ratio num_prompts_multiplier - BENCH_CMD="bash $SGLANG_WS_PATH/bench.sh ${xP} ${yD} $((PREFILL_TP_SIZE*xP)) $((DECODE_TP_SIZE*yD)) \ - $MODEL_DIR $MODEL_NAME /run_logs/slurm_job-${SLURM_JOB_ID} ${BENCH_INPUT_LEN} \ - ${BENCH_OUTPUT_LEN} \"${BENCH_MAX_CONCURRENCY}\" ${BENCH_REQUEST_RATE} \ - ${BENCH_RANDOM_RANGE_RATIO} ${BENCH_NUM_PROMPTS_MULTIPLIER}" - echo "Benchmark runner: bench.sh (fixed-seq-len)" - fi - - if [[ "${EVAL_ONLY:-false}" == "true" ]]; then - echo "EVAL_ONLY mode: skipping throughput benchmark" - elif [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $BENCH_CMD" - else - set -x - eval "$BENCH_CMD" - set +x - fi - - # Run evaluation if requested (before killing router) - if [[ "${RUN_EVAL:-false}" == "true" ]]; then - echo "Running lm-eval evaluation on Node 0..." - - # Health check: verify the router is still serving before running eval. - # The throughput benchmark may have crashed/exhausted decode workers. - EVAL_HEALTH_OK=false - for _attempt in 1 2 3; do - if curl -sf --max-time 10 "http://0.0.0.0:30000/readiness" >/dev/null 2>&1; then - EVAL_HEALTH_OK=true - break - fi - echo "Eval health check attempt $_attempt failed, retrying in 10s..." - sleep 10 - done - - if [[ "$EVAL_HEALTH_OK" != "true" ]]; then - echo "WARNING: Router health check failed after 3 attempts. Skipping eval." - else - # Must run from repo root so utils/evals/${task}.yaml resolves - pushd /workspace - - # Source eval functions from benchmark_lib.sh - source /workspace/benchmarks/benchmark_lib.sh - - # Use EVAL_CONC from workflow if set, otherwise fall back to max of conc list - if [[ -n "${EVAL_CONC:-}" ]]; then - export EVAL_CONCURRENT_REQUESTS="${EVAL_CONC}" - else - export EVAL_CONCURRENT_REQUESTS=$(echo "$BENCH_MAX_CONCURRENCY" | tr 'x' '\n' | sort -n | tail -1) - fi - - # Override eval context length with model's configured context_length - if [[ -n "$prefill_context_length" ]]; then - export EVAL_MAX_MODEL_LEN="$prefill_context_length" - fi - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: run_eval --framework lm-eval --port 30000 (conc=${EVAL_CONCURRENT_REQUESTS}, ctx=${EVAL_MAX_MODEL_LEN:-auto})" - else - # Run lm-eval against the router on port 30000 - run_eval --framework lm-eval --port 30000 - eval_rc=$? - - if [[ $eval_rc -ne 0 ]]; then - echo "ERROR: run_eval exited rc=$eval_rc; skipping metadata write and eval artifact staging" >&2 - EVAL_FAILED=1 - else - # Set metadata env vars for append_lm_eval_summary - export TP="${PREFILL_TP_SIZE}" - export CONC="${EVAL_CONCURRENT_REQUESTS}" - export EP_SIZE=1 - [[ "${PREFILL_ENABLE_EP}" == "true" ]] && EP_SIZE="${PREFILL_TP_SIZE}" - export PREFILL_TP="${PREFILL_TP_SIZE}" - export PREFILL_EP=1 - [[ "${PREFILL_ENABLE_EP}" == "true" ]] && PREFILL_EP="${PREFILL_TP_SIZE}" - export PREFILL_NUM_WORKERS="${xP}" - export DECODE_TP="${DECODE_TP_SIZE}" - export DECODE_EP=1 - [[ "${DECODE_ENABLE_EP}" == "true" ]] && DECODE_EP="${DECODE_TP_SIZE}" - export DECODE_NUM_WORKERS="${yD}" - export DP_ATTENTION="${PREFILL_ENABLE_DP}" - export PREFILL_DP_ATTENTION="${PREFILL_ENABLE_DP}" - export DECODE_DP_ATTENTION="${DECODE_ENABLE_DP}" - export ISL="${BENCH_INPUT_LEN}" - export OSL="${BENCH_OUTPUT_LEN}" - # IS_MULTINODE, FRAMEWORK, PRECISION, MODEL_PREFIX, RUNNER_TYPE, - # RESULT_FILENAME are already set via Docker -e flags from job.slurm - - append_lm_eval_summary - # Files (meta_env.json, results*.json, sample*.jsonl) are now in /workspace - - # Copy eval artifacts to run_logs for NFS extraction by runner - EVAL_COPY_DIR="/run_logs/slurm_job-${SLURM_JOB_ID}/eval_results" - mkdir -p "$EVAL_COPY_DIR" - for f in meta_env.json; do - [ -e "/workspace/$f" ] && cp -f "/workspace/$f" "$EVAL_COPY_DIR/" - done - # Use find for glob patterns to avoid "no match" errors - find /workspace -maxdepth 1 -name 'results*.json' -exec cp -f {} "$EVAL_COPY_DIR/" \; - find /workspace -maxdepth 1 -name 'sample*.jsonl' -exec cp -f {} "$EVAL_COPY_DIR/" \; - - echo "Eval completed. Artifacts staged in $EVAL_COPY_DIR" - fi - fi - - popd - fi - fi - - # Copy benchmark results to BENCHMARK_LOGS_DIR (mounted from host) - LOGS_OUTPUT="${BENCHMARK_LOGS_DIR:-/run_logs}/logs" - mkdir -p "$LOGS_OUTPUT" - - if [[ "$DRY_RUN" -eq 0 ]]; then - cp -r /run_logs/slurm_job-${SLURM_JOB_ID} "$LOGS_OUTPUT/" - echo "Copied results to $LOGS_OUTPUT/slurm_job-${SLURM_JOB_ID}" - fi - - echo "Killing the proxy server and prefill server" - - if [[ "$DRY_RUN" -eq 0 ]]; then - kill $proxy_pid - kill $prefill0_pid - fi - - if [[ "${EVAL_FAILED:-0}" -eq 1 ]]; then - echo "ERROR: eval failed; exiting node-0 with rc=1" - exit 1 - fi - -elif [ "$NODE_RANK" -gt 0 ] && [ "$NODE_RANK" -lt "$NODE_OFFSET" ]; then - echo "${host_name}:${host_ip} is Prefill Node (Model: ${MODEL_NAME:-'default'})" - echo "Using prefill config: $PREFILL_SERVER_CONFIG" - echo "Prefill parallelism: TP=${PREFILL_TP_SIZE}, EP enabled: ${PREFILL_ENABLE_EP}, DP enabled: ${PREFILL_ENABLE_DP}" - - CMD_DUMP="/run_logs/slurm_job-${SLURM_JOB_ID}/commands_${host_name}.txt" - dump_cmd() { echo -e "\n# ── $1 ──\n$2" >> "$CMD_DUMP"; } - echo "# Commands dump — $(date -u '+%Y-%m-%d %H:%M:%S UTC')" > "$CMD_DUMP" - echo "# Host: ${host_name} (${host_ip}) Node rank: ${NODE_RANK}" >> "$CMD_DUMP" - - PREFILL_MORI_MOE_ENV="" - set -x - if [[ -n "$MORI_MOE_MAX_INPUT_TOKENS_PREFILL" ]]; then - PREFILL_MORI_MOE_ENV="SGLANG_MORI_MOE_MAX_INPUT_TOKENS=${MORI_MOE_MAX_INPUT_TOKENS_PREFILL}" - fi - set +x - PREFILL_CMD="SGLANG_MORI_COMBINE_DTYPE=${MORI_COMBINE_DTYPE_PREFILL} ${PREFILL_SDMA_ENV} ${PREFILL_MORI_MOE_ENV} SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_PREFILL:-${MORI_MAX_DISPATCH_TOKENS_PREFILL}} MORI_IO_SQ_BACKOFF_TIMEOUT_US=${MORI_IO_SQ_BACKOFF_TIMEOUT_US} MORI_IO_QP_MAX_SEND_WR=${MORI_IO_QP_MAX_SEND_WR} ${LAUNCH_PREFIX:-} python3 -m sglang.launch_server \ - --model-path $MODEL_DIR/${MODEL_NAME} \ - --disaggregation-mode prefill \ - --disaggregation-ib-device ${IBDEVICES} \ - --host 0.0.0.0 \ - --port 8000 \ - --trust-remote-code \ - --log-level ${SGLANG_SERVER_LOG_LEVEL:-warning} \ - ${PREFILL_SERVER_CONFIG} " - - if [ "$PREFILL_NODES_PER_WORKER" -gt 1 ]; then - rank=$((NODE_RANK % PREFILL_NODES_PER_WORKER)) - prefill_idx=$((NODE_RANK / PREFILL_NODES_PER_WORKER)) - PREFILL_CMD="$PREFILL_CMD --dist-init-addr ${PREFILL_HEADNODE_URLS[$prefill_idx]} --nnodes ${PREFILL_NODES_PER_WORKER} --node-rank $rank" - fi - - dump_cmd "PREFILL (rank ${NODE_RANK})" "$PREFILL_CMD" - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $PREFILL_CMD" - else - set -x - eval "$PREFILL_CMD" \ - 2>&1 | tee /run_logs/slurm_job-${SLURM_JOB_ID}/prefill_${host_name}.log & - set +x - prefill_pid=$! - fi - - echo "Waiting for proxy server to be up..." - BARRIER_CMD="python3 $SGLANG_WS_PATH/sync_agentic.py barrier \ - --node-ips ${NODE0_ADDR} \ - --node-ports 30000 \ - --wait-for-all-ports \ - --timeout 1800" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $BARRIER_CMD" - else - eval "$BARRIER_CMD" - fi - - echo "Waiting until proxy server closes..." - WAIT_CMD="python3 $SGLANG_WS_PATH/sync_agentic.py wait \ - --remote-ip ${NODE0_ADDR} \ - --remote-port 30000" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $WAIT_CMD" - else - eval "$WAIT_CMD" - fi - - echo "Killing the rank $NODE_RANK prefill server" - - if [[ "$DRY_RUN" -eq 0 ]]; then - kill $prefill_pid - fi - -else - RANK=$((NODE_RANK - xP * PREFILL_NODES_PER_WORKER)) - echo "${host_name}:${host_ip} is Decode Node (Model: ${MODEL_NAME:-'default'})" - echo "Using decode config: $DECODE_SERVER_CONFIG" - echo "Decode node rank: $RANK" - echo "Decode parallelism: TP=${DECODE_TP_SIZE}, EP enabled: ${DECODE_ENABLE_EP}, DP enabled: ${DECODE_ENABLE_DP}" - - CMD_DUMP="/run_logs/slurm_job-${SLURM_JOB_ID}/commands_${host_name}.txt" - dump_cmd() { echo -e "\n# ── $1 ──\n$2" >> "$CMD_DUMP"; } - echo "# Commands dump — $(date -u '+%Y-%m-%d %H:%M:%S UTC')" > "$CMD_DUMP" - echo "# Host: ${host_name} (${host_ip}) Node rank: ${NODE_RANK}" >> "$CMD_DUMP" - - DECODE_MORI_MOE_ENV="" - set -x - if [[ -n "$MORI_MOE_MAX_INPUT_TOKENS_DECODE" ]]; then - DECODE_MORI_MOE_ENV="SGLANG_MORI_MOE_MAX_INPUT_TOKENS=${MORI_MOE_MAX_INPUT_TOKENS_DECODE}" - fi - set +x - DECODE_CMD="SGLANG_MORI_COMBINE_DTYPE=${MORI_COMBINE_DTYPE_DECODE} ${DECODE_MORI_MOE_ENV} SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_DECODE:-${MORI_MAX_DISPATCH_TOKENS_DECODE}} MORI_IO_SQ_BACKOFF_TIMEOUT_US=${MORI_IO_SQ_BACKOFF_TIMEOUT_US} MORI_IO_QP_MAX_SEND_WR=${MORI_IO_QP_MAX_SEND_WR} ${LAUNCH_PREFIX:-} python3 -m sglang.launch_server \ - --model-path ${MODEL_DIR}/${MODEL_NAME} \ - --disaggregation-mode decode \ - --disaggregation-ib-device ${IBDEVICES} \ - --host 0.0.0.0 \ - --port 8000 \ - --trust-remote-code \ - --log-level ${SGLANG_SERVER_LOG_LEVEL:-warning} \ - ${DECODE_SERVER_CONFIG} " - - if [ "$DECODE_NODES_PER_WORKER" -gt 1 ]; then - rank=$((RANK % DECODE_NODES_PER_WORKER)) - decode_idx=$((RANK / DECODE_NODES_PER_WORKER)) - DECODE_CMD="$DECODE_CMD --dist-init-addr ${DECODE_HEADNODE_URLS[$decode_idx]} --nnodes ${DECODE_NODES_PER_WORKER} --node-rank $rank" - fi - - dump_cmd "DECODE (rank ${NODE_RANK})" "$DECODE_CMD" - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $DECODE_CMD" - else - set -x - eval "$DECODE_CMD" \ - 2>&1 | tee /run_logs/slurm_job-${SLURM_JOB_ID}/decode_${host_name}.log & - - set +x - decode_pid=$! - fi - - - echo "Waiting for proxy server to be up..." - BARRIER_CMD="python3 $SGLANG_WS_PATH/sync_agentic.py barrier \ - --node-ips ${NODE0_ADDR} \ - --node-ports 30000 \ - --wait-for-all-ports \ - --timeout 1800" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $BARRIER_CMD" - else - eval "$BARRIER_CMD" - fi - - - echo "Waiting until proxy server closes..." - WAIT_CMD="python3 $SGLANG_WS_PATH/sync_agentic.py wait \ - --remote-ip ${NODE0_ADDR} \ - --remote-port 30000" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $WAIT_CMD" - else - eval "$WAIT_CMD" - fi - - echo "Killing the rank $RANK decode server" - if [[ "$DRY_RUN" -eq 0 ]]; then - kill $decode_pid - fi - -fi - -echo "Script completed successfully" -exit 0 diff --git a/benchmarks/multi_node/amd_utils/setup_deps_agentic.sh b/benchmarks/multi_node/amd_utils/setup_deps_agentic.sh deleted file mode 100755 index 9f10958de1..0000000000 --- a/benchmarks/multi_node/amd_utils/setup_deps_agentic.sh +++ /dev/null @@ -1,174 +0,0 @@ -#!/bin/bash -# ============================================================================= -# setup_deps.sh — Install missing disagg dependencies at container start. -# -# Dispatched by $ENGINE (set by server.sh dispatcher): -# vllm-disagg -> recipe deps + amd-quark + UCX/RIXL path exports -# (base image: vllm/vllm-openai-rocm:nightly) -# sglang-disagg -> SGLang aiter gluon patch + per-model installs -# (base image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-*) -# -# Sourced by server_vllm.sh and server_sglang.sh so PATH / LD_LIBRARY_PATH -# exports persist. Each patch is idempotent: skipped if already applied. -# -# Build steps run in subshells to avoid CWD pollution between installers. -# ============================================================================= - -ROCM_PATH="${ROCM_PATH:-/opt/rocm}" -UCX_HOME="${UCX_HOME:-/usr/local/ucx}" -RIXL_HOME="${RIXL_HOME:-/usr/local/rixl}" - -_SETUP_START=$(date +%s) -_SETUP_INSTALLED=() - -git_clone_retry() { - local url="$1" dest="$2" max_tries=3 try=1 - while (( try <= max_tries )); do - if git clone --quiet "$url" "$dest" 2>/dev/null; then return 0; fi - echo "[SETUP] git clone attempt $try/$max_tries failed for $url, retrying in 10s..." - rm -rf "$dest" - sleep 10 - (( try++ )) - done - echo "[SETUP] git clone failed after $max_tries attempts: $url" - return 1 -} - -# --------------------------------------------------------------------------- -# 5. Container RDMA/net tools -# - ibv_devinfo comes from ibverbs-utils -# - iproute2 provides the `ip` command -# Used for in-container NIC/RDMA validation and routing checks. -# --------------------------------------------------------------------------- -install_recipe_deps() { - if command -v ibv_devinfo >/dev/null 2>&1 && command -v ip >/dev/null 2>&1; then - echo "[SETUP] Container RDMA/net tools already present" - return 0 - fi - - echo "[SETUP] Installing ibv_devinfo + iproute2 in container..." - apt-get update -q -y && apt-get install -q -y \ - ibverbs-utils iproute2 \ - && rm -rf /var/lib/apt/lists/* - - if ! command -v ibv_devinfo >/dev/null 2>&1 || ! command -v ip >/dev/null 2>&1; then - echo "[SETUP] ERROR: Failed to install ibv_devinfo/iproute2"; exit 1 - fi - _SETUP_INSTALLED+=("ibverbs-utils+iproute2") -} - -# --------------------------------------------------------------------------- -# 6b. amd-quark (MXFP4 quantization support for Kimi-K2.5-MXFP4 and similar) -# Required due to ROCm vLLM missing the quark dependency: -# https://github.com/vllm-project/vllm/issues/35633 -# --------------------------------------------------------------------------- -install_amd_quark() { - if python3 -c "import quark" 2>/dev/null; then - echo "[SETUP] amd-quark already present" - return 0 - fi - - echo "[SETUP] Installing amd-quark for MXFP4 quantization support..." - pip install --quiet amd-quark - - if ! python3 -c "import quark" 2>/dev/null; then - echo "[SETUP] WARN: amd-quark install failed (non-fatal for non-MXFP4 models)" - return 0 - fi - _SETUP_INSTALLED+=("amd-quark") -} - -# --------------------------------------------------------------------------- -# SGLang: prevent TP-rank collective desync deadlock in disaggregation prefill. -# -# resolve_waiting_queue_bootstrap() runs poll_and_all_reduce_attn_cp_tp_group() -# over `candidates`. The upstream candidate set (all non-aborted waiting reqs) -# can differ across TP ranks, so some ranks enter the all_reduce while others -# skip it -> hang. Narrow candidates to optimistic (pending_bootstrap) requests, -# which is consistent across ranks and is the only set finalize_bootstrap acts on. -# --------------------------------------------------------------------------- -patch_disagg_prefill_bootstrap_desync() { - python3 -c ' -import os, sys - -target = "/sgl-workspace/sglang/python/sglang/srt/disaggregation/prefill.py" -if not os.path.isfile(target): - print("[SETUP] disaggregation/prefill.py not found, skipping") - sys.exit(0) - -src = open(target).read() - -old = " candidates = [req for req in self.waiting_queue if not is_aborted(req)]" -new = ( - " candidates = [\n" - " req\n" - " for req in self.waiting_queue\n" - " if req.pending_bootstrap and not is_aborted(req)\n" - " ]" -) - -if new in src: - print("[SETUP] prefill bootstrap-desync patch already applied") - sys.exit(0) - -if old not in src: - print("[SETUP] WARN: resolve_waiting_queue_bootstrap pattern not found — sglang version may have changed") - sys.exit(0) - -open(target, "w").write(src.replace(old, new)) -print("[SETUP] Patched: disaggregation/prefill.py resolve_waiting_queue_bootstrap candidates") -' - _SETUP_INSTALLED+=("prefill-bootstrap-desync-fix") -} - -# --------------------------------------------------------------------------- -# SGLang: Install latest transformers for GLM-5 model type support. -# -# GLM-5 (zai-org/GLM-5-FP8) requires a transformers build that includes -# the glm_moe_dsa model type. The mori images do not ship it. -# Only install if GLM-5 is the active model (avoid overhead otherwise). -# --------------------------------------------------------------------------- -install_transformers_glm5() { - if [[ "$MODEL_NAME" != "GLM-5-FP8" ]]; then - return 0 - fi - - if python3 -c "from transformers import AutoConfig; AutoConfig.from_pretrained('zai-org/GLM-5-FP8', trust_remote_code=True)" 2>/dev/null; then - echo "[SETUP] transformers already supports GLM-5 model type" - return 0 - fi - - echo "[SETUP] Installing transformers with GLM-5 (glm_moe_dsa) support..." - pip install --quiet -U --no-cache-dir \ - "git+https://github.com/huggingface/transformers.git@6ed9ee36f608fd145168377345bfc4a5de12e1e2" - _SETUP_INSTALLED+=("transformers-glm5") -} - -# ============================================================================= -# Run installers (engine-gated) -# ============================================================================= - -if [[ "$ENGINE" == "vllm-disagg" ]]; then - install_recipe_deps - install_amd_quark - - # ========================================================================= - # vLLM: Export UCX/RIXL paths (persists since this file is sourced) - # ========================================================================= - export ROCM_PATH="${ROCM_PATH}" - export UCX_HOME="${UCX_HOME}" - export RIXL_HOME="${RIXL_HOME}" - export PATH="${UCX_HOME}/bin:/usr/local/bin/etcd:/root/.cargo/bin:${PATH}" - export LD_LIBRARY_PATH="${UCX_HOME}/lib:${RIXL_HOME}/lib:${RIXL_HOME}/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH:-}" -else - patch_disagg_prefill_bootstrap_desync - - install_transformers_glm5 -fi - -_SETUP_END=$(date +%s) -if [[ ${#_SETUP_INSTALLED[@]} -eq 0 ]]; then - echo "[SETUP] All dependencies already present ($(( _SETUP_END - _SETUP_START ))s wallclock)" -else - echo "[SETUP] Installed: ${_SETUP_INSTALLED[*]} in $(( _SETUP_END - _SETUP_START ))s" -fi diff --git a/benchmarks/multi_node/amd_utils/submit_agentic.sh b/benchmarks/multi_node/amd_utils/submit_agentic.sh deleted file mode 100755 index 39e6c3f4a1..0000000000 --- a/benchmarks/multi_node/amd_utils/submit_agentic.sh +++ /dev/null @@ -1,246 +0,0 @@ -#!/bin/bash -# -# Cluster Configuration Template for Multi-Node Disaggregated Serving -# -# This script submits a multi-node disaggregated benchmark job to SLURM. -# It must be configured for your specific cluster before use. -# -# ENGINE=sglang (default): SGLang disaggregated serving -# ENGINE=vllm: vLLM disaggregated serving -# -# Router is co-located with the first prefill node (same for both engines), -# so NUM_NODES = PREFILL_NODES + DECODE_NODES. - -usage() { - cat << 'USAGE' -Usage: - bash submit.sh \ - \ - \ - \ - \ - [NODE_LIST] - -Arguments: - PREFILL_NODES Number of prefill nodes - PREFILL_WORKERS Number of prefill workers (usually 1) - DECODE_NODES Number of decode nodes - DECODE_WORKERS Number of decode workers (usually 1) - ISL Input sequence length - OSL Output sequence length - CONCURRENCIES Concurrency levels, delimited by 'x' (e.g., "8x16x32") - REQUEST_RATE Request rate ("inf" for max throughput) - PREFILL_ENABLE_EP true/false or 1/0 (expert parallelism on prefill) - PREFILL_ENABLE_DP true/false or 1/0 (data-parallel attention on prefill) - DECODE_ENABLE_EP true/false or 1/0 (expert parallelism on decode) - DECODE_ENABLE_DP true/false or 1/0 (data-parallel attention on decode) - PREFILL_TP Tensor parallel size per prefill node - DECODE_TP Tensor parallel size per decode node - RANDOM_RANGE_RATIO Random range ratio for benchmark client - NODE_LIST Optional: comma-separated hostnames (must match NUM_NODES) - -Required environment variables: - SLURM_ACCOUNT SLURM account name - SLURM_PARTITION SLURM partition - TIME_LIMIT Job time limit (e.g., "08:00:00") - MODEL_PATH Path to model directory (e.g., /nfsdata) - MODEL_NAME Model name directory - CONTAINER_IMAGE Docker image name (e.g., vllm_disagg_pd:latest) - RUNNER_NAME Runner identifier (for job name) - -Optional environment variables: - DRY_RUN 1 = echo composed server/router launch commands instead of - running them (preview a recipe against a real allocation). -USAGE -} - -check_env() { - local name="$1" - if [[ -z "${!name:-}" ]]; then - echo "Error: ${name} not specified" >&2 - usage >&2 - exit 1 - fi -} - -check_env SLURM_ACCOUNT -check_env SLURM_PARTITION -check_env TIME_LIMIT - -check_env MODEL_PATH -check_env MODEL_NAME -check_env CONTAINER_IMAGE -check_env RUNNER_NAME -check_env FRAMEWORK - -# GPUS_PER_NODE defaults to 8 (MI355X). Set to 4 for MI325X if needed. -GPUS_PER_NODE="${GPUS_PER_NODE:-8}" - -# COMMAND_LINE ARGS -PREFILL_NODES=$1 -PREFILL_WORKERS=${2:-1} -DECODE_NODES=$3 -DECODE_WORKERS=${4:-1} -ISL=$5 -OSL=$6 -CONCURRENCIES=$7 -REQUEST_RATE=$8 -PREFILL_ENABLE_EP=${9:-true} -PREFILL_ENABLE_DP=${10:-true} -DECODE_ENABLE_EP=${11:-true} -DECODE_ENABLE_DP=${12:-true} -PREFILL_TP=${13:-8} -DECODE_TP=${14:-8} -RANDOM_RANGE_RATIO=${15:-0.8} -NODE_LIST=${16} - -NUM_NODES=$((PREFILL_NODES + DECODE_NODES)) -profiler_args="${ISL} ${OSL} ${CONCURRENCIES} ${REQUEST_RATE}" - -# Export variables for the SLURM job -export ENGINE="${FRAMEWORK:-sglang}" -export MODEL_DIR=$MODEL_PATH -export DOCKER_IMAGE_NAME=$CONTAINER_IMAGE -export PROFILER_ARGS=$profiler_args - -# Engine-specific xP/yD semantics and TP exports -if [[ "$ENGINE" == "vllm-disagg" ]]; then - export PROXY_STREAM_IDLE_TIMEOUT=${PROXY_STREAM_IDLE_TIMEOUT:-300} -fi -# xP = prefill workers, yD = decode workers (may span multiple nodes) -export xP=$PREFILL_WORKERS -export yD=$DECODE_WORKERS -export PREFILL_TP_SIZE=$(( $PREFILL_NODES * $PREFILL_TP / $PREFILL_WORKERS )) -export PREFILL_ENABLE_EP=${PREFILL_ENABLE_EP} -export PREFILL_ENABLE_DP=${PREFILL_ENABLE_DP} -export DECODE_TP_SIZE=$(( $DECODE_NODES * $DECODE_TP / $DECODE_WORKERS )) -export DECODE_ENABLE_EP=${DECODE_ENABLE_EP} -export DECODE_ENABLE_DP=${DECODE_ENABLE_DP} -export DECODE_MTP_SIZE=${DECODE_MTP_SIZE} - -export NUM_NODES=$NUM_NODES -export GPUS_PER_NODE=$GPUS_PER_NODE -export MODEL_NAME=$MODEL_NAME -export BENCH_INPUT_LEN=${ISL} -export BENCH_OUTPUT_LEN=${OSL} -export BENCH_NUM_PROMPTS_MULTIPLIER=${BENCH_NUM_PROMPTS_MULTIPLIER:-10} -export BENCH_MAX_CONCURRENCY=${CONCURRENCIES} -export BENCH_REQUEST_RATE=${REQUEST_RATE} -export BENCH_RANDOM_RANGE_RATIO=${RANDOM_RANGE_RATIO:-0.8} - -# DRY_RUN=1 makes server_sglang.sh echo the composed prefill/decode/router launch -# commands instead of executing them (useful for previewing a recipe against a real -# allocation). Threaded here → job_agentic.slurm → Docker (-e DRY_RUN) → server_sglang.sh. -# sbatch defaults to --export=ALL, so exporting it is what carries it into the job. -export DRY_RUN="${DRY_RUN:-0}" - -# Eval-related env vars (threaded from workflow → runner → here → job_agentic.slurm → Docker) -export RUN_EVAL="${RUN_EVAL:-false}" -export EVAL_ONLY="${EVAL_ONLY:-false}" -export EVAL_CONC="${EVAL_CONC:-}" -export FRAMEWORK="${FRAMEWORK:-}" -export PRECISION="${PRECISION:-}" -export MODEL_PREFIX="${MODEL_PREFIX:-}" -export RUNNER_TYPE="${RUNNER_TYPE:-}" -export RESULT_FILENAME="${RESULT_FILENAME:-}" -export SPEC_DECODING="${SPEC_DECODING:-}" -export IS_MULTINODE="${IS_MULTINODE:-false}" - -# Log directory: must be on NFS (shared filesystem) so the submit host can read SLURM output. -export BENCHMARK_LOGS_DIR="${BENCHMARK_LOGS_DIR:-$(pwd)/benchmark_logs}" -mkdir -p "$BENCHMARK_LOGS_DIR" - -# Optional: pass an explicit node list to sbatch. -NODELIST_OPT=() -if [[ -n "${NODE_LIST//[[:space:]]/}" ]]; then - IFS=',' read -r -a NODE_ARR <<< "$NODE_LIST" - if [[ "${#NODE_ARR[@]}" -ne "$NUM_NODES" ]]; then - echo "Error: NODE_LIST has ${#NODE_ARR[@]} nodes but NUM_NODES=${NUM_NODES}" >&2 - echo "Error: NODE_LIST='${NODE_LIST}'" >&2 - exit 1 - fi - NODELIST_CSV="$(IFS=,; echo "${NODE_ARR[*]}")" - NODELIST_OPT=(--nodelist "$NODELIST_CSV") -fi - -# Optional: exclude specific nodes (e.g. nodes with broken Docker sockets). -# Set SLURM_EXCLUDE_NODES env var to a comma-separated list of hostnames. -EXCLUDE_OPT=() -SLURM_EXCLUDE_NODES="${SLURM_EXCLUDE_NODES:-mia1-p01-g11,mia1-p01-g12,mia1-p01-g15}" -if [[ -n "${SLURM_EXCLUDE_NODES:-}" ]]; then - EXCLUDE_OPT=(--exclude "$SLURM_EXCLUDE_NODES") -fi - -# ============================================================================= -# Reuse existing allocation (skip sbatch) -# ============================================================================= -# When SLURM_REUSE_JOBID is set, run job_agentic.slurm directly in the current shell, -# attaching to the existing allocation. Inner `srun` calls pick up the -# allocation via SLURM_JOB_ID; SLURM_OVERLAP=1 lets them share task slots with -# the interactive shell already holding the allocation. -if [[ -n "${SLURM_REUSE_JOBID:-}" ]]; then - REUSE_JID="$SLURM_REUSE_JOBID" - echo "Reusing existing Slurm allocation ${REUSE_JID} (skipping sbatch)" >&2 - - # Resolve allocation's nodelist if not already provided. - ALLOC_NODELIST="${SLURM_JOB_NODELIST:-$(squeue -h -j "$REUSE_JID" -o '%N' 2>/dev/null)}" - if [[ -z "$ALLOC_NODELIST" ]]; then - echo "Error: could not resolve nodelist for job ${REUSE_JID}" >&2 - exit 1 - fi - ALLOC_NNODES=$(scontrol show hostnames "$ALLOC_NODELIST" | wc -l) - if [[ "$ALLOC_NNODES" -lt "$NUM_NODES" ]]; then - echo "Error: allocation ${REUSE_JID} has ${ALLOC_NNODES} nodes, need ${NUM_NODES}" >&2 - exit 1 - fi - - export SLURM_JOB_ID="$REUSE_JID" - export SLURM_JOBID="$REUSE_JID" - export SLURM_JOB_NODELIST="$ALLOC_NODELIST" - export SLURM_NODELIST="$ALLOC_NODELIST" - export SLURM_NNODES="$ALLOC_NNODES" - export SLURM_JOB_NUM_NODES="$ALLOC_NNODES" - export SLURM_NTASKS="$ALLOC_NNODES" - export SLURM_NPROCS="$ALLOC_NNODES" - export SLURM_NTASKS_PER_NODE=1 - export SLURM_TASKS_PER_NODE="1(x${ALLOC_NNODES})" - export SLURM_OVERLAP=1 - export SLURM_SUBMIT_DIR="$(pwd)" - - STDOUT_LOG="${BENCHMARK_LOGS_DIR}/slurm_job-${REUSE_JID}.out" - STDERR_LOG="${BENCHMARK_LOGS_DIR}/slurm_job-${REUSE_JID}.err" - rm -f "$STDOUT_LOG" "$STDERR_LOG" - - nohup bash "$(dirname "$0")/job_agentic.slurm" >"$STDOUT_LOG" 2>"$STDERR_LOG" & - INLINE_PID=$! - echo "$INLINE_PID" > "${BENCHMARK_LOGS_DIR}/slurm_job-${REUSE_JID}.pid" - echo "Started job_agentic.slurm (pid=${INLINE_PID}); logs: ${STDOUT_LOG}" >&2 - - echo "$REUSE_JID" - exit 0 -fi - -# Construct the sbatch command -sbatch_cmd=( - sbatch - --parsable - --exclusive - -N "$NUM_NODES" - -n "$NUM_NODES" - "${NODELIST_OPT[@]}" - "${EXCLUDE_OPT[@]}" - --time "$TIME_LIMIT" - --partition "$SLURM_PARTITION" - --account "$SLURM_ACCOUNT" - --job-name "$RUNNER_NAME" - --output "${BENCHMARK_LOGS_DIR}/slurm_job-%j.out" - --error "${BENCHMARK_LOGS_DIR}/slurm_job-%j.err" - "$(dirname "$0")/job_agentic.slurm" -) - -JOB_ID=$("${sbatch_cmd[@]}") -if [[ $? -ne 0 ]]; then - echo "Error: Failed to submit job with sbatch" >&2 - exit 1 -fi -echo "$JOB_ID" diff --git a/benchmarks/multi_node/amd_utils/sync_agentic.py b/benchmarks/multi_node/amd_utils/sync_agentic.py deleted file mode 100755 index 3678e76148..0000000000 --- a/benchmarks/multi_node/amd_utils/sync_agentic.py +++ /dev/null @@ -1,201 +0,0 @@ -#!/usr/bin/env python3 -""" -Multi-node synchronization utilities for disaggregated inference. - -Subcommands: - barrier - Wait until all specified nodes have opened their ports (TCP barrier) - Optionally wait for HTTP health endpoints to return 200 - wait - Block until a remote port closes (shutdown coordination) -""" - -import socket -import time -import threading -import argparse -import sys -import urllib.request -import urllib.error - - -def is_port_open(ip, port, timeout=2): - """Check if a given IP and port are accessible.""" - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.settimeout(timeout) - return s.connect_ex((ip, port)) == 0 - - -def check_health(ip, port, path="/health", timeout=2): - """Return True if http://ip:port/path returns HTTP 200.""" - try: - url = f"http://{ip}:{port}{path}" - req = urllib.request.Request(url) - with urllib.request.urlopen(req, timeout=timeout) as resp: - return getattr(resp, "status", 200) == 200 - except (urllib.error.URLError, urllib.error.HTTPError, OSError): - return False - - -# ============================================================================= -# barrier subcommand -# ============================================================================= - -def cmd_barrier(args): - """Wait until all nodes have opened the specified ports.""" - NODE_IPS = [ip.strip() for ip in args.node_ips.split(",") if ip.strip()] - NODE_PORTS = [int(p.strip()) for p in args.node_ports.split(",") if p.strip()] - - if not NODE_IPS: - print("Error: NODE_IPS argument is empty or not set.") - sys.exit(1) - - if len(NODE_PORTS) == 1: - NODE_PORTS *= len(NODE_IPS) - elif len(NODE_PORTS) != len(NODE_IPS): - print("Error: Number of ports must match number of node IPs or only one port should be given for all.") - sys.exit(1) - - server_socket = None - - def open_port(): - nonlocal server_socket - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server_socket.bind((args.local_ip, args.local_port)) - server_socket.listen(5) - print(f"Port {args.local_port} is now open on {args.local_ip}.") - while True: - conn, addr = server_socket.accept() - conn.close() - - def close_port(): - nonlocal server_socket - if server_socket: - server_socket.close() - print(f"Port {args.local_port} has been closed on {args.local_ip}.") - - if args.enable_port: - threading.Thread(target=open_port, daemon=True).start() - - # Wait for all ports (TCP check) - if args.wait_for_all_ports: - start_time = time.time() - timeout = args.timeout - - while True: - if timeout > 0: - elapsed = time.time() - start_time - if elapsed >= timeout: - not_open = [(ip, port) for ip, port in zip(NODE_IPS, NODE_PORTS) - if not is_port_open(ip, port)] - print(f"ERROR: Timeout after {timeout} seconds waiting for ports to open.", flush=True) - print("The following nodes/ports are still not responding:", flush=True) - for ip, port in not_open: - print(f" - {ip}:{port}", flush=True) - sys.exit(1) - - all_open = all(is_port_open(ip, port) for ip, port in zip(NODE_IPS, NODE_PORTS)) - if all_open: - break - - if timeout > 0: - remaining = timeout - (time.time() - start_time) - print(f"Waiting for nodes.{NODE_PORTS},{NODE_IPS} . . ({remaining:.0f}s remaining)", flush=True) - else: - print(f"Waiting for nodes.{NODE_PORTS},{NODE_IPS} . .", flush=True) - time.sleep(5) - - # Wait for all health endpoints (HTTP check) - if args.wait_for_all_health: - health_path = args.health_endpoint - start_time = time.time() - timeout = args.timeout - - while True: - if timeout > 0: - elapsed = time.time() - start_time - if elapsed >= timeout: - not_ready = [ - (ip, port) - for ip, port in zip(NODE_IPS, NODE_PORTS) - if not check_health(ip, port, health_path) - ] - print(f"ERROR: Timeout after {timeout} seconds waiting for health endpoints.", flush=True) - print(f"The following (http://ip:port{health_path}) are still not responding:", flush=True) - for ip, port in not_ready: - print(f" - http://{ip}:{port}{health_path}", flush=True) - sys.exit(1) - - all_ready = all( - check_health(ip, port, health_path) - for ip, port in zip(NODE_IPS, NODE_PORTS) - ) - if all_ready: - break - - if timeout > 0: - remaining = timeout - (time.time() - start_time) - print( - f"Waiting for health on {list(zip(NODE_IPS, NODE_PORTS))} ({health_path}) .. ({remaining:.0f}s remaining)", - flush=True, - ) - else: - print(f"Waiting for health on {list(zip(NODE_IPS, NODE_PORTS))} ({health_path}) ..", flush=True) - time.sleep(30) - - if args.enable_port: - # Keep the port open long enough for slow nodes to pass their barrier. - # The previous 30s was too short when setup times vary by minutes. - grace = max(60, args.timeout // 2) if args.timeout > 0 else 300 - time.sleep(grace) - close_port() - - -# ============================================================================= -# wait subcommand -# ============================================================================= - -def cmd_wait(args): - """Wait while a remote port remains open, exit when it closes.""" - print(f"Waiting while port {args.remote_port} on {args.remote_ip} is open...") - while is_port_open(args.remote_ip, args.remote_port): - time.sleep(5) - print(f"Port {args.remote_port} on {args.remote_ip} is now closed.") - - -# ============================================================================= -# CLI -# ============================================================================= - -def main(): - parser = argparse.ArgumentParser(description="Multi-node synchronization utilities.") - subparsers = parser.add_subparsers(dest="command", required=True) - - # barrier subcommand - bp = subparsers.add_parser("barrier", help="Wait for all nodes to open specified ports.") - bp.add_argument("--local-ip", required=False, help="Local IP address to bind the server.") - bp.add_argument("--local-port", type=int, required=False, help="Port number to bind the server.") - bp.add_argument("--enable-port", action="store_true", help="Enable opening and closing of local port.") - bp.add_argument("--node-ips", required=True, help="Comma-separated list of node IPs.") - bp.add_argument("--node-ports", required=True, help="Comma-separated list of ports to check.") - bp.add_argument("--timeout", type=int, default=600, - help="Timeout in seconds (default: 600). Set to 0 for no timeout.") - bp.add_argument("--wait-for-all-ports", action="store_true", - help="Wait until all node ports are open (TCP).") - bp.add_argument("--wait-for-all-health", action="store_true", - help="Wait until http://ip:port/health returns 200 for all nodes.") - bp.add_argument("--health-endpoint", default="/health", - help="Path for health check (default: /health).") - bp.set_defaults(func=cmd_barrier) - - # wait subcommand - wp = subparsers.add_parser("wait", help="Wait while a remote port remains open.") - wp.add_argument("--remote-ip", required=True, help="Remote server IP address.") - wp.add_argument("--remote-port", type=int, required=True, help="Remote port number.") - wp.set_defaults(func=cmd_wait) - - args = parser.parse_args() - args.func(args) - - -if __name__ == "__main__": - main() diff --git a/benchmarks/multi_node/amd_utils/trace_replay.sh b/benchmarks/multi_node/amd_utils/trace_replay.sh deleted file mode 100644 index a1d3428633..0000000000 --- a/benchmarks/multi_node/amd_utils/trace_replay.sh +++ /dev/null @@ -1,173 +0,0 @@ -#!/bin/bash -# Dual-Engine Disaggregated Benchmark Runner -# -# ENGINE=sglang (default): SGLang benchmark -# ENGINE=vllm: vLLM benchmark -# -# Produces JSON result files via benchmark_serving.py so that the CI pipeline -# can collect and process results. -# -# Usage: bash bench.sh \ -# \ -# - -ENGINE="${ENGINE:-sglang-disagg}" - -model_path=$1 -model_name=$2 -concurrency_list=${3:-"1"} -MODEL_PATH="${MODEL_PATH:-${model_path}/${model_name}}" -# vllm-disagg uses --served-model-name MODEL_NAME; sglang defaults to MODEL_PATH -if [[ "$ENGINE" == "vllm-disagg" ]]; then - MODEL="${MODEL_NAME:-${MODEL_PATH}}" -else - MODEL="${MODEL_PATH}" -fi -log_path=${4:-/run_logs} - -# Split BENCH_MAX_CONCURRENCY (x-delimited, e.g. "8x16x32") into an array. -# Falls back to 1 if unset so the loop always runs at least once. -IFS='x' read -r -a chosen_concurrencies <<< "${concurrency_list}" - - -ROUTER_PORT="${ROUTER_PORT:-30000}" - -export TRANSFORMERS_VERBOSITY=error -export TOKENIZERS_PARALLELISM=false - -# echo "Config ${chosen_isl}; ${chosen_osl}; ${chosen_concurrencies[0]}; ${chosen_req_rate}" - -RESULT_DIR="${RESULT_DIR:-${log_path}/agentic}" -mkdir -p "$RESULT_DIR" - -source "$(dirname "$0")/../../benchmark_lib.sh" - -# Preserve the result schema from the original AgentX PR while fixed-sequence -# and other agentic paths use the current shared aggregation module. -write_agentic_result_json() { - local result_dir="$1" - ( - cd "$INFMAX_CONTAINER_WORKSPACE" - RESULT_DIR="$result_dir" AGENTIC_OUTPUT_DIR="${AGENTIC_OUTPUT_DIR:-$INFMAX_CONTAINER_WORKSPACE}" \ - "$AIPERF_PYTHON" -m utils.agentic.aggregation.process_agentic_result_agentic - ) - - "$AIPERF_PYTHON" "$INFMAX_CONTAINER_WORKSPACE/utils/generate_aiperf_plots.py" "$result_dir" 2>&1 || true -} - -# clear_kv_caches — wipe all KV cache tiers on every backend worker before a -# concurrency point, so each conc is measured cold (no prefix reuse bleeding in -# from the previous conc). Mirrors mori-scheduler/scripts/benchmark/lib/ -# clear_caches.sh, but the worker base URLs are already resolved by -# server_sglang.sh (SERVER_FLUSH_URLS_CSV) so no SSH/IP lookup is needed. -# -# Tiers (SGLang server APIs), hit on EACH worker directly (the router does not -# fan /flush_cache out): -# L1 (GPU radix) + L2 (host hicache): POST /flush_cache — NO-OP while any -# request is in flight, so we drain-retry until "Cache flushed" or -# FLUSH_DRAIN_TIMEOUT (default 120s) elapses. -# L3 (umbp / mooncake store): POST /hicache/storage-backend/clear -# — HTTP != 200 when L3 is off, tolerated. -# Best-effort: logs WARN, never hard-fails the sweep. -clear_kv_caches() { - local drain_tmo="${FLUSH_DRAIN_TIMEOUT:-120}" - local urls_csv="${SERVER_FLUSH_URLS_CSV:-}" - if [[ -z "$urls_csv" ]]; then - echo "[clear_caches] WARN: SERVER_FLUSH_URLS_CSV unset; skipping cache flush" >&2 - return 0 - fi - local -a urls - IFS=',' read -r -a urls <<< "$urls_csv" - local url start ok resp code - for url in "${urls[@]}"; do - [[ -n "$url" ]] || continue - # L1 + L2: drain-retry until flushed (no-op while requests in flight). - start=$(date +%s); ok=0; resp="" - while :; do - resp=$(curl -sf -m 10 -X POST "${url}/flush_cache" 2>/dev/null || true) - echo "$resp" | grep -qi "Cache flushed" && { ok=1; break; } - (( $(date +%s) - start >= drain_tmo )) && break - sleep 3 - done - if [[ "$ok" == 1 ]]; then - echo "[clear_caches] ${url}: L1+L2 flushed" - else - echo "[clear_caches] WARN ${url}: L1+L2 flush NOT confirmed after ${drain_tmo}s (resp='${resp:0:80}')" >&2 - fi - # L3: storage-backend clear (umbp / mooncake). 200 when a backend is attached. - code=$(curl -s -m 60 -o /dev/null -w '%{http_code}' -X POST "${url}/hicache/storage-backend/clear" 2>/dev/null || echo 000) - if [[ "$code" == 200 ]]; then - echo "[clear_caches] ${url}: L3 store cleared" - else - echo "[clear_caches] ${url}: L3 clear http=${code} (no storage backend / L3 off — ok)" - fi - done -} - -# REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" - -PORT="${ROUTER_PORT}" -MODEL="${MODEL:-${BENCH_MODEL}}" -DURATION="${DURATION:-1800}" -export MODEL DURATION MAX_MODEL_LEN -RESULT_DIR="${RESULT_DIR:-${profile_folder}}" -# Base name for the per-conc aggregate written by the existing -# utils.agentic.aggregation.process_agentic_result module. -# The workflow guard / upload steps expect a "${RESULT_FILENAME}_conc.json" -# file per concurrency, so each concurrency below is always suffixed with -# _conc (matching agentic_srt.sh on the gb200 path). -RESULT_FILENAME_BASE="${RESULT_FILENAME:-agentic_bench}" - -mkdir -p "$RESULT_DIR" - -resolve_trace_source -install_agentic_deps - -ANY_FAILED=0 -for max_concurrency in "${chosen_concurrencies[@]}"; do - - echo "==========================================" - echo "Agentic trace replay: conc=$max_concurrency" - echo "==========================================" - - # Clear all KV cache tiers on every backend before this conc point so it is - # measured cold (no prefix reuse from the previous conc). Default on; set - # CLEAR_CACHE_BETWEEN_CONC=0 to disable. Best-effort — never fails the run. - if [[ "${CLEAR_CACHE_BETWEEN_CONC:-1}" == "1" ]]; then - echo "conc=$max_concurrency: clearing L1/L2/L3 on all backends (no server restart)" - clear_kv_caches || echo "WARNING: cache clear had issues for conc=$max_concurrency" >&2 - fi - - # Mirror agentic_srt.sh (the srtctl/gb200 path): every concurrency writes - # its artifacts into a conc_/ subdir of RESULT_DIR. The CI matrix explodes - # agentic runs to one concurrency per job, but benchmark-multinode-tmpl.yml - # still expects the per-conc nesting (LOGS/agentic/conc_*/...) and the - # _conc result-file suffix, so we always nest to keep the layout identical - # across runners and avoid overwriting earlier runs in local multi-conc sweeps. - CONC_RESULT_DIR="$RESULT_DIR/conc_${max_concurrency}" - mkdir -p "$CONC_RESULT_DIR" - - CONC="$max_concurrency" - USERS="$max_concurrency" - export CONC USERS - build_replay_cmd "$CONC_RESULT_DIR" - - # Per-conc result name consumed by write_agentic_result_json. Always suffix - # with _conc so the file matches - # the workflow guard's "${RESULT_FILENAME}_conc*.json" glob (and the agg / - # checkpoint upload steps) for both single-conc CI runs and multi-conc sweeps. - export RESULT_FILENAME="${RESULT_FILENAME_BASE}_conc${max_concurrency}" - if ! run_agentic_replay_and_write_outputs "$CONC_RESULT_DIR"; then - echo "WARNING: agentic trace replay for conc=$max_concurrency failed (replay or validation) after writing available results" >&2 - ANY_FAILED=1 - fi - - echo "-----------------------------------------" - -done - -export RESULT_FILENAME="$RESULT_FILENAME_BASE" - -if [ "$ANY_FAILED" -ne 0 ]; then - echo "WARNING: at least one conc had a non-zero exit; per-conc result files were still written when possible." >&2 -fi diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 9ace1ac0a4..03c7a296ca 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -3122,36 +3122,3 @@ minimaxm3-fp8-mi325x-vllm-agentic: - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: native, conc-list: [3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] } - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: native, conc-list: [10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 32] } - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: native, conc-list: [24, 32, 36, 40, 44, 48, 52, 56, 60, 64, 72, 80, 96] } - -dsv4-fp4-mi355x-sglang-disagg-agentic-hicache: - image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: cluster:mi355x-amds - precision: fp4 - framework: sglang-disagg - multinode: true - disagg: true - scenarios: - agentic-coding: - - dram-utilization: 0.80 - search-space: - - spec-decoding: "none" - conc-list: [ 2,4,8,16,32 ] - kv-offloading: dram - kv-offload-backend: hicache - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 028631183e..be28208ab4 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4716,10 +4716,3 @@ - "Clean the export envs" - "Enable two batch overlap" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2093 - -- config-keys: - - dsv4-fp4-mi355x-sglang-disagg-agentic-hicache - description: - - "Bump the SGLang ROCm image from 20260706 to 20260709 and retire the DeepSeek-V4 compress-state, DSA paged-MQA, AITER instruction-shape, and HiCache host-pool shims that are already included upstream." - - "Remove the SWA re-prefill and unified-KV HiCache overlays after their upstream PRs merged; retain only patch code without a merged upstream equivalent." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2127 diff --git a/runners/launch_mi355x-amds-agentic.sh b/runners/launch_mi355x-amds-agentic.sh deleted file mode 100755 index 1ff0dffd59..0000000000 --- a/runners/launch_mi355x-amds-agentic.sh +++ /dev/null @@ -1,338 +0,0 @@ -#!/usr/bin/env bash - -scancel_sync() { - local jobid=$1 - local timeout=${2:-600} - local interval=10 - local start - start=$(date +%s) - - echo "[scancel_sync] Requesting cancel of job $jobid" - scancel "$jobid" || true - - while [[ -n "$(squeue -j "$jobid" --noheader 2>/dev/null)" ]]; do - local now - now=$(date +%s) - if (( now - start >= timeout )); then - echo "[scancel_sync][WARN] job $jobid still present after ${timeout}s" - return 1 - fi - echo "[scancel_sync] waiting for job $jobid to exit. $((timeout-(now-start))) secs remaining..." - sleep "$interval" - done - echo "[scancel_sync] job $jobid exited" - return 0 -} - -if [[ "$IS_MULTINODE" == "true" ]]; then - # This sets up the environment and launches multi-node benchmarks - - set -x - - # Set up environment variables for SLURM - export SLURM_ACCOUNT="$USER" - export SLURM_PARTITION="compute" - export SLURM_JOB_NAME="benchmark-sglang-disagg.job" - - export MODEL_NAME=${MODEL##*/} - export MODEL_PATH="/it-share/data" - export IBDEVICES="rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7" - export MORI_RDMA_TC=104 - - # Set additional required env vars for multi_node scripts - export MODEL_DIR="$MODEL_PATH" # job.slurm uses MODEL_DIR - export GPUS_PER_NODE=8 # MI355X has 8 GPUs (set to 4 for MI325X) - - export ISL="$ISL" - export OSL="$OSL" - - # Logs go to BENCHMARK_LOGS_DIR (NFS-accessible, outside the repo tree) - export BENCHMARK_LOGS_DIR="${BENCHMARK_LOGS_DIR:-$GITHUB_WORKSPACE/benchmark_logs}" - mkdir -p "$BENCHMARK_LOGS_DIR" - sudo rm -rf "$BENCHMARK_LOGS_DIR/logs" 2>/dev/null || true - - # Ensure root-owned files are cleaned up even on early exit to prevent - # EACCES errors when the next GH Actions job checks out on this runner. - # Always preserve slurm logs as CI artifacts for debugging. - # KEEP_LOGS=1 disables the trap entirely (local-debug knob). - cleanup_and_save_logs() { - if [[ -n "${GITHUB_ACTIONS:-}" && -n "${JOB_ID:-}" ]]; then - local art_dir="$GITHUB_WORKSPACE/benchmark_artifacts" - mkdir -p "$art_dir" - cp -r "$BENCHMARK_LOGS_DIR"/slurm_job-${JOB_ID}.{out,err} "$art_dir/" 2>/dev/null || true - fi - # Print .err inline so failures are visible in CI output - local err_file="$BENCHMARK_LOGS_DIR/slurm_job-${JOB_ID:-unknown}.err" - if [[ -s "$err_file" ]]; then - echo "=== Slurm job stderr ===" - tail -100 "$err_file" - echo "========================" - fi - sudo rm -rf "$BENCHMARK_LOGS_DIR" 2>/dev/null || true - } - if [[ "${KEEP_LOGS:-0}" == "1" ]]; then - trap '' EXIT - else - trap cleanup_and_save_logs EXIT - fi - - SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_mi355x_${FRAMEWORK}.sh" - if [[ "$FRAMEWORK" == "sglang-disagg" ]] || [[ "$FRAMEWORK" == "vllm-disagg" ]] || [[ "$FRAMEWORK" == "atom-disagg" ]]; then - # Agentic recipes live under multi_node/agentic/ and export the - # HiCache tunables (page-size, io-backend, ...); fixed-seq-len recipes - # live at the multi_node/ root. Honor SCENARIO_SUBDIR so agentic-coding - # configs pick the agentic recipe instead of the root one. - if [[ "${SCENARIO_SUBDIR}" == "agentic/" ]]; then - BENCHMARK_SUBDIR="multi_node/agentic" - else - BENCHMARK_SUBDIR="multi_node" - fi - else - BENCHMARK_SUBDIR="single_node/fixed_seq_len" - fi - JOB_ID=$(bash "benchmarks/${BENCHMARK_SUBDIR}/${SCRIPT_NAME}") - - # Wait for job to complete - LOG_FILE="$BENCHMARK_LOGS_DIR/slurm_job-${JOB_ID}.out" - - # Give slurm time to start the job and create log file - sleep 10 - - # Wait for log file to appear (also check job is still alive) - while ! ls "$LOG_FILE" &>/dev/null; do - if ! squeue -u "$USER" --noheader --format='%i' | grep -q "$JOB_ID"; then - echo "ERROR: Job $JOB_ID failed before creating log file" - scontrol show job "$JOB_ID" - exit 1 - fi - sleep 5 - done - - set +x - - # Poll for job completion in background - ( - while squeue -u $USER --noheader --format='%i' | grep -q "$JOB_ID"; do - sleep 10 - done - ) & - POLL_PID=$! - - # Tail the log file until job completes (-F follows by name, polls instead of inotify for NFS) - tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null - - wait $POLL_PID - - set -x - - # FIXME: The below is bad and is a result of the indirection of the ways in which - # Dynamo jobs are launched. In a follow-up PR, the location of the result file should not - # depend on the runner, it should always be in the same spot in the GH workspace. - - # Process results from all configurations - - # search for "FRAMEWORK_DIFF_IF_STATEMENT #3" for this if-statement - # Find the latest log directory that contains the data - - if [[ "${EVAL_ONLY:-false}" != "true" && "${IS_AGENTIC:-0}" != "1" ]]; then - cat > collect_latest_results.py <<'PY' -import os, sys -job_dir, isl, osl, nexp, framework = sys.argv[1], int(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4]), sys.argv[5] -logs_root = f"{job_dir}/logs/" -candidates = [] -if os.path.isdir(logs_root): - for name in os.listdir(logs_root): - subdir = f"{logs_root}{name}/{framework}_isl_{isl}_osl_{osl}" - if os.path.isdir(subdir): - candidates.append(subdir) -for path in sorted(candidates, key=os.path.getmtime, reverse=True)[:nexp]: - print(path) -PY - - LOGS_DIR=$(python3 collect_latest_results.py "$BENCHMARK_LOGS_DIR" "$ISL" "$OSL" 1 "$FRAMEWORK") - if [ -z "$LOGS_DIR" ]; then - echo "No logs directory found for ISL=${ISL}, OSL=${OSL}" - exit 1 - fi - - echo "Found logs directory: $LOGS_DIR" - ls -la "$LOGS_DIR" - - # Result JSON are contained within the result directory - for result_file in $(find $LOGS_DIR -type f); do - # result_file should directly be isl_ISL_osl_OSL_concurrency_CONC_req_rate_R_gpus_N_ctx_M_gen_N.json - file_name=$(basename $result_file) - if [ -f $result_file ]; then - # Copy the result file to workspace with a unique name - WORKSPACE_RESULT_FILE="$GITHUB_WORKSPACE/${RESULT_FILENAME}_${file_name}" - echo "Found result file ${result_file}. Copying it to ${WORKSPACE_RESULT_FILE}" - cp $result_file $WORKSPACE_RESULT_FILE - fi - done - fi - - # Extract eval results if eval was requested - if [[ "${RUN_EVAL:-false}" == "true" ]]; then - # Find eval_results in the slurm job logs directory - EVAL_DIR=$(find "$BENCHMARK_LOGS_DIR/logs" -type d -name eval_results 2>/dev/null | head -1) - if [ -n "$EVAL_DIR" ] && [ -d "$EVAL_DIR" ]; then - echo "Extracting eval results from $EVAL_DIR" - shopt -s nullglob - for eval_file in "$EVAL_DIR"/*; do - [ -f "$eval_file" ] || continue - cp "$eval_file" "$GITHUB_WORKSPACE/" - echo "Copied eval artifact: $(basename "$eval_file")" - done - shopt -u nullglob - else - echo "WARNING: RUN_EVAL=true but no eval results found under $BENCHMARK_LOGS_DIR/logs" - fi - fi - - # Stage agentic raw artifacts + server logs for the CI upload steps. - # server_sglang.sh copies /run_logs/slurm_job- to - # $BENCHMARK_LOGS_DIR/logs/slurm_job- on shared storage, and - # trace_replay.sh writes each concurrency's aiperf artifacts under - # agentic/conc_/ (mirroring agentic_srt.sh). benchmark-multinode-tmpl.yml - # uploads them from $GITHUB_WORKSPACE/LOGS/agentic/conc_*/... plus a - # multinode_server_logs.tar.gz, so preserve the conc_/ nesting here - # before the logs dir is removed below. The agg result JSON is already - # written straight to the mounted workspace by the existing agentic - # aggregation module. - if [[ "${IS_AGENTIC:-0}" == "1" ]]; then - JOB_LOGS_DIR="$BENCHMARK_LOGS_DIR/logs/slurm_job-${JOB_ID}" - if [ -d "$JOB_LOGS_DIR" ]; then - # trace_replay.sh always nests artifacts under agentic/conc_/. - # Copy the whole agentic/ tree so the conc_/ subdirs are - # preserved for the LOGS/agentic/conc_*/... upload globs. - AGENTIC_SRC="$JOB_LOGS_DIR/agentic" - if [ -d "$AGENTIC_SRC" ] && find "$AGENTIC_SRC" -mindepth 1 -maxdepth 1 -type d -name 'conc_*' -print -quit 2>/dev/null | grep -q .; then - echo "Staging agentic raw artifacts from $AGENTIC_SRC" - mkdir -p "$GITHUB_WORKSPACE/LOGS/agentic" - cp -r "$AGENTIC_SRC"/. "$GITHUB_WORKSPACE/LOGS/agentic/" - # The source artifacts are created inside the container as root - # (--container-remap-root), so depending on how the runner - # invokes this script the copies can end up root-owned and/or - # read-only (aiperf/server_sglang make some dirs mode 0555). If - # the staged tree isn't owned+writable by the runner user, the - # next checkout's `git clean` fails with - # EACCES: permission denied, rmdir '.../LOGS/agentic'. - # chown to the invoking user (the same one that runs git clean) - # via sudo (already passwordless here for rm -rf). The follow-up - # chmod uses a+rwX (not just u+rwX): the *next* job against this - # same $GITHUB_WORKSPACE may be picked up by a different runner - # process running as a different OS user, which would otherwise - # fall outside the owner bits and still fail the same - # `git clean` with EACCES despite the chown above. - sudo chown -R "$(id -u):$(id -g)" "$GITHUB_WORKSPACE/LOGS" 2>/dev/null || true - chmod -R a+rwX "$GITHUB_WORKSPACE/LOGS" 2>/dev/null || true - ls -laR "$GITHUB_WORKSPACE/LOGS/agentic" - else - echo "WARNING: no agentic conc_*/ artifacts found under $JOB_LOGS_DIR/agentic" - fi - # Server/router/prefill/decode logs for the multinode_server_logs_* artifact. - if tar czf "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz" -C "$JOB_LOGS_DIR" . 2>/dev/null; then - echo "Created multinode_server_logs.tar.gz" - else - echo "WARNING: failed to create multinode_server_logs.tar.gz" - fi - else - echo "WARNING: agentic staging skipped; $JOB_LOGS_DIR not found" - fi - fi - - echo "All result files processed" - # Use sync scancel to ensure nfs file handle is released in time - set +x - scancel_sync $JOB_ID - set -x - echo "Canceled the slurm job $JOB_ID" - - sudo rm -rf "$BENCHMARK_LOGS_DIR/logs" 2>/dev/null || true - - # Log preservation and cleanup handled by EXIT trap (cleanup_and_save_logs) - -else - - export HF_HUB_CACHE_MOUNT="/var/lib/hf-hub-cache/" - export AIPERF_MMAP_CACHE_HOST_PATH="/it-share/aiperf-cache/" - export PORT_OFFSET=${RUNNER_NAME: -1} - export PORT=$(( 8888 + ${PORT_OFFSET} )) - FRAMEWORK_SUFFIX=$([[ "$FRAMEWORK" == "atom" ]] && printf '_atom' || printf '') - SPEC_SUFFIX=$([[ "$SPEC_DECODING" == "mtp" ]] && printf '_mtp' || printf '') - - PARTITION="compute" - SQUASH_FILE="/var/lib/squash/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" - LOCK_FILE="${SQUASH_FILE}.lock" - - export GPU_COUNT="${GPU_COUNT:-${TP:?TP must be set}}" - - set -x - # Exclude known-bad mi355x compute nodes (KLAUD_DEBUG §5.1 / §5.2): - # mia1-p01-g09: pyxis broken (persistently fails to create container filesystem) - # mia1-p01-g11: docker.sock permissions denied (cluster-cleanup step fails) - # Both have been root-caused via #1431/#1432/#1440/#1441/#1443 sweep failures. - salloc --partition=$PARTITION --exclude=mia1-p01-g09,mia1-p01-g11 --gres=gpu:$GPU_COUNT --exclusive --cpus-per-task=128 --time=500 --no-shell --job-name="$RUNNER_NAME" - JOB_ID=$(squeue --name="$RUNNER_NAME" -h -o %A | head -n1) - - srun --jobid=$JOB_ID bash -c "docker stop \$(docker ps -a -q)" - - # Use flock to serialize concurrent imports to the same squash file - srun --jobid=$JOB_ID bash -c " - exec 9>\"$LOCK_FILE\" - flock -w 600 9 || { echo 'Failed to acquire lock for $SQUASH_FILE'; exit 1; } - if unsquashfs -l \"$SQUASH_FILE\" > /dev/null 2>&1; then - echo 'Squash file already exists and is valid, skipping import' - else - rm -f \"$SQUASH_FILE\" - enroot import -o \"$SQUASH_FILE\" docker://$IMAGE - fi - " - - export VLLM_CACHE_ROOT="/it-share/gharunners/.cache/vllm" - #--container-mount-home \ - - if [[ "$FRAMEWORK" == "atom" ]] || [[ "$FRAMEWORK" == "sglang" ]]; then - SLRUM_HOME_MOUNT="" - else - SLRUM_HOME_MOUNT=" --container-mount-home " - fi - - # to prevent reading outdated saved model. use a fresh model from hf repo - if [[ ("$FRAMEWORK" == "vllm" || "$FRAMEWORK" == "atom") ]] && [[ "$MODEL" == "deepseek-ai/DeepSeek-V4-Pro" ]]; then - export HF_HUB_CACHE_MOUNT="/it-share/hf-hub-cache/" - fi - - # MiniMax-M3 weights are not staged on the node-local /var/lib NVMe cache; - # they are pre-downloaded once to the NFS share instead. Covers both the - # MiniMaxAI MXFP8 checkpoint and the amd MXFP4 atom checkpoint. - if [[ "$MODEL" == MiniMaxAI/MiniMax-M3* || "$MODEL" == amd/MiniMax-M3* ]]; then - export HF_HUB_CACHE_MOUNT="/it-share/hf-hub-cache/" - fi - - SCRIPT_BASE="${EXP_NAME%%_*}_${PRECISION}_mi355x" - SCRIPT_FW="benchmarks/single_node/${SCENARIO_SUBDIR:-fixed_seq_len/}${SCRIPT_BASE}_${FRAMEWORK}${SPEC_SUFFIX}.sh" - SCRIPT_FALLBACK="benchmarks/single_node/${SCENARIO_SUBDIR:-fixed_seq_len/}${SCRIPT_BASE}${FRAMEWORK_SUFFIX}${SPEC_SUFFIX}.sh" - if [[ -f "$SCRIPT_FW" ]]; then - BENCHMARK_SCRIPT="$SCRIPT_FW" - else - BENCHMARK_SCRIPT="$SCRIPT_FALLBACK" - fi - - srun --jobid=$JOB_ID \ - --container-image=$SQUASH_FILE \ - --container-mounts=$GITHUB_WORKSPACE:/workspace/,$HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE,$AIPERF_MMAP_CACHE_HOST_PATH:/aiperf_mmap_cache \ - $SLRUM_HOME_MOUNT \ - --container-writable \ - --container-workdir=/workspace/ \ - --container-remap-root \ - --no-container-entrypoint --export=ALL,AIPERF_DATASET_MMAP_CACHE_DIR=/aiperf_mmap_cache \ - bash "$BENCHMARK_SCRIPT" - - scancel $JOB_ID - - if ls gpucore.* 1> /dev/null 2>&1; then - echo "gpucore files exist. not good" - rm -f gpucore.* - fi -fi diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index ce8d5c1022..8cb92b7a16 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -1,9 +1,5 @@ #!/usr/bin/env bash -if [[ "${SCENARIO_SUBDIR:-}" == "agentic/" ]]; then - exec bash "$(dirname "$0")/launch_mi355x-amds-agentic.sh" -fi - scancel_sync() { local jobid=$1 local timeout=${2:-600} diff --git a/utils/agentic/aggregation/backends/sglang.py b/utils/agentic/aggregation/backends/sglang.py index acbbc12b02..2f560f3bf7 100644 --- a/utils/agentic/aggregation/backends/sglang.py +++ b/utils/agentic/aggregation/backends/sglang.py @@ -61,8 +61,6 @@ def populate( flat["server_gpu_cache_hit_rate"] = rate(device_hits, prompt_total) flat["server_cpu_cache_hit_rate"] = rate(host_hits, prompt_total) - # HiCache host hits are external to the GPU cache. - flat["server_external_cache_hit_rate"] = flat["server_cpu_cache_hit_rate"] flat["server_overall_cache_hit_rate"] = rate(total_cached, prompt_total) if flat["server_overall_cache_hit_rate"] is None: diff --git a/utils/agentic/aggregation/process_agentic_result_agentic.py b/utils/agentic/aggregation/process_agentic_result_agentic.py deleted file mode 100644 index ff685e2c75..0000000000 --- a/utils/agentic/aggregation/process_agentic_result_agentic.py +++ /dev/null @@ -1,270 +0,0 @@ -#!/usr/bin/env python3 -"""Process aiperf agentic-replay output into InferenceX aggregate JSON.""" - -from __future__ import annotations - -import json -import os -import sys -from pathlib import Path -from typing import Any - -from .aggregation_common import round_floats -from .request_metrics import compute_request_metrics, load_aggregate, load_records_with_accounting -from .server_log_metrics import find_server_log_paths -from .server_metrics import compute_server_metrics, load_server_metrics - - -def env_int(name: str, default: int = 0) -> int: - value = os.environ.get(name) - if value in (None, ""): - return default - return int(value) - - -def env_bool(name: str, default: bool = False) -> bool: - value = os.environ.get(name) - if value in (None, ""): - return default - return value.lower() in ("1", "true", "yes", "on") - - -def required_env(name: str) -> str: - value = os.environ.get(name) - if value in (None, ""): - raise SystemExit(f"Missing required environment variable: {name}") - return value - - -def _validate_kv_offload_env() -> tuple[str, str]: - kv_offloading = required_env("KV_OFFLOADING") - kv_offload_backend = os.environ.get("KV_OFFLOAD_BACKEND", "") - if kv_offloading == "none": - if kv_offload_backend: - raise SystemExit("KV_OFFLOAD_BACKEND must be empty when KV_OFFLOADING=none") - else: - if not kv_offload_backend or kv_offload_backend == "none": - raise SystemExit("KV_OFFLOAD_BACKEND is required when KV_OFFLOADING is enabled") - return kv_offloading, kv_offload_backend - - -def _gpu_shape() -> tuple[dict[str, Any], int, int, int, str]: - is_multinode = env_bool("IS_MULTINODE") - tp = env_int("TP", 1) - ep = env_int("EP_SIZE", 1) - dp_attention = os.environ.get("DP_ATTENTION", "false") - fields: dict[str, Any] = {} - - if not is_multinode: - dcp_size = env_int("DCP_SIZE", 1) - pcp_size = env_int("PCP_SIZE", 1) - if dcp_size <= 0 or pcp_size <= 0: - raise SystemExit("DCP_SIZE and PCP_SIZE must be positive integers.") - fields.update({"dcp_size": dcp_size, "pcp_size": pcp_size}) - return fields, tp * pcp_size, tp, ep, dp_attention - - prefill_num_workers = env_int("PREFILL_NUM_WORKERS") - prefill_tp = env_int("PREFILL_TP") - prefill_ep = env_int("PREFILL_EP", 1) - prefill_dp_attention = os.environ.get("PREFILL_DP_ATTN", "false") - decode_num_workers = env_int("DECODE_NUM_WORKERS") - decode_tp = env_int("DECODE_TP") - decode_ep = env_int("DECODE_EP", 1) - decode_dp_attention = os.environ.get("DECODE_DP_ATTN", "false") - prefill_hardware = os.environ.get("PREFILL_HARDWARE", "") - decode_hardware = os.environ.get("DECODE_HARDWARE", "") - if bool(prefill_hardware) != bool(decode_hardware): - raise SystemExit( - "PREFILL_HARDWARE and DECODE_HARDWARE must be specified together." - ) - num_prefill_gpu = prefill_num_workers * prefill_tp - num_decode_gpu = decode_num_workers * decode_tp - num_gpus = num_prefill_gpu + num_decode_gpu - tp = prefill_tp + decode_tp - ep = max(prefill_ep, decode_ep) - dp_attention = ( - "true" - if env_bool("PREFILL_DP_ATTN") or env_bool("DECODE_DP_ATTN") - else "false" - ) - fields.update( - { - "prefill_num_workers": prefill_num_workers, - "prefill_tp": prefill_tp, - "prefill_ep": prefill_ep, - "prefill_dp_attention": prefill_dp_attention, - "num_prefill_gpu": num_prefill_gpu, - "decode_num_workers": decode_num_workers, - "decode_tp": decode_tp, - "decode_ep": decode_ep, - "decode_dp_attention": decode_dp_attention, - "num_decode_gpu": num_decode_gpu, - } - ) - if prefill_hardware: - fields["prefill_hw"] = prefill_hardware - fields["decode_hw"] = decode_hardware - return fields, num_gpus, tp, ep, dp_attention - - -def build_agg( - records: list[dict[str, Any]], - aggregate: dict[str, Any], - server_metrics: dict[str, Any], - *, - request_accounting: dict[str, Any] | None = None, - server_log_paths: list[Path] | None = None, -) -> dict[str, Any]: - """Compose the agg_*.json body from the three aiperf inputs.""" - kv_offloading, kv_offload_backend = _validate_kv_offload_env() - multinode_fields, num_gpus, tp, ep, dp_attention = _gpu_shape() - framework = os.environ.get("FRAMEWORK", "") - request_accounting = request_accounting or { - "records_total": len(records), - "records_profiled": len(records), - "records_dropped_total": 0, - "records_warmup_dropped": 0, - "records_error_dropped": 0, - "error_categories": {}, - } - - agg: dict[str, Any] = { - "hw": os.environ.get("RUNNER_TYPE", ""), - "conc": int(os.environ.get("CONC", "0")), - "image": os.environ.get("IMAGE", ""), - "model": os.environ.get("MODEL", ""), - "infmax_model_prefix": os.environ.get("MODEL_PREFIX", ""), - "framework": framework, - "precision": os.environ.get("PRECISION", ""), - "spec_decoding": os.environ.get("SPEC_DECODING", "none"), - "disagg": env_bool("DISAGG"), - "scenario_type": "agentic-coding", - "is_multinode": env_bool("IS_MULTINODE"), - "tp": tp, - "ep": ep, - "dp_attention": dp_attention, - "kv_offloading": kv_offloading, - "kv_offload_backend": kv_offload_backend, - "allocated_cpu_dram_gb": env_int("TOTAL_CPU_DRAM_GB"), - "num_requests_total": request_accounting["records_total"], - "num_requests_successful": len(records), - "request_accounting": request_accounting, - } - agg.update(multinode_fields) - - metadata = aggregate.get("metadata") - if isinstance(metadata, dict): - dataset = metadata.get("dataset") - if isinstance(dataset, dict): - agg["dataset"] = dataset - - request_flat, request_nested = compute_request_metrics(records, aggregate) - _, server_nested, warnings = compute_server_metrics( - server_metrics, - framework=framework, - records=records, - server_log_paths=server_log_paths, - ) - - if "total_tput_tps" in request_flat and num_gpus > 0: - request_nested["throughput"]["per_gpu"] = { - "total_tput_tps": request_flat["total_tput_tps"] / num_gpus, - "output_tput_tps": request_flat.get("output_tput_tps", 0) / num_gpus, - "input_tput_tps": request_flat.get("input_tput_tps", 0) / num_gpus, - } - - agg["request_metrics"] = request_nested - agg["server_metrics"] = server_nested - if warnings: - agg["warnings"] = warnings - return agg - - -def _resolve_artifact_dir(result_dir: Path) -> Path: - """Find the dir containing aiperf's profile_export* files.""" - base = result_dir / "aiperf_artifacts" - if (base / "profile_export.jsonl").is_file(): - return base - if base.is_dir(): - for child in sorted(base.iterdir()): - if child.is_dir() and (child / "profile_export.jsonl").is_file(): - return child - return base - - -def main() -> int: - result_filename = os.environ.get("RESULT_FILENAME", "") - if not result_filename: - print("ERROR: RESULT_FILENAME env var not set", file=sys.stderr) - return 1 - - result_dir = Path(os.environ.get("RESULT_DIR", "results")) - output_dir = Path(os.environ.get("AGENTIC_OUTPUT_DIR", ".")) - - artifact_dir = _resolve_artifact_dir(result_dir) - aggregate_path = artifact_dir / "profile_export_aiperf.json" - jsonl_path = artifact_dir / "profile_export.jsonl" - server_metrics_path = artifact_dir / "server_metrics_export.json" - - if not jsonl_path.exists(): - print(f"ERROR: {jsonl_path} not found", file=sys.stderr) - return 1 - - records, request_accounting = load_records_with_accounting(jsonl_path) - aggregate = load_aggregate(aggregate_path) if aggregate_path.exists() else {} - server_metrics = load_server_metrics(server_metrics_path) - server_log_paths = find_server_log_paths(result_dir) - agg = round_floats( - build_agg( - records, - aggregate, - server_metrics, - request_accounting=request_accounting, - server_log_paths=server_log_paths, - ) - ) - - output_path = output_dir / f"{result_filename}.json" - output_dir.mkdir(parents=True, exist_ok=True) - with open(output_path, "w") as f: - json.dump(agg, f, indent=2) - - print(f"Saved aggregated agentic result to {output_path}") - print( - f" Requests: {len(records)} successful / " - f"{request_accounting['records_total']} total " - f"({request_accounting['records_warmup_dropped']} warmup, " - f"{request_accounting['records_error_dropped']} error dropped)" - ) - request_metrics = agg.get("request_metrics", {}) - qps_metrics = request_metrics.get("qps", {}) - if "mean" in qps_metrics: - print( - f" QPS: mean={qps_metrics['mean']:.2f} " - f"p75={qps_metrics.get('p75', 0):.2f} " - f"p95={qps_metrics.get('p95', 0):.2f}" - ) - server_metrics = agg.get("server_metrics", {}) - server_cache = server_metrics.get("cache", {}) - server_kv_cache = server_metrics.get("kv_cache", {}) - if server_cache.get("gpu_cache_hit_rate") is not None: - print(f" GPU cache hit rate: {server_cache['gpu_cache_hit_rate']:.1%}") - if server_cache.get("cpu_cache_hit_rate") is not None: - print(f" CPU/offload cache hit rate: {server_cache['cpu_cache_hit_rate']:.1%}") - if server_cache.get("external_cache_hit_rate") is not None: - print(f" External cache hit rate: {server_cache['external_cache_hit_rate']:.1%}") - if server_kv_cache.get("gpu_usage_pct") is not None: - print(f" GPU KV cache usage: {server_kv_cache['gpu_usage_pct']:.1%}") - if server_kv_cache.get("gpu_total_tokens") is not None: - print(f" GPU KV cache capacity: {server_kv_cache['gpu_total_tokens']} tokens") - request_cache = request_metrics.get("cache", {}) - if request_cache.get("theoretical_cache_hit_rate") is not None: - print(f" Theoretical cache hit rate: {request_cache['theoretical_cache_hit_rate']:.1%}") - throughput_per_gpu = request_metrics.get("throughput", {}).get("per_gpu", {}) - if throughput_per_gpu.get("total_tput_tps") is not None: - print(f" Throughput per GPU: {throughput_per_gpu['total_tput_tps']:.0f} tok/s") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/utils/agentic/aggregation/test_process_agentic_result.py b/utils/agentic/aggregation/test_process_agentic_result.py index 0b9b6ab49b..45fd9d563e 100644 --- a/utils/agentic/aggregation/test_process_agentic_result.py +++ b/utils/agentic/aggregation/test_process_agentic_result.py @@ -1050,7 +1050,6 @@ def test_processor_normalizes_sglang_server_metrics(tmp_path: Path): _assert_stable_server_metrics_schema(agg) assert agg["server_metrics"]["cache"]["gpu_cache_hit_rate"] == pytest.approx(0.4) assert agg["server_metrics"]["cache"]["cpu_cache_hit_rate"] == pytest.approx(0.1) - assert agg["server_metrics"]["cache"]["external_cache_hit_rate"] == pytest.approx(0.1) assert agg["server_metrics"]["cache"]["overall_cache_hit_rate"] == pytest.approx(0.5) assert agg["server_metrics"]["kv_cache"]["gpu_usage_pct"] == pytest.approx(0.75) assert agg["server_metrics"]["kv_cache"]["cpu_usage_pct"] == pytest.approx(0.3)