From e5923ca701a4a82c746980a73bb50d01ee3d3041 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Sat, 11 Jul 2026 00:04:00 -0400 Subject: [PATCH 1/2] add Kimi-K2.6 NVFP4 B300 EAGLE3 agentic benchmark (kimik2.5-fp4-b300-vllm-agentic-mtp) --- .github/workflows/e2e-tests.yml | 2 +- .github/workflows/run-sweep.yml | 2 +- .../agentic/kimik2.5_fp4_b300_mtp.sh | 119 ++++++++++++++++++ configs/nvidia-master.yaml | 18 +++ perf-changelog.yaml | 7 ++ runners/launch_b300-nv.sh | 1 + utils/matrix_logic/generate_sweep_configs.py | 6 + utils/matrix_logic/validation.py | 3 + 8 files changed, 156 insertions(+), 2 deletions(-) create mode 100755 benchmarks/single_node/agentic/kimik2.5_fp4_b300_mtp.sh diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 790901ea0c..92259ea9b6 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -210,7 +210,7 @@ jobs: isl: '0' osl: '0' max-model-len: '0' - spec-decoding: 'none' + spec-decoding: ${{ matrix.config.spec-decoding }} disagg: 'false' run-eval: false scenario-type: agentic-coding diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index 86930487cb..c5be49a2a3 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -512,7 +512,7 @@ jobs: isl: '0' osl: '0' max-model-len: '0' - spec-decoding: 'none' + spec-decoding: ${{ matrix.config.spec-decoding }} disagg: ${{ 'false' }} run-eval: false scenario-type: agentic-coding diff --git a/benchmarks/single_node/agentic/kimik2.5_fp4_b300_mtp.sh b/benchmarks/single_node/agentic/kimik2.5_fp4_b300_mtp.sh new file mode 100755 index 0000000000..eddd1477b1 --- /dev/null +++ b/benchmarks/single_node/agentic/kimik2.5_fp4_b300_mtp.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +source "$(dirname "$0")/../../benchmark_lib.sh" + +check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION + + +if [[ -n "${SLURM_JOB_ID:-}" ]]; then + echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" +fi + +DRAFT_MODEL="lightseekorg/kimi-k2.6-eagle3-mla" + +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi + DRAFT_MODEL_PATH="/data/models/${DRAFT_MODEL##*/}" + if [[ ! -d "$DRAFT_MODEL_PATH" || -z "$(ls -A "$DRAFT_MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$DRAFT_MODEL" --local-dir "$DRAFT_MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" + hf download "$DRAFT_MODEL" + DRAFT_MODEL_PATH="$DRAFT_MODEL" +fi +nvidia-smi + +resolve_trace_source +install_agentic_deps + +SERVER_LOG="$RESULT_DIR/server.log" +mkdir -p "$RESULT_DIR" + +SERVER_PID="" + +cleanup_agentic_services() { + local exit_code=$? + trap - EXIT INT TERM + set +e + stop_background_process_tree "$SERVER_PID" "vLLM server" 60 + exit "$exit_code" +} +trap cleanup_agentic_services EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +OFFLOAD_ARGS=() + +if agentic_kv_offload_enabled; then + case "$KV_OFFLOAD_BACKEND" in + native) + export VLLM_USE_SIMPLE_KV_OFFLOAD=1 + CPU_OFFLOAD_BYTES=$((TOTAL_CPU_DRAM_GB * 1024 * 1024 * 1024)) + OFFLOAD_ARGS=( + --disable-hybrid-kv-cache-manager + --kv-transfer-config + "{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"cpu_bytes_to_use\":$CPU_OFFLOAD_BYTES,\"lazy_offload\":false}}" + ) + ;; + *) echo "Error: unsupported KV_OFFLOAD_BACKEND value '$KV_OFFLOAD_BACKEND' with EAGLE3 (expected: native)" >&2; exit 1 ;; + esac +fi + +DCP_SIZE="${DCP_SIZE:-1}" +DCP_ARGS=() +ATTN_BACKEND_ARGS=() +if [[ "$DCP_SIZE" -gt 1 ]]; then + DCP_ARGS+=(--decode-context-parallel-size "$DCP_SIZE") + NUM_SPEC_TOKENS=3 + SYNTHETIC_ACCEPT_LEN=2.88 +else + ATTN_BACKEND_ARGS+=(--attention-backend TOKENSPEED_MLA) + NUM_SPEC_TOKENS=4 + SYNTHETIC_ACCEPT_LEN=3.24 +fi + +echo "Starting vllm server..." +export PYTHONNOUSERSITE=1 + +export VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm + +{ set +x; } 2>/dev/null +VLLM_CMD=( + vllm serve "$MODEL_PATH" --served-model-name "$MODEL" + --host 0.0.0.0 + --port "$PORT" + --kv-cache-dtype fp8 + --trust-remote-code + --block-size 64 + --language-model-only + --gpu-memory-utilization 0.90 + --max-num-seqs "$CONC" + "${ATTN_BACKEND_ARGS[@]}" + --attention-config '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' + --max-cudagraph-capture-size 2048 + --max-num-batched-tokens 16384 + --stream-interval 10 + --enable-prefix-caching + --tensor-parallel-size "$TP" + --speculative-config "{\"method\":\"eagle3\",\"model\":\"$DRAFT_MODEL_PATH\",\"num_speculative_tokens\":$NUM_SPEC_TOKENS,\"rejection_sample_method\":\"synthetic\",\"synthetic_acceptance_length\":$SYNTHETIC_ACCEPT_LEN}" + "${DCP_ARGS[@]}" + "${OFFLOAD_ARGS[@]}" +) +printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" +printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" +"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & +SERVER_PID=$! +echo "Server PID: $SERVER_PID" + +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +build_replay_cmd "$RESULT_DIR" + +run_agentic_replay_and_write_outputs "$RESULT_DIR" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 6be8e006f9..43eb5357d3 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -2978,6 +2978,24 @@ kimik2.5-fp4-b300-vllm-agentic: - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32, 40, 48, 56, 64] } - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: native, conc-list: [1, 2, 4, 8, 16, 32, 40, 48, 56, 64] } +kimik2.5-fp4-b300-vllm-agentic-mtp: + image: vllm/vllm-openai:nightly-09663abde0f50944a8d5ea30120666024b503faa + model: Kimi-K2.6-NVFP4 + model-prefix: kimik2.5 + runner: cluster:b300-nv + precision: fp4 + framework: vllm + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1] } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [2, 4, 8] } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: native, conc-list: [8, 16, 32] } + - { tp: 4, ep: 1, spec-decoding: mtp, dcp-size: 4, kv-offloading: none, conc-list: [32, 64] } + - { tp: 4, ep: 1, spec-decoding: mtp, dcp-size: 4, kv-offloading: dram, kv-offload-backend: native, conc-list: [64, 128] } + dsr1-fp8-b200-trt: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 model: deepseek-ai/DeepSeek-R1-0528 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index be28208ab4..58e992d199 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4716,3 +4716,10 @@ - "Clean the export envs" - "Enable two batch overlap" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2093 + +- config-keys: + - kimik2.5-fp4-b300-vllm-agentic-mtp + description: + - "Add EAGLE3 speculative-decoding arm for the Kimi K2.6 NVFP4 B300 AgentX recipe (draft lightseekorg/kimi-k2.6-eagle3-mla, TOKENSPEED_MLA attention backend with TRT-LLM ragged MLA kernel)." + - "TP8/TP4 GPU-only KV points plus a TP4 native CPU-offload ladder via SimpleCPUOffloadConnector with lazy_offload off; TP4/DCP4 high-concurrency points (conc 32/64) using num_speculative_tokens=3 and synthetic_acceptance_length=2.88." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2158 diff --git a/runners/launch_b300-nv.sh b/runners/launch_b300-nv.sh index 6810ee5d85..2c0f93fc8b 100644 --- a/runners/launch_b300-nv.sh +++ b/runners/launch_b300-nv.sh @@ -352,6 +352,7 @@ else Kimi-K2.5 Kimi-K2.5-NVFP4 Kimi-K2.6 + Kimi-K2.6-NVFP4 MiniMax-M2.5 MiniMax-M2.5-NVFP4 MiniMax-M2.7 diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index 5728580cb1..5189919b83 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -630,6 +630,7 @@ def generate_full_sweep(args, all_config_data, runner_data): pcp_size = bmk.get(Fields.PCP_SIZE.value, 1) ep = bmk.get(Fields.EP.value) dp_attn = bmk.get(Fields.DP_ATTN.value) + spec_decoding = bmk.get(Fields.SPEC_DECODING.value, "none") kv_offloading = bmk[Fields.KV_OFFLOADING.value] kv_offload_backend = bmk.get(Fields.KV_OFFLOAD_BACKEND.value) total_cpu_dram_gb = ( @@ -717,6 +718,7 @@ def generate_full_sweep(args, all_config_data, runner_data): Fields.PCP_SIZE.value: pcp_size, Fields.EP.value: ep if ep is not None else 1, Fields.DP_ATTN.value: dp_attn if dp_attn is not None else False, + Fields.SPEC_DECODING.value: spec_decoding, Fields.CONC.value: conc, Fields.KV_OFFLOADING.value: kv_offloading, Fields.TOTAL_CPU_DRAM_GB.value: total_cpu_dram_gb, @@ -724,6 +726,7 @@ def generate_full_sweep(args, all_config_data, runner_data): Fields.EXP_NAME.value: ( f"{model_code}_tp{tp}_conc{conc}_" f"{agentic_kv_offload_suffix(kv_offloading, kv_offload_backend)}" + + (f"_spec-{spec_decoding}" if spec_decoding != "none" else "") ), Fields.SCENARIO_TYPE.value: "agentic-coding", } @@ -929,6 +932,7 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None): pcp_size = bmk.get(Fields.PCP_SIZE.value, 1) ep = bmk.get(Fields.EP.value) dp_attn = bmk.get(Fields.DP_ATTN.value) + spec_decoding = bmk.get(Fields.SPEC_DECODING.value, "none") kv_offloading = bmk[Fields.KV_OFFLOADING.value] kv_offload_backend = bmk.get(Fields.KV_OFFLOAD_BACKEND.value) total_cpu_dram_gb = ( @@ -1009,6 +1013,7 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None): Fields.PCP_SIZE.value: pcp_size, Fields.EP.value: ep if ep is not None else 1, Fields.DP_ATTN.value: dp_attn if dp_attn is not None else False, + Fields.SPEC_DECODING.value: spec_decoding, Fields.CONC.value: conc, Fields.KV_OFFLOADING.value: kv_offloading, Fields.TOTAL_CPU_DRAM_GB.value: total_cpu_dram_gb, @@ -1016,6 +1021,7 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None): Fields.EXP_NAME.value: ( f"{model_code}_tp{tp}_conc{conc}_" f"{agentic_kv_offload_suffix(kv_offloading, kv_offload_backend)}" + + (f"_spec-{spec_decoding}" if spec_decoding != "none" else "") ), Fields.SCENARIO_TYPE.value: "agentic-coding", } diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py index 510cfcdbc4..31426698dc 100644 --- a/utils/matrix_logic/validation.py +++ b/utils/matrix_logic/validation.py @@ -215,6 +215,9 @@ class SingleNodeAgenticMatrixEntry(BaseModel): pcp_size: int = Field(alias=Fields.PCP_SIZE.value, gt=0, strict=True) ep: int dp_attn: bool = Field(alias=Fields.DP_ATTN.value) + spec_decoding: Literal["mtp", "draft_model", "none"] = Field( + default="none", alias=Fields.SPEC_DECODING.value + ) conc: int kv_offloading: Literal["none", "dram"] = Field( alias=Fields.KV_OFFLOADING.value From 1807ba5ae534980b8b152397639d3f8bfbe0e4bc Mon Sep 17 00:00:00 2001 From: Xin Li Date: Sat, 11 Jul 2026 01:00:34 -0400 Subject: [PATCH 2/2] configs --- .../single_node/agentic/kimik2.5_fp4_b300_mtp.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/benchmarks/single_node/agentic/kimik2.5_fp4_b300_mtp.sh b/benchmarks/single_node/agentic/kimik2.5_fp4_b300_mtp.sh index eddd1477b1..6672441be6 100755 --- a/benchmarks/single_node/agentic/kimik2.5_fp4_b300_mtp.sh +++ b/benchmarks/single_node/agentic/kimik2.5_fp4_b300_mtp.sh @@ -67,13 +67,17 @@ fi DCP_SIZE="${DCP_SIZE:-1}" DCP_ARGS=() -ATTN_BACKEND_ARGS=() +ATTN_ARGS=() if [[ "$DCP_SIZE" -gt 1 ]]; then DCP_ARGS+=(--decode-context-parallel-size "$DCP_SIZE") + ATTN_ARGS+=(--attention-config '{"use_prefill_query_quantization":true}') NUM_SPEC_TOKENS=3 SYNTHETIC_ACCEPT_LEN=2.88 else - ATTN_BACKEND_ARGS+=(--attention-backend TOKENSPEED_MLA) + ATTN_ARGS+=( + --attention-backend TOKENSPEED_MLA + --attention-config '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + ) NUM_SPEC_TOKENS=4 SYNTHETIC_ACCEPT_LEN=3.24 fi @@ -94,8 +98,7 @@ VLLM_CMD=( --language-model-only --gpu-memory-utilization 0.90 --max-num-seqs "$CONC" - "${ATTN_BACKEND_ARGS[@]}" - --attention-config '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + "${ATTN_ARGS[@]}" --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' --max-cudagraph-capture-size 2048 --max-num-batched-tokens 16384