diff --git a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_mi355x_atom.sh b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_mi355x_atom.sh index 6730aded2..38d625d1d 100755 --- a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_mi355x_atom.sh +++ b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_mi355x_atom.sh @@ -21,6 +21,10 @@ echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTIO SERVER_LOG=/workspace/server.log + +export ATOM_DISABLE_MMAP=true # Model load faster. +export AITER_QUICK_REDUCE_QUANTIZATION=INT4 +export AITER_MXFP4_INTERMEDIATE=1 export OMP_NUM_THREADS=1 # Calculate max-model-len based on ISL and OSL @@ -45,6 +49,7 @@ python3 -m atom.entrypoints.openai_server \ --model $MODEL \ --server-port $PORT \ -tp $TP \ + --scheduler-delay-factor 1 \ --kv_cache_dtype fp8 $CALCULATED_MAX_MODEL_LEN $EP \ --trust-remote-code \ > $SERVER_LOG 2>&1 & diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 687439a98..192300798 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -809,7 +809,7 @@ kimik2.5-fp4-mi355x-vllm-agentic: - { tp: 4, kv-offloading: dram, kv-offload-backend: native, conc-list: [16, 24, 32, 40] } kimik2.5-fp4-mi355x-atom: - image: rocm/atom:rocm7.2.3_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom20260511 + image: rocm/atom:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.4_202607091539 model: amd/Kimi-K2.5-MXFP4 model-prefix: kimik2.5 runner: mi355x @@ -821,12 +821,10 @@ kimik2.5-fp4-mi355x-atom: - isl: 1024 osl: 1024 search-space: - - { tp: 8, conc-start: 4, conc-end: 128 } - { tp: 4, conc-start: 4, conc-end: 128 } - isl: 8192 osl: 1024 search-space: - - { tp: 8, conc-start: 4, conc-end: 128 } - { tp: 4, conc-start: 4, conc-end: 128 } dsr1-fp8-mi355x-atom: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c1a3c3aee..695976036 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4587,6 +4587,14 @@ - "Add high concurrency configs" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1994 +- config-keys: + - kimik2.5-fp4-mi355x-atom + description: + - "Update ATOM image from rocm/atom:rocm7.2.3_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom20260511 to rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.4_202607091539" + - "Enable ATOM_DISABLE_MMAP=true, AITER_QUICK_REDUCE_QUANTIZATION=INT4, AITER_MXFP4_INTERMEDIATE=1; add --scheduler-delay-factor 1 to server launch" + - "Conc 4 only for both 1k1k and 8k1k" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2132 + - config-keys: - dsv4-fp4-mi355x-vllm-agentic description: