From c46dc19fdbb903c5e3d06835ddf9dac82ddd07a3 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 29 Jun 2026 14:29:20 -0500 Subject: [PATCH 1/2] Allow combined trace and turn idle caps --- docs/cli-options.md | 2 +- src/aiperf/common/config/loadgen_config.py | 9 +++++---- src/aiperf/common/scenario/base.py | 5 +++-- src/aiperf/dataset/loader/weka_trace.py | 15 ++++++++++----- .../common/scenario/test_scenario_validator.py | 2 +- .../loader/test_weka_flat_split_parallel_adv.py | 11 +++++++---- tests/unit/dataset/loader/test_weka_trace.py | 6 +++--- 7 files changed, 30 insertions(+), 20 deletions(-) diff --git a/docs/cli-options.md b/docs/cli-options.md index 00592d999..9fd09bc38 100644 --- a/docs/cli-options.md +++ b/docs/cli-options.md @@ -913,7 +913,7 @@ Hard ceiling (seconds) for inter-turn delays in trace replay. Applies to all tra #### `--trace-idle-gap-cap-seconds` `` -Hard ceiling (seconds) for idle gaps within each individual trace. For Weka trace replay, AIPerf looks at all parent and subagent request submission timestamps within one root trace, compresses long gaps between consecutive request submissions, and derives turn delays from the compressed per-trace timeline. Original request api_time values are not used to decide these idle gaps. When set for Weka, this takes precedence over `--inter-turn-delay-cap-seconds` so individual parent/subagent-line delays are not separately capped. Defaults to None (no per-trace idle-gap compression). +Hard ceiling (seconds) for idle gaps within each individual trace. For Weka trace replay, AIPerf looks at all parent and subagent request submission timestamps within one root trace, compresses long gaps between consecutive request submissions, and derives turn delays from the compressed per-trace timeline. Original request api_time values are not used to decide these idle gaps. When both this and `--inter-turn-delay-cap-seconds` are set for Weka, the per-trace timeline compression runs first and the derived parent/subagent-line delays are then capped by `--inter-turn-delay-cap-seconds`. Defaults to None (no per-trace idle-gap compression).
_Constraints: ≥ 0.0_ #### `--warmup-request-count`, `--num-warmup-requests` `` diff --git a/src/aiperf/common/config/loadgen_config.py b/src/aiperf/common/config/loadgen_config.py index afd988327..0018dcba2 100644 --- a/src/aiperf/common/config/loadgen_config.py +++ b/src/aiperf/common/config/loadgen_config.py @@ -435,10 +435,11 @@ def parse_concurrency_list( "submission timestamps within one root trace, compresses long gaps between " "consecutive request submissions, and derives turn delays from the " "compressed per-trace timeline. Original request api_time values are not " - "used to decide these idle gaps. When set for Weka, this takes precedence over " - "`--inter-turn-delay-cap-seconds` so individual parent/subagent-line " - "delays are not separately capped. Defaults to None (no per-trace " - "idle-gap compression).", + "used to decide these idle gaps. When both this and " + "`--inter-turn-delay-cap-seconds` are set for Weka, the per-trace " + "timeline compression runs first and the derived parent/subagent-line " + "delays are then capped by `--inter-turn-delay-cap-seconds`. Defaults " + "to None (no per-trace idle-gap compression).", ), CLIParameter( name=("--trace-idle-gap-cap-seconds",), diff --git a/src/aiperf/common/scenario/base.py b/src/aiperf/common/scenario/base.py index 0f67348fc..65f8525bf 100644 --- a/src/aiperf/common/scenario/base.py +++ b/src/aiperf/common/scenario/base.py @@ -108,8 +108,9 @@ class ScenarioSpec(AIPerfBaseModel): description=( "Hard ceiling (seconds) for idle gaps within each root trace. For " "Weka, parent + subagent request-start timestamps are compressed " - "per-trace before per-turn delays are derived. Takes precedence over " - "inter_turn_delay_cap_seconds and supersedes use_think_time_only." + "per-trace before per-turn delays are derived. Supersedes " + "use_think_time_only for Weka timestamp derivation; derived delays " + "may still be capped by inter_turn_delay_cap_seconds." ), ) require_cache_bust: CacheBustTarget | None = Field( diff --git a/src/aiperf/dataset/loader/weka_trace.py b/src/aiperf/dataset/loader/weka_trace.py index aaa5d4dbf..09b675a1c 100644 --- a/src/aiperf/dataset/loader/weka_trace.py +++ b/src/aiperf/dataset/loader/weka_trace.py @@ -1586,13 +1586,17 @@ def convert_to_conversations( ignore_delays = self.user_config.input.ignore_trace_delays think_time_only = self.user_config.input.use_think_time_only cap_seconds = self.user_config.loadgen.inter_turn_delay_cap_seconds - trace_idle_gap_cap_seconds = self._trace_idle_gap_cap_seconds() trace_idle_timing_by_trace = self._build_trace_idle_timing_by_trace( parent_plans, child_plans, flat_plans ) - turn_cap_seconds = ( - None if trace_idle_gap_cap_seconds is not None else cap_seconds - ) + # The caps model different things and may be intentionally combined: + # trace_idle_gap_cap_seconds compresses aggregate request-start idle + # within each root trace first; inter_turn_delay_cap_seconds then caps + # the derived per-stream parent/subagent delays. Historically the trace + # warp disabled the turn cap to avoid accidental double-capping, but + # AgentX needs both knobs to independently bound engine-idle gaps and + # per-conversation dormancy. + turn_cap_seconds = cap_seconds self._delay_cap_tracker.cap_seconds = turn_cap_seconds _t0 = _time.monotonic() @@ -2274,7 +2278,8 @@ def _flat_turn_timing( Same precedence as the subagent-child loop: warped per-trace timing when the idle-gap cap is active, else raw per-chain deltas honoring - ``--use-think-time-only`` and the inter-turn delay cap. + ``--use-think-time-only``. The derived delay is then independently + capped by ``--inter-turn-delay-cap-seconds`` when that cap is set. """ if trace_idle_timing is not None: timing = trace_idle_timing.child_by_session_request[(fp.session_id, k)] diff --git a/tests/unit/common/scenario/test_scenario_validator.py b/tests/unit/common/scenario/test_scenario_validator.py index 90ab110be..61d1e7c77 100644 --- a/tests/unit/common/scenario/test_scenario_validator.py +++ b/tests/unit/common/scenario/test_scenario_validator.py @@ -281,7 +281,7 @@ def test_random_seed_unset_auto_injected_and_logged( def test_inter_turn_delay_cap_explicit_other_value_does_not_raise() -> None: """AgentX MVP no longer locks --inter-turn-delay-cap-seconds; - trace_idle_gap_cap_seconds supersedes the per-turn cap in the weka loader.""" + Weka may apply it after trace_idle_gap_cap_seconds rewrites the timeline.""" cfg = _user_config( inter_turn_delay_cap_seconds=30.0, extra_inputs={"ignore_eos": True} ) diff --git a/tests/unit/dataset/loader/test_weka_flat_split_parallel_adv.py b/tests/unit/dataset/loader/test_weka_flat_split_parallel_adv.py index 6ce9b2150..f8dfd6469 100644 --- a/tests/unit/dataset/loader/test_weka_flat_split_parallel_adv.py +++ b/tests/unit/dataset/loader/test_weka_flat_split_parallel_adv.py @@ -303,8 +303,10 @@ def test_convert_fanout_idle_gap_warp_parallel_byte_identical(tmp_path, monkeypa Three compressible gaps, one INSIDE the worker chain (2.5 -> 8.5 across chains, 9 -> 200 on main, 200 -> 210 reaching a worker-chain request whose - api_time is unrecorded/None). Warped timestamps and per-chain delays must - be byte-identical across paths. + api_time is unrecorded/None). The trace-wide idle-gap cap rewrites the + aggregate timeline first; the per-turn cap then clamps derived stream-local + delays. Warped timestamps and capped per-chain delays must be byte-identical + across paths. """ reqs = _fanout_requests() reqs[5]["t"] = 200.0 # main turn 3 after a 191s idle gap @@ -317,18 +319,19 @@ def test_convert_fanout_idle_gap_warp_parallel_byte_identical(tmp_path, monkeypa monkeypatch, [_trace("trace_warp", reqs)], idle_gap_cap_seconds=5.0, + inter_turn_delay_cap_seconds=6.0, ) convs = _by_sid(serial) root = convs["trace_warp"] # Gaps: [2.5, 8.5] excess 1, [9, 200] excess 186, [200, 210] excess 5. assert [t.timestamp for t in root.turns] == pytest.approx([0.0, 8000.0, 13000.0]) - assert root.turns[1].delay == pytest.approx(8000.0) + assert root.turns[1].delay == pytest.approx(6000.0) assert root.turns[2].delay == pytest.approx(5000.0) w0 = convs["trace_warp::fa:000"] assert [t.timestamp for t in w0.turns] == pytest.approx([2000.0, 7500.0, 18000.0]) assert w0.turns[1].delay == pytest.approx(5500.0) - assert w0.turns[2].delay == pytest.approx(10500.0) + assert w0.turns[2].delay == pytest.approx(6000.0) def test_convert_nonmonotonic_parent_delay_floored_parallel_byte_identical( diff --git a/tests/unit/dataset/loader/test_weka_trace.py b/tests/unit/dataset/loader/test_weka_trace.py index c0d5d5f7d..0515cd9b6 100644 --- a/tests/unit/dataset/loader/test_weka_trace.py +++ b/tests/unit/dataset/loader/test_weka_trace.py @@ -1083,9 +1083,9 @@ def normal( trace_a_turns = conv_by_id["trace_idle_a"].turns assert trace_a_turns[0].timestamp == 0.0 assert trace_a_turns[1].timestamp == 80_000.0 - # The trace-wide idle-gap cap takes precedence over the old per-turn cap, - # so this stays 80s rather than being independently clamped to 60s. - assert trace_a_turns[1].delay == 80_000.0 + # The trace-wide idle-gap cap first shifts the parent start to 80s; the + # per-turn cap then independently caps that derived per-stream delay. + assert trace_a_turns[1].delay == 60_000.0 assert conv_by_id["trace_idle_a::sa:agent_idle"].turns[0].timestamp == 20_000.0 # Trace B is compressed against its own request starts only: 150 -> 220 From 230e3d5559bf32ff7a5232a6e5e15fd991278565 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 29 Jun 2026 14:39:39 -0500 Subject: [PATCH 2/2] Revert "Allow combined trace and turn idle caps" This reverts commit c46dc19fdbb903c5e3d06835ddf9dac82ddd07a3. --- docs/cli-options.md | 2 +- src/aiperf/common/config/loadgen_config.py | 9 ++++----- src/aiperf/common/scenario/base.py | 5 ++--- src/aiperf/dataset/loader/weka_trace.py | 15 +++++---------- .../common/scenario/test_scenario_validator.py | 2 +- .../loader/test_weka_flat_split_parallel_adv.py | 11 ++++------- tests/unit/dataset/loader/test_weka_trace.py | 6 +++--- 7 files changed, 20 insertions(+), 30 deletions(-) diff --git a/docs/cli-options.md b/docs/cli-options.md index 9fd09bc38..00592d999 100644 --- a/docs/cli-options.md +++ b/docs/cli-options.md @@ -913,7 +913,7 @@ Hard ceiling (seconds) for inter-turn delays in trace replay. Applies to all tra #### `--trace-idle-gap-cap-seconds` `` -Hard ceiling (seconds) for idle gaps within each individual trace. For Weka trace replay, AIPerf looks at all parent and subagent request submission timestamps within one root trace, compresses long gaps between consecutive request submissions, and derives turn delays from the compressed per-trace timeline. Original request api_time values are not used to decide these idle gaps. When both this and `--inter-turn-delay-cap-seconds` are set for Weka, the per-trace timeline compression runs first and the derived parent/subagent-line delays are then capped by `--inter-turn-delay-cap-seconds`. Defaults to None (no per-trace idle-gap compression). +Hard ceiling (seconds) for idle gaps within each individual trace. For Weka trace replay, AIPerf looks at all parent and subagent request submission timestamps within one root trace, compresses long gaps between consecutive request submissions, and derives turn delays from the compressed per-trace timeline. Original request api_time values are not used to decide these idle gaps. When set for Weka, this takes precedence over `--inter-turn-delay-cap-seconds` so individual parent/subagent-line delays are not separately capped. Defaults to None (no per-trace idle-gap compression).
_Constraints: ≥ 0.0_ #### `--warmup-request-count`, `--num-warmup-requests` `` diff --git a/src/aiperf/common/config/loadgen_config.py b/src/aiperf/common/config/loadgen_config.py index 0018dcba2..afd988327 100644 --- a/src/aiperf/common/config/loadgen_config.py +++ b/src/aiperf/common/config/loadgen_config.py @@ -435,11 +435,10 @@ def parse_concurrency_list( "submission timestamps within one root trace, compresses long gaps between " "consecutive request submissions, and derives turn delays from the " "compressed per-trace timeline. Original request api_time values are not " - "used to decide these idle gaps. When both this and " - "`--inter-turn-delay-cap-seconds` are set for Weka, the per-trace " - "timeline compression runs first and the derived parent/subagent-line " - "delays are then capped by `--inter-turn-delay-cap-seconds`. Defaults " - "to None (no per-trace idle-gap compression).", + "used to decide these idle gaps. When set for Weka, this takes precedence over " + "`--inter-turn-delay-cap-seconds` so individual parent/subagent-line " + "delays are not separately capped. Defaults to None (no per-trace " + "idle-gap compression).", ), CLIParameter( name=("--trace-idle-gap-cap-seconds",), diff --git a/src/aiperf/common/scenario/base.py b/src/aiperf/common/scenario/base.py index 65f8525bf..0f67348fc 100644 --- a/src/aiperf/common/scenario/base.py +++ b/src/aiperf/common/scenario/base.py @@ -108,9 +108,8 @@ class ScenarioSpec(AIPerfBaseModel): description=( "Hard ceiling (seconds) for idle gaps within each root trace. For " "Weka, parent + subagent request-start timestamps are compressed " - "per-trace before per-turn delays are derived. Supersedes " - "use_think_time_only for Weka timestamp derivation; derived delays " - "may still be capped by inter_turn_delay_cap_seconds." + "per-trace before per-turn delays are derived. Takes precedence over " + "inter_turn_delay_cap_seconds and supersedes use_think_time_only." ), ) require_cache_bust: CacheBustTarget | None = Field( diff --git a/src/aiperf/dataset/loader/weka_trace.py b/src/aiperf/dataset/loader/weka_trace.py index 09b675a1c..aaa5d4dbf 100644 --- a/src/aiperf/dataset/loader/weka_trace.py +++ b/src/aiperf/dataset/loader/weka_trace.py @@ -1586,17 +1586,13 @@ def convert_to_conversations( ignore_delays = self.user_config.input.ignore_trace_delays think_time_only = self.user_config.input.use_think_time_only cap_seconds = self.user_config.loadgen.inter_turn_delay_cap_seconds + trace_idle_gap_cap_seconds = self._trace_idle_gap_cap_seconds() trace_idle_timing_by_trace = self._build_trace_idle_timing_by_trace( parent_plans, child_plans, flat_plans ) - # The caps model different things and may be intentionally combined: - # trace_idle_gap_cap_seconds compresses aggregate request-start idle - # within each root trace first; inter_turn_delay_cap_seconds then caps - # the derived per-stream parent/subagent delays. Historically the trace - # warp disabled the turn cap to avoid accidental double-capping, but - # AgentX needs both knobs to independently bound engine-idle gaps and - # per-conversation dormancy. - turn_cap_seconds = cap_seconds + turn_cap_seconds = ( + None if trace_idle_gap_cap_seconds is not None else cap_seconds + ) self._delay_cap_tracker.cap_seconds = turn_cap_seconds _t0 = _time.monotonic() @@ -2278,8 +2274,7 @@ def _flat_turn_timing( Same precedence as the subagent-child loop: warped per-trace timing when the idle-gap cap is active, else raw per-chain deltas honoring - ``--use-think-time-only``. The derived delay is then independently - capped by ``--inter-turn-delay-cap-seconds`` when that cap is set. + ``--use-think-time-only`` and the inter-turn delay cap. """ if trace_idle_timing is not None: timing = trace_idle_timing.child_by_session_request[(fp.session_id, k)] diff --git a/tests/unit/common/scenario/test_scenario_validator.py b/tests/unit/common/scenario/test_scenario_validator.py index 61d1e7c77..90ab110be 100644 --- a/tests/unit/common/scenario/test_scenario_validator.py +++ b/tests/unit/common/scenario/test_scenario_validator.py @@ -281,7 +281,7 @@ def test_random_seed_unset_auto_injected_and_logged( def test_inter_turn_delay_cap_explicit_other_value_does_not_raise() -> None: """AgentX MVP no longer locks --inter-turn-delay-cap-seconds; - Weka may apply it after trace_idle_gap_cap_seconds rewrites the timeline.""" + trace_idle_gap_cap_seconds supersedes the per-turn cap in the weka loader.""" cfg = _user_config( inter_turn_delay_cap_seconds=30.0, extra_inputs={"ignore_eos": True} ) diff --git a/tests/unit/dataset/loader/test_weka_flat_split_parallel_adv.py b/tests/unit/dataset/loader/test_weka_flat_split_parallel_adv.py index f8dfd6469..6ce9b2150 100644 --- a/tests/unit/dataset/loader/test_weka_flat_split_parallel_adv.py +++ b/tests/unit/dataset/loader/test_weka_flat_split_parallel_adv.py @@ -303,10 +303,8 @@ def test_convert_fanout_idle_gap_warp_parallel_byte_identical(tmp_path, monkeypa Three compressible gaps, one INSIDE the worker chain (2.5 -> 8.5 across chains, 9 -> 200 on main, 200 -> 210 reaching a worker-chain request whose - api_time is unrecorded/None). The trace-wide idle-gap cap rewrites the - aggregate timeline first; the per-turn cap then clamps derived stream-local - delays. Warped timestamps and capped per-chain delays must be byte-identical - across paths. + api_time is unrecorded/None). Warped timestamps and per-chain delays must + be byte-identical across paths. """ reqs = _fanout_requests() reqs[5]["t"] = 200.0 # main turn 3 after a 191s idle gap @@ -319,19 +317,18 @@ def test_convert_fanout_idle_gap_warp_parallel_byte_identical(tmp_path, monkeypa monkeypatch, [_trace("trace_warp", reqs)], idle_gap_cap_seconds=5.0, - inter_turn_delay_cap_seconds=6.0, ) convs = _by_sid(serial) root = convs["trace_warp"] # Gaps: [2.5, 8.5] excess 1, [9, 200] excess 186, [200, 210] excess 5. assert [t.timestamp for t in root.turns] == pytest.approx([0.0, 8000.0, 13000.0]) - assert root.turns[1].delay == pytest.approx(6000.0) + assert root.turns[1].delay == pytest.approx(8000.0) assert root.turns[2].delay == pytest.approx(5000.0) w0 = convs["trace_warp::fa:000"] assert [t.timestamp for t in w0.turns] == pytest.approx([2000.0, 7500.0, 18000.0]) assert w0.turns[1].delay == pytest.approx(5500.0) - assert w0.turns[2].delay == pytest.approx(6000.0) + assert w0.turns[2].delay == pytest.approx(10500.0) def test_convert_nonmonotonic_parent_delay_floored_parallel_byte_identical( diff --git a/tests/unit/dataset/loader/test_weka_trace.py b/tests/unit/dataset/loader/test_weka_trace.py index 0515cd9b6..c0d5d5f7d 100644 --- a/tests/unit/dataset/loader/test_weka_trace.py +++ b/tests/unit/dataset/loader/test_weka_trace.py @@ -1083,9 +1083,9 @@ def normal( trace_a_turns = conv_by_id["trace_idle_a"].turns assert trace_a_turns[0].timestamp == 0.0 assert trace_a_turns[1].timestamp == 80_000.0 - # The trace-wide idle-gap cap first shifts the parent start to 80s; the - # per-turn cap then independently caps that derived per-stream delay. - assert trace_a_turns[1].delay == 60_000.0 + # The trace-wide idle-gap cap takes precedence over the old per-turn cap, + # so this stays 80s rather than being independently clamped to 60s. + assert trace_a_turns[1].delay == 80_000.0 assert conv_by_id["trace_idle_a::sa:agent_idle"].turns[0].timestamp == 20_000.0 # Trace B is compressed against its own request starts only: 150 -> 220