From c0b4263deeccb723bc3d2cd26c7f760cc248b0fb Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 19 Jun 2026 20:38:50 -0400 Subject: [PATCH 1/9] Fix EVP flagevaluation system test contract --- tests/ffe/test_flag_eval_evp.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/ffe/test_flag_eval_evp.py b/tests/ffe/test_flag_eval_evp.py index ae6a5843073..b863e62fc19 100644 --- a/tests/ffe/test_flag_eval_evp.py +++ b/tests/ffe/test_flag_eval_evp.py @@ -17,11 +17,11 @@ RC_PRODUCT = "FFE_FLAGS" RC_PATH = f"datadog/2/{RC_PRODUCT}" -EVP_FLAGEVALUATIONS_PATH = "/api/v2/flagevaluations" +EVP_FLAGEVALUATION_PATH = "/api/v2/flagevaluation" EVP_WAIT_TIMEOUT_SECONDS = 30 EVP_LOAD_WAIT_TIMEOUT_SECONDS = 60 EVP_FULL_TIER_PER_FLAG_CAP = 10_000 -EVP_DEGRADATION_OVERFLOW_EVALS = 50 +EVP_DEGRADATION_OVERFLOW_EVALS = 2_000 def make_multi_flag_fixture(flag_keys: list[str]) -> JSON: @@ -54,7 +54,7 @@ def evaluate_flag( def evp_flagevaluation_events_from_data(data: JSON, flag_key: str) -> list[tuple[JSON, JSON]]: - if data.get("path") != EVP_FLAGEVALUATIONS_PATH: + if data.get("path") != EVP_FLAGEVALUATION_PATH: return [] request = data.get("request") @@ -91,7 +91,7 @@ def wait_for_evp_flagevaluation_event(flag_key: str) -> None: def find_evp_flagevaluation_events(flag_key: str) -> list[tuple[JSON, JSON]]: results: list[tuple[JSON, JSON]] = [] - for data in interfaces.agent.get_data(path_filters=EVP_FLAGEVALUATIONS_PATH): + for data in interfaces.agent.get_data(path_filters=EVP_FLAGEVALUATION_PATH): results.extend(evp_flagevaluation_events_from_data(cast("JSON", data), flag_key)) return results From 8b8237a12371d443e06e4feb12f5a2b8041b1944 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 19 Jun 2026 20:50:56 -0400 Subject: [PATCH 2/9] Enable EVP flagevaluation system test for Ruby --- manifests/ruby.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/ruby.yml b/manifests/ruby.yml index d78e537a578..339598e8a77 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -1887,7 +1887,7 @@ manifest: - weblog_declaration: "*": irrelevant rails72: v2.23.0-dev - tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) + tests/ffe/test_flag_eval_evp.py: v2.36.0-dev tests/ffe/test_flag_eval_metrics.py: - weblog_declaration: "*": irrelevant From d132d6242ab9acaedfb6ce2003a48c2ec8e599ba Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 19 Jun 2026 21:17:35 -0400 Subject: [PATCH 3/9] Limit EVP flagevaluation tests to Ruby rails72 --- manifests/ruby.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/manifests/ruby.yml b/manifests/ruby.yml index 339598e8a77..95113849246 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -1887,7 +1887,10 @@ manifest: - weblog_declaration: "*": irrelevant rails72: v2.23.0-dev - tests/ffe/test_flag_eval_evp.py: v2.36.0-dev + tests/ffe/test_flag_eval_evp.py: + - weblog_declaration: + "*": irrelevant + rails72: v2.36.0-dev tests/ffe/test_flag_eval_metrics.py: - weblog_declaration: "*": irrelevant From ebd16c7d8e47dddc4229245570baa78c60160322 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 19 Jun 2026 23:44:54 -0400 Subject: [PATCH 4/9] Stabilize EVP degradation burst test --- tests/ffe/test_flag_eval_evp.py | 39 +++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/tests/ffe/test_flag_eval_evp.py b/tests/ffe/test_flag_eval_evp.py index b863e62fc19..18ee63aa32e 100644 --- a/tests/ffe/test_flag_eval_evp.py +++ b/tests/ffe/test_flag_eval_evp.py @@ -22,6 +22,8 @@ EVP_LOAD_WAIT_TIMEOUT_SECONDS = 60 EVP_FULL_TIER_PER_FLAG_CAP = 10_000 EVP_DEGRADATION_OVERFLOW_EVALS = 2_000 +EVP_DEGRADATION_REQUEST_BATCH_SIZE = 1_000 +EVP_DEGRADATION_REQUEST_WORKERS = 12 def make_multi_flag_fixture(flag_keys: list[str]) -> JSON: @@ -456,21 +458,40 @@ class Test_FFE_EVP_Flagevaluation_Degradation: def setup_ffe_evp_flagevaluation_degradation(self) -> None: config_id = "ffe-evp-degradation" + self.sync_flag_key = "evp-degradation-window-sync-flag" self.flag_key = "evp-degradation-flag" self.eval_count = EVP_FULL_TIER_PER_FLAG_CAP + EVP_DEGRADATION_OVERFLOW_EVALS - rc.tracer_rc_state.reset().set_config(f"{RC_PATH}/{config_id}/config", make_ufc_fixture(self.flag_key)).apply() + rc.tracer_rc_state.reset().set_config( + f"{RC_PATH}/{config_id}/config", make_multi_flag_fixture([self.sync_flag_key, self.flag_key]) + ).apply() - targeting_keys = [f"evp-degradation-user-{index}" for index in range(self.eval_count)] - self.responses = [ - evaluate_flag( - self.flag_key, - targeting_key=targeting_keys[0], - targeting_keys=targeting_keys, - attributes={}, - ) + targeting_keys = [str(index) for index in range(self.eval_count)] + self.batches = [ + targeting_keys[offset : offset + EVP_DEGRADATION_REQUEST_BATCH_SIZE] + for offset in range(0, len(targeting_keys), EVP_DEGRADATION_REQUEST_BATCH_SIZE) ] + self.sync_response = evaluate_flag( + self.sync_flag_key, targeting_key="evp-degradation-sync-user", attributes={} + ) + assert self.sync_response.status_code == 200, f"Window sync request failed: {self.sync_response.text}" + wait_for_evp_flagevaluation_event(self.sync_flag_key) + + with ThreadPoolExecutor(max_workers=EVP_DEGRADATION_REQUEST_WORKERS) as executor: + self.responses = list( + executor.map( + lambda batch: evaluate_flag( + self.flag_key, + targeting_key=batch[0], + targeting_keys=batch, + attributes={}, + ), + self.batches, + ) + ) + def test_ffe_evp_flagevaluation_degradation(self) -> None: + assert self.sync_response.status_code == 200, f"Window sync request failed: {self.sync_response.text}" for index, response in enumerate(self.responses): assert response.status_code == 200, f"Request {index + 1} failed: {response.text}" From 7b30ebb11c68561fbb40422b6a689358fe6dbba4 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Sat, 20 Jun 2026 02:17:29 -0400 Subject: [PATCH 5/9] Fix EVP flagevaluation test formatting --- tests/ffe/test_flag_eval_evp.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/ffe/test_flag_eval_evp.py b/tests/ffe/test_flag_eval_evp.py index 18ee63aa32e..328e9f0cb94 100644 --- a/tests/ffe/test_flag_eval_evp.py +++ b/tests/ffe/test_flag_eval_evp.py @@ -471,9 +471,7 @@ def setup_ffe_evp_flagevaluation_degradation(self) -> None: for offset in range(0, len(targeting_keys), EVP_DEGRADATION_REQUEST_BATCH_SIZE) ] - self.sync_response = evaluate_flag( - self.sync_flag_key, targeting_key="evp-degradation-sync-user", attributes={} - ) + self.sync_response = evaluate_flag(self.sync_flag_key, targeting_key="evp-degradation-sync-user", attributes={}) assert self.sync_response.status_code == 200, f"Window sync request failed: {self.sync_response.text}" wait_for_evp_flagevaluation_event(self.sync_flag_key) From 87bc61600fed03757f4487c7184a8b3ce2adf252 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 30 Jun 2026 18:29:51 -0400 Subject: [PATCH 6/9] Stabilize Ruby EVP flagevaluation system tests --- manifests/ruby.yml | 2 +- tests/ffe/test_flag_eval_evp.py | 48 ++++++++++++++++----------------- 2 files changed, 24 insertions(+), 26 deletions(-) diff --git a/manifests/ruby.yml b/manifests/ruby.yml index b76e0e1bbf3..fc90ec67e14 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -1901,7 +1901,7 @@ manifest: tests/ffe/test_flag_eval_evp.py: - weblog_declaration: "*": irrelevant - rails72: v2.36.0-dev + rails72: v2.37.0-dev tests/ffe/test_flag_eval_metrics.py: - weblog_declaration: "*": irrelevant diff --git a/tests/ffe/test_flag_eval_evp.py b/tests/ffe/test_flag_eval_evp.py index 24a7c62d4de..0d2ffba79b5 100644 --- a/tests/ffe/test_flag_eval_evp.py +++ b/tests/ffe/test_flag_eval_evp.py @@ -1,6 +1,7 @@ """Test server-side feature flag evaluation counts via EVP flagevaluation.""" import json +import time from concurrent.futures import ThreadPoolExecutor from typing import cast @@ -21,9 +22,9 @@ EVP_WAIT_TIMEOUT_SECONDS = 30 EVP_LOAD_WAIT_TIMEOUT_SECONDS = 60 EVP_FULL_TIER_PER_FLAG_CAP = 10_000 -EVP_DEGRADATION_OVERFLOW_EVALS = 2_000 -EVP_DEGRADATION_REQUEST_BATCH_SIZE = 1_000 -EVP_DEGRADATION_REQUEST_WORKERS = 12 +EVP_DEGRADATION_OVERFLOW_EVALS = 50 +EVP_DEGRADATION_BATCH_SIZE = 1_000 +EVP_DEGRADATION_BATCH_PAUSE_SECONDS = 0.2 def make_multi_flag_fixture(flag_keys: list[str]) -> JSON: @@ -458,38 +459,35 @@ class Test_FFE_EVP_Flagevaluation_Degradation: def setup_ffe_evp_flagevaluation_degradation(self) -> None: config_id = "ffe-evp-degradation" - self.sync_flag_key = "evp-degradation-window-sync-flag" self.flag_key = "evp-degradation-flag" self.eval_count = EVP_FULL_TIER_PER_FLAG_CAP + EVP_DEGRADATION_OVERFLOW_EVALS + anchor_flag_key = "evp-degradation-window-anchor" rc.tracer_rc_state.reset().set_config( - f"{RC_PATH}/{config_id}/config", make_multi_flag_fixture([self.sync_flag_key, self.flag_key]) + f"{RC_PATH}/{config_id}/config", make_multi_flag_fixture([self.flag_key, anchor_flag_key]) ).apply() - targeting_keys = [str(index) for index in range(self.eval_count)] - self.batches = [ - targeting_keys[offset : offset + EVP_DEGRADATION_REQUEST_BATCH_SIZE] - for offset in range(0, len(targeting_keys), EVP_DEGRADATION_REQUEST_BATCH_SIZE) - ] - - self.sync_response = evaluate_flag(self.sync_flag_key, targeting_key="evp-degradation-sync-user", attributes={}) - assert self.sync_response.status_code == 200, f"Window sync request failed: {self.sync_response.text}" - wait_for_evp_flagevaluation_event(self.sync_flag_key) + anchor_response = evaluate_flag(anchor_flag_key, targeting_key="evp-degradation-window-anchor") + assert anchor_response.status_code == 200, f"Window anchor request failed: {anchor_response.text}" + wait_for_evp_flagevaluation_event(anchor_flag_key) - with ThreadPoolExecutor(max_workers=EVP_DEGRADATION_REQUEST_WORKERS) as executor: - self.responses = list( - executor.map( - lambda batch: evaluate_flag( - self.flag_key, - targeting_key=batch[0], - targeting_keys=batch, - attributes={}, - ), - self.batches, + self.responses = [] + for batch_start in range(0, self.eval_count, EVP_DEGRADATION_BATCH_SIZE): + targeting_keys = [ + f"evp-degradation-user-{index}" + for index in range(batch_start, min(batch_start + EVP_DEGRADATION_BATCH_SIZE, self.eval_count)) + ] + self.responses.append( + evaluate_flag( + self.flag_key, + targeting_key=targeting_keys[0], + targeting_keys=targeting_keys, + attributes={}, ) ) + if batch_start + EVP_DEGRADATION_BATCH_SIZE < self.eval_count: + time.sleep(EVP_DEGRADATION_BATCH_PAUSE_SECONDS) def test_ffe_evp_flagevaluation_degradation(self) -> None: - assert self.sync_response.status_code == 200, f"Window sync request failed: {self.sync_response.text}" for index, response in enumerate(self.responses): assert response.status_code == 200, f"Request {index + 1} failed: {response.text}" From 7d42d25308c9d480dae9e669d460aec22def9737 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 30 Jun 2026 18:45:46 -0400 Subject: [PATCH 7/9] Explain Ruby EVP degradation batching --- tests/ffe/test_flag_eval_evp.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/ffe/test_flag_eval_evp.py b/tests/ffe/test_flag_eval_evp.py index 0d2ffba79b5..8fc1adeb673 100644 --- a/tests/ffe/test_flag_eval_evp.py +++ b/tests/ffe/test_flag_eval_evp.py @@ -470,6 +470,9 @@ def setup_ffe_evp_flagevaluation_degradation(self) -> None: assert anchor_response.status_code == 200, f"Window anchor request failed: {anchor_response.text}" wait_for_evp_flagevaluation_event(anchor_flag_key) + # Keep this just over the SDK's per-flag full-tier cap. The test is + # asserting degraded EVP aggregation shape, not Rails request throughput + # or async queue pressure from a parallel burst. self.responses = [] for batch_start in range(0, self.eval_count, EVP_DEGRADATION_BATCH_SIZE): targeting_keys = [ From e8608bc54f71208c70f6d6f12e0d39ff774b1ab7 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 00:01:29 -0400 Subject: [PATCH 8/9] Clarify Ruby EVP degradation workload math --- tests/ffe/test_flag_eval_evp.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/tests/ffe/test_flag_eval_evp.py b/tests/ffe/test_flag_eval_evp.py index 8fc1adeb673..51bc651e827 100644 --- a/tests/ffe/test_flag_eval_evp.py +++ b/tests/ffe/test_flag_eval_evp.py @@ -22,6 +22,12 @@ EVP_WAIT_TIMEOUT_SECONDS = 30 EVP_LOAD_WAIT_TIMEOUT_SECONDS = 60 EVP_FULL_TIER_PER_FLAG_CAP = 10_000 + +# The degradation contract is: the first 10,000 evaluations for one flag keep +# full targeting/context detail, and evaluations after that are aggregated into +# degraded buckets. 10,000 + 50 is enough to prove both sides of that boundary. +# Batching keeps the shared system test from depending on one oversized request +# and one async drain window to deliver every evaluation before CI times out. EVP_DEGRADATION_OVERFLOW_EVALS = 50 EVP_DEGRADATION_BATCH_SIZE = 1_000 EVP_DEGRADATION_BATCH_PAUSE_SECONDS = 0.2 @@ -470,9 +476,8 @@ def setup_ffe_evp_flagevaluation_degradation(self) -> None: assert anchor_response.status_code == 200, f"Window anchor request failed: {anchor_response.text}" wait_for_evp_flagevaluation_event(anchor_flag_key) - # Keep this just over the SDK's per-flag full-tier cap. The test is - # asserting degraded EVP aggregation shape, not Rails request throughput - # or async queue pressure from a parallel burst. + # The endpoint accepts multiple targeting keys, so these batches cross + # the cap while keeping each HTTP request and flush window bounded. self.responses = [] for batch_start in range(0, self.eval_count, EVP_DEGRADATION_BATCH_SIZE): targeting_keys = [ @@ -503,9 +508,16 @@ def test_ffe_evp_flagevaluation_degradation(self) -> None: assert_no_duplicate_visible_events(events) assert_total_evaluation_count(events, self.eval_count, self.flag_key) + degraded_count = 0 for event in degraded_events: assert_event_contract(event, self.flag_key) assert "context" not in event, f"degraded event must omit context: {event}" assert "targeting_key" not in event, f"degraded event must omit targeting_key: {event}" assert object_key(event.get("variant"), "variant") == "on" assert object_key(event.get("allocation"), "allocation") == "default-allocation" + degraded_count += cast(int, event["evaluation_count"]) + + assert degraded_count == EVP_DEGRADATION_OVERFLOW_EVALS, ( + f"expected {EVP_DEGRADATION_OVERFLOW_EVALS} degraded evaluations after " + f"the {EVP_FULL_TIER_PER_FLAG_CAP} full-detail cap, got {degraded_count}" + ) From 684efe67c3b72f65d46d9f7332b51d2ec1425ee4 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 1 Jul 2026 00:03:15 -0400 Subject: [PATCH 9/9] Fix flagevaluation degradation lint --- tests/ffe/test_flag_eval_evp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ffe/test_flag_eval_evp.py b/tests/ffe/test_flag_eval_evp.py index 51bc651e827..96444c4371f 100644 --- a/tests/ffe/test_flag_eval_evp.py +++ b/tests/ffe/test_flag_eval_evp.py @@ -515,7 +515,7 @@ def test_ffe_evp_flagevaluation_degradation(self) -> None: assert "targeting_key" not in event, f"degraded event must omit targeting_key: {event}" assert object_key(event.get("variant"), "variant") == "on" assert object_key(event.get("allocation"), "allocation") == "default-allocation" - degraded_count += cast(int, event["evaluation_count"]) + degraded_count += cast("int", event["evaluation_count"]) assert degraded_count == EVP_DEGRADATION_OVERFLOW_EVALS, ( f"expected {EVP_DEGRADATION_OVERFLOW_EVALS} degraded evaluations after "