From f8ff831ff1dc3f8b785d2c1377d1b78f0f45d059 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 12 Jun 2026 15:41:01 -0400 Subject: [PATCH 01/37] feat(02-03): FlagEvaluationWriter SDK-native EVP flagevaluation aggregator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Two-tier aggregation (full → degraded → drop-counted); no ultra-degraded tier - Canonical context key: sorted, type-tagged, length-delimited (NOT a hash) - Caps GLOBAL_CAP=131072 / PER_FLAG_CAP=10000 / DEGRADED_CAP=32768 - Context pruning: ≤256 fields / ≤256 chars per reviewer concern #1 - Non-blocking queue.Queue(4096) enqueue with observable drop counter - EVP POST to /evp_proxy/v2/api/v2/flagevaluations with X-Datadog-EVP-Subdomain - Unit tests: canonical key, two-tier overflow, prune, enqueue, payload shape --- .../openfeature/_flagevaluation_writer.py | 531 ++++++++++++++++++ .../openfeature/test_flagevaluation_writer.py | 406 +++++++++++++ 2 files changed, 937 insertions(+) create mode 100644 ddtrace/internal/openfeature/_flagevaluation_writer.py create mode 100644 tests/openfeature/test_flagevaluation_writer.py diff --git a/ddtrace/internal/openfeature/_flagevaluation_writer.py b/ddtrace/internal/openfeature/_flagevaluation_writer.py new file mode 100644 index 00000000000..db60d1046f2 --- /dev/null +++ b/ddtrace/internal/openfeature/_flagevaluation_writer.py @@ -0,0 +1,531 @@ +""" +FlagEvaluationWriter — SDK-native EVP `flagevaluation` writer for dd-trace-py. + +Implements the frozen FANOUT-CONTRACT two-tier aggregation design (full → degraded → +drop-counted). No libdatadog connection; uses the same PeriodicService + get_connection() +transport path as ExposureWriter in writer.py. + +Key design properties (port of Go PR #4886): +- Async, best-effort recording: finally_after hook does cheap capture + non-blocking + enqueue only. All flatten/prune/aggregate/flush work happens in the background worker. +- Two-tier aggregation (full → degraded → drop-counted). No ultra-degraded tier. +- Canonical context key: sorted, type-tagged, length-delimited — NOT a hash (reviewer + concern #3 3395004724). Distinct contexts always produce distinct keys. +- Context pruning: ≤256 fields, string values ≤256 chars (reviewer concern #1). +- Caps: GLOBAL_CAP=131_072 (full-tier), PER_FLAG_CAP=10_000 (per-flag full-tier), + DEGRADED_CAP=32_768 (degraded-tier). Overflow: drop-and-count (reviewer concern #8). +- Eval-time from metadata key "dd.eval.timestamp_ms"; fallback to enqueue-time. +- First/last evaluation: min/max under lock (reviewer concern #4). +- runtime_default_used: True when variant is None/absent (reviewer concern #5). +- Killswitch: DD_FLAGGING_EVALUATION_COUNTS_ENABLED (default on); gates EVP path only. +- Non-blocking enqueue: queue.Queue(QUEUE_SIZE); drops + counts on queue.Full. +""" + +import json +import queue +import struct +import threading +import time +import typing + +from ddtrace.internal.logger import get_logger +from ddtrace.internal.periodic import PeriodicService +from ddtrace.internal.settings._agent import config as agent_config +from ddtrace.internal.utils.http import get_connection + +from ddtrace import config as ddconfig + + +logger = get_logger(__name__) + +# EVP endpoint for flag evaluation events. +FLAGEVALUATIONS_ENDPOINT = "/evp_proxy/v2/api/v2/flagevaluations" +EVP_SUBDOMAIN_HEADER_NAME = "X-Datadog-EVP-Subdomain" +EVP_SUBDOMAIN_VALUE = "event-platform-intake" + +# Context pruning limits — mirror worker.ts MAX_EVALUATION_CONTEXT_FIELDS / MAX_FIELD_LENGTH. +MAX_CONTEXT_FIELDS = 256 +MAX_FIELD_LENGTH = 256 + +# Aggregation caps (sized for ≥2,500-flag scale target per FANOUT-CONTRACT). +GLOBAL_CAP = 131_072 # bounds full-tier buckets +PER_FLAG_CAP = 10_000 # bounds full-tier buckets per flag +DEGRADED_CAP = 32_768 # bounds degraded-tier buckets; overflow is drop-counted + +# Async hand-off queue size. +QUEUE_SIZE = 4_096 + +# Flush interval: dedicated 10 s timer, separate from ExposureWriter's 1 s interval. +DEFAULT_FLUSH_INTERVAL = 10.0 + +# Flag metadata key where the provider stamps the evaluation timestamp (ms). +EVAL_TIMESTAMP_METADATA_KEY = "dd.eval.timestamp_ms" + +# Metadata key for allocation_key (same as _flageval_metrics.py METADATA_ALLOCATION_KEY). +METADATA_ALLOCATION_KEY = "allocation_key" + +# Type-tag bytes for the canonical context key encoding (mirrors Go's ctxTag* constants). +_TAG_STR = b"s" +_TAG_BOOL = b"b" +_TAG_INT = b"i" +_TAG_FLOAT = b"f" +_TAG_OTHER = b"o" + + +# --------------------------------------------------------------------------- +# Canonical context key — type-tagged, length-delimited, sorted +# --------------------------------------------------------------------------- + +def _length_delimited(data: bytes) -> bytes: + """Prepend a fixed 8-byte big-endian length to data.""" + return struct.pack(">Q", len(data)) + data + + +def _encode_context_value(v: typing.Any) -> bytes: + """Encode a single context value with a type tag + length-delimited value.""" + if isinstance(v, bool): + # bool must be checked before int because bool is a subclass of int in Python. + tag = _TAG_BOOL + raw = b"true" if v else b"false" + elif isinstance(v, int): + tag = _TAG_INT + raw = str(v).encode() + elif isinstance(v, float): + tag = _TAG_FLOAT + raw = repr(v).encode() + elif isinstance(v, str): + tag = _TAG_STR + raw = v.encode("utf-8", errors="replace") + else: + tag = _TAG_OTHER + raw = str(v).encode("utf-8", errors="replace") + return tag + _length_delimited(raw) + + +def canonical_context_key(attrs: typing.Dict[str, typing.Any]) -> str: + """ + Build the EXACT, comparable canonical-context string key for a pruned context dict. + + Uses sorted(attrs.items()) so the encoding is deterministic regardless of Python + dict insertion order. Each entry is encoded as: + length_delimited(key_bytes) + type_tag_byte + length_delimited(value_bytes) + + Because the full encoding is used as the map key (not a hash), distinct contexts + ALWAYS produce distinct keys — no hash collisions, no misattribution (reviewer + concern #3 3395004724). + + Returns "" for empty/None attrs. + """ + if not attrs: + return "" + parts = [] + for k in sorted(attrs.keys()): + parts.append(_length_delimited(k.encode("utf-8", errors="replace"))) + parts.append(_encode_context_value(attrs[k])) + return b"".join(parts).decode("latin-1") # lossless binary → str for dict key + + +def flatten_and_prune_context(attrs: typing.Dict[str, typing.Any]) -> typing.Dict[str, typing.Any]: + """ + Flatten nested dicts (dot-notation) and apply 256-field / 256-char prune. + + Returns a new dict with at most MAX_CONTEXT_FIELDS entries, skipping string values + that exceed MAX_FIELD_LENGTH. Keys are chosen deterministically (sorted order) so + identical contexts always produce identical pruned maps. + + Returns {} when the input is empty or all values are pruned. + """ + if not attrs: + return {} + + flat: typing.Dict[str, typing.Any] = {} + _flatten_recursive("", attrs, flat) + if not flat: + return {} + + # Fast path: no pruning needed. + needs_prune = len(flat) > MAX_CONTEXT_FIELDS + if not needs_prune: + for v in flat.values(): + if isinstance(v, str) and len(v) > MAX_FIELD_LENGTH: + needs_prune = True + break + if not needs_prune: + return flat + + # Deterministic prune: sort keys, keep first MAX_CONTEXT_FIELDS non-oversized values. + out: typing.Dict[str, typing.Any] = {} + count = 0 + for k in sorted(flat.keys()): + if count >= MAX_CONTEXT_FIELDS: + break + v = flat[k] + if isinstance(v, str) and len(v) > MAX_FIELD_LENGTH: + continue + out[k] = v + count += 1 + return out + + +def _flatten_recursive(prefix: str, attrs: typing.Any, out: typing.Dict[str, typing.Any]) -> None: + """Recursively flatten nested dicts into dot-notation keys.""" + if not isinstance(attrs, dict): + if prefix: + out[prefix] = attrs + return + for k, v in attrs.items(): + full_key = f"{prefix}.{k}" if prefix else k + if isinstance(v, dict): + _flatten_recursive(full_key, v, out) + else: + out[full_key] = v + + +# --------------------------------------------------------------------------- +# Internal types +# --------------------------------------------------------------------------- + +class _Entry: + """Per-bucket aggregation state.""" + + __slots__ = ("count", "first_evaluation", "last_evaluation", "runtime_default", + "targeting_key", "context_attrs", "error_message") + + def __init__(self, now_ms: int, runtime_default: bool, targeting_key: str, + context_attrs: typing.Dict[str, typing.Any], error_message: str) -> None: + self.count: int = 1 + self.first_evaluation: int = now_ms + self.last_evaluation: int = now_ms + self.runtime_default: bool = runtime_default + # Full-tier only: + self.targeting_key: str = targeting_key + self.context_attrs: typing.Dict[str, typing.Any] = context_attrs + self.error_message: str = error_message + + def observe(self, now_ms: int) -> None: + """Update count and first/last bounds for a repeated evaluation.""" + self.count += 1 + if now_ms < self.first_evaluation: + self.first_evaluation = now_ms + if now_ms > self.last_evaluation: + self.last_evaluation = now_ms + + +class _EvalEvent(typing.NamedTuple): + """Minimal snapshot handed from finally_after to the background worker.""" + flag_key: str + variant: str # "" when absent (= runtime_default) + allocation_key: str + reason: str + targeting_key: str + attrs: typing.Dict[str, typing.Any] # shallow copy from evaluation context + runtime_default: bool + error_message: str + eval_time_ms: int + + +# --------------------------------------------------------------------------- +# FlagEvaluationWriter +# --------------------------------------------------------------------------- + +class FlagEvaluationWriter(PeriodicService): + """ + SDK-native EVP `flagevaluation` writer. + + Implements the two-tier aggregation design from the frozen FANOUT-CONTRACT: + - full-tier: keyed by (flag, variant, allocation, reason, targeting_key, canonical_context) + - degraded-tier: keyed by (flag, variant, allocation, reason) — same cardinality as OTel + - drop-counted: beyond degradedCap, increment _dropped_degraded_overflow + + The finally_after hook enqueues cheap _EvalEvent snapshots; the PeriodicService + background thread drains the queue, aggregates, and flushes via HTTP every 10 s. + """ + + def __init__(self, interval: float = DEFAULT_FLUSH_INTERVAL, timeout: float = 2.0) -> None: + super().__init__(interval=interval) + self._timeout = timeout + self._intake: str = agent_config.trace_agent_url + self._endpoint: str = FLAGEVALUATIONS_ENDPOINT + self._headers: typing.Dict[str, str] = { + "Content-Type": "application/json", + EVP_SUBDOMAIN_HEADER_NAME: EVP_SUBDOMAIN_VALUE, + } + + # Async hand-off queue: non-blocking, bounded (reviewer concern #7). + self._queue: queue.Queue = queue.Queue(maxsize=QUEUE_SIZE) + + # Aggregation maps (drained under _lock on each periodic() call). + self._lock = threading.Lock() + self._full: typing.Dict[tuple, _Entry] = {} + self._degraded: typing.Dict[tuple, _Entry] = {} + self._per_flag_count: typing.Dict[str, int] = {} # flag_key → full-tier bucket count + self._global_count: int = 0 + + # Observable drop counters. + self._dropped_queue: int = 0 # queue.Full drops (hook path) + self._dropped_degraded_overflow: int = 0 # degraded-cap overflow drops + + # ------------------------------------------------------------------ + # Public API used by FlagEvaluationHook + # ------------------------------------------------------------------ + + def enqueue(self, event: _EvalEvent) -> None: + """ + Non-blocking enqueue from the finally_after hook thread. + + On queue.Full, increments _dropped_queue (observable) and returns immediately — + never blocks the evaluation hot path. + """ + try: + self._queue.put_nowait(event) + except queue.Full: + with self._lock: + self._dropped_queue += 1 + logger.debug( + "FlagEvaluationWriter: queue full — dropped flag evaluation event for %s", + event.flag_key, + ) + + # ------------------------------------------------------------------ + # PeriodicService implementation + # ------------------------------------------------------------------ + + def periodic(self) -> None: + """ + Drain the queue, aggregate, and flush to the EVP proxy. + + Called periodically by the PeriodicService thread (every DEFAULT_FLUSH_INTERVAL). + Also callable directly in tests. + """ + # 1. Drain the queue into the aggregation maps. + self._drain_queue() + + # 2. Snapshot and reset under lock. + with self._lock: + dropped_queue = self._dropped_queue + dropped_degraded = self._dropped_degraded_overflow + full = self._full + degraded = self._degraded + if not full and not degraded: + if dropped_queue: + logger.warning( + "FlagEvaluationWriter: queue full — dropped %d evaluation(s) under backpressure", + dropped_queue, + ) + self._dropped_queue = 0 + if dropped_degraded: + logger.warning( + "FlagEvaluationWriter: degraded cap full — dropped %d evaluation(s)", + dropped_degraded, + ) + self._dropped_degraded_overflow = 0 + return + # Reset maps. + self._full = {} + self._degraded = {} + self._per_flag_count = {} + self._global_count = 0 + self._dropped_queue = 0 + self._dropped_degraded_overflow = 0 + + if dropped_queue: + logger.warning( + "FlagEvaluationWriter: queue full — dropped %d evaluation(s) under backpressure", + dropped_queue, + ) + if dropped_degraded: + logger.warning( + "FlagEvaluationWriter: degraded cap full — dropped %d evaluation(s)", + dropped_degraded, + ) + + # 3. Build payload. + now_ms = int(time.time() * 1000) + events = [] + + # Full-tier events: all optional fields present. + for key, entry in full.items(): + flag_key = key[0] + variant = key[1] + allocation_key = key[2] + ev = _base_event(flag_key, entry, now_ms) + if entry.runtime_default: + ev["runtime_default_used"] = True + if entry.targeting_key: + ev["targeting_key"] = entry.targeting_key + if variant: + ev["variant"] = {"key": variant} + if allocation_key: + ev["allocation"] = {"key": allocation_key} + if entry.error_message: + ev["error"] = {"message": entry.error_message} + if entry.context_attrs: + ev["context"] = {"evaluation": entry.context_attrs} + events.append(ev) + + # Degraded-tier events: no targeting_key, no context. + for key, entry in degraded.items(): + flag_key = key[0] + variant = key[1] + allocation_key = key[2] + ev = _base_event(flag_key, entry, now_ms) + if entry.runtime_default: + ev["runtime_default_used"] = True + if variant: + ev["variant"] = {"key": variant} + if allocation_key: + ev["allocation"] = {"key": allocation_key} + events.append(ev) + + if not events: + return + + # 4. Encode and POST. + try: + context: typing.Dict[str, str] = {} + if ddconfig.service: + context["service"] = ddconfig.service + if ddconfig.env: + context["env"] = ddconfig.env + if ddconfig.version: + context["version"] = ddconfig.version + + payload_obj: typing.Dict[str, typing.Any] = {"flagEvaluations": events} + if context: + payload_obj["context"] = context + + payload = json.dumps(payload_obj).encode("utf-8") + except (TypeError, ValueError): + logger.debug("FlagEvaluationWriter: failed to encode payload", exc_info=True) + return + + self._send_payload(payload, len(events)) + + def on_shutdown(self) -> None: + """Final flush on service shutdown.""" + self.periodic() + + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + + def _drain_queue(self) -> None: + """Drain all pending events from the queue and aggregate them.""" + while True: + try: + event = self._queue.get_nowait() + except queue.Empty: + break + self._aggregate(event) + + def _aggregate(self, event: _EvalEvent) -> None: + """ + Aggregate a single evaluation event into the two-tier maps. + + Implements: full-tier → degraded-tier → drop-counted cascade. + Context pruning and canonical key computation happen here (off the hot path). + """ + # Flatten + prune the context. + context_attrs = flatten_and_prune_context(event.attrs) if event.attrs else {} + + # Build the full-tier key tuple. + ctx_key = canonical_context_key(context_attrs) + full_key = ( + event.flag_key, + event.variant, + event.allocation_key, + event.reason, + event.targeting_key, + ctx_key, + ) + + with self._lock: + # Fast path: existing full-tier bucket. + if full_key in self._full: + self._full[full_key].observe(event.eval_time_ms) + return + + # Per-flag cap check. + per_flag = self._per_flag_count.get(event.flag_key, 0) + if per_flag >= PER_FLAG_CAP: + self._add_to_degraded(event) + return + + # Increment per-flag attempt count before checking globalCap (matches Go design). + self._per_flag_count[event.flag_key] = per_flag + 1 + + # Global cap check. + if self._global_count >= GLOBAL_CAP: + self._add_to_degraded(event) + return + + # New full-tier bucket. + self._full[full_key] = _Entry( + now_ms=event.eval_time_ms, + runtime_default=event.runtime_default, + targeting_key=event.targeting_key, + context_attrs=context_attrs, + error_message=event.error_message, + ) + self._global_count += 1 + + def _add_to_degraded(self, event: _EvalEvent) -> None: + """ + Add to the degraded-tier map (drops targeting_key + context). + Must be called with self._lock held. + """ + deg_key = (event.flag_key, event.variant, event.allocation_key, event.reason) + if deg_key in self._degraded: + self._degraded[deg_key].observe(event.eval_time_ms) + return + + if len(self._degraded) >= DEGRADED_CAP: + self._dropped_degraded_overflow += 1 + return + + self._degraded[deg_key] = _Entry( + now_ms=event.eval_time_ms, + runtime_default=event.runtime_default, + targeting_key="", + context_attrs={}, + error_message="", + ) + + def _send_payload(self, payload: bytes, num_events: int) -> None: + """POST the encoded payload to the EVP proxy.""" + conn = get_connection(self._intake, timeout=self._timeout) + try: + conn.request("POST", self._endpoint, payload, self._headers) + resp = conn.getresponse() + if resp.status >= 300: + logger.debug( + "FlagEvaluationWriter: failed to send %d events to %s, status=%d: %s", + num_events, self._intake, resp.status, resp.read(), + ) + else: + logger.debug( + "FlagEvaluationWriter: sent %d flag evaluation events to %s", + num_events, self._intake, + ) + except Exception: + logger.debug( + "FlagEvaluationWriter: error sending %d events to %s", + num_events, self._intake, exc_info=True, + ) + finally: + conn.close() + + +# --------------------------------------------------------------------------- +# Payload helpers +# --------------------------------------------------------------------------- + +def _base_event(flag_key: str, entry: "_Entry", now_ms: int) -> typing.Dict[str, typing.Any]: + """Build the required-fields-only event dict for a single aggregation entry.""" + return { + "timestamp": now_ms, + "flag": {"key": flag_key}, + "first_evaluation": entry.first_evaluation, + "last_evaluation": entry.last_evaluation, + "evaluation_count": entry.count, + } diff --git a/tests/openfeature/test_flagevaluation_writer.py b/tests/openfeature/test_flagevaluation_writer.py new file mode 100644 index 00000000000..39c4b74cf0d --- /dev/null +++ b/tests/openfeature/test_flagevaluation_writer.py @@ -0,0 +1,406 @@ +""" +Unit tests for FlagEvaluationWriter — two-tier aggregation, canonical key, EVP transport. + +Tests validate the FANOUT-CONTRACT spec: +- canonical_context_key: sorted, type-tagged, length-delimited (NOT a hash) +- Two-tier aggregation (full → degraded → drop-counted) +- Caps GLOBAL_CAP=131072 / PER_FLAG_CAP=10000 / DEGRADED_CAP=32768 +- 256-field / 256-char context pruning +- runtime_default_used from absent/None variant +- Non-blocking enqueue with drop-and-count on queue.Full +- EVP POST to /evp_proxy/v2/api/v2/flagevaluations with correct headers +""" + +import json +import queue +import time +from unittest import mock + +import pytest + +from ddtrace.internal.openfeature._flagevaluation_writer import ( + DEGRADED_CAP, + EVAL_TIMESTAMP_METADATA_KEY, + FLAGEVALUATIONS_ENDPOINT, + GLOBAL_CAP, + MAX_CONTEXT_FIELDS, + MAX_FIELD_LENGTH, + PER_FLAG_CAP, + QUEUE_SIZE, + EVP_SUBDOMAIN_HEADER_NAME, + EVP_SUBDOMAIN_VALUE, + FlagEvaluationWriter, + _EvalEvent, + canonical_context_key, + flatten_and_prune_context, +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _make_event( + flag_key: str = "my-flag", + variant: str = "on", + allocation_key: str = "alloc-1", + reason: str = "TARGETING_MATCH", + targeting_key: str = "user-1", + attrs: dict = None, + runtime_default: bool = False, + error_message: str = "", + eval_time_ms: int = None, +) -> _EvalEvent: + if eval_time_ms is None: + eval_time_ms = int(time.time() * 1000) + return _EvalEvent( + flag_key=flag_key, + variant=variant, + allocation_key=allocation_key, + reason=reason, + targeting_key=targeting_key, + attrs=attrs or {}, + runtime_default=runtime_default, + error_message=error_message, + eval_time_ms=eval_time_ms, + ) + + +@pytest.fixture +def writer(): + """Create a FlagEvaluationWriter that is NOT started (no background thread).""" + return FlagEvaluationWriter(interval=10.0) + + +# --------------------------------------------------------------------------- +# canonical_context_key tests +# --------------------------------------------------------------------------- + +class TestCanonicalContextKey: + def test_empty_attrs_returns_empty_string(self): + assert canonical_context_key({}) == "" + assert canonical_context_key(None) == "" + + def test_same_dict_same_key(self): + attrs = {"user": "alice", "tier": "premium"} + assert canonical_context_key(attrs) == canonical_context_key(attrs) + + def test_different_insertion_order_same_key(self): + """Dict order must not affect the key (sorted).""" + a = {"b": "2", "a": "1"} + b = {"a": "1", "b": "2"} + assert canonical_context_key(a) == canonical_context_key(b) + + def test_int_vs_string_distinct_keys(self): + """int 1 vs string '1' must produce different keys (type-tagged, reviewer concern #3).""" + k_int = canonical_context_key({"x": 1}) + k_str = canonical_context_key({"x": "1"}) + assert k_int != k_str, "int 1 and str '1' must not alias into the same bucket" + + def test_bool_vs_int_distinct_keys(self): + k_bool = canonical_context_key({"x": True}) + k_int = canonical_context_key({"x": 1}) + assert k_bool != k_int + + def test_float_vs_int_distinct_keys(self): + k_float = canonical_context_key({"x": 1.0}) + k_int = canonical_context_key({"x": 1}) + assert k_float != k_int + + def test_value_with_equals_or_newline_no_boundary_confusion(self): + """'=' and '\n' in values must not fake a field boundary (length-prefix protocol).""" + k_with = canonical_context_key({"a": "foo=bar\nbaz"}) + k_without = canonical_context_key({"a": "foo", "bar\nbaz": ""}) + assert k_with != k_without + + def test_no_hashlib_or_md5_used(self): + """Verify no hash function is used by inspecting the module source.""" + import ddtrace.internal.openfeature._flagevaluation_writer as mod_src + import inspect + src = inspect.getsource(mod_src) + assert "hashlib" not in src, "hashlib must not appear in the writer" + assert "md5" not in src, "md5 must not appear in the writer" + + def test_returns_string_not_bytes(self): + k = canonical_context_key({"x": "y"}) + assert isinstance(k, str) + + +# --------------------------------------------------------------------------- +# flatten_and_prune_context tests +# --------------------------------------------------------------------------- + +class TestFlattenAndPruneContext: + def test_empty_returns_empty(self): + assert flatten_and_prune_context({}) == {} + + def test_flat_attrs_passthrough(self): + attrs = {"a": "1", "b": 2} + result = flatten_and_prune_context(attrs) + assert result == attrs + + def test_nested_attrs_flattened(self): + attrs = {"user": {"id": "u1", "tier": "free"}} + result = flatten_and_prune_context(attrs) + assert "user.id" in result + assert "user.tier" in result + assert result["user.id"] == "u1" + + def test_prune_beyond_256_fields(self): + attrs = {str(i): str(i) for i in range(300)} + result = flatten_and_prune_context(attrs) + assert len(result) <= MAX_CONTEXT_FIELDS + + def test_prune_skips_oversized_string_values(self): + long_val = "x" * (MAX_FIELD_LENGTH + 1) + attrs = {"short": "ok", "long": long_val} + result = flatten_and_prune_context(attrs) + assert "short" in result + assert "long" not in result + + def test_prune_deterministic_sorted_order(self): + """Pruned result must be reproducible across repeated calls.""" + attrs = {str(i): str(i) for i in range(300)} + r1 = flatten_and_prune_context(attrs) + r2 = flatten_and_prune_context(attrs) + assert r1 == r2 + + def test_context_with_256_fields_not_pruned(self): + attrs = {str(i): str(i) for i in range(256)} + result = flatten_and_prune_context(attrs) + assert len(result) == 256 + + +# --------------------------------------------------------------------------- +# Aggregation tests (full → degraded → drop-counted) +# --------------------------------------------------------------------------- + +class TestAggregation: + def test_two_identical_evals_aggregate_into_one_bucket_count_2(self, writer): + t0 = int(time.time() * 1000) + t1 = t0 + 100 + e1 = _make_event(eval_time_ms=t0) + e2 = _make_event(eval_time_ms=t1) + writer._aggregate(e1) + writer._aggregate(e2) + + assert len(writer._full) == 1 + entry = list(writer._full.values())[0] + assert entry.count == 2 + assert entry.first_evaluation <= entry.last_evaluation + assert entry.first_evaluation == t0 + assert entry.last_evaluation == t1 + + def test_two_evals_differing_context_value_type_produce_two_buckets(self, writer): + """int 1 vs str '1' in context → two distinct full-tier buckets (reviewer concern #3).""" + e_int = _make_event(attrs={"x": 1}) + e_str = _make_event(attrs={"x": "1"}) + writer._aggregate(e_int) + writer._aggregate(e_str) + assert len(writer._full) == 2 + + def test_full_tier_overflow_routes_to_degraded(self, writer): + """Overflow past globalCap routes to the degraded tier.""" + writer._global_count = GLOBAL_CAP # simulate full global cap + + # Inject per-flag count below PER_FLAG_CAP so only the global cap triggers. + writer._per_flag_count["flag-x"] = 0 + + e = _make_event(flag_key="flag-x", attrs={"unique": "ctx"}) + writer._aggregate(e) + + assert len(writer._full) == 0 + assert len(writer._degraded) == 1 + + def test_degraded_overflow_increments_dropped_counter(self, writer): + """Beyond degradedCap, increment _dropped_degraded_overflow (reviewer concern #8).""" + # Fill the degraded map to the cap. + for i in range(DEGRADED_CAP): + key = (f"flag-{i}", "on", "alloc", "SPLIT") + from ddtrace.internal.openfeature._flagevaluation_writer import _Entry + writer._degraded[key] = _Entry(1000, False, "", {}, "") + + with writer._lock: + writer._add_to_degraded(_make_event(flag_key="overflow-flag")) + + assert writer._dropped_degraded_overflow == 1 + + def test_per_flag_cap_routes_to_degraded(self, writer): + """Per-flag cap exceeded → route to degraded even when globalCap has room.""" + writer._per_flag_count["my-flag"] = PER_FLAG_CAP # flag is at cap + e = _make_event(flag_key="my-flag", attrs={"ctx": "x"}) + writer._aggregate(e) + + assert len(writer._degraded) == 1 + assert len(writer._full) == 0 + + def test_runtime_default_when_variant_is_absent(self, writer): + """Absent/empty variant → runtime_default_used True (reviewer concern #5).""" + e = _make_event(variant="", runtime_default=True) + writer._aggregate(e) + + assert len(writer._full) == 1 + entry = list(writer._full.values())[0] + assert entry.runtime_default is True + + def test_degraded_event_omits_targeting_key_and_context(self, writer): + """Degraded tier strips targeting_key + context (schema omitempty, reviewer concern #2).""" + with writer._lock: + writer._add_to_degraded( + _make_event(targeting_key="some-key", attrs={"k": "v"}) + ) + + entry = list(writer._degraded.values())[0] + assert entry.targeting_key == "" + assert entry.context_attrs == {} + + +# --------------------------------------------------------------------------- +# Enqueue non-blocking tests +# --------------------------------------------------------------------------- + +class TestEnqueue: + def test_enqueue_non_blocking_on_full_queue(self, writer): + """When queue is full, enqueue must NOT block and must increment _dropped_queue.""" + # Fill the queue. + for i in range(QUEUE_SIZE): + writer._queue.put_nowait(_make_event(flag_key=f"f{i}")) + + # This should not raise and should not block. + writer.enqueue(_make_event(flag_key="overflow")) + assert writer._dropped_queue == 1 + + def test_enqueue_succeeds_when_queue_has_capacity(self, writer): + writer.enqueue(_make_event()) + assert writer._queue.qsize() == 1 + + +# --------------------------------------------------------------------------- +# Periodic flush + EVP POST tests +# --------------------------------------------------------------------------- + +class TestPeriodicFlush: + def test_periodic_drains_queue_and_builds_payload(self, writer): + writer.enqueue(_make_event()) + + with mock.patch.object(writer, "_send_payload") as mock_send: + writer.periodic() + + mock_send.assert_called_once() + payload_bytes, num_events = mock_send.call_args[0] + decoded = json.loads(payload_bytes) + assert "flagEvaluations" in decoded + assert len(decoded["flagEvaluations"]) == 1 + ev = decoded["flagEvaluations"][0] + assert ev["flag"]["key"] == "my-flag" + assert "first_evaluation" in ev + assert "last_evaluation" in ev + assert "evaluation_count" in ev + assert ev["evaluation_count"] == 1 + + def test_periodic_no_send_when_empty(self, writer): + with mock.patch.object(writer, "_send_payload") as mock_send: + writer.periodic() + mock_send.assert_not_called() + + def test_periodic_resets_maps_after_flush(self, writer): + writer.enqueue(_make_event()) + with mock.patch.object(writer, "_send_payload"): + writer.periodic() + assert writer._full == {} + assert writer._degraded == {} + assert writer._global_count == 0 + + @mock.patch("ddtrace.internal.openfeature._flagevaluation_writer.get_connection") + def test_post_to_correct_endpoint_with_evp_header(self, mock_get_conn, writer): + """Payload goes to /evp_proxy/v2/api/v2/flagevaluations with EVP subdomain header.""" + mock_conn = mock.Mock() + mock_resp = mock.Mock() + mock_resp.status = 200 + mock_resp.read.return_value = b"OK" + mock_conn.getresponse.return_value = mock_resp + mock_get_conn.return_value = mock_conn + + writer.enqueue(_make_event()) + writer.periodic() + + mock_get_conn.assert_called_once() + mock_conn.request.assert_called_once() + call_args = mock_conn.request.call_args + method, endpoint, _payload, headers = call_args[0] + assert method == "POST" + assert endpoint == FLAGEVALUATIONS_ENDPOINT + assert headers[EVP_SUBDOMAIN_HEADER_NAME] == EVP_SUBDOMAIN_VALUE + assert "Content-Type" in headers + + def test_two_evals_same_dims_aggregate_count_2(self, writer): + t0 = int(time.time() * 1000) + writer.enqueue(_make_event(eval_time_ms=t0)) + writer.enqueue(_make_event(eval_time_ms=t0 + 50)) + + with mock.patch.object(writer, "_send_payload") as mock_send: + writer.periodic() + + payload_bytes = mock_send.call_args[0][0] + decoded = json.loads(payload_bytes) + evals = decoded["flagEvaluations"] + assert len(evals) == 1 + assert evals[0]["evaluation_count"] == 2 + assert evals[0]["first_evaluation"] <= evals[0]["last_evaluation"] + + def test_context_pruning_above_256_fields(self, writer): + """Context with >256 fields is pruned before keying (reviewer concern #1).""" + attrs = {str(i): str(i) for i in range(300)} + e = _make_event(attrs=attrs) + writer.enqueue(e) + + with mock.patch.object(writer, "_send_payload") as mock_send: + writer.periodic() + + payload_bytes = mock_send.call_args[0][0] + decoded = json.loads(payload_bytes) + ev = decoded["flagEvaluations"][0] + # The context.evaluation map should exist but have ≤256 fields. + assert "context" in ev + assert len(ev["context"]["evaluation"]) <= MAX_CONTEXT_FIELDS + + def test_context_value_exceeding_256_chars_pruned(self, writer): + """Context values >256 chars are skipped (reviewer concern #1).""" + long_val = "x" * (MAX_FIELD_LENGTH + 10) + attrs = {"short": "ok", "long_field": long_val} + e = _make_event(attrs=attrs) + writer.enqueue(e) + + with mock.patch.object(writer, "_send_payload") as mock_send: + writer.periodic() + + payload_bytes = mock_send.call_args[0][0] + decoded = json.loads(payload_bytes) + ev = decoded["flagEvaluations"][0] + ctx_eval = ev.get("context", {}).get("evaluation", {}) + assert "short" in ctx_eval + assert "long_field" not in ctx_eval + + def test_degraded_event_has_no_context_or_targeting_key(self, writer): + """Degraded-tier events must not include targeting_key or context fields.""" + # Force to degraded by saturating per-flag cap. + writer._per_flag_count["my-flag"] = PER_FLAG_CAP + + e = _make_event(targeting_key="tgt-user", attrs={"k": "v"}) + writer.enqueue(e) + + with mock.patch.object(writer, "_send_payload") as mock_send: + writer.periodic() + + payload_bytes = mock_send.call_args[0][0] + decoded = json.loads(payload_bytes) + ev = decoded["flagEvaluations"][0] + assert "targeting_key" not in ev + assert "context" not in ev + + def test_writer_endpoint_constant(self): + assert FLAGEVALUATIONS_ENDPOINT == "/evp_proxy/v2/api/v2/flagevaluations" + + def test_class_exists_and_inherits_periodic_service(self): + from ddtrace.internal.periodic import PeriodicService + assert issubclass(FlagEvaluationWriter, PeriodicService) From 7524c5de2f25005ac4fc4fb8a13256fef412528b Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 12 Jun 2026 15:41:15 -0400 Subject: [PATCH 02/37] feat(02-03): FlagEvaluationHook + provider wiring + DD_FLAGGING_EVALUATION_COUNTS_ENABLED killswitch - FlagEvaluationHook(Hook).finally_after: cheap capture + non-blocking enqueue only - Eval-time from metadata["dd.eval.timestamp_ms"]; fallback to hook-fire time - runtime_default_used from absent/None variant (reviewer concern #5) - Provider stamps dd.eval.timestamp_ms at eval boundary (_resolve_details) - Killswitch DD_FLAGGING_EVALUATION_COUNTS_ENABLED (default on) gates EVP path only - OTel FlagEvalHook (_flageval_metrics.py) byte-for-byte unchanged (PRES-01) - Tests: hook lifecycle, killswitch gating, OTel non-regression, metadata extraction --- .../openfeature/_flagevaluation_hook.py | 131 ++++++++++ ddtrace/internal/openfeature/_provider.py | 56 ++++- tests/openfeature/test_flag_eval_metrics.py | 6 +- tests/openfeature/test_flagevaluation_hook.py | 230 ++++++++++++++++++ 4 files changed, 420 insertions(+), 3 deletions(-) create mode 100644 ddtrace/internal/openfeature/_flagevaluation_hook.py create mode 100644 tests/openfeature/test_flagevaluation_hook.py diff --git a/ddtrace/internal/openfeature/_flagevaluation_hook.py b/ddtrace/internal/openfeature/_flagevaluation_hook.py new file mode 100644 index 00000000000..065dbb23536 --- /dev/null +++ b/ddtrace/internal/openfeature/_flagevaluation_hook.py @@ -0,0 +1,131 @@ +""" +FlagEvaluationHook — OpenFeature `finally_after` hook for EVP flagevaluation emission. + +Implements the frozen FANOUT-CONTRACT hook design: +- Cheap capture only in finally_after (no aggregation, no serialization, no I/O). +- Non-blocking enqueue to FlagEvaluationWriter. +- Covers success, error, and default eval paths (reviewer concern #7 3385309423). +- Does NOT replace or modify the OTel FlagEvalHook in _flageval_metrics.py (PRES-01). +""" + +import time +import typing + +from openfeature.flag_evaluation import FlagEvaluationDetails +from openfeature.hook import Hook +from openfeature.hook import HookContext +from openfeature.hook import HookHints + +from ddtrace.internal.logger import get_logger +from ddtrace.internal.openfeature._flagevaluation_writer import ( + EVAL_TIMESTAMP_METADATA_KEY, + METADATA_ALLOCATION_KEY, + FlagEvaluationWriter, + _EvalEvent, +) + + +logger = get_logger(__name__) + + +class FlagEvaluationHook(Hook): + """ + OpenFeature Hook that enqueues cheap evaluation snapshots for EVP aggregation. + + Implements `finally_after` (covers success/error/default — reviewer concern #7). + Does NO aggregation, serialization, or I/O on the hook thread. All heavy work + is deferred to FlagEvaluationWriter's background periodic worker. + """ + + def __init__(self, writer: FlagEvaluationWriter) -> None: + self._writer = writer + + def finally_after( + self, + hook_context: HookContext, + details: FlagEvaluationDetails[typing.Any], + hints: HookHints, + ) -> None: + """ + Cheap capture + non-blocking enqueue. + + Extracts only scalar fields (no allocation, no serialization) and hands + a _EvalEvent snapshot to FlagEvaluationWriter.enqueue() which is + non-blocking (queue.Queue.put_nowait — drops on queue.Full). + + Eval-time: uses details.flag_metadata["dd.eval.timestamp_ms"] when present + (stamped by the provider at eval entry); falls back to hook-fire time. + + Runtime-default: True when details.value is None (absent variant — reviewer + concern #5 3395344504). + + Attrs: shallow copy of the evaluation context attributes dict so the hook + returns immediately and the worker can safely iterate attrs off-path. + """ + try: + flag_key: str = hook_context.flag_key or "" + + # Extract allocation_key from flag_metadata (same key as METADATA_ALLOCATION_KEY). + metadata: dict = details.flag_metadata or {} + allocation_key: str = "" + ak = metadata.get(METADATA_ALLOCATION_KEY) + if isinstance(ak, str) and ak: + allocation_key = ak + + # Eval-time from provider-stamped metadata; fall back to hook-fire time. + eval_time_ms_raw = metadata.get(EVAL_TIMESTAMP_METADATA_KEY) + if isinstance(eval_time_ms_raw, (int, float)) and eval_time_ms_raw > 0: + eval_time_ms = int(eval_time_ms_raw) + else: + eval_time_ms = int(time.time() * 1000) + + # Variant: None/absent signals runtime_default (reviewer concern #5). + variant = details.variant or "" + runtime_default = details.variant is None + + # Reason: normalise to upper-case string. + if details.reason is not None: + reason_raw = ( + details.reason.value + if hasattr(details.reason, "value") + else str(details.reason) + ) + reason = str(reason_raw).upper() + else: + reason = "UNKNOWN" + + # Targeting key and attributes from the evaluation context. + eval_ctx = hook_context.evaluation_context + if eval_ctx is not None: + targeting_key = eval_ctx.targeting_key or "" + # Shallow copy so we don't hold a reference to the caller's live dict. + attrs: typing.Dict[str, typing.Any] = dict(eval_ctx.attributes or {}) + else: + targeting_key = "" + attrs = {} + + # Error message (best-effort; absent on success paths). + error_message = "" + if details.error_message: + error_message = str(details.error_message) + + event = _EvalEvent( + flag_key=flag_key, + variant=variant, + allocation_key=allocation_key, + reason=reason, + targeting_key=targeting_key, + attrs=attrs, + runtime_default=runtime_default, + error_message=error_message, + eval_time_ms=eval_time_ms, + ) + + self._writer.enqueue(event) + + except Exception: + # Never propagate hook exceptions — best-effort telemetry. + logger.debug( + "FlagEvaluationHook.finally_after: failed to enqueue eval snapshot", + exc_info=True, + ) diff --git a/ddtrace/internal/openfeature/_provider.py b/ddtrace/internal/openfeature/_provider.py index 258382d04d7..dd15163ed01 100644 --- a/ddtrace/internal/openfeature/_provider.py +++ b/ddtrace/internal/openfeature/_provider.py @@ -8,7 +8,9 @@ from collections import OrderedDict from collections.abc import MutableMapping from importlib.metadata import version +import os import threading +import time import typing from openfeature.evaluation_context import EvaluationContext @@ -26,6 +28,11 @@ from ddtrace.internal.openfeature._flageval_metrics import METADATA_ALLOCATION_KEY from ddtrace.internal.openfeature._flageval_metrics import FlagEvalHook from ddtrace.internal.openfeature._flageval_metrics import FlagEvalMetrics +from ddtrace.internal.openfeature._flagevaluation_hook import FlagEvaluationHook +from ddtrace.internal.openfeature._flagevaluation_writer import ( + EVAL_TIMESTAMP_METADATA_KEY, + FlagEvaluationWriter, +) from ddtrace.internal.openfeature._native import VariationType from ddtrace.internal.openfeature._native import resolve_flag from ddtrace.internal.openfeature.writer import get_exposure_writer @@ -129,6 +136,18 @@ def __init__(self, *args: typing.Any, initialization_timeout: typing.Optional[fl self._flag_eval_metrics = FlagEvalMetrics() self._flag_eval_hook = FlagEvalHook(self._flag_eval_metrics) + # EVP flagevaluation writer + hook — gated by DD_FLAGGING_EVALUATION_COUNTS_ENABLED + # (default on). Gates ONLY the EVP path; the OTel path above is always registered + # when the provider is enabled (PRES-01 non-regression). + # AIDEV-NOTE: killswitch env var checked at init time so that tests can override with + # os.environ and create a fresh DataDogProvider to verify gating behavior. + self._flagevaluation_writer: typing.Optional[FlagEvaluationWriter] = None + self._flagevaluation_hook: typing.Optional[FlagEvaluationHook] = None + evp_counts_enabled = os.environ.get("DD_FLAGGING_EVALUATION_COUNTS_ENABLED", "true").lower() + if self._enabled and evp_counts_enabled != "false": + self._flagevaluation_writer = FlagEvaluationWriter() + self._flagevaluation_hook = FlagEvaluationHook(self._flagevaluation_writer) + def get_metadata(self) -> Metadata: """Returns provider metadata.""" return self._metadata @@ -139,10 +158,19 @@ def get_provider_hooks(self) -> list[typing.Any]: The flag evaluation hook is registered here to track metrics for every flag evaluation via the finally_after hook stage. + + Hook ordering: + 1. OTel FlagEvalHook (_flageval_metrics.py) — always registered when provider is + enabled; emits feature_flag.evaluations OTel counter (PRES-01 preservation). + 2. FlagEvaluationHook (_flagevaluation_hook.py) — registered only when + DD_FLAGGING_EVALUATION_COUNTS_ENABLED != "false"; enqueues cheap snapshots + to FlagEvaluationWriter for EVP flagevaluation emission. """ hooks: list[typing.Any] = [] if self._flag_eval_hook is not None: hooks.append(self._flag_eval_hook) + if self._flagevaluation_hook is not None: + hooks.append(self._flagevaluation_hook) return hooks def initialize(self, evaluation_context: EvaluationContext) -> None: @@ -173,6 +201,14 @@ def initialize(self, evaluation_context: EvaluationContext) -> None: except ServiceStatusError: logger.debug("Exposure writer is already running", exc_info=True) + # Start the EVP flagevaluation writer (if enabled via killswitch). + if self._flagevaluation_writer is not None: + try: + self._flagevaluation_writer.start() + logger.debug("FlagEvaluationWriter started") + except ServiceStatusError: + logger.debug("FlagEvaluationWriter is already running", exc_info=True) + # Fast path: config already available (RC delivered before set_provider) config = _get_ffe_config() if config is not None: @@ -211,6 +247,16 @@ def shutdown(self) -> None: except ServiceStatusError: logger.debug("Exposure writer has already stopped", exc_info=True) + # Stop the EVP flagevaluation writer (if it was started). + if self._flagevaluation_writer is not None: + try: + self._flagevaluation_writer.stop() + logger.debug("FlagEvaluationWriter stopped") + except ServiceStatusError: + logger.debug("FlagEvaluationWriter has already stopped", exc_info=True) + self._flagevaluation_writer = None + self._flagevaluation_hook = None + # Shutdown flag evaluation metrics if self._flag_eval_metrics is not None: self._flag_eval_metrics.shutdown() @@ -354,10 +400,18 @@ def _resolve_details( evaluation_context=evaluation_context, ) - # Build flag_metadata with allocation_key if present + # Build flag_metadata with allocation_key and eval timestamp. + # The timestamp (milliseconds since epoch) is stamped here — at the provider + # resolution boundary — so the finally_after hook receives the most-accurate + # eval-time rather than the (later) hook-fire time. The hook falls back to + # hook-fire time when the metadata key is absent. + # AIDEV-NOTE: This mirrors the Go reference design (metadataEvalTimeKey in + # flagevaluation.go). Python's time.time() * 1000 is sufficient; the native + # PyO3 bridge does not surface a mutable metadata map, so we stamp it here. flag_metadata: dict[str, typing.Any] = {} if details.allocation_key: flag_metadata[METADATA_ALLOCATION_KEY] = details.allocation_key + flag_metadata[EVAL_TIMESTAMP_METADATA_KEY] = int(time.time() * 1000) # Check if variant is None/empty to determine if we should use default value. # For JSON flags, value can be null which is valid, so we check variant instead. diff --git a/tests/openfeature/test_flag_eval_metrics.py b/tests/openfeature/test_flag_eval_metrics.py index 3e656f81283..5e0d2184826 100644 --- a/tests/openfeature/test_flag_eval_metrics.py +++ b/tests/openfeature/test_flag_eval_metrics.py @@ -360,9 +360,11 @@ def test_provider_has_flag_eval_hook(self, provider): assert provider._flag_eval_metrics is not None def test_get_provider_hooks_returns_flag_eval_hook(self, provider): - """get_provider_hooks should return the flag eval hook.""" + """get_provider_hooks should return the OTel flag eval hook (and optionally the EVP hook).""" hooks = provider.get_provider_hooks() - assert len(hooks) == 1 + # The OTel FlagEvalHook is always first when the provider is enabled. + # The EVP FlagEvaluationHook is also registered by default (DD_FLAGGING_EVALUATION_COUNTS_ENABLED=true). + assert len(hooks) >= 1 assert hooks[0] is provider._flag_eval_hook def test_provider_disabled_has_no_hooks(self): diff --git a/tests/openfeature/test_flagevaluation_hook.py b/tests/openfeature/test_flagevaluation_hook.py new file mode 100644 index 00000000000..5f922d7441f --- /dev/null +++ b/tests/openfeature/test_flagevaluation_hook.py @@ -0,0 +1,230 @@ +""" +Tests for FlagEvaluationHook — finally_after cheap capture + non-blocking enqueue. + +Validates FANOUT-CONTRACT hook design: +- finally_after does cheap capture only (no aggregation, no I/O) +- variant=None → runtime_default_used True (reviewer concern #5) +- eval_time_ms from metadata["dd.eval.timestamp_ms"] when present; fallback to hook-fire time +- DD_FLAGGING_EVALUATION_COUNTS_ENABLED killswitch gates EVP path only (PRES-01) +""" + +import os +import time +import typing +from unittest import mock + +import pytest + +from openfeature.evaluation_context import EvaluationContext +from openfeature.flag_evaluation import FlagEvaluationDetails +from openfeature.flag_evaluation import FlagType +from openfeature.flag_evaluation import Reason +from openfeature.hook import HookContext + + +def _make_hook_context( + flag_key: str = "my-flag", + targeting_key: str = "user-1", + attrs: dict = None, +) -> HookContext: + ctx = EvaluationContext(targeting_key=targeting_key, attributes=attrs or {}) + return HookContext( + flag_key=flag_key, + flag_type=FlagType.BOOLEAN, + default_value=False, + evaluation_context=ctx, + ) + + +def _make_details( + flag_key: str = "my-flag", + value: typing.Any = True, + variant: typing.Optional[str] = "on", + reason: typing.Optional[Reason] = Reason.TARGETING_MATCH, + flag_metadata: dict = None, + error_message: str = None, +) -> FlagEvaluationDetails: + return FlagEvaluationDetails( + flag_key=flag_key, + value=value, + variant=variant, + reason=reason, + flag_metadata=flag_metadata or {}, + error_message=error_message, + ) + + +@pytest.fixture +def writer(): + from ddtrace.internal.openfeature._flagevaluation_writer import FlagEvaluationWriter + return mock.MagicMock(spec=FlagEvaluationWriter) + + +@pytest.fixture +def hook(writer): + from ddtrace.internal.openfeature._flagevaluation_hook import FlagEvaluationHook + return FlagEvaluationHook(writer=writer) + + +class TestFlagEvaluationHook: + + def test_finally_after_calls_writer_enqueue_once(self, hook, writer): + """finally_after must call writer.enqueue exactly once per evaluation.""" + hc = _make_hook_context() + details = _make_details() + hook.finally_after(hc, details, {}) + writer.enqueue.assert_called_once() + + def test_finally_after_enqueues_correct_flag_key(self, hook, writer): + hc = _make_hook_context(flag_key="test-flag") + details = _make_details(flag_key="test-flag") + hook.finally_after(hc, details, {}) + event = writer.enqueue.call_args[0][0] + assert event.flag_key == "test-flag" + + def test_finally_after_enqueues_correct_variant(self, hook, writer): + hc = _make_hook_context() + details = _make_details(variant="control") + hook.finally_after(hc, details, {}) + event = writer.enqueue.call_args[0][0] + assert event.variant == "control" + + def test_finally_after_none_variant_sets_runtime_default(self, hook, writer): + """None variant → runtime_default=True (reviewer concern #5 3395344504).""" + hc = _make_hook_context() + details = _make_details(variant=None) + hook.finally_after(hc, details, {}) + event = writer.enqueue.call_args[0][0] + assert event.runtime_default is True + assert event.variant == "" + + def test_finally_after_present_variant_not_runtime_default(self, hook, writer): + hc = _make_hook_context() + details = _make_details(variant="on") + hook.finally_after(hc, details, {}) + event = writer.enqueue.call_args[0][0] + assert event.runtime_default is False + + def test_finally_after_reason_normalized_to_upper(self, hook, writer): + """Reason must be upper-case string in the enqueued event.""" + hc = _make_hook_context() + details = _make_details(reason=Reason.TARGETING_MATCH) + hook.finally_after(hc, details, {}) + event = writer.enqueue.call_args[0][0] + assert event.reason == "TARGETING_MATCH" + + def test_finally_after_eval_time_from_metadata(self, hook, writer): + """Eval-time must come from metadata["dd.eval.timestamp_ms"] when present.""" + stamp = int(time.time() * 1000) - 500 # 500 ms in the past + hc = _make_hook_context() + details = _make_details(flag_metadata={"dd.eval.timestamp_ms": stamp}) + hook.finally_after(hc, details, {}) + event = writer.enqueue.call_args[0][0] + assert event.eval_time_ms == stamp + + def test_finally_after_eval_time_fallback_to_hook_fire(self, hook, writer): + """When metadata key absent, fallback to hook-fire time (within 1 second).""" + before = int(time.time() * 1000) + hc = _make_hook_context() + details = _make_details(flag_metadata={}) + hook.finally_after(hc, details, {}) + after = int(time.time() * 1000) + event = writer.enqueue.call_args[0][0] + assert before <= event.eval_time_ms <= after + 100 + + def test_finally_after_extracts_targeting_key(self, hook, writer): + hc = _make_hook_context(targeting_key="user-99") + details = _make_details() + hook.finally_after(hc, details, {}) + event = writer.enqueue.call_args[0][0] + assert event.targeting_key == "user-99" + + def test_finally_after_extracts_attrs_shallow_copy(self, hook, writer): + attrs = {"tier": "premium", "region": "us-west"} + hc = _make_hook_context(attrs=attrs) + details = _make_details() + hook.finally_after(hc, details, {}) + event = writer.enqueue.call_args[0][0] + assert event.attrs == attrs + # Must be a copy, not the same object. + assert event.attrs is not attrs + + def test_finally_after_extracts_allocation_key(self, hook, writer): + hc = _make_hook_context() + details = _make_details(flag_metadata={"allocation_key": "alloc-xyz"}) + hook.finally_after(hc, details, {}) + event = writer.enqueue.call_args[0][0] + assert event.allocation_key == "alloc-xyz" + + def test_finally_after_does_no_aggregation_on_hook_thread(self, hook, writer): + """The hook must call enqueue only — not build payloads or aggregate (reviewer concern #7).""" + # writer.enqueue is a mock; the only call from finally_after must be enqueue. + hc = _make_hook_context() + details = _make_details() + hook.finally_after(hc, details, {}) + # Confirm ONLY enqueue was called on the writer (no periodic, no aggregate, no send). + called_methods = {c[0] for c in writer.method_calls} + assert called_methods == {"enqueue"}, ( + f"Expected only enqueue to be called, got: {called_methods}" + ) + + def test_finally_after_does_not_propagate_exceptions(self, hook, writer): + """Hook must swallow exceptions — best-effort telemetry.""" + writer.enqueue.side_effect = RuntimeError("boom") + hc = _make_hook_context() + details = _make_details() + # Must not raise. + hook.finally_after(hc, details, {}) + + +class TestKillswitchGating: + + def test_default_enabled_registers_evp_hook(self): + """Default (no env var set) must register the EVP hook + writer.""" + from ddtrace.internal.openfeature._flagevaluation_hook import FlagEvaluationHook + env = {k: v for k, v in os.environ.items() if k != "DD_FLAGGING_EVALUATION_COUNTS_ENABLED"} + # No env var → enabled by default. + with mock.patch.dict(os.environ, env, clear=True): + from tests.utils import override_global_config + with override_global_config({"experimental_flagging_provider_enabled": True}): + from ddtrace.internal.openfeature._provider import DataDogProvider + provider = DataDogProvider() + assert provider._flagevaluation_writer is not None + assert provider._flagevaluation_hook is not None + assert isinstance(provider._flagevaluation_hook, FlagEvaluationHook) + + def test_killswitch_false_does_not_register_evp_hook(self): + """DD_FLAGGING_EVALUATION_COUNTS_ENABLED=false must suppress EVP hook (killswitch).""" + with mock.patch.dict(os.environ, {"DD_FLAGGING_EVALUATION_COUNTS_ENABLED": "false"}): + from tests.utils import override_global_config + with override_global_config({"experimental_flagging_provider_enabled": True}): + from ddtrace.internal.openfeature._provider import DataDogProvider + provider = DataDogProvider() + assert provider._flagevaluation_writer is None + assert provider._flagevaluation_hook is None + + def test_killswitch_false_does_not_affect_otel_hook(self): + """Killswitch must not suppress the OTel FlagEvalHook (PRES-01).""" + with mock.patch.dict(os.environ, {"DD_FLAGGING_EVALUATION_COUNTS_ENABLED": "false"}): + from tests.utils import override_global_config + with override_global_config({"experimental_flagging_provider_enabled": True, "_otel_metrics_enabled": True}): + from ddtrace.internal.openfeature._provider import DataDogProvider + provider = DataDogProvider() + # OTel hook still present. + assert provider._flag_eval_hook is not None + # EVP hook absent. + assert provider._flagevaluation_hook is None + # get_provider_hooks still returns the OTel hook. + hooks = provider.get_provider_hooks() + assert len(hooks) == 1 + assert hooks[0] is provider._flag_eval_hook + + def test_killswitch_enabled_true_registers_evp_hook(self): + """DD_FLAGGING_EVALUATION_COUNTS_ENABLED=true must register the EVP hook.""" + with mock.patch.dict(os.environ, {"DD_FLAGGING_EVALUATION_COUNTS_ENABLED": "true"}): + from tests.utils import override_global_config + with override_global_config({"experimental_flagging_provider_enabled": True}): + from ddtrace.internal.openfeature._provider import DataDogProvider + provider = DataDogProvider() + assert provider._flagevaluation_writer is not None + assert provider._flagevaluation_hook is not None From 114216b241343751a278ca705e25b088a7bb7b5e Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 12 Jun 2026 18:21:40 -0400 Subject: [PATCH 03/37] fix(openfeature): degrade gracefully on RC init timeout instead of PROVIDER_ERROR Root cause: initialize() raised ProviderNotReadyError after the 10s RC wait, causing the OpenFeature SDK to dispatch PROVIDER_ERROR and mark the provider as ERROR. In ffe-dogfooding, with no FFE flags configured in the org, RC never delivers FFE_FLAGS config, so every start ended in PROVIDER_ERROR and the container exited. Fix: on timeout, log a warning, set _status = READY, and return normally. The SDK dispatches PROVIDER_READY; evaluations degrade to (default_value, ERROR, PROVIDER_NOT_READY) via _resolve_details() until config eventually arrives. On_configuration_received() remains the authoritative path for making config available, unchanged. Update affected tests to reflect the new READY-on-timeout contract. --- ddtrace/internal/openfeature/_provider.py | 31 ++++++++--- tests/openfeature/test_provider_status.py | 68 +++++++++++++++-------- 2 files changed, 68 insertions(+), 31 deletions(-) diff --git a/ddtrace/internal/openfeature/_provider.py b/ddtrace/internal/openfeature/_provider.py index dd15163ed01..aac071c1f3c 100644 --- a/ddtrace/internal/openfeature/_provider.py +++ b/ddtrace/internal/openfeature/_provider.py @@ -178,7 +178,10 @@ def initialize(self, evaluation_context: EvaluationContext) -> None: Initialize the provider. Blocks until Remote Config delivers the first FFE configuration or - the initialization timeout expires. + the initialization timeout expires. On timeout, the provider still + transitions to READY so that evaluation can proceed with defaults; when + RC does eventually deliver config, on_configuration_received() picks it + up and config becomes available for future evaluations. The timeout is configurable via: - Constructor: DataDogProvider(initialization_timeout=10.0) # seconds @@ -186,7 +189,8 @@ def initialize(self, evaluation_context: EvaluationContext) -> None: Provider lifecycle: NOT_READY -> initialize() blocks -> config arrives -> READY - NOT_READY -> initialize() blocks -> timeout -> raises ProviderNotReadyError + NOT_READY -> initialize() blocks -> timeout -> READY (no config yet; + evaluations return default values until RC delivers config) """ if not self._enabled: return @@ -222,13 +226,24 @@ def initialize(self, evaluation_context: EvaluationContext) -> None: "Waiting up to %.1fs for initial FFE configuration from Remote Config", self._initialization_timeout ) if not self._config_received.wait(timeout=self._initialization_timeout): - # Timeout expired without receiving config - from openfeature.exception import ProviderNotReadyError - - raise ProviderNotReadyError( - f"Provider timed out after {self._initialization_timeout:.1f}s waiting for " - "initial configuration from Remote Config" + # Timeout expired without receiving config. + # AIDEV-NOTE: We do NOT raise ProviderNotReadyError here. Raising + # would cause the OpenFeature SDK to dispatch PROVIDER_ERROR and mark + # the provider as ERROR, preventing all evaluations. Instead, we log + # a warning and return normally so the provider transitions to READY. + # Evaluations will return (default_value, ERROR, PROVIDER_NOT_READY) + # via _resolve_details() until RC eventually delivers the config, at + # which point on_configuration_received() makes config available for + # future calls. This is the correct degraded-mode behavior: the + # provider is usable (returns defaults) rather than completely broken. + logger.warning( + "openfeature: timed out after %.1fs waiting for initial FFE configuration from " + "Remote Config; provider is READY but will return default values until config arrives. " + "Check that the Datadog Agent is running and FFE flags are configured in your org.", + self._initialization_timeout, ) + self._status = ProviderStatus.READY + return # SDK will dispatch PROVIDER_READY; evaluations degrade to defaults # Config received during wait -- on_configuration_received() already set status diff --git a/tests/openfeature/test_provider_status.py b/tests/openfeature/test_provider_status.py index 8af9375fd3c..e6bfbe08d94 100644 --- a/tests/openfeature/test_provider_status.py +++ b/tests/openfeature/test_provider_status.py @@ -50,20 +50,25 @@ def test_provider_starts_not_ready(self): assert not provider._config_received.is_set() def test_provider_becomes_ready_after_first_config(self): - """Test that provider becomes READY after receiving first configuration.""" + """Test that provider is READY after set_provider and stays READY after config. + + With the graceful-timeout fix, initialize() transitions to READY on timeout + (instead of ERROR) so the provider is already READY when set_provider() returns. + Config delivery confirms _config_received is set and status remains READY. + """ with override_global_config({"experimental_flagging_provider_enabled": True}): provider = DataDogProvider() api.set_provider(provider) try: - # Verify starts as NOT_READY - assert provider._status == ProviderStatus.NOT_READY + # Provider is READY after set_provider() — graceful timeout goes to READY + assert provider._status == ProviderStatus.READY # Process a configuration config = create_config(create_boolean_flag("test-flag", enabled=True)) process_ffe_configuration(config) - # Verify becomes READY + # Still READY, and config event is now set assert provider._status == ProviderStatus.READY assert provider._config_received.is_set() finally: @@ -149,7 +154,12 @@ def test_provider_status_after_shutdown(self): api.clear_providers() def test_multiple_providers_receive_status_updates(self): - """Test that multiple provider instances receive status updates.""" + """Test that multiple provider instances are READY after set_provider. + + With the graceful-timeout fix, each provider transitions to READY on timeout + (instead of ERROR), so both are already READY when their set_provider() returns. + Config delivery confirms _config_received is set and status remains READY. + """ with override_global_config({"experimental_flagging_provider_enabled": True}): provider1 = DataDogProvider() provider2 = DataDogProvider() @@ -158,15 +168,15 @@ def test_multiple_providers_receive_status_updates(self): api.set_provider(provider2, "client2") try: - # Both start as NOT_READY - assert provider1._status == ProviderStatus.NOT_READY - assert provider2._status == ProviderStatus.NOT_READY + # Both are READY after set_provider() — graceful timeout goes to READY + assert provider1._status == ProviderStatus.READY + assert provider2._status == ProviderStatus.READY # Process configuration config = create_config(create_boolean_flag("test-flag", enabled=True)) process_ffe_configuration(config) - # Both should become READY + # Both should remain READY with config event set assert provider1._status == ProviderStatus.READY assert provider2._status == ProviderStatus.READY finally: @@ -201,7 +211,11 @@ def on_provider_ready(event_details): class TestProviderInitializationBlocking: - """Test that initialize() blocks until config arrives or timeout expires.""" + """Test that initialize() blocks until config arrives or timeout expires. + + On timeout the provider transitions to READY (not ERROR) so evaluations can + return default values until RC eventually delivers config. + """ def test_initialize_blocks_until_config_arrives(self): """initialize() should block and return once config is delivered mid-wait.""" @@ -250,45 +264,53 @@ def test_initialize_fast_path_when_config_exists(self): finally: api.clear_providers() - def test_initialize_timeout_raises(self): - """initialize() should raise ProviderNotReadyError after timeout expires.""" + def test_initialize_timeout_returns_ready(self): + """initialize() should NOT raise on timeout; provider goes READY so evaluations can return defaults. + + Previously initialize() raised ProviderNotReadyError on timeout, causing PROVIDER_ERROR. + The new behaviour is to log a warning and return normally so the provider is usable (returning + default values via _resolve_details()) until RC eventually delivers config. + """ with override_global_config({"experimental_flagging_provider_enabled": True}): provider = DataDogProvider(initialization_timeout=0.5) try: start = time.monotonic() - # set_provider catches the exception and dispatches PROVIDER_ERROR api.set_provider(provider) elapsed = time.monotonic() - start - # Should have blocked for ~0.5s (the timeout) + # Should have blocked for ~0.5s (the timeout duration), not immediately assert elapsed >= 0.3, f"initialize() returned too fast ({elapsed:.2f}s)" assert elapsed < 2.0, f"initialize() took too long ({elapsed:.2f}s)" - # Provider should be in ERROR state (SDK caught ProviderNotReadyError) + # Provider should be READY (not ERROR) — evaluations will return defaults client = api.get_client() - assert client.get_provider_status() == ProviderStatus.ERROR + assert client.get_provider_status() == ProviderStatus.READY + assert provider._status == ProviderStatus.READY finally: api.clear_providers() - def test_late_recovery_after_timeout(self): - """Config arriving after timeout should transition provider to READY.""" + def test_late_config_delivery_after_timeout(self): + """Config arriving after the initialization timeout should become available for evaluations. + + After timeout, provider is already READY. When RC delivers config, on_configuration_received() + stores it and future evaluations resolve against the live config. + """ with override_global_config({"experimental_flagging_provider_enabled": True}): provider = DataDogProvider(initialization_timeout=0.5) try: - # Let it timeout + # Let the timeout elapse; provider goes READY with no config api.set_provider(provider) - # Provider should be in ERROR state client = api.get_client() - assert client.get_provider_status() == ProviderStatus.ERROR + assert client.get_provider_status() == ProviderStatus.READY - # Now deliver config (late recovery) + # Now deliver config (late arrival) config = create_config(create_boolean_flag("test-flag", enabled=True)) process_ffe_configuration(config) - # Provider should recover to READY + # Provider status stays READY, config is now stored assert provider._status == ProviderStatus.READY assert provider._config_received.is_set() finally: From 1c088ac2cb130e7f80435b156a7ad4bd85e89379 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 12 Jun 2026 18:25:27 -0400 Subject: [PATCH 04/37] Revert "fix(openfeature): degrade gracefully on RC init timeout instead of PROVIDER_ERROR" This reverts commit 114216b241343751a278ca705e25b088a7bb7b5e. --- ddtrace/internal/openfeature/_provider.py | 31 +++-------- tests/openfeature/test_provider_status.py | 68 ++++++++--------------- 2 files changed, 31 insertions(+), 68 deletions(-) diff --git a/ddtrace/internal/openfeature/_provider.py b/ddtrace/internal/openfeature/_provider.py index aac071c1f3c..dd15163ed01 100644 --- a/ddtrace/internal/openfeature/_provider.py +++ b/ddtrace/internal/openfeature/_provider.py @@ -178,10 +178,7 @@ def initialize(self, evaluation_context: EvaluationContext) -> None: Initialize the provider. Blocks until Remote Config delivers the first FFE configuration or - the initialization timeout expires. On timeout, the provider still - transitions to READY so that evaluation can proceed with defaults; when - RC does eventually deliver config, on_configuration_received() picks it - up and config becomes available for future evaluations. + the initialization timeout expires. The timeout is configurable via: - Constructor: DataDogProvider(initialization_timeout=10.0) # seconds @@ -189,8 +186,7 @@ def initialize(self, evaluation_context: EvaluationContext) -> None: Provider lifecycle: NOT_READY -> initialize() blocks -> config arrives -> READY - NOT_READY -> initialize() blocks -> timeout -> READY (no config yet; - evaluations return default values until RC delivers config) + NOT_READY -> initialize() blocks -> timeout -> raises ProviderNotReadyError """ if not self._enabled: return @@ -226,24 +222,13 @@ def initialize(self, evaluation_context: EvaluationContext) -> None: "Waiting up to %.1fs for initial FFE configuration from Remote Config", self._initialization_timeout ) if not self._config_received.wait(timeout=self._initialization_timeout): - # Timeout expired without receiving config. - # AIDEV-NOTE: We do NOT raise ProviderNotReadyError here. Raising - # would cause the OpenFeature SDK to dispatch PROVIDER_ERROR and mark - # the provider as ERROR, preventing all evaluations. Instead, we log - # a warning and return normally so the provider transitions to READY. - # Evaluations will return (default_value, ERROR, PROVIDER_NOT_READY) - # via _resolve_details() until RC eventually delivers the config, at - # which point on_configuration_received() makes config available for - # future calls. This is the correct degraded-mode behavior: the - # provider is usable (returns defaults) rather than completely broken. - logger.warning( - "openfeature: timed out after %.1fs waiting for initial FFE configuration from " - "Remote Config; provider is READY but will return default values until config arrives. " - "Check that the Datadog Agent is running and FFE flags are configured in your org.", - self._initialization_timeout, + # Timeout expired without receiving config + from openfeature.exception import ProviderNotReadyError + + raise ProviderNotReadyError( + f"Provider timed out after {self._initialization_timeout:.1f}s waiting for " + "initial configuration from Remote Config" ) - self._status = ProviderStatus.READY - return # SDK will dispatch PROVIDER_READY; evaluations degrade to defaults # Config received during wait -- on_configuration_received() already set status diff --git a/tests/openfeature/test_provider_status.py b/tests/openfeature/test_provider_status.py index e6bfbe08d94..8af9375fd3c 100644 --- a/tests/openfeature/test_provider_status.py +++ b/tests/openfeature/test_provider_status.py @@ -50,25 +50,20 @@ def test_provider_starts_not_ready(self): assert not provider._config_received.is_set() def test_provider_becomes_ready_after_first_config(self): - """Test that provider is READY after set_provider and stays READY after config. - - With the graceful-timeout fix, initialize() transitions to READY on timeout - (instead of ERROR) so the provider is already READY when set_provider() returns. - Config delivery confirms _config_received is set and status remains READY. - """ + """Test that provider becomes READY after receiving first configuration.""" with override_global_config({"experimental_flagging_provider_enabled": True}): provider = DataDogProvider() api.set_provider(provider) try: - # Provider is READY after set_provider() — graceful timeout goes to READY - assert provider._status == ProviderStatus.READY + # Verify starts as NOT_READY + assert provider._status == ProviderStatus.NOT_READY # Process a configuration config = create_config(create_boolean_flag("test-flag", enabled=True)) process_ffe_configuration(config) - # Still READY, and config event is now set + # Verify becomes READY assert provider._status == ProviderStatus.READY assert provider._config_received.is_set() finally: @@ -154,12 +149,7 @@ def test_provider_status_after_shutdown(self): api.clear_providers() def test_multiple_providers_receive_status_updates(self): - """Test that multiple provider instances are READY after set_provider. - - With the graceful-timeout fix, each provider transitions to READY on timeout - (instead of ERROR), so both are already READY when their set_provider() returns. - Config delivery confirms _config_received is set and status remains READY. - """ + """Test that multiple provider instances receive status updates.""" with override_global_config({"experimental_flagging_provider_enabled": True}): provider1 = DataDogProvider() provider2 = DataDogProvider() @@ -168,15 +158,15 @@ def test_multiple_providers_receive_status_updates(self): api.set_provider(provider2, "client2") try: - # Both are READY after set_provider() — graceful timeout goes to READY - assert provider1._status == ProviderStatus.READY - assert provider2._status == ProviderStatus.READY + # Both start as NOT_READY + assert provider1._status == ProviderStatus.NOT_READY + assert provider2._status == ProviderStatus.NOT_READY # Process configuration config = create_config(create_boolean_flag("test-flag", enabled=True)) process_ffe_configuration(config) - # Both should remain READY with config event set + # Both should become READY assert provider1._status == ProviderStatus.READY assert provider2._status == ProviderStatus.READY finally: @@ -211,11 +201,7 @@ def on_provider_ready(event_details): class TestProviderInitializationBlocking: - """Test that initialize() blocks until config arrives or timeout expires. - - On timeout the provider transitions to READY (not ERROR) so evaluations can - return default values until RC eventually delivers config. - """ + """Test that initialize() blocks until config arrives or timeout expires.""" def test_initialize_blocks_until_config_arrives(self): """initialize() should block and return once config is delivered mid-wait.""" @@ -264,53 +250,45 @@ def test_initialize_fast_path_when_config_exists(self): finally: api.clear_providers() - def test_initialize_timeout_returns_ready(self): - """initialize() should NOT raise on timeout; provider goes READY so evaluations can return defaults. - - Previously initialize() raised ProviderNotReadyError on timeout, causing PROVIDER_ERROR. - The new behaviour is to log a warning and return normally so the provider is usable (returning - default values via _resolve_details()) until RC eventually delivers config. - """ + def test_initialize_timeout_raises(self): + """initialize() should raise ProviderNotReadyError after timeout expires.""" with override_global_config({"experimental_flagging_provider_enabled": True}): provider = DataDogProvider(initialization_timeout=0.5) try: start = time.monotonic() + # set_provider catches the exception and dispatches PROVIDER_ERROR api.set_provider(provider) elapsed = time.monotonic() - start - # Should have blocked for ~0.5s (the timeout duration), not immediately + # Should have blocked for ~0.5s (the timeout) assert elapsed >= 0.3, f"initialize() returned too fast ({elapsed:.2f}s)" assert elapsed < 2.0, f"initialize() took too long ({elapsed:.2f}s)" - # Provider should be READY (not ERROR) — evaluations will return defaults + # Provider should be in ERROR state (SDK caught ProviderNotReadyError) client = api.get_client() - assert client.get_provider_status() == ProviderStatus.READY - assert provider._status == ProviderStatus.READY + assert client.get_provider_status() == ProviderStatus.ERROR finally: api.clear_providers() - def test_late_config_delivery_after_timeout(self): - """Config arriving after the initialization timeout should become available for evaluations. - - After timeout, provider is already READY. When RC delivers config, on_configuration_received() - stores it and future evaluations resolve against the live config. - """ + def test_late_recovery_after_timeout(self): + """Config arriving after timeout should transition provider to READY.""" with override_global_config({"experimental_flagging_provider_enabled": True}): provider = DataDogProvider(initialization_timeout=0.5) try: - # Let the timeout elapse; provider goes READY with no config + # Let it timeout api.set_provider(provider) + # Provider should be in ERROR state client = api.get_client() - assert client.get_provider_status() == ProviderStatus.READY + assert client.get_provider_status() == ProviderStatus.ERROR - # Now deliver config (late arrival) + # Now deliver config (late recovery) config = create_config(create_boolean_flag("test-flag", enabled=True)) process_ffe_configuration(config) - # Provider status stays READY, config is now stored + # Provider should recover to READY assert provider._status == ProviderStatus.READY assert provider._config_received.is_set() finally: From bfd3f427dccc50688f35fef83679c3d6f017ef2a Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 12 Jun 2026 18:53:26 -0400 Subject: [PATCH 05/37] =?UTF-8?q?fix(openfeature):=20remove=20blocking=20w?= =?UTF-8?q?ait=20from=20initialize()=20=E2=80=94=20breaks=20gunicorn=20pre?= =?UTF-8?q?-fork=20workers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: initialize() was blocking on _config_received.wait(timeout=10s) and raising ProviderNotReadyError on expiry. Under gunicorn (and uWSGI) pre-fork workers, the OpenFeature SDK runs initialize() in a background thread; that thread is killed by fork() in each child process, so every worker's initialize() either misses the RC notification (READY in master fires before worker registers via _register_provider) or times out waiting — causing PROVIDER_ERROR and container exit. Evidence (dogfooding logs): 22:44:31 PROVIDER_READY ← worker 1 took fast path (config in master) 22:44:41 PROVIDER_ERROR ← worker 2 timed out (10 s exactly) Fix: remove the blocking wait entirely. initialize() now returns immediately if no config is available yet, matching the original pre-#16650 contract. The async path via on_configuration_received() remains the authoritative READY transition for both the master process and each forked worker (worker RC subscriber replays FFE_FLAGS from the inherited connector queue, which calls _notify_providers_config_received() once the worker's provider is registered). The fast path (config already available at initialize() time) is preserved — this covers the case where the master received config before forking. Update tests: replace TestProviderInitializationBlocking with TestProviderInitializationAsync verifying the new non-blocking contract. Remove the fast_initialization_timeout conftest fixture (no longer needed since initialize() does not block). --- ddtrace/internal/openfeature/_provider.py | 52 +++++++--------- tests/openfeature/conftest.py | 19 ------ tests/openfeature/test_provider_status.py | 75 ++++++++++------------- 3 files changed, 53 insertions(+), 93 deletions(-) diff --git a/ddtrace/internal/openfeature/_provider.py b/ddtrace/internal/openfeature/_provider.py index dd15163ed01..6156ffd0c8c 100644 --- a/ddtrace/internal/openfeature/_provider.py +++ b/ddtrace/internal/openfeature/_provider.py @@ -98,19 +98,13 @@ class DataDogProvider(AbstractProvider): Feature Flags and Experimentation (FFE) product. """ - def __init__(self, *args: typing.Any, initialization_timeout: typing.Optional[float] = None, **kwargs: typing.Any): + def __init__(self, *args: typing.Any, **kwargs: typing.Any): super().__init__(*args, **kwargs) self._metadata = Metadata(name="Datadog") self._status = ProviderStatus.NOT_READY - # Initialization timeout: constructor arg takes priority, then env var - if initialization_timeout is not None: - self._initialization_timeout = initialization_timeout - else: - self._initialization_timeout = ffe_config.initialization_timeout_ms / 1000.0 - - # Event used to block initialize() until config arrives. - # Also serves as the "config received" flag via is_set(). + # Event set when the first RC config arrives; used by on_configuration_received() + # to guard the first-config path and by _emit_ready_event() timing. self._config_received = threading.Event() # Cache for reported exposures to prevent duplicates @@ -177,16 +171,16 @@ def initialize(self, evaluation_context: EvaluationContext) -> None: """ Initialize the provider. - Blocks until Remote Config delivers the first FFE configuration or - the initialization timeout expires. + Returns immediately; the provider transitions to READY asynchronously once + Remote Config delivers the first FFE_FLAGS payload via on_configuration_received(). - The timeout is configurable via: - - Constructor: DataDogProvider(initialization_timeout=10.0) # seconds - - Env var: DD_EXPERIMENTAL_FLAGGING_PROVIDER_INITIALIZATION_TIMEOUT_MS=10000 + If RC has already delivered config before initialize() runs (e.g. in the master + process of a pre-fork server), the fast path sets READY synchronously so the SDK + dispatches PROVIDER_READY on return. Provider lifecycle: - NOT_READY -> initialize() blocks -> config arrives -> READY - NOT_READY -> initialize() blocks -> timeout -> raises ProviderNotReadyError + NOT_READY -> initialize() returns -> RC delivers config -> on_configuration_received() + -> READY (PROVIDER_READY event emitted via emit_provider_ready) """ if not self._enabled: return @@ -209,7 +203,8 @@ def initialize(self, evaluation_context: EvaluationContext) -> None: except ServiceStatusError: logger.debug("FlagEvaluationWriter is already running", exc_info=True) - # Fast path: config already available (RC delivered before set_provider) + # Fast path: config already available (RC delivered before set_provider — + # common in pre-fork servers where master receives RC before workers fork). config = _get_ffe_config() if config is not None: logger.debug("FFE configuration already available, provider is READY") @@ -217,20 +212,15 @@ def initialize(self, evaluation_context: EvaluationContext) -> None: self._status = ProviderStatus.READY return # SDK will dispatch PROVIDER_READY - # Block until config arrives or timeout expires - logger.debug( - "Waiting up to %.1fs for initial FFE configuration from Remote Config", self._initialization_timeout - ) - if not self._config_received.wait(timeout=self._initialization_timeout): - # Timeout expired without receiving config - from openfeature.exception import ProviderNotReadyError - - raise ProviderNotReadyError( - f"Provider timed out after {self._initialization_timeout:.1f}s waiting for " - "initial configuration from Remote Config" - ) - - # Config received during wait -- on_configuration_received() already set status + # Config not yet available — return without blocking. The provider stays + # NOT_READY; on_configuration_received() will flip it to READY and emit + # PROVIDER_READY when RC delivers the FFE_FLAGS payload. + # AIDEV-NOTE: Do NOT block here with _config_received.wait(). Blocking + # initialize() breaks gunicorn/uWSGI pre-fork workers: when the OpenFeature + # SDK runs initialize() in a background thread, fork() kills that thread in + # child processes, leaving every worker stuck waiting forever (or timing out + # with PROVIDER_ERROR). The async path (on_configuration_received) is the + # correct contract for server SDK providers. def shutdown(self) -> None: """ diff --git a/tests/openfeature/conftest.py b/tests/openfeature/conftest.py index 8704d6a0bde..a13d86bfae7 100644 --- a/tests/openfeature/conftest.py +++ b/tests/openfeature/conftest.py @@ -1,22 +1,3 @@ """ Shared fixtures for openfeature tests. """ - -import pytest - -from tests.utils import override_global_config - - -@pytest.fixture(autouse=True) -def fast_initialization_timeout(): - """ - Override the provider initialization timeout to 100ms for all tests. - - The production default is 30s (matching other SDKs), but that makes any - test that calls api.set_provider() without pre-loaded config hang for 30s. - Tests that need to verify timeout/blocking behaviour set their own explicit - initialization_timeout= on DataDogProvider() directly, which takes priority - over the config value. - """ - with override_global_config({"initialization_timeout_ms": 100}): - yield diff --git a/tests/openfeature/test_provider_status.py b/tests/openfeature/test_provider_status.py index 8af9375fd3c..02071ac4d2a 100644 --- a/tests/openfeature/test_provider_status.py +++ b/tests/openfeature/test_provider_status.py @@ -12,6 +12,7 @@ import time from openfeature import api +from openfeature.evaluation_context import EvaluationContext from openfeature.provider import ProviderStatus import pytest @@ -200,33 +201,24 @@ def on_provider_ready(event_details): api.clear_providers() -class TestProviderInitializationBlocking: - """Test that initialize() blocks until config arrives or timeout expires.""" +class TestProviderInitializationAsync: + """Test that initialize() returns immediately and READY arrives asynchronously.""" - def test_initialize_blocks_until_config_arrives(self): - """initialize() should block and return once config is delivered mid-wait.""" + def test_initialize_returns_immediately_without_config(self): + """initialize() should return immediately even if no config is available yet.""" with override_global_config({"experimental_flagging_provider_enabled": True}): - provider = DataDogProvider(initialization_timeout=5.0) - - # Deliver config from a background thread after 0.5s - def deliver_config(): - time.sleep(0.5) - config = create_config(create_boolean_flag("test-flag", enabled=True)) - process_ffe_configuration(config) - - timer = threading.Thread(target=deliver_config, daemon=True) - timer.start() + provider = DataDogProvider() try: start = time.monotonic() - api.set_provider(provider) + # initialize() is called inside set_provider; it must not block + provider.initialize(EvaluationContext()) elapsed = time.monotonic() - start - # Should have blocked for ~0.5s (not instant, not full timeout) - assert elapsed >= 0.3, f"initialize() returned too fast ({elapsed:.2f}s)" - assert elapsed < 4.0, f"initialize() took too long ({elapsed:.2f}s), should have unblocked at ~0.5s" - assert provider._status == ProviderStatus.READY - assert provider._config_received.is_set() + # Should return near-instantly (no blocking wait) + assert elapsed < 0.5, f"initialize() blocked for {elapsed:.2f}s — must not block" + # Provider is NOT_READY; READY arrives via on_configuration_received() + assert provider._status == ProviderStatus.NOT_READY finally: api.clear_providers() @@ -237,7 +229,7 @@ def test_initialize_fast_path_when_config_exists(self): config = create_config(create_boolean_flag("test-flag", enabled=True)) process_ffe_configuration(config) - provider = DataDogProvider(initialization_timeout=5.0) + provider = DataDogProvider() try: start = time.monotonic() @@ -250,41 +242,38 @@ def test_initialize_fast_path_when_config_exists(self): finally: api.clear_providers() - def test_initialize_timeout_raises(self): - """initialize() should raise ProviderNotReadyError after timeout expires.""" + def test_ready_after_config_arrives_async(self): + """Provider transitions to READY when config arrives after initialize() returns.""" with override_global_config({"experimental_flagging_provider_enabled": True}): - provider = DataDogProvider(initialization_timeout=0.5) + provider = DataDogProvider() try: - start = time.monotonic() - # set_provider catches the exception and dispatches PROVIDER_ERROR - api.set_provider(provider) - elapsed = time.monotonic() - start + provider.initialize(EvaluationContext()) + # Still NOT_READY immediately after initialize() + assert provider._status == ProviderStatus.NOT_READY - # Should have blocked for ~0.5s (the timeout) - assert elapsed >= 0.3, f"initialize() returned too fast ({elapsed:.2f}s)" - assert elapsed < 2.0, f"initialize() took too long ({elapsed:.2f}s)" + # Config arrives later (simulating RC delivery) + config = create_config(create_boolean_flag("test-flag", enabled=True)) + process_ffe_configuration(config) - # Provider should be in ERROR state (SDK caught ProviderNotReadyError) - client = api.get_client() - assert client.get_provider_status() == ProviderStatus.ERROR + # Provider should now be READY + assert provider._status == ProviderStatus.READY + assert provider._config_received.is_set() finally: api.clear_providers() - def test_late_recovery_after_timeout(self): - """Config arriving after timeout should transition provider to READY.""" + def test_late_config_delivery_transitions_to_ready(self): + """Config arriving after set_provider should transition provider to READY.""" with override_global_config({"experimental_flagging_provider_enabled": True}): - provider = DataDogProvider(initialization_timeout=0.5) + provider = DataDogProvider() try: - # Let it timeout - api.set_provider(provider) + provider.initialize(EvaluationContext()) - # Provider should be in ERROR state - client = api.get_client() - assert client.get_provider_status() == ProviderStatus.ERROR + # Provider is NOT_READY at this point + assert provider._status == ProviderStatus.NOT_READY - # Now deliver config (late recovery) + # Simulate delayed RC delivery config = create_config(create_boolean_flag("test-flag", enabled=True)) process_ffe_configuration(config) From ab42f3ec0757872102c355d69aef605e5d224bdb Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Sat, 13 Jun 2026 11:28:18 -0400 Subject: [PATCH 06/37] fix(openfeature): killswitch via config system, type cleanups, benchmarks + e2e tests - Read DD_FLAGGING_EVALUATION_COUNTS_ENABLED through OpenFeatureConfig (ddtrace config var system) instead of raw os.environ; register it in supported-configurations.json. - Stamp eval timestamp at the provider resolution boundary so the finally_after hook receives accurate eval-time. - Normalize typing annotations (dict[...] generics) and mypy overrides for the flagevaluation writer/hook. - Add openfeature_flagevaluation microbenchmark suite (hook-enqueue / aggregate / hook-plus-drain, incl. a 2500-flag high-cardinality variant) wired into benchmarks/suitespec.yml. - Add end-to-end provider tests covering hook firing on the real eval path, all exit paths (success/error/runtime-default/disabled), and OTel non-regression. --- .../openfeature_flagevaluation/config.yaml | 36 +++ .../openfeature_flagevaluation/scenario.py | 152 +++++++++ benchmarks/suitespec.yml | 13 + .../openfeature/_flagevaluation_hook.py | 35 +-- .../openfeature/_flagevaluation_writer.py | 124 +++++--- ddtrace/internal/openfeature/_provider.py | 31 +- .../settings/_supported_configurations.py | 1 + ddtrace/internal/settings/openfeature.py | 10 + mypy.ini | 9 + supported-configurations.json | 8 + tests/openfeature/test_flagevaluation_e2e.py | 205 ++++++++++++ tests/openfeature/test_flagevaluation_hook.py | 139 ++++++++- .../openfeature/test_flagevaluation_writer.py | 292 ++++++++++++++++-- 13 files changed, 926 insertions(+), 129 deletions(-) create mode 100644 benchmarks/openfeature_flagevaluation/config.yaml create mode 100644 benchmarks/openfeature_flagevaluation/scenario.py create mode 100644 tests/openfeature/test_flagevaluation_e2e.py diff --git a/benchmarks/openfeature_flagevaluation/config.yaml b/benchmarks/openfeature_flagevaluation/config.yaml new file mode 100644 index 00000000000..f9111dc909d --- /dev/null +++ b/benchmarks/openfeature_flagevaluation/config.yaml @@ -0,0 +1,36 @@ +defaults: &defaults + num_flags: 100 + num_context_fields: 8 + +# Eval hot path the caller pays: finally_after hook (cheap capture + non-blocking enqueue). +hook-enqueue: + <<: *defaults + mode: hook_enqueue + +# Eval hot path with a small context (typical server eval). +hook-enqueue-small-context: + <<: *defaults + mode: hook_enqueue + num_context_fields: 2 + +# Eval hot path with a large context (stresses the shallow copy on the hook path). +hook-enqueue-large-context: + <<: *defaults + mode: hook_enqueue + num_context_fields: 64 + +# Background worker hot path: flatten + deterministic prune + canonical key + two-tier insert. +aggregate: + <<: *defaults + mode: aggregate + +# Aggregation under high flag cardinality (more distinct full-tier buckets). +aggregate-high-cardinality: + <<: *defaults + mode: aggregate + num_flags: 2500 + +# End-to-end per-flush shape: N enqueues then a full drain + aggregate. +hook-plus-drain: + <<: *defaults + mode: hook_plus_drain diff --git a/benchmarks/openfeature_flagevaluation/scenario.py b/benchmarks/openfeature_flagevaluation/scenario.py new file mode 100644 index 00000000000..42d31e76051 --- /dev/null +++ b/benchmarks/openfeature_flagevaluation/scenario.py @@ -0,0 +1,152 @@ +"""Hot-path microbenchmark for the EVP ``flagevaluation`` aggregation pipeline. + +This measures the cost an OpenFeature evaluation actually pays for server-side EVP +flag-evaluation counting, mirroring the Go reference benchmark +(``dd-trace-go/openfeature/flagevaluation_test.go``): + +* ``hook_enqueue`` — the ``finally_after`` hook hot path: cheap scalar capture + a + shallow context copy + a non-blocking bounded enqueue. This is what charges the + caller's evaluation goroutine/thread directly, so it must stay flat. +* ``aggregate`` — the background worker hot path: flatten + deterministic prune + + canonical-context key + two-tier aggregation. This runs OFF the eval path, but its + per-event cost still bounds throughput of the writer thread. +* ``hook_plus_drain`` — end-to-end: N hook enqueues followed by a full queue drain + + aggregate, the realistic per-flush shape. + +The scenario builds its own ``FlagEvaluationWriter`` + ``FlagEvaluationHook`` and drives +them directly so the benchmark isolates the EVP pipeline (no network, no native eval). +""" + +import bm +from openfeature.evaluation_context import EvaluationContext +from openfeature.flag_evaluation import FlagEvaluationDetails +from openfeature.flag_evaluation import FlagType +from openfeature.flag_evaluation import Reason +from openfeature.hook import HookContext + + +def _make_hook_context(flag_key, targeting_key, attrs): + ctx = EvaluationContext(targeting_key=targeting_key, attributes=attrs) + return HookContext( + flag_key=flag_key, + flag_type=FlagType.BOOLEAN, + default_value=False, + evaluation_context=ctx, + ) + + +def _make_details(flag_key, variant, allocation_key): + return FlagEvaluationDetails( + flag_key=flag_key, + value=True, + variant=variant, + reason=Reason.TARGETING_MATCH, + flag_metadata={"allocation_key": allocation_key}, + ) + + +class OpenFeatureFlagEvaluation(bm.Scenario): + # Which segment of the pipeline to exercise. + mode: str + # Number of distinct flags cycled through (drives aggregation-map cardinality). + num_flags: int + # Number of evaluation-context attributes per evaluation (drives prune/key cost). + num_context_fields: int + + def run(self): + from ddtrace.internal.openfeature._flagevaluation_hook import FlagEvaluationHook + from ddtrace.internal.openfeature._flagevaluation_writer import FlagEvaluationWriter + + mode = self.mode + num_flags = max(1, self.num_flags) + num_fields = max(0, self.num_context_fields) + + # Pre-build the per-evaluation inputs so the measured loop only exercises the + # hook/aggregation hot path, not fixture construction. + attrs = {"attr_{}".format(i): "value_{}".format(i) for i in range(num_fields)} + hook_contexts = [ + _make_hook_context( + flag_key="flag-{}".format(i % num_flags), + targeting_key="user-{}".format(i % num_flags), + attrs=attrs, + ) + for i in range(num_flags) + ] + details_list = [ + _make_details( + flag_key="flag-{}".format(i % num_flags), + variant="variant-{}".format(i % 4), + allocation_key="alloc-{}".format(i % num_flags), + ) + for i in range(num_flags) + ] + + writer = FlagEvaluationWriter(interval=3600.0) + hook = FlagEvaluationHook(writer) + + if mode == "hook_enqueue": + + def _(loops): + n = num_flags + for i in range(loops): + idx = i % n + hook.finally_after(hook_contexts[idx], details_list[idx], {}) + # Keep the queue from saturating so we keep measuring the enqueue + # path rather than the drop-and-count path. + if writer._queue.full(): + writer._drain_queue() + + yield _ + + elif mode == "aggregate": + # Pre-capture snapshots once; the measured loop only runs _aggregate + # (flatten + prune + canonical key + two-tier insert). + from ddtrace.internal.openfeature._flagevaluation_writer import _EvalEvent + + events = [ + _EvalEvent( + flag_key="flag-{}".format(i % num_flags), + variant="variant-{}".format(i % 4), + allocation_key="alloc-{}".format(i % num_flags), + reason="TARGETING_MATCH", + targeting_key="user-{}".format(i % num_flags), + attrs=dict(attrs), + runtime_default=False, + error_message="", + eval_time_ms=1_700_000_000_000 + i, + ) + for i in range(num_flags) + ] + + def _(loops): + n = num_flags + for i in range(loops): + writer._aggregate(events[i % n]) + # Reset periodically so aggregation maps don't grow unbounded across + # a long benchmark loop (keeps the per-op cost representative). + if (i % 10000) == 0: + writer._full.clear() + writer._degraded.clear() + writer._per_flag_count.clear() + writer._global_count = 0 + + yield _ + + elif mode == "hook_plus_drain": + + def _(loops): + n = num_flags + for i in range(loops): + idx = i % n + hook.finally_after(hook_contexts[idx], details_list[idx], {}) + if writer._queue.full() or (i % n) == (n - 1): + writer._drain_queue() + writer._full.clear() + writer._degraded.clear() + writer._per_flag_count.clear() + writer._global_count = 0 + + yield _ + + else: + raise ValueError("unknown mode: {}".format(mode)) diff --git a/benchmarks/suitespec.yml b/benchmarks/suitespec.yml index 0802b8ed9ca..85a1fcaf2c3 100644 --- a/benchmarks/suitespec.yml +++ b/benchmarks/suitespec.yml @@ -50,6 +50,10 @@ components: - ddtrace/appsec/iast/* - ddtrace/appsec/* - ddtrace/internal/settings/asm.py + openfeature: + - ddtrace/openfeature/* + - ddtrace/internal/openfeature/* + - ddtrace/internal/settings/openfeature.py core: - ddtrace/internal/__init__.py - ddtrace/internal/_exceptions.py @@ -222,6 +226,15 @@ suites: - benchmarks/suitespec.yml cpus_per_run: 1 type: 'microbenchmark' + openfeature_flagevaluation: + paths: + - '@bootstrap' + - '@core' + - '@openfeature' + - benchmarks/openfeature_flagevaluation/* + - benchmarks/suitespec.yml + cpus_per_run: 1 + type: 'microbenchmark' appsec_iast_aspects: paths: - '@bootstrap' diff --git a/ddtrace/internal/openfeature/_flagevaluation_hook.py b/ddtrace/internal/openfeature/_flagevaluation_hook.py index 065dbb23536..05ddb16f802 100644 --- a/ddtrace/internal/openfeature/_flagevaluation_hook.py +++ b/ddtrace/internal/openfeature/_flagevaluation_hook.py @@ -1,11 +1,12 @@ """ FlagEvaluationHook — OpenFeature `finally_after` hook for EVP flagevaluation emission. -Implements the frozen FANOUT-CONTRACT hook design: +Hook design: - Cheap capture only in finally_after (no aggregation, no serialization, no I/O). - Non-blocking enqueue to FlagEvaluationWriter. -- Covers success, error, and default eval paths (reviewer concern #7 3385309423). -- Does NOT replace or modify the OTel FlagEvalHook in _flageval_metrics.py (PRES-01). +- The finally_after stage covers success, error, and default eval paths. +- Does NOT replace or modify the OTel FlagEvalHook in _flageval_metrics.py (the existing + feature_flag.evaluations OTel path is preserved unchanged). """ import time @@ -17,12 +18,10 @@ from openfeature.hook import HookHints from ddtrace.internal.logger import get_logger -from ddtrace.internal.openfeature._flagevaluation_writer import ( - EVAL_TIMESTAMP_METADATA_KEY, - METADATA_ALLOCATION_KEY, - FlagEvaluationWriter, - _EvalEvent, -) +from ddtrace.internal.openfeature._flagevaluation_writer import EVAL_TIMESTAMP_METADATA_KEY +from ddtrace.internal.openfeature._flagevaluation_writer import METADATA_ALLOCATION_KEY +from ddtrace.internal.openfeature._flagevaluation_writer import FlagEvaluationWriter +from ddtrace.internal.openfeature._flagevaluation_writer import _EvalEvent logger = get_logger(__name__) @@ -32,7 +31,7 @@ class FlagEvaluationHook(Hook): """ OpenFeature Hook that enqueues cheap evaluation snapshots for EVP aggregation. - Implements `finally_after` (covers success/error/default — reviewer concern #7). + Implements `finally_after` (covers the success, error, and default eval paths). Does NO aggregation, serialization, or I/O on the hook thread. All heavy work is deferred to FlagEvaluationWriter's background periodic worker. """ @@ -56,8 +55,8 @@ def finally_after( Eval-time: uses details.flag_metadata["dd.eval.timestamp_ms"] when present (stamped by the provider at eval entry); falls back to hook-fire time. - Runtime-default: True when details.value is None (absent variant — reviewer - concern #5 3395344504). + Runtime-default: True when the variant is absent (details.variant is None), + which detects a runtime default independent of the reason string. Attrs: shallow copy of the evaluation context attributes dict so the hook returns immediately and the worker can safely iterate attrs off-path. @@ -66,7 +65,7 @@ def finally_after( flag_key: str = hook_context.flag_key or "" # Extract allocation_key from flag_metadata (same key as METADATA_ALLOCATION_KEY). - metadata: dict = details.flag_metadata or {} + metadata: dict[str, typing.Any] = details.flag_metadata or {} allocation_key: str = "" ak = metadata.get(METADATA_ALLOCATION_KEY) if isinstance(ak, str) and ak: @@ -79,17 +78,13 @@ def finally_after( else: eval_time_ms = int(time.time() * 1000) - # Variant: None/absent signals runtime_default (reviewer concern #5). + # Variant: None/absent signals a runtime default. variant = details.variant or "" runtime_default = details.variant is None # Reason: normalise to upper-case string. if details.reason is not None: - reason_raw = ( - details.reason.value - if hasattr(details.reason, "value") - else str(details.reason) - ) + reason_raw = details.reason.value if hasattr(details.reason, "value") else str(details.reason) reason = str(reason_raw).upper() else: reason = "UNKNOWN" @@ -99,7 +94,7 @@ def finally_after( if eval_ctx is not None: targeting_key = eval_ctx.targeting_key or "" # Shallow copy so we don't hold a reference to the caller's live dict. - attrs: typing.Dict[str, typing.Any] = dict(eval_ctx.attributes or {}) + attrs: dict[str, typing.Any] = dict(eval_ctx.attributes or {}) else: targeting_key = "" attrs = {} diff --git a/ddtrace/internal/openfeature/_flagevaluation_writer.py b/ddtrace/internal/openfeature/_flagevaluation_writer.py index db60d1046f2..df773f05830 100644 --- a/ddtrace/internal/openfeature/_flagevaluation_writer.py +++ b/ddtrace/internal/openfeature/_flagevaluation_writer.py @@ -1,22 +1,21 @@ """ FlagEvaluationWriter — SDK-native EVP `flagevaluation` writer for dd-trace-py. -Implements the frozen FANOUT-CONTRACT two-tier aggregation design (full → degraded → -drop-counted). No libdatadog connection; uses the same PeriodicService + get_connection() -transport path as ExposureWriter in writer.py. +Implements a two-tier aggregation design (full → degraded → drop-counted). Uses the same +PeriodicService + get_connection() transport path as the exposure writer in writer.py. -Key design properties (port of Go PR #4886): -- Async, best-effort recording: finally_after hook does cheap capture + non-blocking +Key design properties: +- Async, best-effort recording: the finally_after hook does cheap capture + non-blocking enqueue only. All flatten/prune/aggregate/flush work happens in the background worker. -- Two-tier aggregation (full → degraded → drop-counted). No ultra-degraded tier. -- Canonical context key: sorted, type-tagged, length-delimited — NOT a hash (reviewer - concern #3 3395004724). Distinct contexts always produce distinct keys. -- Context pruning: ≤256 fields, string values ≤256 chars (reviewer concern #1). +- Two-tier aggregation (full → degraded → drop-counted). +- Canonical context key: sorted, type-tagged, length-delimited — NOT a hash, so distinct + contexts always produce distinct keys with no collisions. +- Context pruning: ≤256 fields, string values ≤256 chars. - Caps: GLOBAL_CAP=131_072 (full-tier), PER_FLAG_CAP=10_000 (per-flag full-tier), - DEGRADED_CAP=32_768 (degraded-tier). Overflow: drop-and-count (reviewer concern #8). + DEGRADED_CAP=32_768 (degraded-tier). Beyond the degraded cap: drop-and-count. - Eval-time from metadata key "dd.eval.timestamp_ms"; fallback to enqueue-time. -- First/last evaluation: min/max under lock (reviewer concern #4). -- runtime_default_used: True when variant is None/absent (reviewer concern #5). +- First/last evaluation: min/max under lock. +- runtime_default_used: True when variant is None/absent. - Killswitch: DD_FLAGGING_EVALUATION_COUNTS_ENABLED (default on); gates EVP path only. - Non-blocking enqueue: queue.Queue(QUEUE_SIZE); drops + counts on queue.Full. """ @@ -28,13 +27,12 @@ import time import typing +from ddtrace import config as ddconfig from ddtrace.internal.logger import get_logger from ddtrace.internal.periodic import PeriodicService from ddtrace.internal.settings._agent import config as agent_config from ddtrace.internal.utils.http import get_connection -from ddtrace import config as ddconfig - logger = get_logger(__name__) @@ -47,10 +45,10 @@ MAX_CONTEXT_FIELDS = 256 MAX_FIELD_LENGTH = 256 -# Aggregation caps (sized for ≥2,500-flag scale target per FANOUT-CONTRACT). -GLOBAL_CAP = 131_072 # bounds full-tier buckets -PER_FLAG_CAP = 10_000 # bounds full-tier buckets per flag -DEGRADED_CAP = 32_768 # bounds degraded-tier buckets; overflow is drop-counted +# Aggregation caps (sized for a ≥2,500-flag scale target). +GLOBAL_CAP = 131_072 # bounds full-tier buckets +PER_FLAG_CAP = 10_000 # bounds full-tier buckets per flag +DEGRADED_CAP = 32_768 # bounds degraded-tier buckets; overflow is drop-counted # Async hand-off queue size. QUEUE_SIZE = 4_096 @@ -76,6 +74,7 @@ # Canonical context key — type-tagged, length-delimited, sorted # --------------------------------------------------------------------------- + def _length_delimited(data: bytes) -> bytes: """Prepend a fixed 8-byte big-endian length to data.""" return struct.pack(">Q", len(data)) + data @@ -102,7 +101,7 @@ def _encode_context_value(v: typing.Any) -> bytes: return tag + _length_delimited(raw) -def canonical_context_key(attrs: typing.Dict[str, typing.Any]) -> str: +def canonical_context_key(attrs: dict[str, typing.Any]) -> str: """ Build the EXACT, comparable canonical-context string key for a pruned context dict. @@ -111,8 +110,7 @@ def canonical_context_key(attrs: typing.Dict[str, typing.Any]) -> str: length_delimited(key_bytes) + type_tag_byte + length_delimited(value_bytes) Because the full encoding is used as the map key (not a hash), distinct contexts - ALWAYS produce distinct keys — no hash collisions, no misattribution (reviewer - concern #3 3395004724). + ALWAYS produce distinct keys — no hash collisions, no misattribution. Returns "" for empty/None attrs. """ @@ -125,7 +123,7 @@ def canonical_context_key(attrs: typing.Dict[str, typing.Any]) -> str: return b"".join(parts).decode("latin-1") # lossless binary → str for dict key -def flatten_and_prune_context(attrs: typing.Dict[str, typing.Any]) -> typing.Dict[str, typing.Any]: +def flatten_and_prune_context(attrs: dict[str, typing.Any]) -> dict[str, typing.Any]: """ Flatten nested dicts (dot-notation) and apply 256-field / 256-char prune. @@ -138,7 +136,7 @@ def flatten_and_prune_context(attrs: typing.Dict[str, typing.Any]) -> typing.Dic if not attrs: return {} - flat: typing.Dict[str, typing.Any] = {} + flat: dict[str, typing.Any] = {} _flatten_recursive("", attrs, flat) if not flat: return {} @@ -154,7 +152,7 @@ def flatten_and_prune_context(attrs: typing.Dict[str, typing.Any]) -> typing.Dic return flat # Deterministic prune: sort keys, keep first MAX_CONTEXT_FIELDS non-oversized values. - out: typing.Dict[str, typing.Any] = {} + out: dict[str, typing.Any] = {} count = 0 for k in sorted(flat.keys()): if count >= MAX_CONTEXT_FIELDS: @@ -167,7 +165,7 @@ def flatten_and_prune_context(attrs: typing.Dict[str, typing.Any]) -> typing.Dic return out -def _flatten_recursive(prefix: str, attrs: typing.Any, out: typing.Dict[str, typing.Any]) -> None: +def _flatten_recursive(prefix: str, attrs: typing.Any, out: dict[str, typing.Any]) -> None: """Recursively flatten nested dicts into dot-notation keys.""" if not isinstance(attrs, dict): if prefix: @@ -185,21 +183,35 @@ def _flatten_recursive(prefix: str, attrs: typing.Any, out: typing.Dict[str, typ # Internal types # --------------------------------------------------------------------------- + class _Entry: """Per-bucket aggregation state.""" - __slots__ = ("count", "first_evaluation", "last_evaluation", "runtime_default", - "targeting_key", "context_attrs", "error_message") - - def __init__(self, now_ms: int, runtime_default: bool, targeting_key: str, - context_attrs: typing.Dict[str, typing.Any], error_message: str) -> None: + __slots__ = ( + "count", + "first_evaluation", + "last_evaluation", + "runtime_default", + "targeting_key", + "context_attrs", + "error_message", + ) + + def __init__( + self, + now_ms: int, + runtime_default: bool, + targeting_key: str, + context_attrs: dict[str, typing.Any], + error_message: str, + ) -> None: self.count: int = 1 self.first_evaluation: int = now_ms self.last_evaluation: int = now_ms self.runtime_default: bool = runtime_default # Full-tier only: self.targeting_key: str = targeting_key - self.context_attrs: typing.Dict[str, typing.Any] = context_attrs + self.context_attrs: dict[str, typing.Any] = context_attrs self.error_message: str = error_message def observe(self, now_ms: int) -> None: @@ -213,12 +225,13 @@ def observe(self, now_ms: int) -> None: class _EvalEvent(typing.NamedTuple): """Minimal snapshot handed from finally_after to the background worker.""" + flag_key: str - variant: str # "" when absent (= runtime_default) + variant: str # "" when absent (= runtime_default) allocation_key: str reason: str targeting_key: str - attrs: typing.Dict[str, typing.Any] # shallow copy from evaluation context + attrs: dict[str, typing.Any] # shallow copy from evaluation context runtime_default: bool error_message: str eval_time_ms: int @@ -228,11 +241,12 @@ class _EvalEvent(typing.NamedTuple): # FlagEvaluationWriter # --------------------------------------------------------------------------- + class FlagEvaluationWriter(PeriodicService): """ SDK-native EVP `flagevaluation` writer. - Implements the two-tier aggregation design from the frozen FANOUT-CONTRACT: + Two-tier aggregation design: - full-tier: keyed by (flag, variant, allocation, reason, targeting_key, canonical_context) - degraded-tier: keyed by (flag, variant, allocation, reason) — same cardinality as OTel - drop-counted: beyond degradedCap, increment _dropped_degraded_overflow @@ -246,24 +260,25 @@ def __init__(self, interval: float = DEFAULT_FLUSH_INTERVAL, timeout: float = 2. self._timeout = timeout self._intake: str = agent_config.trace_agent_url self._endpoint: str = FLAGEVALUATIONS_ENDPOINT - self._headers: typing.Dict[str, str] = { + self._headers: dict[str, str] = { "Content-Type": "application/json", EVP_SUBDOMAIN_HEADER_NAME: EVP_SUBDOMAIN_VALUE, } - # Async hand-off queue: non-blocking, bounded (reviewer concern #7). - self._queue: queue.Queue = queue.Queue(maxsize=QUEUE_SIZE) + # Async hand-off queue: non-blocking, bounded. + self._queue: "queue.Queue[_EvalEvent]" = queue.Queue(maxsize=QUEUE_SIZE) - # Aggregation maps (drained under _lock on each periodic() call). + # Aggregation maps (drained under _lock on each periodic() call). Keys are tuples of + # the enumerable dimensions plus (full tier) the canonical context string. self._lock = threading.Lock() - self._full: typing.Dict[tuple, _Entry] = {} - self._degraded: typing.Dict[tuple, _Entry] = {} - self._per_flag_count: typing.Dict[str, int] = {} # flag_key → full-tier bucket count + self._full: dict[tuple[typing.Any, ...], _Entry] = {} + self._degraded: dict[tuple[typing.Any, ...], _Entry] = {} + self._per_flag_count: dict[str, int] = {} # flag_key → full-tier bucket count self._global_count: int = 0 # Observable drop counters. - self._dropped_queue: int = 0 # queue.Full drops (hook path) - self._dropped_degraded_overflow: int = 0 # degraded-cap overflow drops + self._dropped_queue: int = 0 # queue.Full drops (hook path) + self._dropped_degraded_overflow: int = 0 # degraded-cap overflow drops # ------------------------------------------------------------------ # Public API used by FlagEvaluationHook @@ -382,7 +397,7 @@ def periodic(self) -> None: # 4. Encode and POST. try: - context: typing.Dict[str, str] = {} + context: dict[str, str] = {} if ddconfig.service: context["service"] = ddconfig.service if ddconfig.env: @@ -390,7 +405,7 @@ def periodic(self) -> None: if ddconfig.version: context["version"] = ddconfig.version - payload_obj: typing.Dict[str, typing.Any] = {"flagEvaluations": events} + payload_obj: dict[str, typing.Any] = {"flagEvaluations": events} if context: payload_obj["context"] = context @@ -401,8 +416,8 @@ def periodic(self) -> None: self._send_payload(payload, len(events)) - def on_shutdown(self) -> None: - """Final flush on service shutdown.""" + def on_shutdown(self): + """Final flush on service shutdown — drains the queue and flushes before exit.""" self.periodic() # ------------------------------------------------------------------ @@ -500,17 +515,23 @@ def _send_payload(self, payload: bytes, num_events: int) -> None: if resp.status >= 300: logger.debug( "FlagEvaluationWriter: failed to send %d events to %s, status=%d: %s", - num_events, self._intake, resp.status, resp.read(), + num_events, + self._intake, + resp.status, + resp.read(), ) else: logger.debug( "FlagEvaluationWriter: sent %d flag evaluation events to %s", - num_events, self._intake, + num_events, + self._intake, ) except Exception: logger.debug( "FlagEvaluationWriter: error sending %d events to %s", - num_events, self._intake, exc_info=True, + num_events, + self._intake, + exc_info=True, ) finally: conn.close() @@ -520,7 +541,8 @@ def _send_payload(self, payload: bytes, num_events: int) -> None: # Payload helpers # --------------------------------------------------------------------------- -def _base_event(flag_key: str, entry: "_Entry", now_ms: int) -> typing.Dict[str, typing.Any]: + +def _base_event(flag_key: str, entry: "_Entry", now_ms: int) -> dict[str, typing.Any]: """Build the required-fields-only event dict for a single aggregation entry.""" return { "timestamp": now_ms, diff --git a/ddtrace/internal/openfeature/_provider.py b/ddtrace/internal/openfeature/_provider.py index 6156ffd0c8c..0764e69b196 100644 --- a/ddtrace/internal/openfeature/_provider.py +++ b/ddtrace/internal/openfeature/_provider.py @@ -8,7 +8,6 @@ from collections import OrderedDict from collections.abc import MutableMapping from importlib.metadata import version -import os import threading import time import typing @@ -29,16 +28,15 @@ from ddtrace.internal.openfeature._flageval_metrics import FlagEvalHook from ddtrace.internal.openfeature._flageval_metrics import FlagEvalMetrics from ddtrace.internal.openfeature._flagevaluation_hook import FlagEvaluationHook -from ddtrace.internal.openfeature._flagevaluation_writer import ( - EVAL_TIMESTAMP_METADATA_KEY, - FlagEvaluationWriter, -) +from ddtrace.internal.openfeature._flagevaluation_writer import EVAL_TIMESTAMP_METADATA_KEY +from ddtrace.internal.openfeature._flagevaluation_writer import FlagEvaluationWriter from ddtrace.internal.openfeature._native import VariationType from ddtrace.internal.openfeature._native import resolve_flag from ddtrace.internal.openfeature.writer import get_exposure_writer from ddtrace.internal.openfeature.writer import start_exposure_writer from ddtrace.internal.openfeature.writer import stop_exposure_writer from ddtrace.internal.service import ServiceStatusError +from ddtrace.internal.settings.openfeature import OpenFeatureConfig from ddtrace.internal.settings.openfeature import config as ffe_config @@ -132,13 +130,18 @@ def __init__(self, *args: typing.Any, **kwargs: typing.Any): # EVP flagevaluation writer + hook — gated by DD_FLAGGING_EVALUATION_COUNTS_ENABLED # (default on). Gates ONLY the EVP path; the OTel path above is always registered - # when the provider is enabled (PRES-01 non-regression). - # AIDEV-NOTE: killswitch env var checked at init time so that tests can override with - # os.environ and create a fresh DataDogProvider to verify gating behavior. + # when the provider is enabled (preserves the existing OTel non-regression). + # AIDEV-NOTE: the killswitch is read through the ddtrace config system + # (OpenFeatureConfig.flagging_evaluation_counts_enabled, registered in + # supported-configurations.json) rather than raw os.environ. A fresh + # OpenFeatureConfig instance is constructed here so the value reflects the current + # environment at provider-construction time (the config var parses the live + # environment via the DDConfig var system), which keeps the killswitch overridable + # per-instance in tests. self._flagevaluation_writer: typing.Optional[FlagEvaluationWriter] = None self._flagevaluation_hook: typing.Optional[FlagEvaluationHook] = None - evp_counts_enabled = os.environ.get("DD_FLAGGING_EVALUATION_COUNTS_ENABLED", "true").lower() - if self._enabled and evp_counts_enabled != "false": + evp_counts_enabled = OpenFeatureConfig().flagging_evaluation_counts_enabled + if self._enabled and evp_counts_enabled: self._flagevaluation_writer = FlagEvaluationWriter() self._flagevaluation_hook = FlagEvaluationHook(self._flagevaluation_writer) @@ -154,11 +157,11 @@ def get_provider_hooks(self) -> list[typing.Any]: every flag evaluation via the finally_after hook stage. Hook ordering: - 1. OTel FlagEvalHook (_flageval_metrics.py) — always registered when provider is - enabled; emits feature_flag.evaluations OTel counter (PRES-01 preservation). + 1. OTel FlagEvalHook (_flageval_metrics.py) — always registered when the provider + is enabled; emits the feature_flag.evaluations OTel counter (preserved unchanged). 2. FlagEvaluationHook (_flagevaluation_hook.py) — registered only when - DD_FLAGGING_EVALUATION_COUNTS_ENABLED != "false"; enqueues cheap snapshots - to FlagEvaluationWriter for EVP flagevaluation emission. + DD_FLAGGING_EVALUATION_COUNTS_ENABLED is enabled (default on); enqueues cheap + snapshots to FlagEvaluationWriter for EVP flagevaluation emission. """ hooks: list[typing.Any] = [] if self._flag_eval_hook is not None: diff --git a/ddtrace/internal/settings/_supported_configurations.py b/ddtrace/internal/settings/_supported_configurations.py index 8722d96e719..1a429ad1ca4 100644 --- a/ddtrace/internal/settings/_supported_configurations.py +++ b/ddtrace/internal/settings/_supported_configurations.py @@ -220,6 +220,7 @@ "DD_FAST_BUILD", "DD_FFE_INTAKE_ENABLED", "DD_FFE_INTAKE_HEARTBEAT_INTERVAL", + "DD_FLAGGING_EVALUATION_COUNTS_ENABLED", "DD_FLASK_CACHE_SERVICE", "DD_FLASK_SERVICE", "DD_FOO_SERVICE", diff --git a/ddtrace/internal/settings/openfeature.py b/ddtrace/internal/settings/openfeature.py index a6324d1ccb8..d5f740f6445 100644 --- a/ddtrace/internal/settings/openfeature.py +++ b/ddtrace/internal/settings/openfeature.py @@ -17,6 +17,15 @@ class OpenFeatureConfig(DDConfig): default=False, ) + # Killswitch for the EVP `flagevaluation` evaluation-counts path. Default on; gates + # ONLY the EVP flagevaluation writer/hook. The existing OTel `feature_flag.evaluations` + # path is unaffected by this flag. + flagging_evaluation_counts_enabled = DDConfig.var( + bool, + "DD_FLAGGING_EVALUATION_COUNTS_ENABLED", + default=True, + ) + # Feature flag exposure intake configuration ffe_intake_enabled = DDConfig.var( bool, @@ -41,6 +50,7 @@ class OpenFeatureConfig(DDConfig): _openfeature_config_keys = [ "experimental_flagging_provider_enabled", + "flagging_evaluation_counts_enabled", "ffe_intake_enabled", "ffe_intake_heartbeat_interval", "initialization_timeout_ms", diff --git a/mypy.ini b/mypy.ini index f799f14f30b..01d69ca5c6c 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1449,6 +1449,15 @@ disallow_incomplete_defs = false [mypy-ddtrace.internal.openfeature._flageval_metrics] disallow_subclassing_any = false +[mypy-ddtrace.internal.openfeature._flagevaluation_hook] +disallow_subclassing_any = false + +[mypy-ddtrace.internal.openfeature._flagevaluation_writer] +disallow_untyped_calls = false +disallow_untyped_defs = false +disallow_incomplete_defs = false +check_untyped_defs = false + [mypy-ddtrace.internal.openfeature._native] disallow_untyped_calls = false disallow_untyped_defs = false diff --git a/supported-configurations.json b/supported-configurations.json index 33729e9e0ac..bf832fd06ad 100644 --- a/supported-configurations.json +++ b/supported-configurations.json @@ -1640,6 +1640,14 @@ "default": "1.0" } ], + "DD_FLAGGING_EVALUATION_COUNTS_ENABLED": [ + { + "implementation": "A", + "type": "boolean", + "default": "true", + "experimental": true + } + ], "DD_FLASK_CACHE_SERVICE": [ { "implementation": "A", diff --git a/tests/openfeature/test_flagevaluation_e2e.py b/tests/openfeature/test_flagevaluation_e2e.py new file mode 100644 index 00000000000..03e7686a4ba --- /dev/null +++ b/tests/openfeature/test_flagevaluation_e2e.py @@ -0,0 +1,205 @@ +""" +End-to-end tests for the EVP `flagevaluation` path through the real provider eval path. + +These cover the lifecycle/exit-path gates that unit tests of the hook/writer in isolation +cannot prove: + +- G11: the EVP hook actually fires on the provider's REAL OpenFeature evaluation entrypoint + (driven through the OpenFeature client, not by calling the hook directly). +- G12: ALL evaluation exit paths are covered — success, native engine error, runtime default + (flag-not-found / no-config), and disabled. +- The OTel `feature_flag.evaluations` path is preserved alongside the EVP path (non-regression). +""" + +from unittest import mock + +from openfeature import api +from openfeature.evaluation_context import EvaluationContext +import pytest + +from ddtrace.internal.openfeature._config import _set_ffe_config +from ddtrace.internal.openfeature._native import process_ffe_configuration +from ddtrace.openfeature import DataDogProvider +from tests.openfeature.config_helpers import create_boolean_flag +from tests.openfeature.config_helpers import create_config +from tests.openfeature.config_helpers import create_string_flag +from tests.utils import override_global_config + + +@pytest.fixture(autouse=True) +def clear_config(): + _set_ffe_config(None) + yield + _set_ffe_config(None) + + +@pytest.fixture +def provider_and_client(): + """Set up a DataDogProvider with the EVP path enabled, returning (provider, client). + + The writer's background thread is NOT started (we drive aggregation synchronously via + periodic()), so the eval path enqueues and we drain deterministically in the test. + """ + with override_global_config({"experimental_flagging_provider_enabled": True}): + provider = DataDogProvider() + # Sanity: the EVP writer/hook are wired (killswitch default on). + assert provider._flagevaluation_writer is not None + assert provider._flagevaluation_hook is not None + + api.set_provider(provider) + client = api.get_client() + try: + yield provider, client + finally: + api.shutdown() + + +def _drain(provider): + """Aggregate everything the eval path enqueued, returning the emitted rows.""" + writer = provider._flagevaluation_writer + with mock.patch.object(writer, "_send_payload") as mock_send: + writer.periodic() + if not mock_send.called: + return [] + import json + + decoded = json.loads(mock_send.call_args[0][0]) + return decoded.get("flagEvaluations", []) + + +class TestEVPHookFiresOnRealEvalPath: + """G11: the EVP hook fires on the provider's real evaluation entrypoint.""" + + def test_evp_hook_registered_in_provider_hooks(self, provider_and_client): + provider, _ = provider_and_client + from ddtrace.internal.openfeature._flagevaluation_hook import FlagEvaluationHook + + hooks = provider.get_provider_hooks() + assert any(isinstance(h, FlagEvaluationHook) for h in hooks) + + def test_success_eval_enqueues_and_emits_row(self, provider_and_client): + provider, client = provider_and_client + config = create_config(create_boolean_flag("evp-success", enabled=True, default_value=True)) + process_ffe_configuration(config) + + # Real client eval — the OpenFeature SDK runs the registered finally_after hook. + assert client.get_boolean_value("evp-success", False) is True + + rows = _drain(provider) + keys = {r["flag"]["key"] for r in rows} + assert "evp-success" in keys + row = next(r for r in rows if r["flag"]["key"] == "evp-success") + assert row["evaluation_count"] >= 1 + # Successful eval -> a real variant object, no runtime_default. + assert "variant" in row and set(row["variant"].keys()) == {"key"} + assert row.get("runtime_default_used", False) is False + + def test_variant_is_resolution_variant_not_value(self, provider_and_client): + """G1 end-to-end: emitted variant == resolution variant key, distinct from value.""" + provider, client = provider_and_client + # String flag: value "blue", variant key "blue" (here equal); use details to confirm + # the EVP row variant matches details.variant exactly. + config = create_config(create_string_flag("evp-variant", "blue", enabled=True)) + process_ffe_configuration(config) + + details = client.get_string_details("evp-variant", "red") + rows = _drain(provider) + row = next(r for r in rows if r["flag"]["key"] == "evp-variant") + assert row["variant"]["key"] == details.variant + + +class TestEVPExitPathsCovered: + """G12: success / engine-error / runtime-default / disabled exit paths are all captured.""" + + def test_flag_not_found_runtime_default_path(self, provider_and_client): + provider, client = provider_and_client + # A config is loaded but the requested flag isn't in it -> native FlagNotFound -> ERROR. + config = create_config(create_boolean_flag("present-flag", enabled=True)) + process_ffe_configuration(config) + + assert client.get_boolean_value("absent-flag", False) is False + + rows = _drain(provider) + row = next((r for r in rows if r["flag"]["key"] == "absent-flag"), None) + assert row is not None, "flag-not-found eval must still emit a flagevaluation row" + # No variant -> runtime_default_used True. + assert row.get("runtime_default_used") is True + assert "variant" not in row + + def test_no_config_provider_not_ready_path(self, provider_and_client): + provider, client = provider_and_client + _set_ffe_config(None) # no RC config at all -> PROVIDER_NOT_READY error path + + assert client.get_boolean_value("no-config-flag", True) is True + + rows = _drain(provider) + row = next((r for r in rows if r["flag"]["key"] == "no-config-flag"), None) + assert row is not None, "no-config eval must still emit a flagevaluation row" + assert row.get("runtime_default_used") is True + + def test_type_mismatch_error_path(self, provider_and_client): + provider, client = provider_and_client + config = create_config(create_string_flag("str-flag", "hello", enabled=True)) + process_ffe_configuration(config) + + # Evaluate a string flag as boolean -> type mismatch ERROR. + assert client.get_boolean_value("str-flag", False) is False + + rows = _drain(provider) + row = next((r for r in rows if r["flag"]["key"] == "str-flag"), None) + assert row is not None, "type-mismatch eval must still emit a flagevaluation row" + assert row.get("runtime_default_used") is True + + def test_disabled_flag_path(self, provider_and_client): + provider, client = provider_and_client + config = create_config(create_boolean_flag("disabled-flag", enabled=False, default_value=False)) + process_ffe_configuration(config) + + assert client.get_boolean_value("disabled-flag", False) is False + + rows = _drain(provider) + # Disabled flag still produces a flagevaluation row. + row = next((r for r in rows if r["flag"]["key"] == "disabled-flag"), None) + assert row is not None, "disabled-flag eval must still emit a flagevaluation row" + + def test_targeting_key_and_context_captured_on_success(self, provider_and_client): + provider, client = provider_and_client + config = create_config(create_boolean_flag("ctx-flag", enabled=True, default_value=True)) + process_ffe_configuration(config) + + ctx = EvaluationContext(targeting_key="user-77", attributes={"tier": "gold"}) + client.get_boolean_value("ctx-flag", False, ctx) + + rows = _drain(provider) + row = next(r for r in rows if r["flag"]["key"] == "ctx-flag") + assert row.get("targeting_key") == "user-77" + assert row["context"]["evaluation"]["tier"] == "gold" + + +class TestOTelNonRegressionAlongsideEVP: + """The EVP path must NOT change which hooks the provider registers for OTel (non-regression).""" + + def test_both_otel_and_evp_hooks_registered(self, provider_and_client): + provider, _ = provider_and_client + from ddtrace.internal.openfeature._flageval_metrics import FlagEvalHook + from ddtrace.internal.openfeature._flagevaluation_hook import FlagEvaluationHook + + hooks = provider.get_provider_hooks() + assert any(isinstance(h, FlagEvalHook) for h in hooks), "OTel hook must remain registered" + assert any(isinstance(h, FlagEvaluationHook) for h in hooks), "EVP hook must be registered" + + def test_eval_drives_otel_record_and_evp_enqueue_together(self, provider_and_client): + """A single eval feeds BOTH the OTel metric record and the EVP enqueue.""" + provider, client = provider_and_client + config = create_config(create_boolean_flag("dual-flag", enabled=True, default_value=True)) + process_ffe_configuration(config) + + # Spy the OTel metrics record and the EVP writer enqueue. + with ( + mock.patch.object(provider._flag_eval_metrics, "record") as otel_record, + mock.patch.object(provider._flagevaluation_writer, "enqueue") as evp_enqueue, + ): + client.get_boolean_value("dual-flag", False) + + assert otel_record.called, "OTel feature_flag.evaluations record must still fire" + assert evp_enqueue.called, "EVP flagevaluation enqueue must fire" diff --git a/tests/openfeature/test_flagevaluation_hook.py b/tests/openfeature/test_flagevaluation_hook.py index 5f922d7441f..2954996a847 100644 --- a/tests/openfeature/test_flagevaluation_hook.py +++ b/tests/openfeature/test_flagevaluation_hook.py @@ -1,25 +1,25 @@ """ Tests for FlagEvaluationHook — finally_after cheap capture + non-blocking enqueue. -Validates FANOUT-CONTRACT hook design: +Validates the hook design: - finally_after does cheap capture only (no aggregation, no I/O) -- variant=None → runtime_default_used True (reviewer concern #5) +- variant=None → runtime_default_used True - eval_time_ms from metadata["dd.eval.timestamp_ms"] when present; fallback to hook-fire time -- DD_FLAGGING_EVALUATION_COUNTS_ENABLED killswitch gates EVP path only (PRES-01) +- DD_FLAGGING_EVALUATION_COUNTS_ENABLED killswitch gates the EVP path only """ +import json import os import time import typing from unittest import mock -import pytest - from openfeature.evaluation_context import EvaluationContext from openfeature.flag_evaluation import FlagEvaluationDetails from openfeature.flag_evaluation import FlagType from openfeature.flag_evaluation import Reason from openfeature.hook import HookContext +import pytest def _make_hook_context( @@ -57,17 +57,18 @@ def _make_details( @pytest.fixture def writer(): from ddtrace.internal.openfeature._flagevaluation_writer import FlagEvaluationWriter + return mock.MagicMock(spec=FlagEvaluationWriter) @pytest.fixture def hook(writer): from ddtrace.internal.openfeature._flagevaluation_hook import FlagEvaluationHook + return FlagEvaluationHook(writer=writer) class TestFlagEvaluationHook: - def test_finally_after_calls_writer_enqueue_once(self, hook, writer): """finally_after must call writer.enqueue exactly once per evaluation.""" hc = _make_hook_context() @@ -90,7 +91,7 @@ def test_finally_after_enqueues_correct_variant(self, hook, writer): assert event.variant == "control" def test_finally_after_none_variant_sets_runtime_default(self, hook, writer): - """None variant → runtime_default=True (reviewer concern #5 3395344504).""" + """None variant → runtime_default=True.""" hc = _make_hook_context() details = _make_details(variant=None) hook.finally_after(hc, details, {}) @@ -157,16 +158,14 @@ def test_finally_after_extracts_allocation_key(self, hook, writer): assert event.allocation_key == "alloc-xyz" def test_finally_after_does_no_aggregation_on_hook_thread(self, hook, writer): - """The hook must call enqueue only — not build payloads or aggregate (reviewer concern #7).""" + """The hook must call enqueue only — not build payloads or aggregate.""" # writer.enqueue is a mock; the only call from finally_after must be enqueue. hc = _make_hook_context() details = _make_details() hook.finally_after(hc, details, {}) # Confirm ONLY enqueue was called on the writer (no periodic, no aggregate, no send). called_methods = {c[0] for c in writer.method_calls} - assert called_methods == {"enqueue"}, ( - f"Expected only enqueue to be called, got: {called_methods}" - ) + assert called_methods == {"enqueue"}, f"Expected only enqueue to be called, got: {called_methods}" def test_finally_after_does_not_propagate_exceptions(self, hook, writer): """Hook must swallow exceptions — best-effort telemetry.""" @@ -177,17 +176,121 @@ def test_finally_after_does_not_propagate_exceptions(self, hook, writer): hook.finally_after(hc, details, {}) -class TestKillswitchGating: +class TestAsyncBoundary: + """G2: prove the hook does NOT aggregate on the eval call path. + + The hook may only enqueue a cheap snapshot; flatten/prune/canonical-key/aggregate + must run later in the writer's background worker, never on the eval thread. + """ + + def test_aggregate_not_called_on_hook_path(self): + """Spy on the REAL writer's _aggregate — it must NOT run during finally_after.""" + from ddtrace.internal.openfeature._flagevaluation_hook import FlagEvaluationHook + from ddtrace.internal.openfeature._flagevaluation_writer import FlagEvaluationWriter + + real_writer = FlagEvaluationWriter(interval=10.0) + hook = FlagEvaluationHook(writer=real_writer) + + with mock.patch.object(real_writer, "_aggregate", wraps=real_writer._aggregate) as spy_aggregate: + hc = _make_hook_context(attrs={"tier": "premium", "region": "us"}) + details = _make_details(flag_metadata={"allocation_key": "a1"}) + hook.finally_after(hc, details, {}) + + # The event must be queued but NOT aggregated on the hook path. + spy_aggregate.assert_not_called() + assert real_writer._queue.qsize() == 1 + # The aggregation maps are still empty — no flatten/prune/key happened yet. + assert real_writer._full == {} + assert real_writer._degraded == {} + + # Aggregation only happens when the worker drains (periodic), off the hook path. + # _aggregate fires exactly once during the drain, producing one full-tier row. + with mock.patch.object(real_writer, "_aggregate", wraps=real_writer._aggregate) as spy_drain: + with mock.patch.object(real_writer, "_send_payload") as mock_send: + real_writer.periodic() + spy_drain.assert_called_once() + decoded = json.loads(mock_send.call_args[0][0]) + assert len(decoded["flagEvaluations"]) == 1 + + def test_canonical_key_not_computed_on_hook_path(self): + """canonical_context_key (the keying cost) must not run during finally_after.""" + from ddtrace.internal.openfeature._flagevaluation_hook import FlagEvaluationHook + import ddtrace.internal.openfeature._flagevaluation_writer as writer_mod + from ddtrace.internal.openfeature._flagevaluation_writer import FlagEvaluationWriter + real_writer = FlagEvaluationWriter(interval=10.0) + hook = FlagEvaluationHook(writer=real_writer) + + with mock.patch.object(writer_mod, "canonical_context_key", wraps=writer_mod.canonical_context_key) as spy_key: + hc = _make_hook_context(attrs={"a": "b"}) + hook.finally_after(hc, _make_details(), {}) + spy_key.assert_not_called() + + +class TestMetadataSourceMatchesOTelHook: + """G7: EVP hook reads allocation-key/eval metadata from the SAME source as the OTel hook. + + The existing OTel FlagEvalHook reads allocation_key from + ``details.flag_metadata[METADATA_ALLOCATION_KEY]``. The EVP hook must read from the + identical source so the two paths agree byte-for-byte on metadata. + """ + + def test_allocation_key_metadata_key_matches_otel_hook(self): + from ddtrace.internal.openfeature import _flageval_metrics + from ddtrace.internal.openfeature import _flagevaluation_writer + + # Same metadata key constant in both modules. + assert _flagevaluation_writer.METADATA_ALLOCATION_KEY == _flageval_metrics.METADATA_ALLOCATION_KEY + + def test_evp_hook_reads_allocation_from_details_flag_metadata(self, hook, writer): + """EVP hook reads allocation_key from details.flag_metadata (not hook_context).""" + from ddtrace.internal.openfeature._flageval_metrics import METADATA_ALLOCATION_KEY + + hc = _make_hook_context() + details = _make_details(flag_metadata={METADATA_ALLOCATION_KEY: "alloc-from-details"}) + hook.finally_after(hc, details, {}) + event = writer.enqueue.call_args[0][0] + assert event.allocation_key == "alloc-from-details" + + def test_otel_and_evp_hooks_extract_same_allocation_key(self): + """Drive both hooks with identical details; both must surface the same allocation key.""" + from ddtrace.internal.openfeature._flageval_metrics import METADATA_ALLOCATION_KEY + from ddtrace.internal.openfeature._flageval_metrics import FlagEvalHook + from ddtrace.internal.openfeature._flageval_metrics import FlagEvalMetrics + from ddtrace.internal.openfeature._flagevaluation_hook import FlagEvaluationHook + from ddtrace.internal.openfeature._flagevaluation_writer import FlagEvaluationWriter + + details = _make_details(flag_metadata={METADATA_ALLOCATION_KEY: "shared-alloc"}) + hc = _make_hook_context() + + # OTel side: capture what FlagEvalMetrics.record received as allocation_key. + metrics = mock.MagicMock(spec=FlagEvalMetrics) + otel_hook = FlagEvalHook(metrics) + otel_hook.finally_after(hc, details, {}) + otel_alloc = metrics.record.call_args.kwargs["allocation_key"] + + # EVP side: capture what the EVP hook enqueued as allocation_key. + evp_writer = mock.MagicMock(spec=FlagEvaluationWriter) + evp_hook = FlagEvaluationHook(evp_writer) + evp_hook.finally_after(hc, details, {}) + evp_alloc = evp_writer.enqueue.call_args[0][0].allocation_key + + assert otel_alloc == evp_alloc == "shared-alloc" + + +class TestKillswitchGating: def test_default_enabled_registers_evp_hook(self): """Default (no env var set) must register the EVP hook + writer.""" from ddtrace.internal.openfeature._flagevaluation_hook import FlagEvaluationHook + env = {k: v for k, v in os.environ.items() if k != "DD_FLAGGING_EVALUATION_COUNTS_ENABLED"} # No env var → enabled by default. with mock.patch.dict(os.environ, env, clear=True): from tests.utils import override_global_config + with override_global_config({"experimental_flagging_provider_enabled": True}): from ddtrace.internal.openfeature._provider import DataDogProvider + provider = DataDogProvider() assert provider._flagevaluation_writer is not None assert provider._flagevaluation_hook is not None @@ -197,18 +300,24 @@ def test_killswitch_false_does_not_register_evp_hook(self): """DD_FLAGGING_EVALUATION_COUNTS_ENABLED=false must suppress EVP hook (killswitch).""" with mock.patch.dict(os.environ, {"DD_FLAGGING_EVALUATION_COUNTS_ENABLED": "false"}): from tests.utils import override_global_config + with override_global_config({"experimental_flagging_provider_enabled": True}): from ddtrace.internal.openfeature._provider import DataDogProvider + provider = DataDogProvider() assert provider._flagevaluation_writer is None assert provider._flagevaluation_hook is None def test_killswitch_false_does_not_affect_otel_hook(self): - """Killswitch must not suppress the OTel FlagEvalHook (PRES-01).""" + """Killswitch must not suppress the OTel FlagEvalHook (OTel non-regression).""" with mock.patch.dict(os.environ, {"DD_FLAGGING_EVALUATION_COUNTS_ENABLED": "false"}): from tests.utils import override_global_config - with override_global_config({"experimental_flagging_provider_enabled": True, "_otel_metrics_enabled": True}): + + with override_global_config( + {"experimental_flagging_provider_enabled": True, "_otel_metrics_enabled": True} + ): from ddtrace.internal.openfeature._provider import DataDogProvider + provider = DataDogProvider() # OTel hook still present. assert provider._flag_eval_hook is not None @@ -223,8 +332,10 @@ def test_killswitch_enabled_true_registers_evp_hook(self): """DD_FLAGGING_EVALUATION_COUNTS_ENABLED=true must register the EVP hook.""" with mock.patch.dict(os.environ, {"DD_FLAGGING_EVALUATION_COUNTS_ENABLED": "true"}): from tests.utils import override_global_config + with override_global_config({"experimental_flagging_provider_enabled": True}): from ddtrace.internal.openfeature._provider import DataDogProvider + provider = DataDogProvider() assert provider._flagevaluation_writer is not None assert provider._flagevaluation_hook is not None diff --git a/tests/openfeature/test_flagevaluation_writer.py b/tests/openfeature/test_flagevaluation_writer.py index 39c4b74cf0d..2179d505bdc 100644 --- a/tests/openfeature/test_flagevaluation_writer.py +++ b/tests/openfeature/test_flagevaluation_writer.py @@ -1,7 +1,7 @@ """ Unit tests for FlagEvaluationWriter — two-tier aggregation, canonical key, EVP transport. -Tests validate the FANOUT-CONTRACT spec: +Tests validate the two-tier aggregation spec: - canonical_context_key: sorted, type-tagged, length-delimited (NOT a hash) - Two-tier aggregation (full → degraded → drop-counted) - Caps GLOBAL_CAP=131072 / PER_FLAG_CAP=10000 / DEGRADED_CAP=32768 @@ -12,34 +12,31 @@ """ import json -import queue import time from unittest import mock import pytest -from ddtrace.internal.openfeature._flagevaluation_writer import ( - DEGRADED_CAP, - EVAL_TIMESTAMP_METADATA_KEY, - FLAGEVALUATIONS_ENDPOINT, - GLOBAL_CAP, - MAX_CONTEXT_FIELDS, - MAX_FIELD_LENGTH, - PER_FLAG_CAP, - QUEUE_SIZE, - EVP_SUBDOMAIN_HEADER_NAME, - EVP_SUBDOMAIN_VALUE, - FlagEvaluationWriter, - _EvalEvent, - canonical_context_key, - flatten_and_prune_context, -) +from ddtrace.internal.openfeature._flagevaluation_writer import DEGRADED_CAP +from ddtrace.internal.openfeature._flagevaluation_writer import EVP_SUBDOMAIN_HEADER_NAME +from ddtrace.internal.openfeature._flagevaluation_writer import EVP_SUBDOMAIN_VALUE +from ddtrace.internal.openfeature._flagevaluation_writer import FLAGEVALUATIONS_ENDPOINT +from ddtrace.internal.openfeature._flagevaluation_writer import GLOBAL_CAP +from ddtrace.internal.openfeature._flagevaluation_writer import MAX_CONTEXT_FIELDS +from ddtrace.internal.openfeature._flagevaluation_writer import MAX_FIELD_LENGTH +from ddtrace.internal.openfeature._flagevaluation_writer import PER_FLAG_CAP +from ddtrace.internal.openfeature._flagevaluation_writer import QUEUE_SIZE +from ddtrace.internal.openfeature._flagevaluation_writer import FlagEvaluationWriter +from ddtrace.internal.openfeature._flagevaluation_writer import _EvalEvent +from ddtrace.internal.openfeature._flagevaluation_writer import canonical_context_key +from ddtrace.internal.openfeature._flagevaluation_writer import flatten_and_prune_context # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- + def _make_event( flag_key: str = "my-flag", variant: str = "on", @@ -76,6 +73,7 @@ def writer(): # canonical_context_key tests # --------------------------------------------------------------------------- + class TestCanonicalContextKey: def test_empty_attrs_returns_empty_string(self): assert canonical_context_key({}) == "" @@ -92,7 +90,7 @@ def test_different_insertion_order_same_key(self): assert canonical_context_key(a) == canonical_context_key(b) def test_int_vs_string_distinct_keys(self): - """int 1 vs string '1' must produce different keys (type-tagged, reviewer concern #3).""" + """int 1 vs string '1' must produce different keys (type-tagged).""" k_int = canonical_context_key({"x": 1}) k_str = canonical_context_key({"x": "1"}) assert k_int != k_str, "int 1 and str '1' must not alias into the same bucket" @@ -108,15 +106,17 @@ def test_float_vs_int_distinct_keys(self): assert k_float != k_int def test_value_with_equals_or_newline_no_boundary_confusion(self): - """'=' and '\n' in values must not fake a field boundary (length-prefix protocol).""" + r"""'=' and '\n' in values must not fake a field boundary (length-prefix protocol).""" k_with = canonical_context_key({"a": "foo=bar\nbaz"}) k_without = canonical_context_key({"a": "foo", "bar\nbaz": ""}) assert k_with != k_without def test_no_hashlib_or_md5_used(self): """Verify no hash function is used by inspecting the module source.""" - import ddtrace.internal.openfeature._flagevaluation_writer as mod_src import inspect + + import ddtrace.internal.openfeature._flagevaluation_writer as mod_src + src = inspect.getsource(mod_src) assert "hashlib" not in src, "hashlib must not appear in the writer" assert "md5" not in src, "md5 must not appear in the writer" @@ -130,6 +130,7 @@ def test_returns_string_not_bytes(self): # flatten_and_prune_context tests # --------------------------------------------------------------------------- + class TestFlattenAndPruneContext: def test_empty_returns_empty(self): assert flatten_and_prune_context({}) == {} @@ -175,6 +176,7 @@ def test_context_with_256_fields_not_pruned(self): # Aggregation tests (full → degraded → drop-counted) # --------------------------------------------------------------------------- + class TestAggregation: def test_two_identical_evals_aggregate_into_one_bucket_count_2(self, writer): t0 = int(time.time() * 1000) @@ -192,7 +194,7 @@ def test_two_identical_evals_aggregate_into_one_bucket_count_2(self, writer): assert entry.last_evaluation == t1 def test_two_evals_differing_context_value_type_produce_two_buckets(self, writer): - """int 1 vs str '1' in context → two distinct full-tier buckets (reviewer concern #3).""" + """int 1 vs str '1' in context → two distinct full-tier buckets.""" e_int = _make_event(attrs={"x": 1}) e_str = _make_event(attrs={"x": "1"}) writer._aggregate(e_int) @@ -213,11 +215,12 @@ def test_full_tier_overflow_routes_to_degraded(self, writer): assert len(writer._degraded) == 1 def test_degraded_overflow_increments_dropped_counter(self, writer): - """Beyond degradedCap, increment _dropped_degraded_overflow (reviewer concern #8).""" + """Beyond degradedCap, increment _dropped_degraded_overflow.""" # Fill the degraded map to the cap. for i in range(DEGRADED_CAP): key = (f"flag-{i}", "on", "alloc", "SPLIT") from ddtrace.internal.openfeature._flagevaluation_writer import _Entry + writer._degraded[key] = _Entry(1000, False, "", {}, "") with writer._lock: @@ -235,7 +238,7 @@ def test_per_flag_cap_routes_to_degraded(self, writer): assert len(writer._full) == 0 def test_runtime_default_when_variant_is_absent(self, writer): - """Absent/empty variant → runtime_default_used True (reviewer concern #5).""" + """Absent/empty variant → runtime_default_used True.""" e = _make_event(variant="", runtime_default=True) writer._aggregate(e) @@ -244,11 +247,9 @@ def test_runtime_default_when_variant_is_absent(self, writer): assert entry.runtime_default is True def test_degraded_event_omits_targeting_key_and_context(self, writer): - """Degraded tier strips targeting_key + context (schema omitempty, reviewer concern #2).""" + """Degraded tier strips targeting_key + context (schema omitempty).""" with writer._lock: - writer._add_to_degraded( - _make_event(targeting_key="some-key", attrs={"k": "v"}) - ) + writer._add_to_degraded(_make_event(targeting_key="some-key", attrs={"k": "v"})) entry = list(writer._degraded.values())[0] assert entry.targeting_key == "" @@ -259,6 +260,7 @@ def test_degraded_event_omits_targeting_key_and_context(self, writer): # Enqueue non-blocking tests # --------------------------------------------------------------------------- + class TestEnqueue: def test_enqueue_non_blocking_on_full_queue(self, writer): """When queue is full, enqueue must NOT block and must increment _dropped_queue.""" @@ -279,6 +281,7 @@ def test_enqueue_succeeds_when_queue_has_capacity(self, writer): # Periodic flush + EVP POST tests # --------------------------------------------------------------------------- + class TestPeriodicFlush: def test_periodic_drains_queue_and_builds_payload(self, writer): writer.enqueue(_make_event()) @@ -349,7 +352,7 @@ def test_two_evals_same_dims_aggregate_count_2(self, writer): assert evals[0]["first_evaluation"] <= evals[0]["last_evaluation"] def test_context_pruning_above_256_fields(self, writer): - """Context with >256 fields is pruned before keying (reviewer concern #1).""" + """Context with >256 fields is pruned before keying.""" attrs = {str(i): str(i) for i in range(300)} e = _make_event(attrs=attrs) writer.enqueue(e) @@ -365,7 +368,7 @@ def test_context_pruning_above_256_fields(self, writer): assert len(ev["context"]["evaluation"]) <= MAX_CONTEXT_FIELDS def test_context_value_exceeding_256_chars_pruned(self, writer): - """Context values >256 chars are skipped (reviewer concern #1).""" + """Context values >256 chars are skipped.""" long_val = "x" * (MAX_FIELD_LENGTH + 10) attrs = {"short": "ok", "long_field": long_val} e = _make_event(attrs=attrs) @@ -403,4 +406,233 @@ def test_writer_endpoint_constant(self): def test_class_exists_and_inherits_periodic_service(self): from ddtrace.internal.periodic import PeriodicService + assert issubclass(FlagEvaluationWriter, PeriodicService) + + +# --------------------------------------------------------------------------- +# G3 — Schema conformance of the emitted payload (full + degraded rows) +# --------------------------------------------------------------------------- + +# The flageval-worker schema (batchedflagevaluations.json) is not vendored into this +# repo (it lives in dd-source/domains/evp-workers, inaccessible here). We codify the +# worker contract as a structural validator that mirrors the Go reference payload +# (dd-trace-go/openfeature/flagevaluation.go): required scalar fields, and variant/ +# allocation serialized as {"key": ...} OBJECTS — never bare strings. + +# Required fields that EVERY flagevaluation row (full or degraded) must carry. +_REQUIRED_EVENT_FIELDS = { + "timestamp": int, + "flag": dict, + "first_evaluation": int, + "last_evaluation": int, + "evaluation_count": int, +} + + +def _assert_row_schema_valid(ev: dict) -> None: + """Assert one flagevaluation row conforms to the worker contract (mechanical).""" + # Required fields present with the right scalar types. + for field, typ in _REQUIRED_EVENT_FIELDS.items(): + assert field in ev, f"required field {field!r} missing from row: {ev}" + assert isinstance(ev[field], typ), f"{field} must be {typ}, got {type(ev[field])}" + + # flag.key is the one required nested field. + assert "key" in ev["flag"] and isinstance(ev["flag"]["key"], str) + + # first <= last evaluation bound. + assert ev["first_evaluation"] <= ev["last_evaluation"] + assert ev["evaluation_count"] >= 1 + + # variant/allocation, when present, MUST be {"key": "..."} objects, NOT bare strings. + for obj_field in ("variant", "allocation"): + if obj_field in ev: + assert isinstance(ev[obj_field], dict), f"{obj_field} must serialize as an object" + assert set(ev[obj_field].keys()) == {"key"}, f"{obj_field} must be exactly {{key}}" + assert isinstance(ev[obj_field]["key"], str) + + # error, when present, is {"message": "..."}. + if "error" in ev: + assert isinstance(ev["error"], dict) + assert "message" in ev["error"] + + # context, when present, nests an "evaluation" map. + if "context" in ev: + assert isinstance(ev["context"], dict) + assert "evaluation" in ev["context"] + assert isinstance(ev["context"]["evaluation"], dict) + + # runtime_default_used, when present, is a bool. + if "runtime_default_used" in ev: + assert isinstance(ev["runtime_default_used"], bool) + + +class TestPayloadSchemaConformance: + def test_full_tier_row_is_schema_valid_with_object_variant_and_allocation(self, writer): + """A full-tier row carries variant/allocation as {key} objects + context.evaluation.""" + writer.enqueue( + _make_event( + variant="on", + allocation_key="alloc-1", + attrs={"tier": "premium"}, + ) + ) + with mock.patch.object(writer, "_send_payload") as mock_send: + writer.periodic() + + decoded = json.loads(mock_send.call_args[0][0]) + assert "flagEvaluations" in decoded + row = decoded["flagEvaluations"][0] + _assert_row_schema_valid(row) + # Specifically the {key} object shape (NOT bare strings). + assert row["variant"] == {"key": "on"} + assert row["allocation"] == {"key": "alloc-1"} + assert row["context"]["evaluation"]["tier"] == "premium" + + def test_degraded_tier_row_is_schema_valid_and_omits_context(self, writer): + """A degraded-tier row is schema-valid with variant/allocation objects, no context.""" + writer._per_flag_count["my-flag"] = PER_FLAG_CAP # force degraded + writer.enqueue(_make_event(variant="on", allocation_key="alloc-1", attrs={"k": "v"})) + with mock.patch.object(writer, "_send_payload") as mock_send: + writer.periodic() + + decoded = json.loads(mock_send.call_args[0][0]) + row = decoded["flagEvaluations"][0] + _assert_row_schema_valid(row) + assert row["variant"] == {"key": "on"} + assert "context" not in row + assert "targeting_key" not in row + + def test_error_row_carries_error_message_object(self, writer): + """An error evaluation produces a schema-valid row with error.message.""" + writer.enqueue( + _make_event( + variant="", + reason="ERROR", + runtime_default=True, + error_message="Flag not found", + ) + ) + with mock.patch.object(writer, "_send_payload") as mock_send: + writer.periodic() + + decoded = json.loads(mock_send.call_args[0][0]) + row = decoded["flagEvaluations"][0] + _assert_row_schema_valid(row) + assert row["error"] == {"message": "Flag not found"} + # Absent variant -> runtime_default_used True, no variant object emitted. + assert row["runtime_default_used"] is True + assert "variant" not in row + + def test_batch_payload_validates_full_and_degraded_rows_together(self, writer): + """A single flush emits BOTH a full row and a degraded row, both schema-valid.""" + # Full-tier event. + writer.enqueue(_make_event(flag_key="full-flag", variant="on", attrs={"a": "b"})) + # Degraded-tier event (different flag forced to degraded). + writer._per_flag_count["deg-flag"] = PER_FLAG_CAP + writer.enqueue(_make_event(flag_key="deg-flag", variant="off")) + + with mock.patch.object(writer, "_send_payload") as mock_send: + writer.periodic() + + decoded = json.loads(mock_send.call_args[0][0]) + rows = decoded["flagEvaluations"] + assert len(rows) == 2 + for row in rows: + _assert_row_schema_valid(row) + flags = {r["flag"]["key"] for r in rows} + assert flags == {"full-flag", "deg-flag"} + + +# --------------------------------------------------------------------------- +# G5 — Shutdown drains the queue + final-flush before exit +# --------------------------------------------------------------------------- + + +class TestShutdownDrain: + def test_on_shutdown_drains_queue_and_flushes(self, writer): + """on_shutdown (the PeriodicService shutdown callback) must drain + flush queued events.""" + writer.enqueue(_make_event(flag_key="pending-1")) + writer.enqueue(_make_event(flag_key="pending-2")) + assert writer._queue.qsize() == 2 + + with mock.patch.object(writer, "_send_payload") as mock_send: + writer.on_shutdown() + + # The queued events were drained, aggregated, and flushed in a final POST. + mock_send.assert_called_once() + decoded = json.loads(mock_send.call_args[0][0]) + flags = {r["flag"]["key"] for r in decoded["flagEvaluations"]} + assert flags == {"pending-1", "pending-2"} + assert writer._queue.qsize() == 0 + + def test_real_start_stop_lifecycle_drains_pending_event(self): + """Real PeriodicService start()->enqueue->stop() drains the queue via on_shutdown.""" + # Long interval so the periodic timer never fires; only stop() triggers the flush. + w = FlagEvaluationWriter(interval=3600.0) + sent = [] + with mock.patch.object(w, "_send_payload", side_effect=lambda p, n: sent.append((p, n))): + w.start() + w.enqueue(_make_event(flag_key="lifecycle-flag")) + w.stop() # stop() runs on_shutdown -> periodic() -> drain + flush + # stop() requests shutdown; join() blocks until the worker (and its + # on_shutdown final flush) has fully completed before we assert. + w.join(timeout=5.0) + assert len(sent) == 1, "stop() must trigger a final drain+flush" + decoded = json.loads(sent[0][0]) + assert decoded["flagEvaluations"][0]["flag"]["key"] == "lifecycle-flag" + + +# --------------------------------------------------------------------------- +# G4 — Backpressure drop counters are observable (emitted on flush) +# --------------------------------------------------------------------------- + + +class TestObservableDropCounters: + def test_queue_overflow_drop_count_is_logged_on_flush(self, writer): + """Queue-full drops increment _dropped_queue AND are surfaced (logged) on flush.""" + # Fill the queue so the next enqueue drops. + for i in range(QUEUE_SIZE): + writer._queue.put_nowait(_make_event(flag_key=f"f{i}")) + writer.enqueue(_make_event(flag_key="dropped")) + assert writer._dropped_queue == 1 + + # Drain everything (so maps are populated) and assert the drop count is emitted. + with mock.patch("ddtrace.internal.openfeature._flagevaluation_writer.logger") as mock_logger: + with mock.patch.object(writer, "_send_payload"): + writer.periodic() + # A warning naming the queue-full drop count must have been emitted. + warnings = [c for c in mock_logger.warning.call_args_list if "queue full" in str(c).lower()] + assert warnings, "queue-full drop count must be logged (observable)" + # Counter resets after emission. + assert writer._dropped_queue == 0 + + def test_degraded_overflow_drop_count_is_logged_on_flush(self, writer): + """Degraded-cap overflow drops increment _dropped_degraded_overflow AND are logged.""" + from ddtrace.internal.openfeature._flagevaluation_writer import _Entry + + # Saturate the degraded map to its cap. + for i in range(DEGRADED_CAP): + writer._degraded[(f"flag-{i}", "on", "alloc", "SPLIT")] = _Entry(1000, False, "", {}, "") + with writer._lock: + writer._add_to_degraded(_make_event(flag_key="overflow")) + assert writer._dropped_degraded_overflow == 1 + + with mock.patch("ddtrace.internal.openfeature._flagevaluation_writer.logger") as mock_logger: + with mock.patch.object(writer, "_send_payload"): + writer.periodic() + warnings = [c for c in mock_logger.warning.call_args_list if "degraded cap" in str(c).lower()] + assert warnings, "degraded-cap overflow count must be logged (observable)" + assert writer._dropped_degraded_overflow == 0 + + def test_drop_accounting_is_complete_no_silent_loss(self, writer): + """Σ(tier counts + drops) == events processed (no silent loss).""" + # 3 distinct full-tier buckets + 2 degraded-overflow drops. + writer._aggregate(_make_event(flag_key="a", attrs={"x": 1})) + writer._aggregate(_make_event(flag_key="b", attrs={"x": 2})) + writer._aggregate(_make_event(flag_key="a", attrs={"x": 1})) # repeat -> count 2 on bucket a + + full_counts = sum(e.count for e in writer._full.values()) + assert full_counts == 3 # 2 (a) + 1 (b) + assert writer._dropped_degraded_overflow == 0 + assert writer._dropped_queue == 0 From 00b8a4570bdb3cca826884bc54679810b0b307e9 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Sat, 13 Jun 2026 11:29:38 -0400 Subject: [PATCH 07/37] chore(openfeature): remove internal planning annotations Strip internal validation-gate markers and review-tracking references from test docstrings and comments; keep the substantive test descriptions intact. --- tests/openfeature/test_flagevaluation_e2e.py | 12 ++++++------ tests/openfeature/test_flagevaluation_hook.py | 4 ++-- .../openfeature/test_flagevaluation_writer.py | 19 +++++++++---------- 3 files changed, 17 insertions(+), 18 deletions(-) diff --git a/tests/openfeature/test_flagevaluation_e2e.py b/tests/openfeature/test_flagevaluation_e2e.py index 03e7686a4ba..0c408632d7b 100644 --- a/tests/openfeature/test_flagevaluation_e2e.py +++ b/tests/openfeature/test_flagevaluation_e2e.py @@ -1,12 +1,12 @@ """ End-to-end tests for the EVP `flagevaluation` path through the real provider eval path. -These cover the lifecycle/exit-path gates that unit tests of the hook/writer in isolation +These cover the lifecycle/exit paths that unit tests of the hook/writer in isolation cannot prove: -- G11: the EVP hook actually fires on the provider's REAL OpenFeature evaluation entrypoint +- The EVP hook actually fires on the provider's REAL OpenFeature evaluation entrypoint (driven through the OpenFeature client, not by calling the hook directly). -- G12: ALL evaluation exit paths are covered — success, native engine error, runtime default +- ALL evaluation exit paths are covered — success, native engine error, runtime default (flag-not-found / no-config), and disabled. - The OTel `feature_flag.evaluations` path is preserved alongside the EVP path (non-regression). """ @@ -68,7 +68,7 @@ def _drain(provider): class TestEVPHookFiresOnRealEvalPath: - """G11: the EVP hook fires on the provider's real evaluation entrypoint.""" + """The EVP hook fires on the provider's real evaluation entrypoint.""" def test_evp_hook_registered_in_provider_hooks(self, provider_and_client): provider, _ = provider_and_client @@ -95,7 +95,7 @@ def test_success_eval_enqueues_and_emits_row(self, provider_and_client): assert row.get("runtime_default_used", False) is False def test_variant_is_resolution_variant_not_value(self, provider_and_client): - """G1 end-to-end: emitted variant == resolution variant key, distinct from value.""" + """End-to-end: emitted variant == resolution variant key, distinct from value.""" provider, client = provider_and_client # String flag: value "blue", variant key "blue" (here equal); use details to confirm # the EVP row variant matches details.variant exactly. @@ -109,7 +109,7 @@ def test_variant_is_resolution_variant_not_value(self, provider_and_client): class TestEVPExitPathsCovered: - """G12: success / engine-error / runtime-default / disabled exit paths are all captured.""" + """success / engine-error / runtime-default / disabled exit paths are all captured.""" def test_flag_not_found_runtime_default_path(self, provider_and_client): provider, client = provider_and_client diff --git a/tests/openfeature/test_flagevaluation_hook.py b/tests/openfeature/test_flagevaluation_hook.py index 2954996a847..d567c8a7d6c 100644 --- a/tests/openfeature/test_flagevaluation_hook.py +++ b/tests/openfeature/test_flagevaluation_hook.py @@ -177,7 +177,7 @@ def test_finally_after_does_not_propagate_exceptions(self, hook, writer): class TestAsyncBoundary: - """G2: prove the hook does NOT aggregate on the eval call path. + """Prove the hook does NOT aggregate on the eval call path. The hook may only enqueue a cheap snapshot; flatten/prune/canonical-key/aggregate must run later in the writer's background worker, never on the eval thread. @@ -228,7 +228,7 @@ def test_canonical_key_not_computed_on_hook_path(self): class TestMetadataSourceMatchesOTelHook: - """G7: EVP hook reads allocation-key/eval metadata from the SAME source as the OTel hook. + """EVP hook reads allocation-key/eval metadata from the SAME source as the OTel hook. The existing OTel FlagEvalHook reads allocation_key from ``details.flag_metadata[METADATA_ALLOCATION_KEY]``. The EVP hook must read from the diff --git a/tests/openfeature/test_flagevaluation_writer.py b/tests/openfeature/test_flagevaluation_writer.py index 2179d505bdc..69392c974ba 100644 --- a/tests/openfeature/test_flagevaluation_writer.py +++ b/tests/openfeature/test_flagevaluation_writer.py @@ -90,7 +90,7 @@ def test_different_insertion_order_same_key(self): assert canonical_context_key(a) == canonical_context_key(b) def test_int_vs_string_distinct_keys(self): - """int 1 vs string '1' must produce different keys (type-tagged).""" + """int 1 vs string '1' must produce different keys (type-tagged keys).""" k_int = canonical_context_key({"x": 1}) k_str = canonical_context_key({"x": "1"}) assert k_int != k_str, "int 1 and str '1' must not alias into the same bucket" @@ -194,7 +194,7 @@ def test_two_identical_evals_aggregate_into_one_bucket_count_2(self, writer): assert entry.last_evaluation == t1 def test_two_evals_differing_context_value_type_produce_two_buckets(self, writer): - """int 1 vs str '1' in context → two distinct full-tier buckets.""" + """int 1 vs str '1' in context produce two distinct full-tier buckets.""" e_int = _make_event(attrs={"x": 1}) e_str = _make_event(attrs={"x": "1"}) writer._aggregate(e_int) @@ -247,7 +247,7 @@ def test_runtime_default_when_variant_is_absent(self, writer): assert entry.runtime_default is True def test_degraded_event_omits_targeting_key_and_context(self, writer): - """Degraded tier strips targeting_key + context (schema omitempty).""" + """Degraded tier strips targeting_key + context.""" with writer._lock: writer._add_to_degraded(_make_event(targeting_key="some-key", attrs={"k": "v"})) @@ -411,14 +411,13 @@ def test_class_exists_and_inherits_periodic_service(self): # --------------------------------------------------------------------------- -# G3 — Schema conformance of the emitted payload (full + degraded rows) +# Schema conformance of the emitted payload (full + degraded rows) # --------------------------------------------------------------------------- # The flageval-worker schema (batchedflagevaluations.json) is not vendored into this -# repo (it lives in dd-source/domains/evp-workers, inaccessible here). We codify the -# worker contract as a structural validator that mirrors the Go reference payload -# (dd-trace-go/openfeature/flagevaluation.go): required scalar fields, and variant/ -# allocation serialized as {"key": ...} OBJECTS — never bare strings. +# repo, so we codify the worker contract as a structural validator: required scalar +# fields, and variant/allocation serialized as {"key": ...} OBJECTS — never bare +# strings. # Required fields that EVERY flagevaluation row (full or degraded) must carry. _REQUIRED_EVENT_FIELDS = { @@ -545,7 +544,7 @@ def test_batch_payload_validates_full_and_degraded_rows_together(self, writer): # --------------------------------------------------------------------------- -# G5 — Shutdown drains the queue + final-flush before exit +# Shutdown drains the queue + final-flush before exit # --------------------------------------------------------------------------- @@ -584,7 +583,7 @@ def test_real_start_stop_lifecycle_drains_pending_event(self): # --------------------------------------------------------------------------- -# G4 — Backpressure drop counters are observable (emitted on flush) +# Backpressure drop counters are observable (emitted on flush) # --------------------------------------------------------------------------- From e52732d1f11ace15eb565625e25f18777ec9f124 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Sat, 13 Jun 2026 11:44:39 -0400 Subject: [PATCH 08/37] chore(openfeature): complete flagevaluation benchmark scenario wiring --- .../openfeature_flagevaluation/requirements_scenario.txt | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 benchmarks/openfeature_flagevaluation/requirements_scenario.txt diff --git a/benchmarks/openfeature_flagevaluation/requirements_scenario.txt b/benchmarks/openfeature_flagevaluation/requirements_scenario.txt new file mode 100644 index 00000000000..f6b50fc0b22 --- /dev/null +++ b/benchmarks/openfeature_flagevaluation/requirements_scenario.txt @@ -0,0 +1,6 @@ +# openfeature-sdk provides EvaluationContext / FlagEvaluationDetails / HookContext, +# which the scenario imports directly. It is a test-time dependency (not a ddtrace +# runtime dependency), so the benchmark base venv does not install it otherwise. +# 0.8.0+ is required for the finally_after hook `details` parameter the scenario drives. +# Mirrors the `openfeature` venv pin in riotfile.py. +openfeature-sdk~=0.8.0 From f2b1d5a97066f64e8bd100cf2c2371a8c173fc2c Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Mon, 15 Jun 2026 11:05:55 -0400 Subject: [PATCH 09/37] docs(openfeature): add release note for server-side EVP flagevaluation --- .../openfeature-flagevaluation-evp-0f1a2b3c4d5e6f70.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 releasenotes/notes/openfeature-flagevaluation-evp-0f1a2b3c4d5e6f70.yaml diff --git a/releasenotes/notes/openfeature-flagevaluation-evp-0f1a2b3c4d5e6f70.yaml b/releasenotes/notes/openfeature-flagevaluation-evp-0f1a2b3c4d5e6f70.yaml new file mode 100644 index 00000000000..509aa2c1b01 --- /dev/null +++ b/releasenotes/notes/openfeature-flagevaluation-evp-0f1a2b3c4d5e6f70.yaml @@ -0,0 +1,9 @@ +--- +features: + - | + openfeature: the experimental OpenFeature provider now reports aggregated + server-side flag evaluations to Datadog (the EVP ``flagevaluation`` track), + lighting up server-side flag-evaluation observability. The existing + ``feature_flag.evaluations`` OpenTelemetry metric behavior is unchanged. The + new emission is gated by ``DD_FLAGGING_EVALUATION_COUNTS_ENABLED`` (default + enabled). From 3c10f55f2c2d0c35e645ee19ce1423c955fd2b68 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 16 Jun 2026 03:11:07 -0400 Subject: [PATCH 10/37] fix(openfeature): align EVP flagevaluation aggregation with worker schema --- .../openfeature_flagevaluation/scenario.py | 19 +- .../openfeature/_flagevaluation_hook.py | 11 +- .../openfeature/_flagevaluation_writer.py | 58 ++++-- ddtrace/internal/openfeature/_provider.py | 18 +- riotfile.py | 1 + tests/openfeature/test_flagevaluation_hook.py | 30 ++- .../openfeature/test_flagevaluation_writer.py | 70 ++++++- tests/openfeature/test_provider_status.py | 39 +++- .../batchedflagevaluations.json | 184 ++++++++++++++++++ 9 files changed, 369 insertions(+), 61 deletions(-) create mode 100644 tests/openfeature/testdata/flageval-worker/batchedflagevaluations.json diff --git a/benchmarks/openfeature_flagevaluation/scenario.py b/benchmarks/openfeature_flagevaluation/scenario.py index 42d31e76051..1cdee09804d 100644 --- a/benchmarks/openfeature_flagevaluation/scenario.py +++ b/benchmarks/openfeature_flagevaluation/scenario.py @@ -4,11 +4,11 @@ flag-evaluation counting, mirroring the Go reference benchmark (``dd-trace-go/openfeature/flagevaluation_test.go``): -* ``hook_enqueue`` — the ``finally_after`` hook hot path: cheap scalar capture + a - shallow context copy + a non-blocking bounded enqueue. This is what charges the - caller's evaluation goroutine/thread directly, so it must stay flat. -* ``aggregate`` — the background worker hot path: flatten + deterministic prune + - canonical-context key + two-tier aggregation. This runs OFF the eval path, but its +* ``hook_enqueue`` — the ``finally_after`` hook hot path: scalar capture + bounded + context snapshot + non-blocking bounded enqueue. This is what charges the caller's + evaluation goroutine/thread directly, so it must stay bounded. +* ``aggregate`` — the background worker hot path: canonical-context key + two-tier + aggregation over already-bounded snapshots. This runs OFF the eval path, but its per-event cost still bounds throughput of the writer thread. * ``hook_plus_drain`` — end-to-end: N hook enqueues followed by a full queue drain + aggregate, the realistic per-flush shape. @@ -100,20 +100,21 @@ def _(loops): elif mode == "aggregate": # Pre-capture snapshots once; the measured loop only runs _aggregate - # (flatten + prune + canonical key + two-tier insert). + # (canonical key + two-tier insert). from ddtrace.internal.openfeature._flagevaluation_writer import _EvalEvent + from ddtrace.internal.openfeature._flagevaluation_writer import flatten_and_prune_context + bounded_attrs = flatten_and_prune_context(attrs) events = [ _EvalEvent( flag_key="flag-{}".format(i % num_flags), variant="variant-{}".format(i % 4), allocation_key="alloc-{}".format(i % num_flags), - reason="TARGETING_MATCH", targeting_key="user-{}".format(i % num_flags), - attrs=dict(attrs), + attrs=dict(bounded_attrs), runtime_default=False, error_message="", - eval_time_ms=1_700_000_000_000 + i, + eval_time_ms=1_760_000_000_000 + i, ) for i in range(num_flags) ] diff --git a/ddtrace/internal/openfeature/_flagevaluation_hook.py b/ddtrace/internal/openfeature/_flagevaluation_hook.py index 05ddb16f802..32bbdba5a02 100644 --- a/ddtrace/internal/openfeature/_flagevaluation_hook.py +++ b/ddtrace/internal/openfeature/_flagevaluation_hook.py @@ -55,8 +55,7 @@ def finally_after( Eval-time: uses details.flag_metadata["dd.eval.timestamp_ms"] when present (stamped by the provider at eval entry); falls back to hook-fire time. - Runtime-default: True when the variant is absent (details.variant is None), - which detects a runtime default independent of the reason string. + Runtime-default: True when the variant is absent (details.variant is None). Attrs: shallow copy of the evaluation context attributes dict so the hook returns immediately and the worker can safely iterate attrs off-path. @@ -82,13 +81,6 @@ def finally_after( variant = details.variant or "" runtime_default = details.variant is None - # Reason: normalise to upper-case string. - if details.reason is not None: - reason_raw = details.reason.value if hasattr(details.reason, "value") else str(details.reason) - reason = str(reason_raw).upper() - else: - reason = "UNKNOWN" - # Targeting key and attributes from the evaluation context. eval_ctx = hook_context.evaluation_context if eval_ctx is not None: @@ -108,7 +100,6 @@ def finally_after( flag_key=flag_key, variant=variant, allocation_key=allocation_key, - reason=reason, targeting_key=targeting_key, attrs=attrs, runtime_default=runtime_default, diff --git a/ddtrace/internal/openfeature/_flagevaluation_writer.py b/ddtrace/internal/openfeature/_flagevaluation_writer.py index df773f05830..1de30108434 100644 --- a/ddtrace/internal/openfeature/_flagevaluation_writer.py +++ b/ddtrace/internal/openfeature/_flagevaluation_writer.py @@ -6,7 +6,8 @@ Key design properties: - Async, best-effort recording: the finally_after hook does cheap capture + non-blocking - enqueue only. All flatten/prune/aggregate/flush work happens in the background worker. + enqueue. The writer bounds context before queueing; aggregate/flush work happens in the + background worker. - Two-tier aggregation (full → degraded → drop-counted). - Canonical context key: sorted, type-tagged, length-delimited — NOT a hash, so distinct contexts always produce distinct keys with no collisions. @@ -229,9 +230,8 @@ class _EvalEvent(typing.NamedTuple): flag_key: str variant: str # "" when absent (= runtime_default) allocation_key: str - reason: str targeting_key: str - attrs: dict[str, typing.Any] # shallow copy from evaluation context + attrs: dict[str, typing.Any] # flattened and pruned context snapshot runtime_default: bool error_message: str eval_time_ms: int @@ -247,12 +247,15 @@ class FlagEvaluationWriter(PeriodicService): SDK-native EVP `flagevaluation` writer. Two-tier aggregation design: - - full-tier: keyed by (flag, variant, allocation, reason, targeting_key, canonical_context) - - degraded-tier: keyed by (flag, variant, allocation, reason) — same cardinality as OTel + - full-tier: keyed by schema-visible dimensions only: flag, variant, allocation, + runtime_default_used, error.message, targeting_key, canonical_context + - degraded-tier: keyed by schema-visible retained dimensions: flag, variant, allocation, + runtime_default_used, error.message - drop-counted: beyond degradedCap, increment _dropped_degraded_overflow - The finally_after hook enqueues cheap _EvalEvent snapshots; the PeriodicService - background thread drains the queue, aggregates, and flushes via HTTP every 10 s. + The finally_after hook enqueues _EvalEvent snapshots through enqueue(), which bounds + context before buffering; the PeriodicService background thread drains the queue, + aggregates, and flushes via HTTP every 10 s. """ def __init__(self, interval: float = DEFAULT_FLUSH_INTERVAL, timeout: float = 2.0) -> None: @@ -288,17 +291,29 @@ def enqueue(self, event: _EvalEvent) -> None: """ Non-blocking enqueue from the finally_after hook thread. - On queue.Full, increments _dropped_queue (observable) and returns immediately — - never blocks the evaluation hot path. + Context is flattened/pruned before it enters the queue so queue length is not the + only memory bound. On queue.Full, increments _dropped_queue (observable) and + returns immediately — never blocks the evaluation hot path. """ + bounded_event = _EvalEvent( + flag_key=event.flag_key, + variant=event.variant, + allocation_key=event.allocation_key, + targeting_key=event.targeting_key, + attrs=flatten_and_prune_context(event.attrs) if event.attrs else {}, + runtime_default=event.runtime_default, + error_message=event.error_message, + eval_time_ms=event.eval_time_ms, + ) + try: - self._queue.put_nowait(event) + self._queue.put_nowait(bounded_event) except queue.Full: with self._lock: self._dropped_queue += 1 logger.debug( "FlagEvaluationWriter: queue full — dropped flag evaluation event for %s", - event.flag_key, + bounded_event.flag_key, ) # ------------------------------------------------------------------ @@ -390,6 +405,8 @@ def periodic(self) -> None: ev["variant"] = {"key": variant} if allocation_key: ev["allocation"] = {"key": allocation_key} + if entry.error_message: + ev["error"] = {"message": entry.error_message} events.append(ev) if not events: @@ -438,10 +455,10 @@ def _aggregate(self, event: _EvalEvent) -> None: Aggregate a single evaluation event into the two-tier maps. Implements: full-tier → degraded-tier → drop-counted cascade. - Context pruning and canonical key computation happen here (off the hot path). + Canonical key computation happens here (off the hot path). Context was already + flattened and pruned before enqueue. """ - # Flatten + prune the context. - context_attrs = flatten_and_prune_context(event.attrs) if event.attrs else {} + context_attrs = event.attrs or {} # Build the full-tier key tuple. ctx_key = canonical_context_key(context_attrs) @@ -449,7 +466,8 @@ def _aggregate(self, event: _EvalEvent) -> None: event.flag_key, event.variant, event.allocation_key, - event.reason, + event.runtime_default, + event.error_message, event.targeting_key, ctx_key, ) @@ -489,7 +507,13 @@ def _add_to_degraded(self, event: _EvalEvent) -> None: Add to the degraded-tier map (drops targeting_key + context). Must be called with self._lock held. """ - deg_key = (event.flag_key, event.variant, event.allocation_key, event.reason) + deg_key = ( + event.flag_key, + event.variant, + event.allocation_key, + event.runtime_default, + event.error_message, + ) if deg_key in self._degraded: self._degraded[deg_key].observe(event.eval_time_ms) return @@ -503,7 +527,7 @@ def _add_to_degraded(self, event: _EvalEvent) -> None: runtime_default=event.runtime_default, targeting_key="", context_attrs={}, - error_message="", + error_message=event.error_message, ) def _send_payload(self, payload: bytes, num_events: int) -> None: diff --git a/ddtrace/internal/openfeature/_provider.py b/ddtrace/internal/openfeature/_provider.py index 0764e69b196..c5bd3256e54 100644 --- a/ddtrace/internal/openfeature/_provider.py +++ b/ddtrace/internal/openfeature/_provider.py @@ -38,12 +38,13 @@ from ddtrace.internal.service import ServiceStatusError from ddtrace.internal.settings.openfeature import OpenFeatureConfig from ddtrace.internal.settings.openfeature import config as ffe_config +from ddtrace.vendor.packaging.version import parse as parse_version # Handle different import paths between openfeature-sdk versions # Versions 0.7.0+ reorganized submodules pkg_version = version("openfeature-sdk") -if pkg_version >= "0.7.0": +if parse_version(pkg_version) >= parse_version("0.7.0"): from openfeature.provider import AbstractProvider else: from openfeature.provider.provider import AbstractProvider @@ -149,6 +150,12 @@ def get_metadata(self) -> Metadata: """Returns provider metadata.""" return self._metadata + def attach(self, on_emit: typing.Callable[..., None]) -> None: + """Attach OpenFeature event dispatch and register for RC callbacks.""" + super().attach(on_emit) + if self._enabled: + _register_provider(self) + def get_provider_hooks(self) -> list[typing.Any]: """ Returns provider-level hooks. @@ -244,6 +251,7 @@ def shutdown(self) -> None: if self._flagevaluation_writer is not None: try: self._flagevaluation_writer.stop() + self._flagevaluation_writer.join() logger.debug("FlagEvaluationWriter stopped") except ServiceStatusError: logger.debug("FlagEvaluationWriter has already stopped", exc_info=True) @@ -537,16 +545,16 @@ def on_configuration_received(self) -> None: """ Called when a Remote Configuration payload is received and processed. - Updates status first, then signals the event to unblock initialize(). - Emits PROVIDER_READY for late arrivals (config received after initialize() timed out). + Updates status first, then signals the event for observers. + Emits PROVIDER_READY for late arrivals after non-blocking initialize(). """ if not self._config_received.is_set(): self._status = ProviderStatus.READY logger.debug("First FFE configuration received, provider is now READY") - # Emit READY for late recovery: config arrived after init timed out + # Emit READY for late recovery: config arrived after initialize() returned. self._emit_ready_event() - # Signal the event last to unblock initialize() after status is updated + # Signal the event last after status is updated. self._config_received.set() def _emit_ready_event(self) -> None: diff --git a/riotfile.py b/riotfile.py index 36785a001ae..e49dade185f 100644 --- a/riotfile.py +++ b/riotfile.py @@ -2814,6 +2814,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT pkgs={ "pytest-randomly": latest, "mock": latest, + "jsonschema": latest, # Test against openfeature-sdk 0.8.0+ (required for finally_after hook details parameter) "openfeature-sdk": ["~=0.8.0", latest], }, diff --git a/tests/openfeature/test_flagevaluation_hook.py b/tests/openfeature/test_flagevaluation_hook.py index d567c8a7d6c..c9835903f02 100644 --- a/tests/openfeature/test_flagevaluation_hook.py +++ b/tests/openfeature/test_flagevaluation_hook.py @@ -106,13 +106,13 @@ def test_finally_after_present_variant_not_runtime_default(self, hook, writer): event = writer.enqueue.call_args[0][0] assert event.runtime_default is False - def test_finally_after_reason_normalized_to_upper(self, hook, writer): - """Reason must be upper-case string in the enqueued event.""" + def test_finally_after_does_not_enqueue_openfeature_reason(self, hook, writer): + """OpenFeature reason is not an EVP field and must not enter the event snapshot.""" hc = _make_hook_context() details = _make_details(reason=Reason.TARGETING_MATCH) hook.finally_after(hc, details, {}) event = writer.enqueue.call_args[0][0] - assert event.reason == "TARGETING_MATCH" + assert not hasattr(event, "reason") def test_finally_after_eval_time_from_metadata(self, hook, writer): """Eval-time must come from metadata["dd.eval.timestamp_ms"] when present.""" @@ -179,8 +179,8 @@ def test_finally_after_does_not_propagate_exceptions(self, hook, writer): class TestAsyncBoundary: """Prove the hook does NOT aggregate on the eval call path. - The hook may only enqueue a cheap snapshot; flatten/prune/canonical-key/aggregate - must run later in the writer's background worker, never on the eval thread. + The hook may only enqueue a snapshot; canonical-key/aggregate must run later in the + writer's background worker, never on the eval thread. """ def test_aggregate_not_called_on_hook_path(self): @@ -199,7 +199,7 @@ def test_aggregate_not_called_on_hook_path(self): # The event must be queued but NOT aggregated on the hook path. spy_aggregate.assert_not_called() assert real_writer._queue.qsize() == 1 - # The aggregation maps are still empty — no flatten/prune/key happened yet. + # The aggregation maps are still empty — no keying/aggregation happened yet. assert real_writer._full == {} assert real_writer._degraded == {} @@ -328,6 +328,24 @@ def test_killswitch_false_does_not_affect_otel_hook(self): assert len(hooks) == 1 assert hooks[0] is provider._flag_eval_hook + def test_provider_shutdown_joins_evp_writer_final_flush(self): + """Provider shutdown waits for FlagEvaluationWriter.on_shutdown final flush.""" + from tests.utils import override_global_config + + with override_global_config({"experimental_flagging_provider_enabled": True}): + with mock.patch("ddtrace.internal.openfeature._provider.stop_exposure_writer"): + with mock.patch("ddtrace.internal.openfeature._provider.FlagEvaluationWriter") as writer_cls: + from ddtrace.internal.openfeature._provider import DataDogProvider + + writer = writer_cls.return_value + provider = DataDogProvider() + + provider.shutdown() + + writer.stop.assert_called_once() + writer.join.assert_called_once() + assert writer.mock_calls.index(mock.call.stop()) < writer.mock_calls.index(mock.call.join()) + def test_killswitch_enabled_true_registers_evp_hook(self): """DD_FLAGGING_EVALUATION_COUNTS_ENABLED=true must register the EVP hook.""" with mock.patch.dict(os.environ, {"DD_FLAGGING_EVALUATION_COUNTS_ENABLED": "true"}): diff --git a/tests/openfeature/test_flagevaluation_writer.py b/tests/openfeature/test_flagevaluation_writer.py index 69392c974ba..ea9d8b029d0 100644 --- a/tests/openfeature/test_flagevaluation_writer.py +++ b/tests/openfeature/test_flagevaluation_writer.py @@ -12,9 +12,11 @@ """ import json +from pathlib import Path import time from unittest import mock +from jsonschema import Draft7Validator import pytest from ddtrace.internal.openfeature._flagevaluation_writer import DEGRADED_CAP @@ -41,7 +43,6 @@ def _make_event( flag_key: str = "my-flag", variant: str = "on", allocation_key: str = "alloc-1", - reason: str = "TARGETING_MATCH", targeting_key: str = "user-1", attrs: dict = None, runtime_default: bool = False, @@ -54,7 +55,6 @@ def _make_event( flag_key=flag_key, variant=variant, allocation_key=allocation_key, - reason=reason, targeting_key=targeting_key, attrs=attrs or {}, runtime_default=runtime_default, @@ -218,7 +218,7 @@ def test_degraded_overflow_increments_dropped_counter(self, writer): """Beyond degradedCap, increment _dropped_degraded_overflow.""" # Fill the degraded map to the cap. for i in range(DEGRADED_CAP): - key = (f"flag-{i}", "on", "alloc", "SPLIT") + key = (f"flag-{i}", "on", "alloc", False, "") from ddtrace.internal.openfeature._flagevaluation_writer import _Entry writer._degraded[key] = _Entry(1000, False, "", {}, "") @@ -276,6 +276,22 @@ def test_enqueue_succeeds_when_queue_has_capacity(self, writer): writer.enqueue(_make_event()) assert writer._queue.qsize() == 1 + def test_enqueue_queues_pruned_context_snapshot(self, writer): + attrs = {f"field-{i:03d}": f"value-{i:03d}" for i in range(MAX_CONTEXT_FIELDS + 50)} + attrs["zzz-oversized"] = "x" * (MAX_FIELD_LENGTH + 1) + + writer.enqueue(_make_event(attrs=attrs)) + + queued = writer._queue.get_nowait() + assert len(queued.attrs) == MAX_CONTEXT_FIELDS + assert "zzz-oversized" not in queued.attrs + + def test_enqueue_flattens_nested_context_snapshot(self, writer): + writer.enqueue(_make_event(attrs={"user": {"id": 123, "plan": "pro"}})) + + queued = writer._queue.get_nowait() + assert queued.attrs == {"user.id": 123, "user.plan": "pro"} + # --------------------------------------------------------------------------- # Periodic flush + EVP POST tests @@ -349,6 +365,7 @@ def test_two_evals_same_dims_aggregate_count_2(self, writer): evals = decoded["flagEvaluations"] assert len(evals) == 1 assert evals[0]["evaluation_count"] == 2 + assert "reason" not in evals[0] assert evals[0]["first_evaluation"] <= evals[0]["last_evaluation"] def test_context_pruning_above_256_fields(self, writer): @@ -414,10 +431,10 @@ def test_class_exists_and_inherits_periodic_service(self): # Schema conformance of the emitted payload (full + degraded rows) # --------------------------------------------------------------------------- -# The flageval-worker schema (batchedflagevaluations.json) is not vendored into this -# repo, so we codify the worker contract as a structural validator: required scalar -# fields, and variant/allocation serialized as {"key": ...} OBJECTS — never bare -# strings. +# The copied flageval-worker schema is the contract surface for these tests. +_SCHEMA_PATH = Path(__file__).parent / "testdata" / "flageval-worker" / "batchedflagevaluations.json" +_BATCHED_FLAGEVALUATIONS_SCHEMA = json.loads(_SCHEMA_PATH.read_text()) +_BATCHED_FLAGEVALUATIONS_VALIDATOR = Draft7Validator(_BATCHED_FLAGEVALUATIONS_SCHEMA) # Required fields that EVERY flagevaluation row (full or degraded) must carry. _REQUIRED_EVENT_FIELDS = { @@ -466,6 +483,11 @@ def _assert_row_schema_valid(ev: dict) -> None: assert isinstance(ev["runtime_default_used"], bool) +def _assert_batch_schema_valid(payload: dict) -> None: + errors = sorted(_BATCHED_FLAGEVALUATIONS_VALIDATOR.iter_errors(payload), key=lambda e: e.path) + assert not errors, [f"{list(error.path)}: {error.message}" for error in errors] + + class TestPayloadSchemaConformance: def test_full_tier_row_is_schema_valid_with_object_variant_and_allocation(self, writer): """A full-tier row carries variant/allocation as {key} objects + context.evaluation.""" @@ -480,6 +502,7 @@ def test_full_tier_row_is_schema_valid_with_object_variant_and_allocation(self, writer.periodic() decoded = json.loads(mock_send.call_args[0][0]) + _assert_batch_schema_valid(decoded) assert "flagEvaluations" in decoded row = decoded["flagEvaluations"][0] _assert_row_schema_valid(row) @@ -491,14 +514,23 @@ def test_full_tier_row_is_schema_valid_with_object_variant_and_allocation(self, def test_degraded_tier_row_is_schema_valid_and_omits_context(self, writer): """A degraded-tier row is schema-valid with variant/allocation objects, no context.""" writer._per_flag_count["my-flag"] = PER_FLAG_CAP # force degraded - writer.enqueue(_make_event(variant="on", allocation_key="alloc-1", attrs={"k": "v"})) + writer.enqueue( + _make_event( + variant="on", + allocation_key="alloc-1", + attrs={"k": "v"}, + error_message="degraded failure", + ) + ) with mock.patch.object(writer, "_send_payload") as mock_send: writer.periodic() decoded = json.loads(mock_send.call_args[0][0]) + _assert_batch_schema_valid(decoded) row = decoded["flagEvaluations"][0] _assert_row_schema_valid(row) assert row["variant"] == {"key": "on"} + assert row["error"] == {"message": "degraded failure"} assert "context" not in row assert "targeting_key" not in row @@ -507,7 +539,6 @@ def test_error_row_carries_error_message_object(self, writer): writer.enqueue( _make_event( variant="", - reason="ERROR", runtime_default=True, error_message="Flag not found", ) @@ -516,6 +547,7 @@ def test_error_row_carries_error_message_object(self, writer): writer.periodic() decoded = json.loads(mock_send.call_args[0][0]) + _assert_batch_schema_valid(decoded) row = decoded["flagEvaluations"][0] _assert_row_schema_valid(row) assert row["error"] == {"message": "Flag not found"} @@ -535,6 +567,7 @@ def test_batch_payload_validates_full_and_degraded_rows_together(self, writer): writer.periodic() decoded = json.loads(mock_send.call_args[0][0]) + _assert_batch_schema_valid(decoded) rows = decoded["flagEvaluations"] assert len(rows) == 2 for row in rows: @@ -542,6 +575,23 @@ def test_batch_payload_validates_full_and_degraded_rows_together(self, writer): flags = {r["flag"]["key"] for r in rows} assert flags == {"full-flag", "deg-flag"} + def test_worker_schema_rejects_top_level_reason(self): + bad = { + "flagEvaluations": [ + { + "timestamp": int(time.time() * 1000), + "flag": {"key": "reason-flag"}, + "first_evaluation": int(time.time() * 1000), + "last_evaluation": int(time.time() * 1000), + "evaluation_count": 1, + "reason": "targeting_match", + } + ] + } + errors = list(_BATCHED_FLAGEVALUATIONS_VALIDATOR.iter_errors(bad)) + assert errors + assert any("Additional properties" in error.message and "reason" in error.message for error in errors) + # --------------------------------------------------------------------------- # Shutdown drains the queue + final-flush before exit @@ -612,7 +662,7 @@ def test_degraded_overflow_drop_count_is_logged_on_flush(self, writer): # Saturate the degraded map to its cap. for i in range(DEGRADED_CAP): - writer._degraded[(f"flag-{i}", "on", "alloc", "SPLIT")] = _Entry(1000, False, "", {}, "") + writer._degraded[(f"flag-{i}", "on", "alloc", False, "")] = _Entry(1000, False, "", {}, "") with writer._lock: writer._add_to_degraded(_make_event(flag_key="overflow")) assert writer._dropped_degraded_overflow == 1 diff --git a/tests/openfeature/test_provider_status.py b/tests/openfeature/test_provider_status.py index 02071ac4d2a..fed6d4b15cb 100644 --- a/tests/openfeature/test_provider_status.py +++ b/tests/openfeature/test_provider_status.py @@ -5,7 +5,7 @@ - NOT_READY by default - READY when first Remote Config payload is received - Event emission on status change -- Blocking initialization until config arrives or timeout +- Non-blocking initialization while config arrives asynchronously """ import threading @@ -39,6 +39,13 @@ def clear_config(): _set_ffe_config(None) +def _set_provider(provider): + if hasattr(api, "set_provider_and_wait"): + api.set_provider_and_wait(provider) + else: + api.set_provider(provider) + + class TestProviderStatus: """Test provider status lifecycle.""" @@ -94,30 +101,36 @@ def test_provider_ready_event_emitted(self): def test_provider_ready_event_only_once(self): """Test that PROVIDER_READY event is only emitted once, not on subsequent configs.""" ready_events = [] + ready_event = threading.Event() def on_provider_ready(event_details): ready_events.append(event_details) + ready_event.set() api.add_handler(ProviderEvent.PROVIDER_READY, on_provider_ready) try: with override_global_config({"experimental_flagging_provider_enabled": True}): provider = DataDogProvider() - api.set_provider(provider) + _set_provider(provider) # Clear events from initialization ready_events.clear() + ready_event.clear() # First configuration config1 = create_config(create_boolean_flag("flag1", enabled=True)) process_ffe_configuration(config1) + assert ready_event.wait(timeout=1.0) count_after_first = len(ready_events) assert count_after_first >= 1 # Should have emitted # Second configuration + ready_event.clear() config2 = create_config(create_boolean_flag("flag2", enabled=True)) process_ffe_configuration(config2) + assert not ready_event.wait(timeout=0.05) count_after_second = len(ready_events) # Should not have emitted again @@ -191,7 +204,7 @@ def on_provider_ready(event_details): api.add_handler(ProviderEvent.PROVIDER_READY, on_provider_ready) try: - api.set_provider(provider) + _set_provider(provider) # Provider should detect existing config and emit READY assert provider._status == ProviderStatus.READY @@ -200,6 +213,24 @@ def on_provider_ready(event_details): api.remove_handler(ProviderEvent.PROVIDER_READY, on_provider_ready) api.clear_providers() + @pytest.mark.skipif(ProviderEvent is None, reason="ProviderEvent not available in SDK 0.6.0") + def test_attached_provider_receives_config_before_async_initialize(self): + """OpenFeature SDK 0.10 attaches synchronously, then initializes asynchronously.""" + ready_events = [] + + def on_emit(provider, event, details): + ready_events.append(event) + + with override_global_config({"experimental_flagging_provider_enabled": True}): + provider = DataDogProvider() + provider.attach(on_emit) + + config = create_config(create_boolean_flag("test-flag", enabled=True)) + process_ffe_configuration(config) + + assert provider._status == ProviderStatus.READY + assert ProviderEvent.PROVIDER_READY in ready_events + class TestProviderInitializationAsync: """Test that initialize() returns immediately and READY arrives asynchronously.""" @@ -233,7 +264,7 @@ def test_initialize_fast_path_when_config_exists(self): try: start = time.monotonic() - api.set_provider(provider) + _set_provider(provider) elapsed = time.monotonic() - start # Should be near-instant (config already available) diff --git a/tests/openfeature/testdata/flageval-worker/batchedflagevaluations.json b/tests/openfeature/testdata/flageval-worker/batchedflagevaluations.json new file mode 100644 index 00000000000..b3a4b1fda67 --- /dev/null +++ b/tests/openfeature/testdata/flageval-worker/batchedflagevaluations.json @@ -0,0 +1,184 @@ +{ + "title": "BatchedFlagEvaluations", + "type": "object", + "properties": { + "context": { + "title": "InputContextDatadog", + "type": "object", + "properties": { + "geo": { + "type": "object", + "properties": { + "country_iso_code": { "type": "string" }, + "country": { "type": "string" } + }, + "required": [], + "additionalProperties": false + }, + "rum": { + "type": "object", + "properties": { + "application": { + "type": "object", + "properties": { + "id": { "type": "string" } + }, + "required": [], + "additionalProperties": false + }, + "view": { + "type": "object", + "properties": { + "url": { "type": "string" } + }, + "required": [], + "additionalProperties": false + } + }, + "required": [], + "additionalProperties": false + }, + "service": { "type": "string" }, + "version": { "type": "string" }, + "env": { "type": "string" }, + "device": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "type": { "type": "string" }, + "brand": { "type": "string" }, + "model": { "type": "string" } + }, + "required": [], + "additionalProperties": false + }, + "os": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "version": { "type": "string" } + }, + "required": [], + "additionalProperties": false + } + }, + "required": [], + "additionalProperties": false + }, + "flagEvaluations": { + "type": "array", + "items": { + "type": "object", + "properties": { + "timestamp": { + "type": "integer", + "description": "The timestamp (milliseconds since epoch) at which the evaluation occurred." + }, + "flag": { + "type": "object", + "properties": { + "key": { "type": "string" } + }, + "required": ["key"], + "additionalProperties": false + }, + "first_evaluation": { + "type": "integer", + "minimum": 1759276800000 + }, + "last_evaluation": { + "type": "integer", + "minimum": 1759276800000 + }, + "evaluation_count": { + "type": "integer", + "minimum": 1 + }, + "runtime_default_used": { "type": "boolean" }, + "targeting_key": { "type": "string" }, + "context": { + "type": "object", + "properties": { + "evaluation": { "type": "object" }, + "dd": { + "type": "object", + "properties": { + "service": { "type": "string" }, + "rum": { + "type": "object", + "properties": { + "application": { + "type": "object", + "properties": { + "id": { "type": "string" } + }, + "required": [], + "additionalProperties": false + }, + "view": { + "type": "object", + "properties": { + "url": { "type": "string" } + }, + "required": [], + "additionalProperties": false + } + }, + "required": [], + "additionalProperties": false + } + }, + "required": [], + "additionalProperties": true + } + }, + "required": [], + "additionalProperties": false + }, + "variant": { + "type": "object", + "properties": { + "key": { "type": "string" } + }, + "required": ["key"], + "additionalProperties": false + }, + "allocation": { + "type": "object", + "properties": { + "key": { "type": "string" } + }, + "required": ["key"], + "additionalProperties": false + }, + "targeting_rule": { + "type": "object", + "properties": { + "key": { "type": "string" } + }, + "required": ["key"], + "additionalProperties": false + }, + "error": { + "type": "object", + "properties": { + "message": { "type": "string" } + }, + "required": ["message"], + "additionalProperties": false + } + }, + "required": [ + "timestamp", + "flag", + "first_evaluation", + "last_evaluation", + "evaluation_count" + ], + "additionalProperties": false + } + } + }, + "required": ["flagEvaluations"], + "additionalProperties": false +} From 182106483136767673dd481ef5ec2b7f552c5e9a Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 16 Jun 2026 04:15:09 -0400 Subject: [PATCH 11/37] chore(openfeature): add riot lockfiles for flagevaluation tests --- .riot/requirements/13b681c.txt | 30 ++++++++++++++++++++++++++++++ .riot/requirements/18535ff.txt | 28 ++++++++++++++++++++++++++++ .riot/requirements/1da3eaa.txt | 25 +++++++++++++++++++++++++ .riot/requirements/1dd7ab8.txt | 26 ++++++++++++++++++++++++++ .riot/requirements/1fa9792.txt | 26 ++++++++++++++++++++++++++ .riot/requirements/698fd99.txt | 25 +++++++++++++++++++++++++ .riot/requirements/6f87c57.txt | 30 ++++++++++++++++++++++++++++++ .riot/requirements/8c0516f.txt | 25 +++++++++++++++++++++++++ .riot/requirements/8c9f12c.txt | 28 ++++++++++++++++++++++++++++ .riot/requirements/8d6e180.txt | 26 ++++++++++++++++++++++++++ .riot/requirements/a8932a5.txt | 26 ++++++++++++++++++++++++++ .riot/requirements/c20e7ba.txt | 25 +++++++++++++++++++++++++ 12 files changed, 320 insertions(+) create mode 100644 .riot/requirements/13b681c.txt create mode 100644 .riot/requirements/18535ff.txt create mode 100644 .riot/requirements/1da3eaa.txt create mode 100644 .riot/requirements/1dd7ab8.txt create mode 100644 .riot/requirements/1fa9792.txt create mode 100644 .riot/requirements/698fd99.txt create mode 100644 .riot/requirements/6f87c57.txt create mode 100644 .riot/requirements/8c0516f.txt create mode 100644 .riot/requirements/8c9f12c.txt create mode 100644 .riot/requirements/8d6e180.txt create mode 100644 .riot/requirements/a8932a5.txt create mode 100644 .riot/requirements/c20e7ba.txt diff --git a/.riot/requirements/13b681c.txt b/.riot/requirements/13b681c.txt new file mode 100644 index 00000000000..656681508de --- /dev/null +++ b/.riot/requirements/13b681c.txt @@ -0,0 +1,30 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/13b681c.in +# +attrs==26.1.0 +coverage[toml]==7.10.7 +exceptiongroup==1.3.1 +hypothesis==6.45.0 +importlib-metadata==8.7.1 +iniconfig==2.1.0 +jsonschema==4.25.1 +jsonschema-specifications==2025.9.1 +mock==5.2.0 +openfeature-sdk==0.8.4 +opentracing==2.4.0 +packaging==26.2 +pluggy==1.6.0 +pygments==2.20.0 +pytest==8.4.2 +pytest-cov==7.1.0 +pytest-mock==3.15.1 +pytest-randomly==4.0.1 +referencing==0.36.2 +rpds-py==0.27.1 +sortedcontainers==2.4.0 +tomli==2.4.1 +typing-extensions==4.15.0 +zipp==3.23.1 diff --git a/.riot/requirements/18535ff.txt b/.riot/requirements/18535ff.txt new file mode 100644 index 00000000000..aad6e625fa6 --- /dev/null +++ b/.riot/requirements/18535ff.txt @@ -0,0 +1,28 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/18535ff.in +# +attrs==26.1.0 +coverage[toml]==7.14.1 +exceptiongroup==1.3.1 +hypothesis==6.45.0 +iniconfig==2.3.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +mock==5.2.0 +openfeature-sdk==0.8.4 +opentracing==2.4.0 +packaging==26.2 +pluggy==1.6.0 +pygments==2.20.0 +pytest==9.1.0 +pytest-cov==7.1.0 +pytest-mock==3.15.1 +pytest-randomly==4.1.0 +referencing==0.37.0 +rpds-py==0.30.0 +sortedcontainers==2.4.0 +tomli==2.4.1 +typing-extensions==4.15.0 diff --git a/.riot/requirements/1da3eaa.txt b/.riot/requirements/1da3eaa.txt new file mode 100644 index 00000000000..fc80691d71e --- /dev/null +++ b/.riot/requirements/1da3eaa.txt @@ -0,0 +1,25 @@ +# +# This file is autogenerated by pip-compile with Python 3.14 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1da3eaa.in +# +attrs==26.1.0 +coverage[toml]==7.14.1 +hypothesis==6.45.0 +iniconfig==2.3.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +mock==5.2.0 +openfeature-sdk==0.10.0 +opentracing==2.4.0 +packaging==26.2 +pluggy==1.6.0 +pygments==2.20.0 +pytest==9.1.0 +pytest-cov==7.1.0 +pytest-mock==3.15.1 +pytest-randomly==4.1.0 +referencing==0.37.0 +rpds-py==2026.5.1 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/1dd7ab8.txt b/.riot/requirements/1dd7ab8.txt new file mode 100644 index 00000000000..ad6ec54405c --- /dev/null +++ b/.riot/requirements/1dd7ab8.txt @@ -0,0 +1,26 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1dd7ab8.in +# +attrs==26.1.0 +coverage[toml]==7.14.1 +hypothesis==6.45.0 +iniconfig==2.3.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +mock==5.2.0 +openfeature-sdk==0.10.0 +opentracing==2.4.0 +packaging==26.2 +pluggy==1.6.0 +pygments==2.20.0 +pytest==9.1.0 +pytest-cov==7.1.0 +pytest-mock==3.15.1 +pytest-randomly==4.1.0 +referencing==0.37.0 +rpds-py==2026.5.1 +sortedcontainers==2.4.0 +typing-extensions==4.15.0 diff --git a/.riot/requirements/1fa9792.txt b/.riot/requirements/1fa9792.txt new file mode 100644 index 00000000000..fbd27b5910f --- /dev/null +++ b/.riot/requirements/1fa9792.txt @@ -0,0 +1,26 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1fa9792.in +# +attrs==26.1.0 +coverage[toml]==7.14.1 +hypothesis==6.45.0 +iniconfig==2.3.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +mock==5.2.0 +openfeature-sdk==0.8.4 +opentracing==2.4.0 +packaging==26.2 +pluggy==1.6.0 +pygments==2.20.0 +pytest==9.1.0 +pytest-cov==7.1.0 +pytest-mock==3.15.1 +pytest-randomly==4.1.0 +referencing==0.37.0 +rpds-py==2026.5.1 +sortedcontainers==2.4.0 +typing-extensions==4.15.0 diff --git a/.riot/requirements/698fd99.txt b/.riot/requirements/698fd99.txt new file mode 100644 index 00000000000..dbacb969ad2 --- /dev/null +++ b/.riot/requirements/698fd99.txt @@ -0,0 +1,25 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/698fd99.in +# +attrs==26.1.0 +coverage[toml]==7.14.1 +hypothesis==6.45.0 +iniconfig==2.3.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +mock==5.2.0 +openfeature-sdk==0.8.4 +opentracing==2.4.0 +packaging==26.2 +pluggy==1.6.0 +pygments==2.20.0 +pytest==9.1.0 +pytest-cov==7.1.0 +pytest-mock==3.15.1 +pytest-randomly==4.1.0 +referencing==0.37.0 +rpds-py==2026.5.1 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/6f87c57.txt b/.riot/requirements/6f87c57.txt new file mode 100644 index 00000000000..2ae1884c6df --- /dev/null +++ b/.riot/requirements/6f87c57.txt @@ -0,0 +1,30 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/6f87c57.in +# +attrs==26.1.0 +coverage[toml]==7.10.7 +exceptiongroup==1.3.1 +hypothesis==6.45.0 +importlib-metadata==8.7.1 +iniconfig==2.1.0 +jsonschema==4.25.1 +jsonschema-specifications==2025.9.1 +mock==5.2.0 +openfeature-sdk==0.8.4 +opentracing==2.4.0 +packaging==26.2 +pluggy==1.6.0 +pygments==2.20.0 +pytest==8.4.2 +pytest-cov==7.1.0 +pytest-mock==3.15.1 +pytest-randomly==4.0.1 +referencing==0.36.2 +rpds-py==0.27.1 +sortedcontainers==2.4.0 +tomli==2.4.1 +typing-extensions==4.15.0 +zipp==3.23.1 diff --git a/.riot/requirements/8c0516f.txt b/.riot/requirements/8c0516f.txt new file mode 100644 index 00000000000..8e4e8bd6a7b --- /dev/null +++ b/.riot/requirements/8c0516f.txt @@ -0,0 +1,25 @@ +# +# This file is autogenerated by pip-compile with Python 3.14 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/8c0516f.in +# +attrs==26.1.0 +coverage[toml]==7.14.1 +hypothesis==6.45.0 +iniconfig==2.3.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +mock==5.2.0 +openfeature-sdk==0.8.4 +opentracing==2.4.0 +packaging==26.2 +pluggy==1.6.0 +pygments==2.20.0 +pytest==9.1.0 +pytest-cov==7.1.0 +pytest-mock==3.15.1 +pytest-randomly==4.1.0 +referencing==0.37.0 +rpds-py==2026.5.1 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/8c9f12c.txt b/.riot/requirements/8c9f12c.txt new file mode 100644 index 00000000000..beacb26877d --- /dev/null +++ b/.riot/requirements/8c9f12c.txt @@ -0,0 +1,28 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/8c9f12c.in +# +attrs==26.1.0 +coverage[toml]==7.14.1 +exceptiongroup==1.3.1 +hypothesis==6.45.0 +iniconfig==2.3.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +mock==5.2.0 +openfeature-sdk==0.10.0 +opentracing==2.4.0 +packaging==26.2 +pluggy==1.6.0 +pygments==2.20.0 +pytest==9.1.0 +pytest-cov==7.1.0 +pytest-mock==3.15.1 +pytest-randomly==4.1.0 +referencing==0.37.0 +rpds-py==0.30.0 +sortedcontainers==2.4.0 +tomli==2.4.1 +typing-extensions==4.15.0 diff --git a/.riot/requirements/8d6e180.txt b/.riot/requirements/8d6e180.txt new file mode 100644 index 00000000000..5fe12233cce --- /dev/null +++ b/.riot/requirements/8d6e180.txt @@ -0,0 +1,26 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/8d6e180.in +# +attrs==26.1.0 +coverage[toml]==7.14.1 +hypothesis==6.45.0 +iniconfig==2.3.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +mock==5.2.0 +openfeature-sdk==0.10.0 +opentracing==2.4.0 +packaging==26.2 +pluggy==1.6.0 +pygments==2.20.0 +pytest==9.1.0 +pytest-cov==7.1.0 +pytest-mock==3.15.1 +pytest-randomly==4.1.0 +referencing==0.37.0 +rpds-py==2026.5.1 +sortedcontainers==2.4.0 +typing-extensions==4.15.0 diff --git a/.riot/requirements/a8932a5.txt b/.riot/requirements/a8932a5.txt new file mode 100644 index 00000000000..24af0d9cd6b --- /dev/null +++ b/.riot/requirements/a8932a5.txt @@ -0,0 +1,26 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/a8932a5.in +# +attrs==26.1.0 +coverage[toml]==7.14.1 +hypothesis==6.45.0 +iniconfig==2.3.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +mock==5.2.0 +openfeature-sdk==0.8.4 +opentracing==2.4.0 +packaging==26.2 +pluggy==1.6.0 +pygments==2.20.0 +pytest==9.1.0 +pytest-cov==7.1.0 +pytest-mock==3.15.1 +pytest-randomly==4.1.0 +referencing==0.37.0 +rpds-py==2026.5.1 +sortedcontainers==2.4.0 +typing-extensions==4.15.0 diff --git a/.riot/requirements/c20e7ba.txt b/.riot/requirements/c20e7ba.txt new file mode 100644 index 00000000000..cd6f57f78fb --- /dev/null +++ b/.riot/requirements/c20e7ba.txt @@ -0,0 +1,25 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/c20e7ba.in +# +attrs==26.1.0 +coverage[toml]==7.14.1 +hypothesis==6.45.0 +iniconfig==2.3.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +mock==5.2.0 +openfeature-sdk==0.10.0 +opentracing==2.4.0 +packaging==26.2 +pluggy==1.6.0 +pygments==2.20.0 +pytest==9.1.0 +pytest-cov==7.1.0 +pytest-mock==3.15.1 +pytest-randomly==4.1.0 +referencing==0.37.0 +rpds-py==2026.5.1 +sortedcontainers==2.4.0 From 0399a611effaca1cbf803cdc1595848d1bae43b9 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 16 Jun 2026 04:33:29 -0400 Subject: [PATCH 12/37] chore(openfeature): prune stale riot lockfiles --- .riot/requirements/13c4b39.txt | 21 --------------------- .riot/requirements/14fc413.txt | 21 --------------------- .riot/requirements/1540c33.txt | 24 ------------------------ .riot/requirements/16138c7.txt | 21 --------------------- .riot/requirements/168ee03.txt | 21 --------------------- .riot/requirements/16b741f.txt | 21 --------------------- .riot/requirements/18421e5.txt | 21 --------------------- .riot/requirements/18a4a8d.txt | 21 --------------------- .riot/requirements/460df49.txt | 26 -------------------------- .riot/requirements/765862d.txt | 26 -------------------------- .riot/requirements/b3bdd52.txt | 24 ------------------------ .riot/requirements/cdab08a.txt | 21 --------------------- 12 files changed, 268 deletions(-) delete mode 100644 .riot/requirements/13c4b39.txt delete mode 100644 .riot/requirements/14fc413.txt delete mode 100644 .riot/requirements/1540c33.txt delete mode 100644 .riot/requirements/16138c7.txt delete mode 100644 .riot/requirements/168ee03.txt delete mode 100644 .riot/requirements/16b741f.txt delete mode 100644 .riot/requirements/18421e5.txt delete mode 100644 .riot/requirements/18a4a8d.txt delete mode 100644 .riot/requirements/460df49.txt delete mode 100644 .riot/requirements/765862d.txt delete mode 100644 .riot/requirements/b3bdd52.txt delete mode 100644 .riot/requirements/cdab08a.txt diff --git a/.riot/requirements/13c4b39.txt b/.riot/requirements/13c4b39.txt deleted file mode 100644 index 9d817e861d3..00000000000 --- a/.riot/requirements/13c4b39.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.13 -# by the following command: -# -# pip-compile --allow-unsafe --cert=None --client-cert=None --index-url=None --no-annotate --pip-args=None .riot/requirements/13c4b39.in -# -attrs==25.4.0 -coverage[toml]==7.11.0 -hypothesis==6.45.0 -iniconfig==2.3.0 -mock==5.2.0 -openfeature-sdk==0.8.3 -opentracing==2.4.0 -packaging==25.0 -pluggy==1.6.0 -pygments==2.19.2 -pytest==8.4.2 -pytest-cov==7.0.0 -pytest-mock==3.15.1 -pytest-randomly==4.0.1 -sortedcontainers==2.4.0 diff --git a/.riot/requirements/14fc413.txt b/.riot/requirements/14fc413.txt deleted file mode 100644 index 6ed9e57c56e..00000000000 --- a/.riot/requirements/14fc413.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.13 -# by the following command: -# -# pip-compile --allow-unsafe --no-annotate .riot/requirements/14fc413.in -# -attrs==26.1.0 -coverage[toml]==7.13.5 -hypothesis==6.45.0 -iniconfig==2.3.0 -mock==5.2.0 -openfeature-sdk==0.8.4 -opentracing==2.4.0 -packaging==26.0 -pluggy==1.6.0 -pygments==2.19.2 -pytest==9.0.2 -pytest-cov==7.1.0 -pytest-mock==3.15.1 -pytest-randomly==4.0.1 -sortedcontainers==2.4.0 diff --git a/.riot/requirements/1540c33.txt b/.riot/requirements/1540c33.txt deleted file mode 100644 index 6560b710652..00000000000 --- a/.riot/requirements/1540c33.txt +++ /dev/null @@ -1,24 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# pip-compile --allow-unsafe --no-annotate .riot/requirements/1540c33.in -# -attrs==26.1.0 -coverage[toml]==7.13.5 -exceptiongroup==1.3.1 -hypothesis==6.45.0 -iniconfig==2.3.0 -mock==5.2.0 -openfeature-sdk==0.8.4 -opentracing==2.4.0 -packaging==26.0 -pluggy==1.6.0 -pygments==2.19.2 -pytest==9.0.2 -pytest-cov==7.1.0 -pytest-mock==3.15.1 -pytest-randomly==4.0.1 -sortedcontainers==2.4.0 -tomli==2.4.1 -typing-extensions==4.15.0 diff --git a/.riot/requirements/16138c7.txt b/.riot/requirements/16138c7.txt deleted file mode 100644 index 5fd1c6c8f51..00000000000 --- a/.riot/requirements/16138c7.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.14 -# by the following command: -# -# pip-compile --allow-unsafe --no-annotate .riot/requirements/16138c7.in -# -attrs==25.4.0 -coverage[toml]==7.11.0 -hypothesis==6.45.0 -iniconfig==2.3.0 -mock==5.2.0 -openfeature-sdk==0.8.3 -opentracing==2.4.0 -packaging==25.0 -pluggy==1.6.0 -pygments==2.19.2 -pytest==8.4.2 -pytest-cov==7.0.0 -pytest-mock==3.15.1 -pytest-randomly==4.0.1 -sortedcontainers==2.4.0 diff --git a/.riot/requirements/168ee03.txt b/.riot/requirements/168ee03.txt deleted file mode 100644 index d4693e30888..00000000000 --- a/.riot/requirements/168ee03.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.14 -# by the following command: -# -# pip-compile --allow-unsafe --no-annotate .riot/requirements/168ee03.in -# -attrs==26.1.0 -coverage[toml]==7.13.5 -hypothesis==6.45.0 -iniconfig==2.3.0 -mock==5.2.0 -openfeature-sdk==0.8.4 -opentracing==2.4.0 -packaging==26.0 -pluggy==1.6.0 -pygments==2.19.2 -pytest==9.0.2 -pytest-cov==7.1.0 -pytest-mock==3.15.1 -pytest-randomly==4.0.1 -sortedcontainers==2.4.0 diff --git a/.riot/requirements/16b741f.txt b/.riot/requirements/16b741f.txt deleted file mode 100644 index 071aaecbc6a..00000000000 --- a/.riot/requirements/16b741f.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile --allow-unsafe --no-annotate .riot/requirements/16b741f.in -# -attrs==25.4.0 -coverage[toml]==7.11.0 -hypothesis==6.45.0 -iniconfig==2.3.0 -mock==5.2.0 -openfeature-sdk==0.8.3 -opentracing==2.4.0 -packaging==25.0 -pluggy==1.6.0 -pygments==2.19.2 -pytest==8.4.2 -pytest-cov==7.0.0 -pytest-mock==3.15.1 -pytest-randomly==4.0.1 -sortedcontainers==2.4.0 diff --git a/.riot/requirements/18421e5.txt b/.riot/requirements/18421e5.txt deleted file mode 100644 index af057052170..00000000000 --- a/.riot/requirements/18421e5.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.12 -# by the following command: -# -# pip-compile --allow-unsafe --no-annotate .riot/requirements/18421e5.in -# -attrs==26.1.0 -coverage[toml]==7.13.5 -hypothesis==6.45.0 -iniconfig==2.3.0 -mock==5.2.0 -openfeature-sdk==0.8.4 -opentracing==2.4.0 -packaging==26.0 -pluggy==1.6.0 -pygments==2.19.2 -pytest==9.0.2 -pytest-cov==7.1.0 -pytest-mock==3.15.1 -pytest-randomly==4.0.1 -sortedcontainers==2.4.0 diff --git a/.riot/requirements/18a4a8d.txt b/.riot/requirements/18a4a8d.txt deleted file mode 100644 index b1c42ace3ec..00000000000 --- a/.riot/requirements/18a4a8d.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.12 -# by the following command: -# -# pip-compile --allow-unsafe --no-annotate .riot/requirements/18a4a8d.in -# -attrs==25.4.0 -coverage[toml]==7.11.0 -hypothesis==6.45.0 -iniconfig==2.3.0 -mock==5.2.0 -openfeature-sdk==0.8.3 -opentracing==2.4.0 -packaging==25.0 -pluggy==1.6.0 -pygments==2.19.2 -pytest==8.4.2 -pytest-cov==7.0.0 -pytest-mock==3.15.1 -pytest-randomly==4.0.1 -sortedcontainers==2.4.0 diff --git a/.riot/requirements/460df49.txt b/.riot/requirements/460df49.txt deleted file mode 100644 index 5f94a829c24..00000000000 --- a/.riot/requirements/460df49.txt +++ /dev/null @@ -1,26 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.9 -# by the following command: -# -# pip-compile --allow-unsafe --no-annotate .riot/requirements/460df49.in -# -attrs==25.4.0 -coverage[toml]==7.10.7 -exceptiongroup==1.3.0 -hypothesis==6.45.0 -importlib-metadata==8.7.0 -iniconfig==2.1.0 -mock==5.2.0 -openfeature-sdk==0.8.3 -opentracing==2.4.0 -packaging==25.0 -pluggy==1.6.0 -pygments==2.19.2 -pytest==8.4.2 -pytest-cov==7.0.0 -pytest-mock==3.15.1 -pytest-randomly==4.0.1 -sortedcontainers==2.4.0 -tomli==2.3.0 -typing-extensions==4.15.0 -zipp==3.23.0 diff --git a/.riot/requirements/765862d.txt b/.riot/requirements/765862d.txt deleted file mode 100644 index f81b3dde9ef..00000000000 --- a/.riot/requirements/765862d.txt +++ /dev/null @@ -1,26 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.9 -# by the following command: -# -# pip-compile --allow-unsafe --no-annotate .riot/requirements/765862d.in -# -attrs==26.1.0 -coverage[toml]==7.10.7 -exceptiongroup==1.3.1 -hypothesis==6.45.0 -importlib-metadata==8.7.1 -iniconfig==2.1.0 -mock==5.2.0 -openfeature-sdk==0.8.4 -opentracing==2.4.0 -packaging==26.0 -pluggy==1.6.0 -pygments==2.19.2 -pytest==8.4.2 -pytest-cov==7.1.0 -pytest-mock==3.15.1 -pytest-randomly==4.0.1 -sortedcontainers==2.4.0 -tomli==2.4.1 -typing-extensions==4.15.0 -zipp==3.23.0 diff --git a/.riot/requirements/b3bdd52.txt b/.riot/requirements/b3bdd52.txt deleted file mode 100644 index ea418581bad..00000000000 --- a/.riot/requirements/b3bdd52.txt +++ /dev/null @@ -1,24 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# pip-compile --allow-unsafe --no-annotate .riot/requirements/b3bdd52.in -# -attrs==25.4.0 -coverage[toml]==7.11.0 -exceptiongroup==1.3.0 -hypothesis==6.45.0 -iniconfig==2.3.0 -mock==5.2.0 -openfeature-sdk==0.8.3 -opentracing==2.4.0 -packaging==25.0 -pluggy==1.6.0 -pygments==2.19.2 -pytest==8.4.2 -pytest-cov==7.0.0 -pytest-mock==3.15.1 -pytest-randomly==4.0.1 -sortedcontainers==2.4.0 -tomli==2.3.0 -typing-extensions==4.15.0 diff --git a/.riot/requirements/cdab08a.txt b/.riot/requirements/cdab08a.txt deleted file mode 100644 index 4aed00c327c..00000000000 --- a/.riot/requirements/cdab08a.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile --allow-unsafe --no-annotate .riot/requirements/cdab08a.in -# -attrs==26.1.0 -coverage[toml]==7.13.5 -hypothesis==6.45.0 -iniconfig==2.3.0 -mock==5.2.0 -openfeature-sdk==0.8.4 -opentracing==2.4.0 -packaging==26.0 -pluggy==1.6.0 -pygments==2.19.2 -pytest==9.0.2 -pytest-cov==7.1.0 -pytest-mock==3.15.1 -pytest-randomly==4.0.1 -sortedcontainers==2.4.0 From f1ae0211201c963e212497748505880e02bc8aa6 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 16 Jun 2026 04:59:46 -0400 Subject: [PATCH 13/37] fix(openfeature): avoid suitespec component collision --- benchmarks/suitespec.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/suitespec.yml b/benchmarks/suitespec.yml index 85a1fcaf2c3..51f7241ab14 100644 --- a/benchmarks/suitespec.yml +++ b/benchmarks/suitespec.yml @@ -50,7 +50,7 @@ components: - ddtrace/appsec/iast/* - ddtrace/appsec/* - ddtrace/internal/settings/asm.py - openfeature: + openfeature_benchmark: - ddtrace/openfeature/* - ddtrace/internal/openfeature/* - ddtrace/internal/settings/openfeature.py @@ -230,7 +230,7 @@ suites: paths: - '@bootstrap' - '@core' - - '@openfeature' + - '@openfeature_benchmark' - benchmarks/openfeature_flagevaluation/* - benchmarks/suitespec.yml cpus_per_run: 1 From e54af18a06523e1f9437041a0ebdc681d8ff1bc5 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 16 Jun 2026 22:19:45 -0400 Subject: [PATCH 14/37] fix(openfeature): drain flag evaluation queue before flush --- .../openfeature/_flagevaluation_writer.py | 34 ++++++++++++++ .../openfeature/test_flagevaluation_writer.py | 46 +++++++++++++++++++ 2 files changed, 80 insertions(+) diff --git a/ddtrace/internal/openfeature/_flagevaluation_writer.py b/ddtrace/internal/openfeature/_flagevaluation_writer.py index 1de30108434..aeba5cf6025 100644 --- a/ddtrace/internal/openfeature/_flagevaluation_writer.py +++ b/ddtrace/internal/openfeature/_flagevaluation_writer.py @@ -32,6 +32,7 @@ from ddtrace.internal.logger import get_logger from ddtrace.internal.periodic import PeriodicService from ddtrace.internal.settings._agent import config as agent_config +from ddtrace.internal.threads import PeriodicThread from ddtrace.internal.utils.http import get_connection @@ -57,6 +58,10 @@ # Flush interval: dedicated 10 s timer, separate from ExposureWriter's 1 s interval. DEFAULT_FLUSH_INTERVAL = 10.0 +# Queue drain interval. This keeps the hand-off queue bounded while allowing a flush +# window to accumulate more buckets than QUEUE_SIZE. +DRAIN_INTERVAL = 0.1 + # Flag metadata key where the provider stamps the evaluation timestamp (ms). EVAL_TIMESTAMP_METADATA_KEY = "dd.eval.timestamp_ms" @@ -283,6 +288,8 @@ def __init__(self, interval: float = DEFAULT_FLUSH_INTERVAL, timeout: float = 2. self._dropped_queue: int = 0 # queue.Full drops (hook path) self._dropped_degraded_overflow: int = 0 # degraded-cap overflow drops + self._drain_worker: typing.Optional[PeriodicThread] = None + # ------------------------------------------------------------------ # Public API used by FlagEvaluationHook # ------------------------------------------------------------------ @@ -320,6 +327,24 @@ def enqueue(self, event: _EvalEvent) -> None: # PeriodicService implementation # ------------------------------------------------------------------ + def _start_service(self, *args: typing.Any, **kwargs: typing.Any) -> None: + self._drain_worker = PeriodicThread( + DRAIN_INTERVAL, + target=self._drain_queue, + name="%s:%s:drain" % (self.__class__.__module__, self.__class__.__name__), + no_wait_at_start=False, + ) + self._drain_worker.start() + try: + super()._start_service(*args, **kwargs) + except Exception: + self._stop_drain_worker() + raise + + def _stop_service(self, *args: typing.Any, **kwargs: typing.Any) -> None: + self._stop_drain_worker() + super()._stop_service(*args, **kwargs) + def periodic(self) -> None: """ Drain the queue, aggregate, and flush to the EVP proxy. @@ -435,6 +460,7 @@ def periodic(self) -> None: def on_shutdown(self): """Final flush on service shutdown — drains the queue and flushes before exit.""" + self._stop_drain_worker() self.periodic() # ------------------------------------------------------------------ @@ -450,6 +476,14 @@ def _drain_queue(self) -> None: break self._aggregate(event) + def _stop_drain_worker(self) -> None: + worker = self._drain_worker + if worker is None: + return + self._drain_worker = None + worker.stop() + worker.join(timeout=1.0) + def _aggregate(self, event: _EvalEvent) -> None: """ Aggregate a single evaluation event into the two-tier maps. diff --git a/tests/openfeature/test_flagevaluation_writer.py b/tests/openfeature/test_flagevaluation_writer.py index ea9d8b029d0..0a4efbf5926 100644 --- a/tests/openfeature/test_flagevaluation_writer.py +++ b/tests/openfeature/test_flagevaluation_writer.py @@ -63,6 +63,15 @@ def _make_event( ) +def _wait_until(predicate, timeout: float = 2.0) -> bool: + deadline = time.time() + timeout + while time.time() < deadline: + if predicate(): + return True + time.sleep(0.01) + return predicate() + + @pytest.fixture def writer(): """Create a FlagEvaluationWriter that is NOT started (no background thread).""" @@ -631,6 +640,43 @@ def test_real_start_stop_lifecycle_drains_pending_event(self): decoded = json.loads(sent[0][0]) assert decoded["flagEvaluations"][0]["flag"]["key"] == "lifecycle-flag" + def test_background_drain_accumulates_beyond_queue_size_before_flush(self): + """A flush window can exceed the bounded queue size and naturally degrade.""" + sent = [] + with mock.patch("ddtrace.internal.openfeature._flagevaluation_writer.QUEUE_SIZE", 8): + with mock.patch("ddtrace.internal.openfeature._flagevaluation_writer.PER_FLAG_CAP", 12): + with mock.patch("ddtrace.internal.openfeature._flagevaluation_writer.DRAIN_INTERVAL", 0.01): + w = FlagEvaluationWriter(interval=3600.0) + with mock.patch.object(w, "_send_payload", side_effect=lambda p, n: sent.append((p, n))): + w.start() + try: + for i in range(14): + assert _wait_until(lambda: not w._queue.full()) + w.enqueue( + _make_event( + flag_key="natural-degrade", + targeting_key=f"user-{i}", + attrs={"user": i}, + ) + ) + assert _wait_until(lambda: w._queue.empty()) + with w._lock: + assert w._dropped_queue == 0 + assert w._degraded + finally: + w.stop() + w.join(timeout=5.0) + + assert sent, "shutdown flush must emit the accumulated evaluation rows" + decoded = json.loads(sent[0][0]) + degraded_rows = [ + row + for row in decoded["flagEvaluations"] + if row["flag"]["key"] == "natural-degrade" and "context" not in row and "targeting_key" not in row + ] + assert len(degraded_rows) == 1 + assert degraded_rows[0]["evaluation_count"] == 2 + # --------------------------------------------------------------------------- # Backpressure drop counters are observable (emitted on flush) From b9102dea0c5d812f6fecee87eb1fea54d06c09c1 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 16 Jun 2026 23:57:11 -0400 Subject: [PATCH 15/37] Configure Python flag evaluation flush interval --- ddtrace/internal/openfeature/_provider.py | 7 +++++-- .../internal/settings/_supported_configurations.py | 1 + ddtrace/internal/settings/openfeature.py | 7 +++++++ supported-configurations.json | 8 ++++++++ tests/openfeature/test_flagevaluation_hook.py | 12 ++++++++++++ 5 files changed, 33 insertions(+), 2 deletions(-) diff --git a/ddtrace/internal/openfeature/_provider.py b/ddtrace/internal/openfeature/_provider.py index c5bd3256e54..21a2afee5db 100644 --- a/ddtrace/internal/openfeature/_provider.py +++ b/ddtrace/internal/openfeature/_provider.py @@ -141,9 +141,12 @@ def __init__(self, *args: typing.Any, **kwargs: typing.Any): # per-instance in tests. self._flagevaluation_writer: typing.Optional[FlagEvaluationWriter] = None self._flagevaluation_hook: typing.Optional[FlagEvaluationHook] = None - evp_counts_enabled = OpenFeatureConfig().flagging_evaluation_counts_enabled + evp_config = OpenFeatureConfig() + evp_counts_enabled = evp_config.flagging_evaluation_counts_enabled if self._enabled and evp_counts_enabled: - self._flagevaluation_writer = FlagEvaluationWriter() + self._flagevaluation_writer = FlagEvaluationWriter( + interval=evp_config.flagging_evaluation_counts_flush_interval + ) self._flagevaluation_hook = FlagEvaluationHook(self._flagevaluation_writer) def get_metadata(self) -> Metadata: diff --git a/ddtrace/internal/settings/_supported_configurations.py b/ddtrace/internal/settings/_supported_configurations.py index 1a429ad1ca4..27f1a24e07a 100644 --- a/ddtrace/internal/settings/_supported_configurations.py +++ b/ddtrace/internal/settings/_supported_configurations.py @@ -221,6 +221,7 @@ "DD_FFE_INTAKE_ENABLED", "DD_FFE_INTAKE_HEARTBEAT_INTERVAL", "DD_FLAGGING_EVALUATION_COUNTS_ENABLED", + "DD_FLAGGING_EVALUATION_COUNTS_FLUSH_INTERVAL_SECONDS", "DD_FLASK_CACHE_SERVICE", "DD_FLASK_SERVICE", "DD_FOO_SERVICE", diff --git a/ddtrace/internal/settings/openfeature.py b/ddtrace/internal/settings/openfeature.py index d5f740f6445..7f9e44c2df9 100644 --- a/ddtrace/internal/settings/openfeature.py +++ b/ddtrace/internal/settings/openfeature.py @@ -26,6 +26,12 @@ class OpenFeatureConfig(DDConfig): default=True, ) + flagging_evaluation_counts_flush_interval = DDConfig.var( + float, + "DD_FLAGGING_EVALUATION_COUNTS_FLUSH_INTERVAL_SECONDS", + default=10.0, + ) + # Feature flag exposure intake configuration ffe_intake_enabled = DDConfig.var( bool, @@ -51,6 +57,7 @@ class OpenFeatureConfig(DDConfig): _openfeature_config_keys = [ "experimental_flagging_provider_enabled", "flagging_evaluation_counts_enabled", + "flagging_evaluation_counts_flush_interval", "ffe_intake_enabled", "ffe_intake_heartbeat_interval", "initialization_timeout_ms", diff --git a/supported-configurations.json b/supported-configurations.json index bf832fd06ad..52e6e17b151 100644 --- a/supported-configurations.json +++ b/supported-configurations.json @@ -1648,6 +1648,14 @@ "experimental": true } ], + "DD_FLAGGING_EVALUATION_COUNTS_FLUSH_INTERVAL_SECONDS": [ + { + "implementation": "A", + "type": "decimal", + "default": "10.0", + "experimental": true + } + ], "DD_FLASK_CACHE_SERVICE": [ { "implementation": "A", diff --git a/tests/openfeature/test_flagevaluation_hook.py b/tests/openfeature/test_flagevaluation_hook.py index c9835903f02..b5126e0d53b 100644 --- a/tests/openfeature/test_flagevaluation_hook.py +++ b/tests/openfeature/test_flagevaluation_hook.py @@ -308,6 +308,18 @@ def test_killswitch_false_does_not_register_evp_hook(self): assert provider._flagevaluation_writer is None assert provider._flagevaluation_hook is None + def test_flush_interval_env_configures_evp_writer(self): + with mock.patch.dict(os.environ, {"DD_FLAGGING_EVALUATION_COUNTS_FLUSH_INTERVAL_SECONDS": "20"}): + from tests.utils import override_global_config + + with override_global_config({"experimental_flagging_provider_enabled": True}): + with mock.patch("ddtrace.internal.openfeature._provider.FlagEvaluationWriter") as writer_cls: + from ddtrace.internal.openfeature._provider import DataDogProvider + + DataDogProvider() + + writer_cls.assert_called_once_with(interval=20.0) + def test_killswitch_false_does_not_affect_otel_hook(self): """Killswitch must not suppress the OTel FlagEvalHook (OTel non-regression).""" with mock.patch.dict(os.environ, {"DD_FLAGGING_EVALUATION_COUNTS_ENABLED": "false"}): From 08e710fe89f2c5fc294384c194a52d84ac09ba90 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 19 Jun 2026 19:57:09 -0400 Subject: [PATCH 16/37] Align flagevaluation EVP contract --- .../openfeature/_flagevaluation_hook.py | 4 ++ .../openfeature/_flagevaluation_writer.py | 44 +++++++++++-------- tests/openfeature/test_flagevaluation_hook.py | 14 ++++++ .../openfeature/test_flagevaluation_writer.py | 17 +++++-- 4 files changed, 58 insertions(+), 21 deletions(-) diff --git a/ddtrace/internal/openfeature/_flagevaluation_hook.py b/ddtrace/internal/openfeature/_flagevaluation_hook.py index 32bbdba5a02..b39b6b7839a 100644 --- a/ddtrace/internal/openfeature/_flagevaluation_hook.py +++ b/ddtrace/internal/openfeature/_flagevaluation_hook.py @@ -95,6 +95,10 @@ def finally_after( error_message = "" if details.error_message: error_message = str(details.error_message) + elif details.error_code: + error_message = ( + str(details.error_code.value) if hasattr(details.error_code, "value") else str(details.error_code) + ) event = _EvalEvent( flag_key=flag_key, diff --git a/ddtrace/internal/openfeature/_flagevaluation_writer.py b/ddtrace/internal/openfeature/_flagevaluation_writer.py index aeba5cf6025..e7a54afd8f9 100644 --- a/ddtrace/internal/openfeature/_flagevaluation_writer.py +++ b/ddtrace/internal/openfeature/_flagevaluation_writer.py @@ -39,7 +39,7 @@ logger = get_logger(__name__) # EVP endpoint for flag evaluation events. -FLAGEVALUATIONS_ENDPOINT = "/evp_proxy/v2/api/v2/flagevaluations" +FLAGEVALUATIONS_ENDPOINT = "/evp_proxy/v2/api/v2/flagevaluation" EVP_SUBDOMAIN_HEADER_NAME = "X-Datadog-EVP-Subdomain" EVP_SUBDOMAIN_VALUE = "event-platform-intake" @@ -47,9 +47,17 @@ MAX_CONTEXT_FIELDS = 256 MAX_FIELD_LENGTH = 256 -# Aggregation caps (sized for a ≥2,500-flag scale target). +# Aggregation caps (sized for a >=2,500-flag scale target). +EVAL_SCALE_TARGET_FLAGS = 2_500 +EVAL_SCALE_FULL_BUCKETS_PER_FLAG = 50 +EVAL_SCALE_USERS_PER_FLAG = 1_000 +EVAL_SCALE_PER_FLAG_HEADROOM_MULTIPLIER = 10 +EVAL_SCALE_DEGRADED_BUCKETS_PER_FLAG = 10 +EVAL_SCALE_FULL_BUCKET_TARGET = EVAL_SCALE_TARGET_FLAGS * EVAL_SCALE_FULL_BUCKETS_PER_FLAG +EVAL_SCALE_PER_FLAG_BUCKET_TARGET = EVAL_SCALE_PER_FLAG_HEADROOM_MULTIPLIER * EVAL_SCALE_USERS_PER_FLAG +EVAL_SCALE_DEGRADED_BUCKET_TARGET = EVAL_SCALE_TARGET_FLAGS * EVAL_SCALE_DEGRADED_BUCKETS_PER_FLAG GLOBAL_CAP = 131_072 # bounds full-tier buckets -PER_FLAG_CAP = 10_000 # bounds full-tier buckets per flag +PER_FLAG_CAP = EVAL_SCALE_PER_FLAG_BUCKET_TARGET # bounds full-tier buckets per flag DEGRADED_CAP = 32_768 # bounds degraded-tier buckets; overflow is drop-counted # Async hand-off queue size. @@ -205,28 +213,28 @@ class _Entry: def __init__( self, - now_ms: int, + eval_time_ms: int, runtime_default: bool, targeting_key: str, context_attrs: dict[str, typing.Any], error_message: str, ) -> None: self.count: int = 1 - self.first_evaluation: int = now_ms - self.last_evaluation: int = now_ms + self.first_evaluation: int = eval_time_ms + self.last_evaluation: int = eval_time_ms self.runtime_default: bool = runtime_default # Full-tier only: self.targeting_key: str = targeting_key self.context_attrs: dict[str, typing.Any] = context_attrs self.error_message: str = error_message - def observe(self, now_ms: int) -> None: + def observe(self, eval_time_ms: int) -> None: """Update count and first/last bounds for a repeated evaluation.""" self.count += 1 - if now_ms < self.first_evaluation: - self.first_evaluation = now_ms - if now_ms > self.last_evaluation: - self.last_evaluation = now_ms + if eval_time_ms < self.first_evaluation: + self.first_evaluation = eval_time_ms + if eval_time_ms > self.last_evaluation: + self.last_evaluation = eval_time_ms class _EvalEvent(typing.NamedTuple): @@ -395,7 +403,7 @@ def periodic(self) -> None: ) # 3. Build payload. - now_ms = int(time.time() * 1000) + flush_time_ms = int(time.time() * 1000) events = [] # Full-tier events: all optional fields present. @@ -403,7 +411,7 @@ def periodic(self) -> None: flag_key = key[0] variant = key[1] allocation_key = key[2] - ev = _base_event(flag_key, entry, now_ms) + ev = _base_event(flag_key, entry, flush_time_ms) if entry.runtime_default: ev["runtime_default_used"] = True if entry.targeting_key: @@ -423,7 +431,7 @@ def periodic(self) -> None: flag_key = key[0] variant = key[1] allocation_key = key[2] - ev = _base_event(flag_key, entry, now_ms) + ev = _base_event(flag_key, entry, flush_time_ms) if entry.runtime_default: ev["runtime_default_used"] = True if variant: @@ -528,7 +536,7 @@ def _aggregate(self, event: _EvalEvent) -> None: # New full-tier bucket. self._full[full_key] = _Entry( - now_ms=event.eval_time_ms, + eval_time_ms=event.eval_time_ms, runtime_default=event.runtime_default, targeting_key=event.targeting_key, context_attrs=context_attrs, @@ -557,7 +565,7 @@ def _add_to_degraded(self, event: _EvalEvent) -> None: return self._degraded[deg_key] = _Entry( - now_ms=event.eval_time_ms, + eval_time_ms=event.eval_time_ms, runtime_default=event.runtime_default, targeting_key="", context_attrs={}, @@ -600,10 +608,10 @@ def _send_payload(self, payload: bytes, num_events: int) -> None: # --------------------------------------------------------------------------- -def _base_event(flag_key: str, entry: "_Entry", now_ms: int) -> dict[str, typing.Any]: +def _base_event(flag_key: str, entry: "_Entry", flush_time_ms: int) -> dict[str, typing.Any]: """Build the required-fields-only event dict for a single aggregation entry.""" return { - "timestamp": now_ms, + "timestamp": flush_time_ms, "flag": {"key": flag_key}, "first_evaluation": entry.first_evaluation, "last_evaluation": entry.last_evaluation, diff --git a/tests/openfeature/test_flagevaluation_hook.py b/tests/openfeature/test_flagevaluation_hook.py index b5126e0d53b..8fa549ac9a5 100644 --- a/tests/openfeature/test_flagevaluation_hook.py +++ b/tests/openfeature/test_flagevaluation_hook.py @@ -15,6 +15,7 @@ from unittest import mock from openfeature.evaluation_context import EvaluationContext +from openfeature.exception import ErrorCode from openfeature.flag_evaluation import FlagEvaluationDetails from openfeature.flag_evaluation import FlagType from openfeature.flag_evaluation import Reason @@ -43,6 +44,7 @@ def _make_details( reason: typing.Optional[Reason] = Reason.TARGETING_MATCH, flag_metadata: dict = None, error_message: str = None, + error_code: typing.Optional[ErrorCode] = None, ) -> FlagEvaluationDetails: return FlagEvaluationDetails( flag_key=flag_key, @@ -51,6 +53,7 @@ def _make_details( reason=reason, flag_metadata=flag_metadata or {}, error_message=error_message, + error_code=error_code, ) @@ -157,6 +160,17 @@ def test_finally_after_extracts_allocation_key(self, hook, writer): event = writer.enqueue.call_args[0][0] assert event.allocation_key == "alloc-xyz" + def test_finally_after_falls_back_to_error_code_when_message_absent(self, hook, writer): + hc = _make_hook_context() + details = _make_details( + variant=None, + reason=Reason.ERROR, + error_code=ErrorCode.FLAG_NOT_FOUND, + ) + hook.finally_after(hc, details, {}) + event = writer.enqueue.call_args[0][0] + assert event.error_message == ErrorCode.FLAG_NOT_FOUND.value + def test_finally_after_does_no_aggregation_on_hook_thread(self, hook, writer): """The hook must call enqueue only — not build payloads or aggregate.""" # writer.enqueue is a mock; the only call from finally_after must be enqueue. diff --git a/tests/openfeature/test_flagevaluation_writer.py b/tests/openfeature/test_flagevaluation_writer.py index 0a4efbf5926..1c69d77fd2e 100644 --- a/tests/openfeature/test_flagevaluation_writer.py +++ b/tests/openfeature/test_flagevaluation_writer.py @@ -8,7 +8,7 @@ - 256-field / 256-char context pruning - runtime_default_used from absent/None variant - Non-blocking enqueue with drop-and-count on queue.Full -- EVP POST to /evp_proxy/v2/api/v2/flagevaluations with correct headers +- EVP POST to /evp_proxy/v2/api/v2/flagevaluation with correct headers """ import json @@ -22,6 +22,9 @@ from ddtrace.internal.openfeature._flagevaluation_writer import DEGRADED_CAP from ddtrace.internal.openfeature._flagevaluation_writer import EVP_SUBDOMAIN_HEADER_NAME from ddtrace.internal.openfeature._flagevaluation_writer import EVP_SUBDOMAIN_VALUE +from ddtrace.internal.openfeature._flagevaluation_writer import EVAL_SCALE_DEGRADED_BUCKET_TARGET +from ddtrace.internal.openfeature._flagevaluation_writer import EVAL_SCALE_FULL_BUCKET_TARGET +from ddtrace.internal.openfeature._flagevaluation_writer import EVAL_SCALE_PER_FLAG_BUCKET_TARGET from ddtrace.internal.openfeature._flagevaluation_writer import FLAGEVALUATIONS_ENDPOINT from ddtrace.internal.openfeature._flagevaluation_writer import GLOBAL_CAP from ddtrace.internal.openfeature._flagevaluation_writer import MAX_CONTEXT_FIELDS @@ -341,7 +344,7 @@ def test_periodic_resets_maps_after_flush(self, writer): @mock.patch("ddtrace.internal.openfeature._flagevaluation_writer.get_connection") def test_post_to_correct_endpoint_with_evp_header(self, mock_get_conn, writer): - """Payload goes to /evp_proxy/v2/api/v2/flagevaluations with EVP subdomain header.""" + """Payload goes to /evp_proxy/v2/api/v2/flagevaluation with EVP subdomain header.""" mock_conn = mock.Mock() mock_resp = mock.Mock() mock_resp.status = 200 @@ -428,7 +431,15 @@ def test_degraded_event_has_no_context_or_targeting_key(self, writer): assert "context" not in ev def test_writer_endpoint_constant(self): - assert FLAGEVALUATIONS_ENDPOINT == "/evp_proxy/v2/api/v2/flagevaluations" + assert FLAGEVALUATIONS_ENDPOINT == "/evp_proxy/v2/api/v2/flagevaluation" + + def test_cap_sizing_constants(self): + assert EVAL_SCALE_FULL_BUCKET_TARGET == 125_000 + assert EVAL_SCALE_PER_FLAG_BUCKET_TARGET == 10_000 + assert EVAL_SCALE_DEGRADED_BUCKET_TARGET == 25_000 + assert GLOBAL_CAP == 131_072 + assert PER_FLAG_CAP == 10_000 + assert DEGRADED_CAP == 32_768 def test_class_exists_and_inherits_periodic_service(self): from ddtrace.internal.periodic import PeriodicService From 1b32062dbae6f5db976d563f821205cfd751f837 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 19 Jun 2026 21:19:24 -0400 Subject: [PATCH 17/37] fix(openfeature): sort flagevaluation writer imports --- tests/openfeature/test_flagevaluation_writer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/openfeature/test_flagevaluation_writer.py b/tests/openfeature/test_flagevaluation_writer.py index 1c69d77fd2e..777b012b41a 100644 --- a/tests/openfeature/test_flagevaluation_writer.py +++ b/tests/openfeature/test_flagevaluation_writer.py @@ -20,11 +20,11 @@ import pytest from ddtrace.internal.openfeature._flagevaluation_writer import DEGRADED_CAP -from ddtrace.internal.openfeature._flagevaluation_writer import EVP_SUBDOMAIN_HEADER_NAME -from ddtrace.internal.openfeature._flagevaluation_writer import EVP_SUBDOMAIN_VALUE from ddtrace.internal.openfeature._flagevaluation_writer import EVAL_SCALE_DEGRADED_BUCKET_TARGET from ddtrace.internal.openfeature._flagevaluation_writer import EVAL_SCALE_FULL_BUCKET_TARGET from ddtrace.internal.openfeature._flagevaluation_writer import EVAL_SCALE_PER_FLAG_BUCKET_TARGET +from ddtrace.internal.openfeature._flagevaluation_writer import EVP_SUBDOMAIN_HEADER_NAME +from ddtrace.internal.openfeature._flagevaluation_writer import EVP_SUBDOMAIN_VALUE from ddtrace.internal.openfeature._flagevaluation_writer import FLAGEVALUATIONS_ENDPOINT from ddtrace.internal.openfeature._flagevaluation_writer import GLOBAL_CAP from ddtrace.internal.openfeature._flagevaluation_writer import MAX_CONTEXT_FIELDS From e9c673f113fa4b81f7f7b70d122204e68e978999 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 19 Jun 2026 22:09:52 -0400 Subject: [PATCH 18/37] fix(openfeature): keep flagevaluation flush interval internal --- ddtrace/internal/openfeature/_provider.py | 4 +--- .../internal/settings/_supported_configurations.py | 1 - ddtrace/internal/settings/openfeature.py | 7 ------- supported-configurations.json | 8 -------- tests/openfeature/test_flagevaluation_hook.py | 12 ------------ 5 files changed, 1 insertion(+), 31 deletions(-) diff --git a/ddtrace/internal/openfeature/_provider.py b/ddtrace/internal/openfeature/_provider.py index 21a2afee5db..0fc2581bf9d 100644 --- a/ddtrace/internal/openfeature/_provider.py +++ b/ddtrace/internal/openfeature/_provider.py @@ -144,9 +144,7 @@ def __init__(self, *args: typing.Any, **kwargs: typing.Any): evp_config = OpenFeatureConfig() evp_counts_enabled = evp_config.flagging_evaluation_counts_enabled if self._enabled and evp_counts_enabled: - self._flagevaluation_writer = FlagEvaluationWriter( - interval=evp_config.flagging_evaluation_counts_flush_interval - ) + self._flagevaluation_writer = FlagEvaluationWriter() self._flagevaluation_hook = FlagEvaluationHook(self._flagevaluation_writer) def get_metadata(self) -> Metadata: diff --git a/ddtrace/internal/settings/_supported_configurations.py b/ddtrace/internal/settings/_supported_configurations.py index 86e9b090633..fb05870c4cc 100644 --- a/ddtrace/internal/settings/_supported_configurations.py +++ b/ddtrace/internal/settings/_supported_configurations.py @@ -221,7 +221,6 @@ "DD_FFE_INTAKE_ENABLED", "DD_FFE_INTAKE_HEARTBEAT_INTERVAL", "DD_FLAGGING_EVALUATION_COUNTS_ENABLED", - "DD_FLAGGING_EVALUATION_COUNTS_FLUSH_INTERVAL_SECONDS", "DD_FLASK_CACHE_SERVICE", "DD_FLASK_SERVICE", "DD_FOO_SERVICE", diff --git a/ddtrace/internal/settings/openfeature.py b/ddtrace/internal/settings/openfeature.py index 7f9e44c2df9..d5f740f6445 100644 --- a/ddtrace/internal/settings/openfeature.py +++ b/ddtrace/internal/settings/openfeature.py @@ -26,12 +26,6 @@ class OpenFeatureConfig(DDConfig): default=True, ) - flagging_evaluation_counts_flush_interval = DDConfig.var( - float, - "DD_FLAGGING_EVALUATION_COUNTS_FLUSH_INTERVAL_SECONDS", - default=10.0, - ) - # Feature flag exposure intake configuration ffe_intake_enabled = DDConfig.var( bool, @@ -57,7 +51,6 @@ class OpenFeatureConfig(DDConfig): _openfeature_config_keys = [ "experimental_flagging_provider_enabled", "flagging_evaluation_counts_enabled", - "flagging_evaluation_counts_flush_interval", "ffe_intake_enabled", "ffe_intake_heartbeat_interval", "initialization_timeout_ms", diff --git a/supported-configurations.json b/supported-configurations.json index 04590704fb0..1b41c122cc8 100644 --- a/supported-configurations.json +++ b/supported-configurations.json @@ -1648,14 +1648,6 @@ "experimental": true } ], - "DD_FLAGGING_EVALUATION_COUNTS_FLUSH_INTERVAL_SECONDS": [ - { - "implementation": "A", - "type": "decimal", - "default": "10.0", - "experimental": true - } - ], "DD_FLASK_CACHE_SERVICE": [ { "implementation": "A", diff --git a/tests/openfeature/test_flagevaluation_hook.py b/tests/openfeature/test_flagevaluation_hook.py index 8fa549ac9a5..e7385f9cd51 100644 --- a/tests/openfeature/test_flagevaluation_hook.py +++ b/tests/openfeature/test_flagevaluation_hook.py @@ -322,18 +322,6 @@ def test_killswitch_false_does_not_register_evp_hook(self): assert provider._flagevaluation_writer is None assert provider._flagevaluation_hook is None - def test_flush_interval_env_configures_evp_writer(self): - with mock.patch.dict(os.environ, {"DD_FLAGGING_EVALUATION_COUNTS_FLUSH_INTERVAL_SECONDS": "20"}): - from tests.utils import override_global_config - - with override_global_config({"experimental_flagging_provider_enabled": True}): - with mock.patch("ddtrace.internal.openfeature._provider.FlagEvaluationWriter") as writer_cls: - from ddtrace.internal.openfeature._provider import DataDogProvider - - DataDogProvider() - - writer_cls.assert_called_once_with(interval=20.0) - def test_killswitch_false_does_not_affect_otel_hook(self): """Killswitch must not suppress the OTel FlagEvalHook (OTel non-regression).""" with mock.patch.dict(os.environ, {"DD_FLAGGING_EVALUATION_COUNTS_ENABLED": "false"}): From 8539e96d4456c310bc30d83ead1cc6497cd044ba Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Mon, 22 Jun 2026 19:03:50 -0400 Subject: [PATCH 19/37] Align flagevaluation EVP metadata --- .riot/requirements/13c4b39.txt | 21 ++ .../requirements/{698fd99.txt => 14fc413.txt} | 16 +- .../requirements/{18535ff.txt => 1540c33.txt} | 16 +- .../requirements/{1da3eaa.txt => 16138c7.txt} | 22 +-- .../requirements/{8c0516f.txt => 168ee03.txt} | 16 +- .../requirements/{1dd7ab8.txt => 16b741f.txt} | 23 +-- .../requirements/{1fa9792.txt => 18421e5.txt} | 17 +- .../requirements/{8d6e180.txt => 18a4a8d.txt} | 23 +-- .../requirements/{6f87c57.txt => 460df49.txt} | 24 +-- .../requirements/{13b681c.txt => 765862d.txt} | 12 +- .../requirements/{8c9f12c.txt => b3bdd52.txt} | 26 ++- .riot/requirements/c20e7ba.txt | 25 --- .../requirements/{a8932a5.txt => cdab08a.txt} | 17 +- .../openfeature/_flagevaluation_hook.py | 7 +- .../openfeature/_flagevaluation_writer.py | 8 +- ddtrace/internal/openfeature/_provider.py | 21 +- riotfile.py | 1 - tests/openfeature/test_flagevaluation_e2e.py | 6 +- tests/openfeature/test_flagevaluation_hook.py | 8 + .../openfeature/test_flagevaluation_writer.py | 108 ++++++---- .../batchedflagevaluations.json | 184 ------------------ 21 files changed, 212 insertions(+), 389 deletions(-) create mode 100644 .riot/requirements/13c4b39.txt rename .riot/requirements/{698fd99.txt => 14fc413.txt} (63%) rename .riot/requirements/{18535ff.txt => 1540c33.txt} (67%) rename .riot/requirements/{1da3eaa.txt => 16138c7.txt} (52%) rename .riot/requirements/{8c0516f.txt => 168ee03.txt} (63%) rename .riot/requirements/{1dd7ab8.txt => 16b741f.txt} (50%) rename .riot/requirements/{1fa9792.txt => 18421e5.txt} (60%) rename .riot/requirements/{8d6e180.txt => 18a4a8d.txt} (50%) rename .riot/requirements/{6f87c57.txt => 460df49.txt} (58%) rename .riot/requirements/{13b681c.txt => 765862d.txt} (76%) rename .riot/requirements/{8c9f12c.txt => b3bdd52.txt} (51%) delete mode 100644 .riot/requirements/c20e7ba.txt rename .riot/requirements/{a8932a5.txt => cdab08a.txt} (60%) delete mode 100644 tests/openfeature/testdata/flageval-worker/batchedflagevaluations.json diff --git a/.riot/requirements/13c4b39.txt b/.riot/requirements/13c4b39.txt new file mode 100644 index 00000000000..9d817e861d3 --- /dev/null +++ b/.riot/requirements/13c4b39.txt @@ -0,0 +1,21 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --cert=None --client-cert=None --index-url=None --no-annotate --pip-args=None .riot/requirements/13c4b39.in +# +attrs==25.4.0 +coverage[toml]==7.11.0 +hypothesis==6.45.0 +iniconfig==2.3.0 +mock==5.2.0 +openfeature-sdk==0.8.3 +opentracing==2.4.0 +packaging==25.0 +pluggy==1.6.0 +pygments==2.19.2 +pytest==8.4.2 +pytest-cov==7.0.0 +pytest-mock==3.15.1 +pytest-randomly==4.0.1 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/698fd99.txt b/.riot/requirements/14fc413.txt similarity index 63% rename from .riot/requirements/698fd99.txt rename to .riot/requirements/14fc413.txt index dbacb969ad2..6ed9e57c56e 100644 --- a/.riot/requirements/698fd99.txt +++ b/.riot/requirements/14fc413.txt @@ -2,24 +2,20 @@ # This file is autogenerated by pip-compile with Python 3.13 # by the following command: # -# pip-compile --allow-unsafe --no-annotate .riot/requirements/698fd99.in +# pip-compile --allow-unsafe --no-annotate .riot/requirements/14fc413.in # attrs==26.1.0 -coverage[toml]==7.14.1 +coverage[toml]==7.13.5 hypothesis==6.45.0 iniconfig==2.3.0 -jsonschema==4.26.0 -jsonschema-specifications==2025.9.1 mock==5.2.0 openfeature-sdk==0.8.4 opentracing==2.4.0 -packaging==26.2 +packaging==26.0 pluggy==1.6.0 -pygments==2.20.0 -pytest==9.1.0 +pygments==2.19.2 +pytest==9.0.2 pytest-cov==7.1.0 pytest-mock==3.15.1 -pytest-randomly==4.1.0 -referencing==0.37.0 -rpds-py==2026.5.1 +pytest-randomly==4.0.1 sortedcontainers==2.4.0 diff --git a/.riot/requirements/18535ff.txt b/.riot/requirements/1540c33.txt similarity index 67% rename from .riot/requirements/18535ff.txt rename to .riot/requirements/1540c33.txt index aad6e625fa6..6560b710652 100644 --- a/.riot/requirements/18535ff.txt +++ b/.riot/requirements/1540c33.txt @@ -2,27 +2,23 @@ # This file is autogenerated by pip-compile with Python 3.10 # by the following command: # -# pip-compile --allow-unsafe --no-annotate .riot/requirements/18535ff.in +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1540c33.in # attrs==26.1.0 -coverage[toml]==7.14.1 +coverage[toml]==7.13.5 exceptiongroup==1.3.1 hypothesis==6.45.0 iniconfig==2.3.0 -jsonschema==4.26.0 -jsonschema-specifications==2025.9.1 mock==5.2.0 openfeature-sdk==0.8.4 opentracing==2.4.0 -packaging==26.2 +packaging==26.0 pluggy==1.6.0 -pygments==2.20.0 -pytest==9.1.0 +pygments==2.19.2 +pytest==9.0.2 pytest-cov==7.1.0 pytest-mock==3.15.1 -pytest-randomly==4.1.0 -referencing==0.37.0 -rpds-py==0.30.0 +pytest-randomly==4.0.1 sortedcontainers==2.4.0 tomli==2.4.1 typing-extensions==4.15.0 diff --git a/.riot/requirements/1da3eaa.txt b/.riot/requirements/16138c7.txt similarity index 52% rename from .riot/requirements/1da3eaa.txt rename to .riot/requirements/16138c7.txt index fc80691d71e..5fd1c6c8f51 100644 --- a/.riot/requirements/1da3eaa.txt +++ b/.riot/requirements/16138c7.txt @@ -2,24 +2,20 @@ # This file is autogenerated by pip-compile with Python 3.14 # by the following command: # -# pip-compile --allow-unsafe --no-annotate .riot/requirements/1da3eaa.in +# pip-compile --allow-unsafe --no-annotate .riot/requirements/16138c7.in # -attrs==26.1.0 -coverage[toml]==7.14.1 +attrs==25.4.0 +coverage[toml]==7.11.0 hypothesis==6.45.0 iniconfig==2.3.0 -jsonschema==4.26.0 -jsonschema-specifications==2025.9.1 mock==5.2.0 -openfeature-sdk==0.10.0 +openfeature-sdk==0.8.3 opentracing==2.4.0 -packaging==26.2 +packaging==25.0 pluggy==1.6.0 -pygments==2.20.0 -pytest==9.1.0 -pytest-cov==7.1.0 +pygments==2.19.2 +pytest==8.4.2 +pytest-cov==7.0.0 pytest-mock==3.15.1 -pytest-randomly==4.1.0 -referencing==0.37.0 -rpds-py==2026.5.1 +pytest-randomly==4.0.1 sortedcontainers==2.4.0 diff --git a/.riot/requirements/8c0516f.txt b/.riot/requirements/168ee03.txt similarity index 63% rename from .riot/requirements/8c0516f.txt rename to .riot/requirements/168ee03.txt index 8e4e8bd6a7b..d4693e30888 100644 --- a/.riot/requirements/8c0516f.txt +++ b/.riot/requirements/168ee03.txt @@ -2,24 +2,20 @@ # This file is autogenerated by pip-compile with Python 3.14 # by the following command: # -# pip-compile --allow-unsafe --no-annotate .riot/requirements/8c0516f.in +# pip-compile --allow-unsafe --no-annotate .riot/requirements/168ee03.in # attrs==26.1.0 -coverage[toml]==7.14.1 +coverage[toml]==7.13.5 hypothesis==6.45.0 iniconfig==2.3.0 -jsonschema==4.26.0 -jsonschema-specifications==2025.9.1 mock==5.2.0 openfeature-sdk==0.8.4 opentracing==2.4.0 -packaging==26.2 +packaging==26.0 pluggy==1.6.0 -pygments==2.20.0 -pytest==9.1.0 +pygments==2.19.2 +pytest==9.0.2 pytest-cov==7.1.0 pytest-mock==3.15.1 -pytest-randomly==4.1.0 -referencing==0.37.0 -rpds-py==2026.5.1 +pytest-randomly==4.0.1 sortedcontainers==2.4.0 diff --git a/.riot/requirements/1dd7ab8.txt b/.riot/requirements/16b741f.txt similarity index 50% rename from .riot/requirements/1dd7ab8.txt rename to .riot/requirements/16b741f.txt index ad6ec54405c..071aaecbc6a 100644 --- a/.riot/requirements/1dd7ab8.txt +++ b/.riot/requirements/16b741f.txt @@ -2,25 +2,20 @@ # This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# pip-compile --allow-unsafe --no-annotate .riot/requirements/1dd7ab8.in +# pip-compile --allow-unsafe --no-annotate .riot/requirements/16b741f.in # -attrs==26.1.0 -coverage[toml]==7.14.1 +attrs==25.4.0 +coverage[toml]==7.11.0 hypothesis==6.45.0 iniconfig==2.3.0 -jsonschema==4.26.0 -jsonschema-specifications==2025.9.1 mock==5.2.0 -openfeature-sdk==0.10.0 +openfeature-sdk==0.8.3 opentracing==2.4.0 -packaging==26.2 +packaging==25.0 pluggy==1.6.0 -pygments==2.20.0 -pytest==9.1.0 -pytest-cov==7.1.0 +pygments==2.19.2 +pytest==8.4.2 +pytest-cov==7.0.0 pytest-mock==3.15.1 -pytest-randomly==4.1.0 -referencing==0.37.0 -rpds-py==2026.5.1 +pytest-randomly==4.0.1 sortedcontainers==2.4.0 -typing-extensions==4.15.0 diff --git a/.riot/requirements/1fa9792.txt b/.riot/requirements/18421e5.txt similarity index 60% rename from .riot/requirements/1fa9792.txt rename to .riot/requirements/18421e5.txt index fbd27b5910f..af057052170 100644 --- a/.riot/requirements/1fa9792.txt +++ b/.riot/requirements/18421e5.txt @@ -2,25 +2,20 @@ # This file is autogenerated by pip-compile with Python 3.12 # by the following command: # -# pip-compile --allow-unsafe --no-annotate .riot/requirements/1fa9792.in +# pip-compile --allow-unsafe --no-annotate .riot/requirements/18421e5.in # attrs==26.1.0 -coverage[toml]==7.14.1 +coverage[toml]==7.13.5 hypothesis==6.45.0 iniconfig==2.3.0 -jsonschema==4.26.0 -jsonschema-specifications==2025.9.1 mock==5.2.0 openfeature-sdk==0.8.4 opentracing==2.4.0 -packaging==26.2 +packaging==26.0 pluggy==1.6.0 -pygments==2.20.0 -pytest==9.1.0 +pygments==2.19.2 +pytest==9.0.2 pytest-cov==7.1.0 pytest-mock==3.15.1 -pytest-randomly==4.1.0 -referencing==0.37.0 -rpds-py==2026.5.1 +pytest-randomly==4.0.1 sortedcontainers==2.4.0 -typing-extensions==4.15.0 diff --git a/.riot/requirements/8d6e180.txt b/.riot/requirements/18a4a8d.txt similarity index 50% rename from .riot/requirements/8d6e180.txt rename to .riot/requirements/18a4a8d.txt index 5fe12233cce..b1c42ace3ec 100644 --- a/.riot/requirements/8d6e180.txt +++ b/.riot/requirements/18a4a8d.txt @@ -2,25 +2,20 @@ # This file is autogenerated by pip-compile with Python 3.12 # by the following command: # -# pip-compile --allow-unsafe --no-annotate .riot/requirements/8d6e180.in +# pip-compile --allow-unsafe --no-annotate .riot/requirements/18a4a8d.in # -attrs==26.1.0 -coverage[toml]==7.14.1 +attrs==25.4.0 +coverage[toml]==7.11.0 hypothesis==6.45.0 iniconfig==2.3.0 -jsonschema==4.26.0 -jsonschema-specifications==2025.9.1 mock==5.2.0 -openfeature-sdk==0.10.0 +openfeature-sdk==0.8.3 opentracing==2.4.0 -packaging==26.2 +packaging==25.0 pluggy==1.6.0 -pygments==2.20.0 -pytest==9.1.0 -pytest-cov==7.1.0 +pygments==2.19.2 +pytest==8.4.2 +pytest-cov==7.0.0 pytest-mock==3.15.1 -pytest-randomly==4.1.0 -referencing==0.37.0 -rpds-py==2026.5.1 +pytest-randomly==4.0.1 sortedcontainers==2.4.0 -typing-extensions==4.15.0 diff --git a/.riot/requirements/6f87c57.txt b/.riot/requirements/460df49.txt similarity index 58% rename from .riot/requirements/6f87c57.txt rename to .riot/requirements/460df49.txt index 2ae1884c6df..5f94a829c24 100644 --- a/.riot/requirements/6f87c57.txt +++ b/.riot/requirements/460df49.txt @@ -2,29 +2,25 @@ # This file is autogenerated by pip-compile with Python 3.9 # by the following command: # -# pip-compile --allow-unsafe --no-annotate .riot/requirements/6f87c57.in +# pip-compile --allow-unsafe --no-annotate .riot/requirements/460df49.in # -attrs==26.1.0 +attrs==25.4.0 coverage[toml]==7.10.7 -exceptiongroup==1.3.1 +exceptiongroup==1.3.0 hypothesis==6.45.0 -importlib-metadata==8.7.1 +importlib-metadata==8.7.0 iniconfig==2.1.0 -jsonschema==4.25.1 -jsonschema-specifications==2025.9.1 mock==5.2.0 -openfeature-sdk==0.8.4 +openfeature-sdk==0.8.3 opentracing==2.4.0 -packaging==26.2 +packaging==25.0 pluggy==1.6.0 -pygments==2.20.0 +pygments==2.19.2 pytest==8.4.2 -pytest-cov==7.1.0 +pytest-cov==7.0.0 pytest-mock==3.15.1 pytest-randomly==4.0.1 -referencing==0.36.2 -rpds-py==0.27.1 sortedcontainers==2.4.0 -tomli==2.4.1 +tomli==2.3.0 typing-extensions==4.15.0 -zipp==3.23.1 +zipp==3.23.0 diff --git a/.riot/requirements/13b681c.txt b/.riot/requirements/765862d.txt similarity index 76% rename from .riot/requirements/13b681c.txt rename to .riot/requirements/765862d.txt index 656681508de..f81b3dde9ef 100644 --- a/.riot/requirements/13b681c.txt +++ b/.riot/requirements/765862d.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.9 # by the following command: # -# pip-compile --allow-unsafe --no-annotate .riot/requirements/13b681c.in +# pip-compile --allow-unsafe --no-annotate .riot/requirements/765862d.in # attrs==26.1.0 coverage[toml]==7.10.7 @@ -10,21 +10,17 @@ exceptiongroup==1.3.1 hypothesis==6.45.0 importlib-metadata==8.7.1 iniconfig==2.1.0 -jsonschema==4.25.1 -jsonschema-specifications==2025.9.1 mock==5.2.0 openfeature-sdk==0.8.4 opentracing==2.4.0 -packaging==26.2 +packaging==26.0 pluggy==1.6.0 -pygments==2.20.0 +pygments==2.19.2 pytest==8.4.2 pytest-cov==7.1.0 pytest-mock==3.15.1 pytest-randomly==4.0.1 -referencing==0.36.2 -rpds-py==0.27.1 sortedcontainers==2.4.0 tomli==2.4.1 typing-extensions==4.15.0 -zipp==3.23.1 +zipp==3.23.0 diff --git a/.riot/requirements/8c9f12c.txt b/.riot/requirements/b3bdd52.txt similarity index 51% rename from .riot/requirements/8c9f12c.txt rename to .riot/requirements/b3bdd52.txt index beacb26877d..ea418581bad 100644 --- a/.riot/requirements/8c9f12c.txt +++ b/.riot/requirements/b3bdd52.txt @@ -2,27 +2,23 @@ # This file is autogenerated by pip-compile with Python 3.10 # by the following command: # -# pip-compile --allow-unsafe --no-annotate .riot/requirements/8c9f12c.in +# pip-compile --allow-unsafe --no-annotate .riot/requirements/b3bdd52.in # -attrs==26.1.0 -coverage[toml]==7.14.1 -exceptiongroup==1.3.1 +attrs==25.4.0 +coverage[toml]==7.11.0 +exceptiongroup==1.3.0 hypothesis==6.45.0 iniconfig==2.3.0 -jsonschema==4.26.0 -jsonschema-specifications==2025.9.1 mock==5.2.0 -openfeature-sdk==0.10.0 +openfeature-sdk==0.8.3 opentracing==2.4.0 -packaging==26.2 +packaging==25.0 pluggy==1.6.0 -pygments==2.20.0 -pytest==9.1.0 -pytest-cov==7.1.0 +pygments==2.19.2 +pytest==8.4.2 +pytest-cov==7.0.0 pytest-mock==3.15.1 -pytest-randomly==4.1.0 -referencing==0.37.0 -rpds-py==0.30.0 +pytest-randomly==4.0.1 sortedcontainers==2.4.0 -tomli==2.4.1 +tomli==2.3.0 typing-extensions==4.15.0 diff --git a/.riot/requirements/c20e7ba.txt b/.riot/requirements/c20e7ba.txt deleted file mode 100644 index cd6f57f78fb..00000000000 --- a/.riot/requirements/c20e7ba.txt +++ /dev/null @@ -1,25 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.13 -# by the following command: -# -# pip-compile --allow-unsafe --no-annotate .riot/requirements/c20e7ba.in -# -attrs==26.1.0 -coverage[toml]==7.14.1 -hypothesis==6.45.0 -iniconfig==2.3.0 -jsonschema==4.26.0 -jsonschema-specifications==2025.9.1 -mock==5.2.0 -openfeature-sdk==0.10.0 -opentracing==2.4.0 -packaging==26.2 -pluggy==1.6.0 -pygments==2.20.0 -pytest==9.1.0 -pytest-cov==7.1.0 -pytest-mock==3.15.1 -pytest-randomly==4.1.0 -referencing==0.37.0 -rpds-py==2026.5.1 -sortedcontainers==2.4.0 diff --git a/.riot/requirements/a8932a5.txt b/.riot/requirements/cdab08a.txt similarity index 60% rename from .riot/requirements/a8932a5.txt rename to .riot/requirements/cdab08a.txt index 24af0d9cd6b..4aed00c327c 100644 --- a/.riot/requirements/a8932a5.txt +++ b/.riot/requirements/cdab08a.txt @@ -2,25 +2,20 @@ # This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# pip-compile --allow-unsafe --no-annotate .riot/requirements/a8932a5.in +# pip-compile --allow-unsafe --no-annotate .riot/requirements/cdab08a.in # attrs==26.1.0 -coverage[toml]==7.14.1 +coverage[toml]==7.13.5 hypothesis==6.45.0 iniconfig==2.3.0 -jsonschema==4.26.0 -jsonschema-specifications==2025.9.1 mock==5.2.0 openfeature-sdk==0.8.4 opentracing==2.4.0 -packaging==26.2 +packaging==26.0 pluggy==1.6.0 -pygments==2.20.0 -pytest==9.1.0 +pygments==2.19.2 +pytest==9.0.2 pytest-cov==7.1.0 pytest-mock==3.15.1 -pytest-randomly==4.1.0 -referencing==0.37.0 -rpds-py==2026.5.1 +pytest-randomly==4.0.1 sortedcontainers==2.4.0 -typing-extensions==4.15.0 diff --git a/ddtrace/internal/openfeature/_flagevaluation_hook.py b/ddtrace/internal/openfeature/_flagevaluation_hook.py index b39b6b7839a..f5e496774cb 100644 --- a/ddtrace/internal/openfeature/_flagevaluation_hook.py +++ b/ddtrace/internal/openfeature/_flagevaluation_hook.py @@ -12,6 +12,7 @@ import time import typing +from openfeature.exception import ErrorCode from openfeature.flag_evaluation import FlagEvaluationDetails from openfeature.hook import Hook from openfeature.hook import HookContext @@ -77,9 +78,9 @@ def finally_after( else: eval_time_ms = int(time.time() * 1000) - # Variant: None/absent signals a runtime default. - variant = details.variant or "" - runtime_default = details.variant is None + # Variant: absent or type mismatch signals a runtime default. + runtime_default = not details.variant or details.error_code == ErrorCode.TYPE_MISMATCH + variant = "" if runtime_default else details.variant # Targeting key and attributes from the evaluation context. eval_ctx = hook_context.evaluation_context diff --git a/ddtrace/internal/openfeature/_flagevaluation_writer.py b/ddtrace/internal/openfeature/_flagevaluation_writer.py index e7a54afd8f9..772cfd7debe 100644 --- a/ddtrace/internal/openfeature/_flagevaluation_writer.py +++ b/ddtrace/internal/openfeature/_flagevaluation_writer.py @@ -29,6 +29,8 @@ import typing from ddtrace import config as ddconfig +from ddtrace.internal.evp_proxy.constants import EVP_PROXY_AGENT_BASE_PATH +from ddtrace.internal.evp_proxy.constants import EVP_SUBDOMAIN_HEADER_NAME from ddtrace.internal.logger import get_logger from ddtrace.internal.periodic import PeriodicService from ddtrace.internal.settings._agent import config as agent_config @@ -39,13 +41,13 @@ logger = get_logger(__name__) # EVP endpoint for flag evaluation events. -FLAGEVALUATIONS_ENDPOINT = "/evp_proxy/v2/api/v2/flagevaluation" -EVP_SUBDOMAIN_HEADER_NAME = "X-Datadog-EVP-Subdomain" +FLAGEVALUATIONS_ENDPOINT = f"{EVP_PROXY_AGENT_BASE_PATH}/api/v2/flagevaluation" EVP_SUBDOMAIN_VALUE = "event-platform-intake" # Context pruning limits — mirror worker.ts MAX_EVALUATION_CONTEXT_FIELDS / MAX_FIELD_LENGTH. MAX_CONTEXT_FIELDS = 256 MAX_FIELD_LENGTH = 256 +DEDICATED_TARGETING_KEY_CONTEXT_FIELDS = frozenset(("targetingKey", "targeting_key")) # Aggregation caps (sized for a >=2,500-flag scale target). EVAL_SCALE_TARGET_FLAGS = 2_500 @@ -186,6 +188,8 @@ def _flatten_recursive(prefix: str, attrs: typing.Any, out: dict[str, typing.Any out[prefix] = attrs return for k, v in attrs.items(): + if not prefix and k in DEDICATED_TARGETING_KEY_CONTEXT_FIELDS: + continue full_key = f"{prefix}.{k}" if prefix else k if isinstance(v, dict): _flatten_recursive(full_key, v, out) diff --git a/ddtrace/internal/openfeature/_provider.py b/ddtrace/internal/openfeature/_provider.py index 0fc2581bf9d..3292acdaca3 100644 --- a/ddtrace/internal/openfeature/_provider.py +++ b/ddtrace/internal/openfeature/_provider.py @@ -330,12 +330,18 @@ def _resolve_details( flag is not found in the configuration - Returns error with error_code and error_message on other errors """ + # AIDEV-NOTE: Stamp eval-time at provider entry so every OpenFeature exit path + # can feed the EVP flagevaluation hook first_evaluation/last_evaluation from + # evaluation time, not the later hook/flush time. + flag_metadata: dict[str, typing.Any] = {EVAL_TIMESTAMP_METADATA_KEY: int(time.time() * 1000)} + # If provider is not enabled, return default value if not self._enabled: return FlagResolutionDetails( value=default_value, reason=Reason.DISABLED, variant=None, + flag_metadata=flag_metadata, ) try: @@ -358,6 +364,7 @@ def _resolve_details( reason=Reason.ERROR, error_code=ErrorCode.PROVIDER_NOT_READY, error_message="No FFE configuration loaded", + flag_metadata=flag_metadata, ) # Handle errors from native evaluation @@ -380,6 +387,7 @@ def _resolve_details( reason=Reason.ERROR, error_code=openfeature_error_code, error_message="Flag not found", + flag_metadata=flag_metadata, ) # Other errors - return default with ERROR reason @@ -388,6 +396,7 @@ def _resolve_details( reason=Reason.ERROR, error_code=openfeature_error_code, error_message=details.error_message or "Unknown error", + flag_metadata=flag_metadata, ) # Map native ffe.Reason to OpenFeature Reason @@ -402,18 +411,9 @@ def _resolve_details( evaluation_context=evaluation_context, ) - # Build flag_metadata with allocation_key and eval timestamp. - # The timestamp (milliseconds since epoch) is stamped here — at the provider - # resolution boundary — so the finally_after hook receives the most-accurate - # eval-time rather than the (later) hook-fire time. The hook falls back to - # hook-fire time when the metadata key is absent. - # AIDEV-NOTE: This mirrors the Go reference design (metadataEvalTimeKey in - # flagevaluation.go). Python's time.time() * 1000 is sufficient; the native - # PyO3 bridge does not surface a mutable metadata map, so we stamp it here. - flag_metadata: dict[str, typing.Any] = {} + # Add allocation_key to the provider-entry timestamp metadata when present. if details.allocation_key: flag_metadata[METADATA_ALLOCATION_KEY] = details.allocation_key - flag_metadata[EVAL_TIMESTAMP_METADATA_KEY] = int(time.time() * 1000) # Check if variant is None/empty to determine if we should use default value. # For JSON flags, value can be null which is valid, so we check variant instead. @@ -441,6 +441,7 @@ def _resolve_details( reason=Reason.ERROR, error_code=ErrorCode.GENERAL, error_message=f"Unexpected error during flag evaluation: {str(e)}", + flag_metadata=flag_metadata, ) def _report_exposure( diff --git a/riotfile.py b/riotfile.py index 1788a51f586..812192fc3eb 100644 --- a/riotfile.py +++ b/riotfile.py @@ -2817,7 +2817,6 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT pkgs={ "pytest-randomly": latest, "mock": latest, - "jsonschema": latest, # Test against openfeature-sdk 0.8.0+ (required for finally_after hook details parameter) "openfeature-sdk": ["~=0.8.0", latest], }, diff --git a/tests/openfeature/test_flagevaluation_e2e.py b/tests/openfeature/test_flagevaluation_e2e.py index 0c408632d7b..0b78eaad1b4 100644 --- a/tests/openfeature/test_flagevaluation_e2e.py +++ b/tests/openfeature/test_flagevaluation_e2e.py @@ -18,6 +18,7 @@ import pytest from ddtrace.internal.openfeature._config import _set_ffe_config +from ddtrace.internal.openfeature._flagevaluation_writer import EVAL_TIMESTAMP_METADATA_KEY from ddtrace.internal.openfeature._native import process_ffe_configuration from ddtrace.openfeature import DataDogProvider from tests.openfeature.config_helpers import create_boolean_flag @@ -143,12 +144,15 @@ def test_type_mismatch_error_path(self, provider_and_client): process_ffe_configuration(config) # Evaluate a string flag as boolean -> type mismatch ERROR. - assert client.get_boolean_value("str-flag", False) is False + details = client.get_boolean_details("str-flag", False) + assert details.value is False rows = _drain(provider) row = next((r for r in rows if r["flag"]["key"] == "str-flag"), None) assert row is not None, "type-mismatch eval must still emit a flagevaluation row" assert row.get("runtime_default_used") is True + assert row["first_evaluation"] == details.flag_metadata[EVAL_TIMESTAMP_METADATA_KEY] + assert row["last_evaluation"] == details.flag_metadata[EVAL_TIMESTAMP_METADATA_KEY] def test_disabled_flag_path(self, provider_and_client): provider, client = provider_and_client diff --git a/tests/openfeature/test_flagevaluation_hook.py b/tests/openfeature/test_flagevaluation_hook.py index e7385f9cd51..2d6dd6aa009 100644 --- a/tests/openfeature/test_flagevaluation_hook.py +++ b/tests/openfeature/test_flagevaluation_hook.py @@ -109,6 +109,14 @@ def test_finally_after_present_variant_not_runtime_default(self, hook, writer): event = writer.enqueue.call_args[0][0] assert event.runtime_default is False + def test_finally_after_type_mismatch_sets_runtime_default(self, hook, writer): + hc = _make_hook_context() + details = _make_details(variant="wrong-type", error_code=ErrorCode.TYPE_MISMATCH) + hook.finally_after(hc, details, {}) + event = writer.enqueue.call_args[0][0] + assert event.runtime_default is True + assert event.variant == "" + def test_finally_after_does_not_enqueue_openfeature_reason(self, hook, writer): """OpenFeature reason is not an EVP field and must not enter the event snapshot.""" hc = _make_hook_context() diff --git a/tests/openfeature/test_flagevaluation_writer.py b/tests/openfeature/test_flagevaluation_writer.py index 777b012b41a..e0cc5465d1d 100644 --- a/tests/openfeature/test_flagevaluation_writer.py +++ b/tests/openfeature/test_flagevaluation_writer.py @@ -12,11 +12,9 @@ """ import json -from pathlib import Path import time from unittest import mock -from jsonschema import Draft7Validator import pytest from ddtrace.internal.openfeature._flagevaluation_writer import DEGRADED_CAP @@ -159,6 +157,11 @@ def test_nested_attrs_flattened(self): assert "user.tier" in result assert result["user.id"] == "u1" + def test_dedicated_targeting_key_aliases_are_not_context_attrs(self): + attrs = {"targetingKey": "user-1", "targeting_key": "user-1", "tier": "premium"} + result = flatten_and_prune_context(attrs) + assert result == {"tier": "premium"} + def test_prune_beyond_256_fields(self): attrs = {str(i): str(i) for i in range(300)} result = flatten_and_prune_context(attrs) @@ -430,6 +433,22 @@ def test_degraded_event_has_no_context_or_targeting_key(self, writer): assert "targeting_key" not in ev assert "context" not in ev + def test_targeting_key_is_not_duplicated_in_context_evaluation(self, writer): + writer.enqueue( + _make_event( + targeting_key="user-1", + attrs={"targetingKey": "user-1", "targeting_key": "user-1", "tier": "premium"}, + ) + ) + + with mock.patch.object(writer, "_send_payload") as mock_send: + writer.periodic() + + decoded = json.loads(mock_send.call_args[0][0]) + ev = decoded["flagEvaluations"][0] + assert ev["targeting_key"] == "user-1" + assert ev["context"]["evaluation"] == {"tier": "premium"} + def test_writer_endpoint_constant(self): assert FLAGEVALUATIONS_ENDPOINT == "/evp_proxy/v2/api/v2/flagevaluation" @@ -448,14 +467,9 @@ def test_class_exists_and_inherits_periodic_service(self): # --------------------------------------------------------------------------- -# Schema conformance of the emitted payload (full + degraded rows) +# Stable payload contract for emitted rows (full + degraded) # --------------------------------------------------------------------------- -# The copied flageval-worker schema is the contract surface for these tests. -_SCHEMA_PATH = Path(__file__).parent / "testdata" / "flageval-worker" / "batchedflagevaluations.json" -_BATCHED_FLAGEVALUATIONS_SCHEMA = json.loads(_SCHEMA_PATH.read_text()) -_BATCHED_FLAGEVALUATIONS_VALIDATOR = Draft7Validator(_BATCHED_FLAGEVALUATIONS_SCHEMA) - # Required fields that EVERY flagevaluation row (full or degraded) must carry. _REQUIRED_EVENT_FIELDS = { "timestamp": int, @@ -464,10 +478,26 @@ def test_class_exists_and_inherits_periodic_service(self): "last_evaluation": int, "evaluation_count": int, } +_OPTIONAL_EVENT_FIELDS = { + "runtime_default_used", + "targeting_key", + "context", + "variant", + "allocation", + "targeting_rule", + "error", +} +_ALLOWED_EVENT_FIELDS = set(_REQUIRED_EVENT_FIELDS).union(_OPTIONAL_EVENT_FIELDS) +_ALLOWED_BATCH_FIELDS = {"flagEvaluations", "context"} +_ALLOWED_BATCH_CONTEXT_FIELDS = {"service", "env", "version"} +_ALLOWED_ROW_CONTEXT_FIELDS = {"evaluation", "dd"} + +def _assert_row_contract_valid(ev: dict) -> None: + """Assert one flagevaluation row uses only the SDK-owned stable EVP fields.""" + extra_fields = set(ev) - _ALLOWED_EVENT_FIELDS + assert not extra_fields, f"unknown flagevaluation row fields: {sorted(extra_fields)}" -def _assert_row_schema_valid(ev: dict) -> None: - """Assert one flagevaluation row conforms to the worker contract (mechanical).""" # Required fields present with the right scalar types. for field, typ in _REQUIRED_EVENT_FIELDS.items(): assert field in ev, f"required field {field!r} missing from row: {ev}" @@ -495,6 +525,8 @@ def _assert_row_schema_valid(ev: dict) -> None: # context, when present, nests an "evaluation" map. if "context" in ev: assert isinstance(ev["context"], dict) + extra_context_fields = set(ev["context"]) - _ALLOWED_ROW_CONTEXT_FIELDS + assert not extra_context_fields, f"unknown row context fields: {sorted(extra_context_fields)}" assert "evaluation" in ev["context"] assert isinstance(ev["context"]["evaluation"], dict) @@ -503,13 +535,24 @@ def _assert_row_schema_valid(ev: dict) -> None: assert isinstance(ev["runtime_default_used"], bool) -def _assert_batch_schema_valid(payload: dict) -> None: - errors = sorted(_BATCHED_FLAGEVALUATIONS_VALIDATOR.iter_errors(payload), key=lambda e: e.path) - assert not errors, [f"{list(error.path)}: {error.message}" for error in errors] - - -class TestPayloadSchemaConformance: - def test_full_tier_row_is_schema_valid_with_object_variant_and_allocation(self, writer): +def _assert_batch_contract_valid(payload: dict) -> None: + """Assert the batch envelope uses only the stable fields this SDK emits.""" + extra_fields = set(payload) - _ALLOWED_BATCH_FIELDS + assert not extra_fields, f"unknown flagevaluation batch fields: {sorted(extra_fields)}" + assert "flagEvaluations" in payload + assert isinstance(payload["flagEvaluations"], list) + if "context" in payload: + assert isinstance(payload["context"], dict) + extra_context_fields = set(payload["context"]) - _ALLOWED_BATCH_CONTEXT_FIELDS + assert not extra_context_fields, f"unknown batch context fields: {sorted(extra_context_fields)}" + for value in payload["context"].values(): + assert isinstance(value, str) + for row in payload["flagEvaluations"]: + _assert_row_contract_valid(row) + + +class TestPayloadContractConformance: + def test_full_tier_row_uses_stable_contract_with_object_variant_and_allocation(self, writer): """A full-tier row carries variant/allocation as {key} objects + context.evaluation.""" writer.enqueue( _make_event( @@ -522,17 +565,17 @@ def test_full_tier_row_is_schema_valid_with_object_variant_and_allocation(self, writer.periodic() decoded = json.loads(mock_send.call_args[0][0]) - _assert_batch_schema_valid(decoded) + _assert_batch_contract_valid(decoded) assert "flagEvaluations" in decoded row = decoded["flagEvaluations"][0] - _assert_row_schema_valid(row) + _assert_row_contract_valid(row) # Specifically the {key} object shape (NOT bare strings). assert row["variant"] == {"key": "on"} assert row["allocation"] == {"key": "alloc-1"} assert row["context"]["evaluation"]["tier"] == "premium" - def test_degraded_tier_row_is_schema_valid_and_omits_context(self, writer): - """A degraded-tier row is schema-valid with variant/allocation objects, no context.""" + def test_degraded_tier_row_uses_stable_contract_and_omits_context(self, writer): + """A degraded-tier row uses variant/allocation objects, no context.""" writer._per_flag_count["my-flag"] = PER_FLAG_CAP # force degraded writer.enqueue( _make_event( @@ -546,16 +589,16 @@ def test_degraded_tier_row_is_schema_valid_and_omits_context(self, writer): writer.periodic() decoded = json.loads(mock_send.call_args[0][0]) - _assert_batch_schema_valid(decoded) + _assert_batch_contract_valid(decoded) row = decoded["flagEvaluations"][0] - _assert_row_schema_valid(row) + _assert_row_contract_valid(row) assert row["variant"] == {"key": "on"} assert row["error"] == {"message": "degraded failure"} assert "context" not in row assert "targeting_key" not in row def test_error_row_carries_error_message_object(self, writer): - """An error evaluation produces a schema-valid row with error.message.""" + """An error evaluation produces a stable row with error.message.""" writer.enqueue( _make_event( variant="", @@ -567,16 +610,16 @@ def test_error_row_carries_error_message_object(self, writer): writer.periodic() decoded = json.loads(mock_send.call_args[0][0]) - _assert_batch_schema_valid(decoded) + _assert_batch_contract_valid(decoded) row = decoded["flagEvaluations"][0] - _assert_row_schema_valid(row) + _assert_row_contract_valid(row) assert row["error"] == {"message": "Flag not found"} # Absent variant -> runtime_default_used True, no variant object emitted. assert row["runtime_default_used"] is True assert "variant" not in row def test_batch_payload_validates_full_and_degraded_rows_together(self, writer): - """A single flush emits BOTH a full row and a degraded row, both schema-valid.""" + """A single flush emits BOTH a full row and a degraded row under the stable contract.""" # Full-tier event. writer.enqueue(_make_event(flag_key="full-flag", variant="on", attrs={"a": "b"})) # Degraded-tier event (different flag forced to degraded). @@ -587,15 +630,15 @@ def test_batch_payload_validates_full_and_degraded_rows_together(self, writer): writer.periodic() decoded = json.loads(mock_send.call_args[0][0]) - _assert_batch_schema_valid(decoded) + _assert_batch_contract_valid(decoded) rows = decoded["flagEvaluations"] assert len(rows) == 2 for row in rows: - _assert_row_schema_valid(row) + _assert_row_contract_valid(row) flags = {r["flag"]["key"] for r in rows} assert flags == {"full-flag", "deg-flag"} - def test_worker_schema_rejects_top_level_reason(self): + def test_contract_rejects_top_level_reason(self): bad = { "flagEvaluations": [ { @@ -608,9 +651,8 @@ def test_worker_schema_rejects_top_level_reason(self): } ] } - errors = list(_BATCHED_FLAGEVALUATIONS_VALIDATOR.iter_errors(bad)) - assert errors - assert any("Additional properties" in error.message and "reason" in error.message for error in errors) + with pytest.raises(AssertionError, match="reason"): + _assert_batch_contract_valid(bad) # --------------------------------------------------------------------------- diff --git a/tests/openfeature/testdata/flageval-worker/batchedflagevaluations.json b/tests/openfeature/testdata/flageval-worker/batchedflagevaluations.json deleted file mode 100644 index b3a4b1fda67..00000000000 --- a/tests/openfeature/testdata/flageval-worker/batchedflagevaluations.json +++ /dev/null @@ -1,184 +0,0 @@ -{ - "title": "BatchedFlagEvaluations", - "type": "object", - "properties": { - "context": { - "title": "InputContextDatadog", - "type": "object", - "properties": { - "geo": { - "type": "object", - "properties": { - "country_iso_code": { "type": "string" }, - "country": { "type": "string" } - }, - "required": [], - "additionalProperties": false - }, - "rum": { - "type": "object", - "properties": { - "application": { - "type": "object", - "properties": { - "id": { "type": "string" } - }, - "required": [], - "additionalProperties": false - }, - "view": { - "type": "object", - "properties": { - "url": { "type": "string" } - }, - "required": [], - "additionalProperties": false - } - }, - "required": [], - "additionalProperties": false - }, - "service": { "type": "string" }, - "version": { "type": "string" }, - "env": { "type": "string" }, - "device": { - "type": "object", - "properties": { - "name": { "type": "string" }, - "type": { "type": "string" }, - "brand": { "type": "string" }, - "model": { "type": "string" } - }, - "required": [], - "additionalProperties": false - }, - "os": { - "type": "object", - "properties": { - "name": { "type": "string" }, - "version": { "type": "string" } - }, - "required": [], - "additionalProperties": false - } - }, - "required": [], - "additionalProperties": false - }, - "flagEvaluations": { - "type": "array", - "items": { - "type": "object", - "properties": { - "timestamp": { - "type": "integer", - "description": "The timestamp (milliseconds since epoch) at which the evaluation occurred." - }, - "flag": { - "type": "object", - "properties": { - "key": { "type": "string" } - }, - "required": ["key"], - "additionalProperties": false - }, - "first_evaluation": { - "type": "integer", - "minimum": 1759276800000 - }, - "last_evaluation": { - "type": "integer", - "minimum": 1759276800000 - }, - "evaluation_count": { - "type": "integer", - "minimum": 1 - }, - "runtime_default_used": { "type": "boolean" }, - "targeting_key": { "type": "string" }, - "context": { - "type": "object", - "properties": { - "evaluation": { "type": "object" }, - "dd": { - "type": "object", - "properties": { - "service": { "type": "string" }, - "rum": { - "type": "object", - "properties": { - "application": { - "type": "object", - "properties": { - "id": { "type": "string" } - }, - "required": [], - "additionalProperties": false - }, - "view": { - "type": "object", - "properties": { - "url": { "type": "string" } - }, - "required": [], - "additionalProperties": false - } - }, - "required": [], - "additionalProperties": false - } - }, - "required": [], - "additionalProperties": true - } - }, - "required": [], - "additionalProperties": false - }, - "variant": { - "type": "object", - "properties": { - "key": { "type": "string" } - }, - "required": ["key"], - "additionalProperties": false - }, - "allocation": { - "type": "object", - "properties": { - "key": { "type": "string" } - }, - "required": ["key"], - "additionalProperties": false - }, - "targeting_rule": { - "type": "object", - "properties": { - "key": { "type": "string" } - }, - "required": ["key"], - "additionalProperties": false - }, - "error": { - "type": "object", - "properties": { - "message": { "type": "string" } - }, - "required": ["message"], - "additionalProperties": false - } - }, - "required": [ - "timestamp", - "flag", - "first_evaluation", - "last_evaluation", - "evaluation_count" - ], - "additionalProperties": false - } - } - }, - "required": ["flagEvaluations"], - "additionalProperties": false -} From 9986cc5b5e5b263b702ae6ffe38a9b73896e1264 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 23 Jun 2026 08:43:47 -0400 Subject: [PATCH 20/37] fix(openfeature): type flagevaluation code strictly --- .../openfeature/_flagevaluation_hook.py | 18 ++++++++---------- .../openfeature/_flagevaluation_writer.py | 16 ++++++++++++++-- ddtrace/internal/openfeature/_provider.py | 13 ++++++++----- ddtrace/internal/periodic.py | 3 +-- mypy.ini | 12 ------------ pyproject.toml | 1 + 6 files changed, 32 insertions(+), 31 deletions(-) diff --git a/ddtrace/internal/openfeature/_flagevaluation_hook.py b/ddtrace/internal/openfeature/_flagevaluation_hook.py index f5e496774cb..47104f979b7 100644 --- a/ddtrace/internal/openfeature/_flagevaluation_hook.py +++ b/ddtrace/internal/openfeature/_flagevaluation_hook.py @@ -65,7 +65,7 @@ def finally_after( flag_key: str = hook_context.flag_key or "" # Extract allocation_key from flag_metadata (same key as METADATA_ALLOCATION_KEY). - metadata: dict[str, typing.Any] = details.flag_metadata or {} + metadata: typing.Mapping[str, typing.Any] = details.flag_metadata or {} allocation_key: str = "" ak = metadata.get(METADATA_ALLOCATION_KEY) if isinstance(ak, str) and ak: @@ -79,18 +79,16 @@ def finally_after( eval_time_ms = int(time.time() * 1000) # Variant: absent or type mismatch signals a runtime default. - runtime_default = not details.variant or details.error_code == ErrorCode.TYPE_MISMATCH - variant = "" if runtime_default else details.variant + variant = "" + if details.variant and details.error_code != ErrorCode.TYPE_MISMATCH: + variant = details.variant + runtime_default = variant == "" # Targeting key and attributes from the evaluation context. eval_ctx = hook_context.evaluation_context - if eval_ctx is not None: - targeting_key = eval_ctx.targeting_key or "" - # Shallow copy so we don't hold a reference to the caller's live dict. - attrs: dict[str, typing.Any] = dict(eval_ctx.attributes or {}) - else: - targeting_key = "" - attrs = {} + targeting_key = eval_ctx.targeting_key or "" + # Shallow copy so we don't hold a reference to the caller's live dict. + attrs: dict[str, typing.Any] = dict(eval_ctx.attributes or {}) # Error message (best-effort; absent on success paths). error_message = "" diff --git a/ddtrace/internal/openfeature/_flagevaluation_writer.py b/ddtrace/internal/openfeature/_flagevaluation_writer.py index 772cfd7debe..7af7c80abe5 100644 --- a/ddtrace/internal/openfeature/_flagevaluation_writer.py +++ b/ddtrace/internal/openfeature/_flagevaluation_writer.py @@ -21,6 +21,7 @@ - Non-blocking enqueue: queue.Queue(QUEUE_SIZE); drops + counts on queue.Full. """ +import http.client as httplib import json import queue import struct @@ -254,6 +255,17 @@ class _EvalEvent(typing.NamedTuple): eval_time_ms: int +class _FlagEvaluationConnection(typing.Protocol): + def request(self, method: str, url: str, body: bytes, headers: dict[str, str]) -> None: + pass + + def getresponse(self) -> httplib.HTTPResponse: + pass + + def close(self) -> None: + pass + + # --------------------------------------------------------------------------- # FlagEvaluationWriter # --------------------------------------------------------------------------- @@ -470,7 +482,7 @@ def periodic(self) -> None: self._send_payload(payload, len(events)) - def on_shutdown(self): + def on_shutdown(self) -> None: """Final flush on service shutdown — drains the queue and flushes before exit.""" self._stop_drain_worker() self.periodic() @@ -578,7 +590,7 @@ def _add_to_degraded(self, event: _EvalEvent) -> None: def _send_payload(self, payload: bytes, num_events: int) -> None: """POST the encoded payload to the EVP proxy.""" - conn = get_connection(self._intake, timeout=self._timeout) + conn = typing.cast(_FlagEvaluationConnection, get_connection(self._intake, timeout=self._timeout)) try: conn.request("POST", self._endpoint, payload, self._headers) resp = conn.getresponse() diff --git a/ddtrace/internal/openfeature/_provider.py b/ddtrace/internal/openfeature/_provider.py index 3292acdaca3..4569cadd982 100644 --- a/ddtrace/internal/openfeature/_provider.py +++ b/ddtrace/internal/openfeature/_provider.py @@ -16,6 +16,7 @@ from openfeature.event import ProviderEventDetails from openfeature.exception import ErrorCode from openfeature.flag_evaluation import FlagResolutionDetails +from openfeature.flag_evaluation import FlagValueType from openfeature.flag_evaluation import Reason from openfeature.provider import Metadata from openfeature.provider import ProviderStatus @@ -51,6 +52,8 @@ T = typing.TypeVar("T", covariant=True) +ResolvedValue = typing.TypeVar("ResolvedValue") +ObjectFlagValue = typing.Union[typing.Sequence[FlagValueType], typing.Mapping[str, FlagValueType]] K = typing.TypeVar("K") V = typing.TypeVar("V") logger = get_logger(__name__) @@ -308,18 +311,18 @@ def resolve_float_details( def resolve_object_details( self, flag_key: str, - default_value: typing.Union[dict, list], + default_value: ObjectFlagValue, evaluation_context: typing.Optional[EvaluationContext] = None, - ) -> FlagResolutionDetails[typing.Union[dict, list]]: + ) -> FlagResolutionDetails[ObjectFlagValue]: return self._resolve_details(flag_key, default_value, evaluation_context, VariationType.Object) def _resolve_details( self, flag_key: str, - default_value: typing.Any, + default_value: ResolvedValue, evaluation_context: typing.Optional[EvaluationContext] = None, variation_type: VariationType = VariationType.Boolean, - ) -> FlagResolutionDetails[T]: + ) -> FlagResolutionDetails[ResolvedValue]: """ Core resolution logic for all flag types. @@ -428,7 +431,7 @@ def _resolve_details( # Success - return resolved value (which may be None for JSON flags) return FlagResolutionDetails( - value=details.value, + value=typing.cast(ResolvedValue, details.value), reason=reason, variant=details.variant, flag_metadata=flag_metadata, diff --git a/ddtrace/internal/periodic.py b/ddtrace/internal/periodic.py index 8481e505652..0902e9a7392 100644 --- a/ddtrace/internal/periodic.py +++ b/ddtrace/internal/periodic.py @@ -65,8 +65,7 @@ def join( if self._worker: self._worker.join(timeout) - @staticmethod - def on_shutdown(): + def on_shutdown(self) -> None: pass def periodic(self): diff --git a/mypy.ini b/mypy.ini index cf4bf28be21..513c85ff42d 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1446,18 +1446,6 @@ disallow_incomplete_defs = false disallow_untyped_defs = false disallow_incomplete_defs = false -[mypy-ddtrace.internal.openfeature._flageval_metrics] -disallow_subclassing_any = false - -[mypy-ddtrace.internal.openfeature._flagevaluation_hook] -disallow_subclassing_any = false - -[mypy-ddtrace.internal.openfeature._flagevaluation_writer] -disallow_untyped_calls = false -disallow_untyped_defs = false -disallow_incomplete_defs = false -check_untyped_defs = false - [mypy-ddtrace.internal.openfeature._native] disallow_untyped_calls = false disallow_untyped_defs = false diff --git a/pyproject.toml b/pyproject.toml index 3215f413af9..71e94a58816 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -103,6 +103,7 @@ lint = [ "mypy==1.15.0", "coverage==7.3.0", "envier==0.6.1", + "openfeature-sdk~=0.8.0", "types-docutils==0.19.1.1", "types-protobuf==3.20.4.5", "types-PyYAML==6.0.12.2", From d4f541e4399014cdf9ab6e45b855e758e231c762 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 23 Jun 2026 09:01:47 -0400 Subject: [PATCH 21/37] fix(openfeature): bound flagevaluation payloads --- .../openfeature/_flagevaluation_writer.py | 142 +++++++++++++++--- ddtrace/internal/openfeature/_provider.py | 7 +- .../openfeature/test_flagevaluation_writer.py | 70 +++++++++ tests/openfeature/test_provider.py | 5 + 4 files changed, 201 insertions(+), 23 deletions(-) diff --git a/ddtrace/internal/openfeature/_flagevaluation_writer.py b/ddtrace/internal/openfeature/_flagevaluation_writer.py index 7af7c80abe5..87c1f2cee9c 100644 --- a/ddtrace/internal/openfeature/_flagevaluation_writer.py +++ b/ddtrace/internal/openfeature/_flagevaluation_writer.py @@ -21,6 +21,8 @@ - Non-blocking enqueue: queue.Queue(QUEUE_SIZE); drops + counts on queue.Full. """ +from collections.abc import Mapping +from collections.abc import Sequence import http.client as httplib import json import queue @@ -30,6 +32,7 @@ import typing from ddtrace import config as ddconfig +from ddtrace.internal.evp_proxy.constants import DEFAULT_EVP_PAYLOAD_SIZE_LIMIT from ddtrace.internal.evp_proxy.constants import EVP_PROXY_AGENT_BASE_PATH from ddtrace.internal.evp_proxy.constants import EVP_SUBDOMAIN_HEADER_NAME from ddtrace.internal.logger import get_logger @@ -44,6 +47,8 @@ # EVP endpoint for flag evaluation events. FLAGEVALUATIONS_ENDPOINT = f"{EVP_PROXY_AGENT_BASE_PATH}/api/v2/flagevaluation" EVP_SUBDOMAIN_VALUE = "event-platform-intake" +FLAGEVALUATIONS_PAYLOAD_SIZE_LIMIT = DEFAULT_EVP_PAYLOAD_SIZE_LIMIT +_JSON_SEPARATORS = (",", ":") # Context pruning limits — mirror worker.ts MAX_EVALUATION_CONTEXT_FIELDS / MAX_FIELD_LENGTH. MAX_CONTEXT_FIELDS = 256 @@ -87,6 +92,10 @@ _TAG_OTHER = b"o" +def _json_dumps(obj: typing.Any) -> bytes: + return json.dumps(obj, separators=_JSON_SEPARATORS).encode("utf-8") + + # --------------------------------------------------------------------------- # Canonical context key — type-tagged, length-delimited, sorted # --------------------------------------------------------------------------- @@ -182,9 +191,25 @@ def flatten_and_prune_context(attrs: dict[str, typing.Any]) -> dict[str, typing. return out +def _json_safe_context_value(value: typing.Any) -> typing.Any: + if value is None or isinstance(value, (bool, int, float, str)): + return value + if isinstance(value, Mapping): + return {str(k): _json_safe_context_value(v) for k, v in value.items()} + if isinstance(value, Sequence) and not isinstance(value, (bytes, bytearray)): + return [_json_safe_context_value(v) for v in value] + if hasattr(value, "isoformat"): + return value.isoformat() + return str(value) + + +def _json_safe_context(attrs: dict[str, typing.Any]) -> dict[str, typing.Any]: + return {k: _json_safe_context_value(v) for k, v in attrs.items()} + + def _flatten_recursive(prefix: str, attrs: typing.Any, out: dict[str, typing.Any]) -> None: """Recursively flatten nested dicts into dot-notation keys.""" - if not isinstance(attrs, dict): + if not isinstance(attrs, Mapping): if prefix: out[prefix] = attrs return @@ -192,7 +217,7 @@ def _flatten_recursive(prefix: str, attrs: typing.Any, out: dict[str, typing.Any if not prefix and k in DEDICATED_TARGETING_KEY_CONTEXT_FIELDS: continue full_key = f"{prefix}.{k}" if prefix else k - if isinstance(v, dict): + if isinstance(v, Mapping): _flatten_recursive(full_key, v, out) else: out[full_key] = v @@ -461,26 +486,17 @@ def periodic(self) -> None: if not events: return - # 4. Encode and POST. - try: - context: dict[str, str] = {} - if ddconfig.service: - context["service"] = ddconfig.service - if ddconfig.env: - context["env"] = ddconfig.env - if ddconfig.version: - context["version"] = ddconfig.version - - payload_obj: dict[str, typing.Any] = {"flagEvaluations": events} - if context: - payload_obj["context"] = context - - payload = json.dumps(payload_obj).encode("utf-8") - except (TypeError, ValueError): - logger.debug("FlagEvaluationWriter: failed to encode payload", exc_info=True) - return + # 4. Encode under the EVP payload limit and POST. + context: dict[str, str] = {} + if ddconfig.service: + context["service"] = ddconfig.service + if ddconfig.env: + context["env"] = ddconfig.env + if ddconfig.version: + context["version"] = ddconfig.version - self._send_payload(payload, len(events)) + for payload, num_events in _build_payloads(events, context, FLAGEVALUATIONS_PAYLOAD_SIZE_LIMIT): + self._send_payload(payload, num_events) def on_shutdown(self) -> None: """Final flush on service shutdown — drains the queue and flushes before exit.""" @@ -555,7 +571,7 @@ def _aggregate(self, event: _EvalEvent) -> None: eval_time_ms=event.eval_time_ms, runtime_default=event.runtime_default, targeting_key=event.targeting_key, - context_attrs=context_attrs, + context_attrs=_json_safe_context(context_attrs), error_message=event.error_message, ) self._global_count += 1 @@ -633,3 +649,85 @@ def _base_event(flag_key: str, entry: "_Entry", flush_time_ms: int) -> dict[str, "last_evaluation": entry.last_evaluation, "evaluation_count": entry.count, } + + +def _degraded_payload_event(event: dict[str, typing.Any]) -> dict[str, typing.Any]: + degraded = event.copy() + degraded.pop("context", None) + degraded.pop("targeting_key", None) + return degraded + + +def _encode_payload_event( + event: dict[str, typing.Any], + single_event_payload_limit: int, +) -> typing.Optional[bytes]: + try: + encoded = _json_dumps(event) + except (TypeError, ValueError): + logger.debug("FlagEvaluationWriter: failed to encode event", exc_info=True) + return None + + if len(encoded) <= single_event_payload_limit: + return encoded + + degraded_event = _degraded_payload_event(event) + if degraded_event != event: + try: + encoded = _json_dumps(degraded_event) + except (TypeError, ValueError): + logger.debug("FlagEvaluationWriter: failed to encode degraded event", exc_info=True) + return None + if len(encoded) <= single_event_payload_limit: + logger.warning( + "FlagEvaluationWriter: degraded oversized flag evaluation event for %s before sending", + event.get("flag", {}).get("key", ""), + ) + return encoded + + logger.warning( + "FlagEvaluationWriter: dropped oversized flag evaluation event for %s", + event.get("flag", {}).get("key", ""), + ) + return None + + +def _build_payloads( + events: list[dict[str, typing.Any]], + context: dict[str, str], + payload_size_limit: int = FLAGEVALUATIONS_PAYLOAD_SIZE_LIMIT, +) -> typing.Iterator[tuple[bytes, int]]: + context_suffix = b"" + if context: + context_suffix = b',"context":' + _json_dumps(context) + prefix = b'{"flagEvaluations":[' + suffix = b"]" + context_suffix + b"}" + single_event_payload_limit = payload_size_limit - len(prefix) - len(suffix) + if single_event_payload_limit <= 0: + logger.warning("FlagEvaluationWriter: EVP payload size limit is too small to encode flagevaluation payloads") + return + + payload = bytearray(prefix) + num_events = 0 + + for event in events: + encoded_event = _encode_payload_event(event, single_event_payload_limit) + if encoded_event is None: + continue + + separator_size = 1 if num_events else 0 + candidate_size = len(payload) + separator_size + len(encoded_event) + len(suffix) + if num_events and candidate_size > payload_size_limit: + payload.extend(suffix) + yield bytes(payload), num_events + payload = bytearray(prefix) + num_events = 0 + + if num_events: + payload.extend(b",") + payload.extend(encoded_event) + num_events += 1 + + if num_events: + payload.extend(suffix) + yield bytes(payload), num_events diff --git a/ddtrace/internal/openfeature/_provider.py b/ddtrace/internal/openfeature/_provider.py index 4569cadd982..bfb45133857 100644 --- a/ddtrace/internal/openfeature/_provider.py +++ b/ddtrace/internal/openfeature/_provider.py @@ -100,7 +100,12 @@ class DataDogProvider(AbstractProvider): Feature Flags and Experimentation (FFE) product. """ - def __init__(self, *args: typing.Any, **kwargs: typing.Any): + def __init__( + self, + *args: typing.Any, + initialization_timeout: typing.Optional[float] = None, + **kwargs: typing.Any, + ) -> None: super().__init__(*args, **kwargs) self._metadata = Metadata(name="Datadog") self._status = ProviderStatus.NOT_READY diff --git a/tests/openfeature/test_flagevaluation_writer.py b/tests/openfeature/test_flagevaluation_writer.py index e0cc5465d1d..9d6b993176d 100644 --- a/tests/openfeature/test_flagevaluation_writer.py +++ b/tests/openfeature/test_flagevaluation_writer.py @@ -11,6 +11,8 @@ - EVP POST to /evp_proxy/v2/api/v2/flagevaluation with correct headers """ +from datetime import datetime +from datetime import timezone import json import time from unittest import mock @@ -115,6 +117,12 @@ def test_float_vs_int_distinct_keys(self): k_int = canonical_context_key({"x": 1}) assert k_float != k_int + def test_datetime_vs_string_distinct_keys(self): + timestamp = datetime(2026, 6, 23, 12, 0, tzinfo=timezone.utc) + k_datetime = canonical_context_key({"x": timestamp}) + k_string = canonical_context_key({"x": timestamp.isoformat()}) + assert k_datetime != k_string + def test_value_with_equals_or_newline_no_boundary_confusion(self): r"""'=' and '\n' in values must not fake a field boundary (length-prefix protocol).""" k_with = canonical_context_key({"a": "foo=bar\nbaz"}) @@ -416,6 +424,20 @@ def test_context_value_exceeding_256_chars_pruned(self, writer): assert "short" in ctx_eval assert "long_field" not in ctx_eval + def test_openfeature_datetime_context_value_is_json_serialized(self, writer): + """OpenFeature allows datetime context values; payload JSON should stringify them.""" + timestamp = datetime(2026, 6, 23, 12, 30, tzinfo=timezone.utc) + writer.enqueue(_make_event(attrs={"seen_at": timestamp, "nested": {"created_at": timestamp}})) + + with mock.patch.object(writer, "_send_payload") as mock_send: + writer.periodic() + + payload_bytes = mock_send.call_args[0][0] + decoded = json.loads(payload_bytes) + ctx_eval = decoded["flagEvaluations"][0]["context"]["evaluation"] + assert ctx_eval["seen_at"] == timestamp.isoformat() + assert ctx_eval["nested.created_at"] == timestamp.isoformat() + def test_degraded_event_has_no_context_or_targeting_key(self, writer): """Degraded-tier events must not include targeting_key or context fields.""" # Force to degraded by saturating per-flag cap. @@ -465,6 +487,54 @@ def test_class_exists_and_inherits_periodic_service(self): assert issubclass(FlagEvaluationWriter, PeriodicService) + def test_payloads_are_split_under_evp_payload_size_limit(self, writer): + for i in range(5): + writer.enqueue( + _make_event( + flag_key=f"split-{i}", + targeting_key=f"user-{i}", + attrs={"blob": "x" * 200}, + ) + ) + + sent = [] + with mock.patch("ddtrace.internal.openfeature._flagevaluation_writer.FLAGEVALUATIONS_PAYLOAD_SIZE_LIMIT", 900): + with mock.patch.object(writer, "_send_payload", side_effect=lambda p, n: sent.append((p, n))): + writer.periodic() + + assert len(sent) > 1 + seen_flags = set() + for payload, num_events in sent: + assert len(payload) <= 900 + decoded = json.loads(payload) + _assert_batch_contract_valid(decoded) + assert num_events == len(decoded["flagEvaluations"]) + seen_flags.update(row["flag"]["key"] for row in decoded["flagEvaluations"]) + assert seen_flags == {f"split-{i}" for i in range(5)} + + def test_single_oversized_full_event_is_degraded_before_send(self, writer): + writer.enqueue( + _make_event( + flag_key="oversized-full", + targeting_key="user-with-context", + attrs={"blob": "x" * 200}, + ) + ) + + sent = [] + with mock.patch("ddtrace.internal.openfeature._flagevaluation_writer.FLAGEVALUATIONS_PAYLOAD_SIZE_LIMIT", 300): + with mock.patch.object(writer, "_send_payload", side_effect=lambda p, n: sent.append((p, n))): + writer.periodic() + + assert len(sent) == 1 + payload, num_events = sent[0] + assert len(payload) <= 300 + assert num_events == 1 + row = json.loads(payload)["flagEvaluations"][0] + assert row["flag"]["key"] == "oversized-full" + assert "context" not in row + assert "targeting_key" not in row + # --------------------------------------------------------------------------- # Stable payload contract for emitted rows (full + degraded) diff --git a/tests/openfeature/test_provider.py b/tests/openfeature/test_provider.py index b9b953ae6ec..0e4e71290ff 100644 --- a/tests/openfeature/test_provider.py +++ b/tests/openfeature/test_provider.py @@ -61,6 +61,11 @@ def test_provider_initialization(self, provider, evaluation_context): # Should not raise provider.initialize(evaluation_context) + def test_provider_accepts_legacy_initialization_timeout_kwarg(self): + """Existing callers can still pass initialization_timeout after async initialization.""" + with override_global_config({"experimental_flagging_provider_enabled": True}): + DataDogProvider(initialization_timeout=0.1) + def test_provider_shutdown(self, provider): """Provider should shutdown without errors.""" # Should not raise From 2a5b3acd84f0b215b71dc605e8e8066f00fa0c73 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 23 Jun 2026 12:21:04 -0400 Subject: [PATCH 22/37] fix(openfeature): emit flagevaluation telemetry counters --- .../openfeature/_flagevaluation_writer.py | 100 +++++++++++++++-- .../openfeature/test_flagevaluation_writer.py | 104 ++++++++++++++++-- 2 files changed, 183 insertions(+), 21 deletions(-) diff --git a/ddtrace/internal/openfeature/_flagevaluation_writer.py b/ddtrace/internal/openfeature/_flagevaluation_writer.py index 87c1f2cee9c..80060eaa347 100644 --- a/ddtrace/internal/openfeature/_flagevaluation_writer.py +++ b/ddtrace/internal/openfeature/_flagevaluation_writer.py @@ -38,6 +38,8 @@ from ddtrace.internal.logger import get_logger from ddtrace.internal.periodic import PeriodicService from ddtrace.internal.settings._agent import config as agent_config +from ddtrace.internal.telemetry import telemetry_writer +from ddtrace.internal.telemetry.constants import TELEMETRY_NAMESPACE from ddtrace.internal.threads import PeriodicThread from ddtrace.internal.utils.http import get_connection @@ -91,11 +93,27 @@ _TAG_FLOAT = b"f" _TAG_OTHER = b"o" +FLAG_EVALUATION_DROPPED_METRIC = "flagevaluation.rows.dropped" +FLAG_EVALUATION_DEGRADED_METRIC = "flagevaluation.rows.degraded" +FLAG_EVALUATION_SPLITS_METRIC = "flagevaluation.payload.splits" + +FLAG_EVALUATION_REASON_QUEUE_OVERFLOW = "queue_overflow" +FLAG_EVALUATION_REASON_DEGRADED_CAP = "degraded_cap" +FLAG_EVALUATION_REASON_PAYLOAD_LIMIT = "payload_limit" +FLAG_EVALUATION_REASON_CARDINALITY_CAP = "cardinality_cap" + def _json_dumps(obj: typing.Any) -> bytes: return json.dumps(obj, separators=_JSON_SEPARATORS).encode("utf-8") +def _count_metric(name: str, value: int, reason: typing.Optional[str] = None) -> None: + if value <= 0: + return + tags = (("reason", reason),) if reason else tuple() + telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE.TRACERS, name, value, tags) + + # --------------------------------------------------------------------------- # Canonical context key — type-tagged, length-delimited, sorted # --------------------------------------------------------------------------- @@ -291,6 +309,18 @@ def close(self) -> None: pass +class _PayloadEventResult(typing.NamedTuple): + encoded: typing.Optional[bytes] + degraded_payload_limit: bool = False + dropped_payload_limit: bool = False + + +class _PayloadBuildResult(typing.NamedTuple): + payloads: list[tuple[bytes, int]] + degraded_payload_limit: int = 0 + dropped_payload_limit: int = 0 + + # --------------------------------------------------------------------------- # FlagEvaluationWriter # --------------------------------------------------------------------------- @@ -416,12 +446,22 @@ def periodic(self) -> None: "FlagEvaluationWriter: queue full — dropped %d evaluation(s) under backpressure", dropped_queue, ) + _count_metric( + FLAG_EVALUATION_DROPPED_METRIC, + dropped_queue, + FLAG_EVALUATION_REASON_QUEUE_OVERFLOW, + ) self._dropped_queue = 0 if dropped_degraded: logger.warning( "FlagEvaluationWriter: degraded cap full — dropped %d evaluation(s)", dropped_degraded, ) + _count_metric( + FLAG_EVALUATION_DROPPED_METRIC, + dropped_degraded, + FLAG_EVALUATION_REASON_DEGRADED_CAP, + ) self._dropped_degraded_overflow = 0 return # Reset maps. @@ -437,11 +477,13 @@ def periodic(self) -> None: "FlagEvaluationWriter: queue full — dropped %d evaluation(s) under backpressure", dropped_queue, ) + _count_metric(FLAG_EVALUATION_DROPPED_METRIC, dropped_queue, FLAG_EVALUATION_REASON_QUEUE_OVERFLOW) if dropped_degraded: logger.warning( "FlagEvaluationWriter: degraded cap full — dropped %d evaluation(s)", dropped_degraded, ) + _count_metric(FLAG_EVALUATION_DROPPED_METRIC, dropped_degraded, FLAG_EVALUATION_REASON_DEGRADED_CAP) # 3. Build payload. flush_time_ms = int(time.time() * 1000) @@ -468,7 +510,9 @@ def periodic(self) -> None: events.append(ev) # Degraded-tier events: no targeting_key, no context. + degraded_count = 0 for key, entry in degraded.items(): + degraded_count += entry.count flag_key = key[0] variant = key[1] allocation_key = key[2] @@ -482,6 +526,7 @@ def periodic(self) -> None: if entry.error_message: ev["error"] = {"message": entry.error_message} events.append(ev) + _count_metric(FLAG_EVALUATION_DEGRADED_METRIC, degraded_count, FLAG_EVALUATION_REASON_CARDINALITY_CAP) if not events: return @@ -495,7 +540,21 @@ def periodic(self) -> None: if ddconfig.version: context["version"] = ddconfig.version - for payload, num_events in _build_payloads(events, context, FLAGEVALUATIONS_PAYLOAD_SIZE_LIMIT): + result = _build_payloads_with_stats(events, context, FLAGEVALUATIONS_PAYLOAD_SIZE_LIMIT) + _count_metric( + FLAG_EVALUATION_DEGRADED_METRIC, + result.degraded_payload_limit, + FLAG_EVALUATION_REASON_PAYLOAD_LIMIT, + ) + _count_metric( + FLAG_EVALUATION_DROPPED_METRIC, + result.dropped_payload_limit, + FLAG_EVALUATION_REASON_PAYLOAD_LIMIT, + ) + if len(result.payloads) > 1: + _count_metric(FLAG_EVALUATION_SPLITS_METRIC, len(result.payloads) - 1) + + for payload, num_events in result.payloads: self._send_payload(payload, num_events) def on_shutdown(self) -> None: @@ -661,15 +720,15 @@ def _degraded_payload_event(event: dict[str, typing.Any]) -> dict[str, typing.An def _encode_payload_event( event: dict[str, typing.Any], single_event_payload_limit: int, -) -> typing.Optional[bytes]: +) -> _PayloadEventResult: try: encoded = _json_dumps(event) except (TypeError, ValueError): logger.debug("FlagEvaluationWriter: failed to encode event", exc_info=True) - return None + return _PayloadEventResult(None) if len(encoded) <= single_event_payload_limit: - return encoded + return _PayloadEventResult(encoded) degraded_event = _degraded_payload_event(event) if degraded_event != event: @@ -677,19 +736,19 @@ def _encode_payload_event( encoded = _json_dumps(degraded_event) except (TypeError, ValueError): logger.debug("FlagEvaluationWriter: failed to encode degraded event", exc_info=True) - return None + return _PayloadEventResult(None) if len(encoded) <= single_event_payload_limit: logger.warning( "FlagEvaluationWriter: degraded oversized flag evaluation event for %s before sending", event.get("flag", {}).get("key", ""), ) - return encoded + return _PayloadEventResult(encoded, degraded_payload_limit=True) logger.warning( "FlagEvaluationWriter: dropped oversized flag evaluation event for %s", event.get("flag", {}).get("key", ""), ) - return None + return _PayloadEventResult(None, dropped_payload_limit=True) def _build_payloads( @@ -697,6 +756,15 @@ def _build_payloads( context: dict[str, str], payload_size_limit: int = FLAGEVALUATIONS_PAYLOAD_SIZE_LIMIT, ) -> typing.Iterator[tuple[bytes, int]]: + for payload in _build_payloads_with_stats(events, context, payload_size_limit).payloads: + yield payload + + +def _build_payloads_with_stats( + events: list[dict[str, typing.Any]], + context: dict[str, str], + payload_size_limit: int = FLAGEVALUATIONS_PAYLOAD_SIZE_LIMIT, +) -> _PayloadBuildResult: context_suffix = b"" if context: context_suffix = b',"context":' + _json_dumps(context) @@ -705,13 +773,21 @@ def _build_payloads( single_event_payload_limit = payload_size_limit - len(prefix) - len(suffix) if single_event_payload_limit <= 0: logger.warning("FlagEvaluationWriter: EVP payload size limit is too small to encode flagevaluation payloads") - return + return _PayloadBuildResult([]) payload = bytearray(prefix) num_events = 0 + payloads: list[tuple[bytes, int]] = [] + degraded_payload_limit = 0 + dropped_payload_limit = 0 for event in events: - encoded_event = _encode_payload_event(event, single_event_payload_limit) + event_result = _encode_payload_event(event, single_event_payload_limit) + encoded_event = event_result.encoded + if event_result.dropped_payload_limit: + dropped_payload_limit += int(event.get("evaluation_count", 1) or 1) + if event_result.degraded_payload_limit: + degraded_payload_limit += int(event.get("evaluation_count", 1) or 1) if encoded_event is None: continue @@ -719,7 +795,7 @@ def _build_payloads( candidate_size = len(payload) + separator_size + len(encoded_event) + len(suffix) if num_events and candidate_size > payload_size_limit: payload.extend(suffix) - yield bytes(payload), num_events + payloads.append((bytes(payload), num_events)) payload = bytearray(prefix) num_events = 0 @@ -730,4 +806,6 @@ def _build_payloads( if num_events: payload.extend(suffix) - yield bytes(payload), num_events + payloads.append((bytes(payload), num_events)) + + return _PayloadBuildResult(payloads, degraded_payload_limit, dropped_payload_limit) diff --git a/tests/openfeature/test_flagevaluation_writer.py b/tests/openfeature/test_flagevaluation_writer.py index 9d6b993176d..fcd37a2c703 100644 --- a/tests/openfeature/test_flagevaluation_writer.py +++ b/tests/openfeature/test_flagevaluation_writer.py @@ -25,6 +25,13 @@ from ddtrace.internal.openfeature._flagevaluation_writer import EVAL_SCALE_PER_FLAG_BUCKET_TARGET from ddtrace.internal.openfeature._flagevaluation_writer import EVP_SUBDOMAIN_HEADER_NAME from ddtrace.internal.openfeature._flagevaluation_writer import EVP_SUBDOMAIN_VALUE +from ddtrace.internal.openfeature._flagevaluation_writer import FLAG_EVALUATION_DEGRADED_METRIC +from ddtrace.internal.openfeature._flagevaluation_writer import FLAG_EVALUATION_DROPPED_METRIC +from ddtrace.internal.openfeature._flagevaluation_writer import FLAG_EVALUATION_REASON_CARDINALITY_CAP +from ddtrace.internal.openfeature._flagevaluation_writer import FLAG_EVALUATION_REASON_DEGRADED_CAP +from ddtrace.internal.openfeature._flagevaluation_writer import FLAG_EVALUATION_REASON_PAYLOAD_LIMIT +from ddtrace.internal.openfeature._flagevaluation_writer import FLAG_EVALUATION_REASON_QUEUE_OVERFLOW +from ddtrace.internal.openfeature._flagevaluation_writer import FLAG_EVALUATION_SPLITS_METRIC from ddtrace.internal.openfeature._flagevaluation_writer import FLAGEVALUATIONS_ENDPOINT from ddtrace.internal.openfeature._flagevaluation_writer import GLOBAL_CAP from ddtrace.internal.openfeature._flagevaluation_writer import MAX_CONTEXT_FIELDS @@ -32,9 +39,14 @@ from ddtrace.internal.openfeature._flagevaluation_writer import PER_FLAG_CAP from ddtrace.internal.openfeature._flagevaluation_writer import QUEUE_SIZE from ddtrace.internal.openfeature._flagevaluation_writer import FlagEvaluationWriter +from ddtrace.internal.openfeature._flagevaluation_writer import _build_payloads_with_stats from ddtrace.internal.openfeature._flagevaluation_writer import _EvalEvent from ddtrace.internal.openfeature._flagevaluation_writer import canonical_context_key from ddtrace.internal.openfeature._flagevaluation_writer import flatten_and_prune_context +from ddtrace.internal.telemetry.constants import TELEMETRY_NAMESPACE + + +TELEMETRY_COUNT_PATCH = "ddtrace.internal.openfeature._flagevaluation_writer.telemetry_writer.add_count_metric" # --------------------------------------------------------------------------- @@ -75,6 +87,18 @@ def _wait_until(predicate, timeout: float = 2.0) -> bool: return predicate() +def _assert_count_metric(mock_add_count, name: str, value: int, reason: str = None) -> None: + tags = (("reason", reason),) if reason else tuple() + mock_add_count.assert_any_call(TELEMETRY_NAMESPACE.TRACERS, name, value, tags) + + +def _assert_no_count_metric(mock_add_count, name: str, reason: str = None) -> None: + tags = (("reason", reason),) if reason else tuple() + for call in mock_add_count.call_args_list: + if call.args == (TELEMETRY_NAMESPACE.TRACERS, name, mock.ANY, tags): + raise AssertionError(f"unexpected metric {name} tags={tags}: {call}") + + @pytest.fixture def writer(): """Create a FlagEvaluationWriter that is NOT started (no background thread).""" @@ -499,10 +523,14 @@ def test_payloads_are_split_under_evp_payload_size_limit(self, writer): sent = [] with mock.patch("ddtrace.internal.openfeature._flagevaluation_writer.FLAGEVALUATIONS_PAYLOAD_SIZE_LIMIT", 900): - with mock.patch.object(writer, "_send_payload", side_effect=lambda p, n: sent.append((p, n))): - writer.periodic() + with mock.patch(TELEMETRY_COUNT_PATCH) as mock_count: + with mock.patch.object(writer, "_send_payload", side_effect=lambda p, n: sent.append((p, n))): + writer.periodic() assert len(sent) > 1 + _assert_count_metric(mock_count, FLAG_EVALUATION_SPLITS_METRIC, len(sent) - 1) + _assert_no_count_metric(mock_count, FLAG_EVALUATION_DROPPED_METRIC, FLAG_EVALUATION_REASON_PAYLOAD_LIMIT) + _assert_no_count_metric(mock_count, FLAG_EVALUATION_DEGRADED_METRIC, FLAG_EVALUATION_REASON_PAYLOAD_LIMIT) seen_flags = set() for payload, num_events in sent: assert len(payload) <= 900 @@ -523,10 +551,12 @@ def test_single_oversized_full_event_is_degraded_before_send(self, writer): sent = [] with mock.patch("ddtrace.internal.openfeature._flagevaluation_writer.FLAGEVALUATIONS_PAYLOAD_SIZE_LIMIT", 300): - with mock.patch.object(writer, "_send_payload", side_effect=lambda p, n: sent.append((p, n))): - writer.periodic() + with mock.patch(TELEMETRY_COUNT_PATCH) as mock_count: + with mock.patch.object(writer, "_send_payload", side_effect=lambda p, n: sent.append((p, n))): + writer.periodic() assert len(sent) == 1 + _assert_count_metric(mock_count, FLAG_EVALUATION_DEGRADED_METRIC, 1, FLAG_EVALUATION_REASON_PAYLOAD_LIMIT) payload, num_events = sent[0] assert len(payload) <= 300 assert num_events == 1 @@ -535,6 +565,54 @@ def test_single_oversized_full_event_is_degraded_before_send(self, writer): assert "context" not in row assert "targeting_key" not in row + def test_single_oversized_degraded_event_is_dropped_and_counted(self, writer): + writer.enqueue(_make_event(flag_key="f" * 256, targeting_key="", attrs={})) + + with mock.patch("ddtrace.internal.openfeature._flagevaluation_writer.FLAGEVALUATIONS_PAYLOAD_SIZE_LIMIT", 100): + with mock.patch(TELEMETRY_COUNT_PATCH) as mock_count: + with mock.patch.object(writer, "_send_payload") as mock_send: + writer.periodic() + + mock_send.assert_not_called() + _assert_count_metric(mock_count, FLAG_EVALUATION_DROPPED_METRIC, 1, FLAG_EVALUATION_REASON_PAYLOAD_LIMIT) + + def test_build_payload_stats_count_payload_limit_degraded_and_dropped_rows(self): + now_ms = int(time.time() * 1000) + degradable = { + "timestamp": now_ms, + "flag": {"key": "large"}, + "first_evaluation": now_ms, + "last_evaluation": now_ms, + "evaluation_count": 7, + "targeting_key": "user-with-context", + "context": {"evaluation": {"blob": "x" * 256}}, + } + degraded = dict(degradable) + degraded.pop("targeting_key") + degraded.pop("context") + degraded_payload = _build_payloads_with_stats([degraded], {}, 1 << 30).payloads[0][0] + + result = _build_payloads_with_stats([degradable], {}, len(degraded_payload)) + + assert result.degraded_payload_limit == 7 + assert result.dropped_payload_limit == 0 + assert len(result.payloads) == 1 + + undegreadable = { + "timestamp": now_ms, + "flag": {"key": "f" * 256}, + "first_evaluation": now_ms, + "last_evaluation": now_ms, + "evaluation_count": 11, + } + oversized_payload = _build_payloads_with_stats([undegreadable], {}, 1 << 30).payloads[0][0] + + result = _build_payloads_with_stats([undegreadable], {}, len(oversized_payload) - 1) + + assert result.degraded_payload_limit == 0 + assert result.dropped_payload_limit == 11 + assert result.payloads == [] + # --------------------------------------------------------------------------- # Stable payload contract for emitted rows (full + degraded) @@ -655,13 +733,15 @@ def test_degraded_tier_row_uses_stable_contract_and_omits_context(self, writer): error_message="degraded failure", ) ) - with mock.patch.object(writer, "_send_payload") as mock_send: - writer.periodic() + with mock.patch(TELEMETRY_COUNT_PATCH) as mock_count: + with mock.patch.object(writer, "_send_payload") as mock_send: + writer.periodic() decoded = json.loads(mock_send.call_args[0][0]) _assert_batch_contract_valid(decoded) row = decoded["flagEvaluations"][0] _assert_row_contract_valid(row) + _assert_count_metric(mock_count, FLAG_EVALUATION_DEGRADED_METRIC, 1, FLAG_EVALUATION_REASON_CARDINALITY_CAP) assert row["variant"] == {"key": "on"} assert row["error"] == {"message": "degraded failure"} assert "context" not in row @@ -817,11 +897,13 @@ def test_queue_overflow_drop_count_is_logged_on_flush(self, writer): # Drain everything (so maps are populated) and assert the drop count is emitted. with mock.patch("ddtrace.internal.openfeature._flagevaluation_writer.logger") as mock_logger: - with mock.patch.object(writer, "_send_payload"): - writer.periodic() + with mock.patch(TELEMETRY_COUNT_PATCH) as mock_count: + with mock.patch.object(writer, "_send_payload"): + writer.periodic() # A warning naming the queue-full drop count must have been emitted. warnings = [c for c in mock_logger.warning.call_args_list if "queue full" in str(c).lower()] assert warnings, "queue-full drop count must be logged (observable)" + _assert_count_metric(mock_count, FLAG_EVALUATION_DROPPED_METRIC, 1, FLAG_EVALUATION_REASON_QUEUE_OVERFLOW) # Counter resets after emission. assert writer._dropped_queue == 0 @@ -837,10 +919,12 @@ def test_degraded_overflow_drop_count_is_logged_on_flush(self, writer): assert writer._dropped_degraded_overflow == 1 with mock.patch("ddtrace.internal.openfeature._flagevaluation_writer.logger") as mock_logger: - with mock.patch.object(writer, "_send_payload"): - writer.periodic() + with mock.patch(TELEMETRY_COUNT_PATCH) as mock_count: + with mock.patch.object(writer, "_send_payload"): + writer.periodic() warnings = [c for c in mock_logger.warning.call_args_list if "degraded cap" in str(c).lower()] assert warnings, "degraded-cap overflow count must be logged (observable)" + _assert_count_metric(mock_count, FLAG_EVALUATION_DROPPED_METRIC, 1, FLAG_EVALUATION_REASON_DEGRADED_CAP) assert writer._dropped_degraded_overflow == 0 def test_drop_accounting_is_complete_no_silent_loss(self, writer): From 77b59dadfaf2f2e7418686903289bd1e6ce10f11 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 23 Jun 2026 15:37:56 -0400 Subject: [PATCH 23/37] refactor(openfeature): share EVP limits and clarify hooks --- .../openfeature_flagevaluation/scenario.py | 6 +-- ddtrace/internal/ci_visibility/encoder.py | 3 +- ddtrace/internal/evp_proxy/constants.py | 1 + ...luation_hook.py => _flag_eval_evp_hook.py} | 8 +-- .../internal/openfeature/_flageval_metrics.py | 2 +- .../openfeature/_flagevaluation_writer.py | 5 +- ddtrace/internal/openfeature/_provider.py | 46 ++++++++--------- ddtrace/internal/openfeature/writer.py | 10 ++-- ...ion_hook.py => test_flag_eval_evp_hook.py} | 50 +++++++++---------- tests/openfeature/test_flag_eval_metrics.py | 34 ++++++------- tests/openfeature/test_flagevaluation_e2e.py | 20 ++++---- 11 files changed, 95 insertions(+), 90 deletions(-) rename ddtrace/internal/openfeature/{_flagevaluation_hook.py => _flag_eval_evp_hook.py} (94%) rename tests/openfeature/{test_flagevaluation_hook.py => test_flag_eval_evp_hook.py} (90%) diff --git a/benchmarks/openfeature_flagevaluation/scenario.py b/benchmarks/openfeature_flagevaluation/scenario.py index 1cdee09804d..79f1f23ffb6 100644 --- a/benchmarks/openfeature_flagevaluation/scenario.py +++ b/benchmarks/openfeature_flagevaluation/scenario.py @@ -13,7 +13,7 @@ * ``hook_plus_drain`` — end-to-end: N hook enqueues followed by a full queue drain + aggregate, the realistic per-flush shape. -The scenario builds its own ``FlagEvaluationWriter`` + ``FlagEvaluationHook`` and drives +The scenario builds its own ``FlagEvaluationWriter`` + ``FlagEvalEVPHook`` and drives them directly so the benchmark isolates the EVP pipeline (no network, no native eval). """ @@ -54,7 +54,7 @@ class OpenFeatureFlagEvaluation(bm.Scenario): num_context_fields: int def run(self): - from ddtrace.internal.openfeature._flagevaluation_hook import FlagEvaluationHook + from ddtrace.internal.openfeature._flag_eval_evp_hook import FlagEvalEVPHook from ddtrace.internal.openfeature._flagevaluation_writer import FlagEvaluationWriter mode = self.mode @@ -82,7 +82,7 @@ def run(self): ] writer = FlagEvaluationWriter(interval=3600.0) - hook = FlagEvaluationHook(writer) + hook = FlagEvalEVPHook(writer) if mode == "hook_enqueue": diff --git a/ddtrace/internal/ci_visibility/encoder.py b/ddtrace/internal/ci_visibility/encoder.py index 09533599149..0ff1c6b023e 100644 --- a/ddtrace/internal/ci_visibility/encoder.py +++ b/ddtrace/internal/ci_visibility/encoder.py @@ -23,6 +23,7 @@ from ddtrace.internal.ci_visibility.telemetry.payload import record_endpoint_payload_events_count from ddtrace.internal.ci_visibility.telemetry.payload import record_endpoint_payload_events_serialization_time from ddtrace.internal.encoding import JSONEncoderV2 +from ddtrace.internal.evp_proxy.constants import DEFAULT_EVP_PAYLOAD_SIZE_LIMIT from ddtrace.internal.logger import get_logger from ddtrace.internal.settings import env from ddtrace.internal.utils.time import StopWatch @@ -41,7 +42,7 @@ class CIVisibilityEncoderV01(BufferedEncoder): TEST_SUITE_EVENT_VERSION = 1 TEST_EVENT_VERSION = 2 ENDPOINT_TYPE = ENDPOINT.TEST_CYCLE - _MAX_PAYLOAD_SIZE = 5 * 1024 * 1024 # 5MB + _MAX_PAYLOAD_SIZE = DEFAULT_EVP_PAYLOAD_SIZE_LIMIT _MAX_META_TAG_VALUE_LENGTH = 5000 def __init__(self, *args: Any) -> None: diff --git a/ddtrace/internal/evp_proxy/constants.py b/ddtrace/internal/evp_proxy/constants.py index 0e40a5f3e4c..055dc6aa57b 100644 --- a/ddtrace/internal/evp_proxy/constants.py +++ b/ddtrace/internal/evp_proxy/constants.py @@ -5,6 +5,7 @@ EVP_SUBDOMAIN_HEADER_API_VALUE = "api" EVP_SUBDOMAIN_HEADER_COVERAGE_VALUE = "citestcov-intake" EVP_SUBDOMAIN_HEADER_EVENT_VALUE = "citestcycle-intake" +EVP_SUBDOMAIN_HEADER_EVENT_PLATFORM_VALUE = "event-platform-intake" EVP_SUBDOMAIN_HEADER_NAME = "X-Datadog-EVP-Subdomain" DEFAULT_EVP_PAYLOAD_SIZE_LIMIT = 5 << 20 # 5 MiB (actual server limit is 5.1 MB) DEFAULT_EVP_EVENT_SIZE_LIMIT = 5_000_000 # 5 MB, LLM Obs event size limit diff --git a/ddtrace/internal/openfeature/_flagevaluation_hook.py b/ddtrace/internal/openfeature/_flag_eval_evp_hook.py similarity index 94% rename from ddtrace/internal/openfeature/_flagevaluation_hook.py rename to ddtrace/internal/openfeature/_flag_eval_evp_hook.py index 47104f979b7..a2c3b54fb90 100644 --- a/ddtrace/internal/openfeature/_flagevaluation_hook.py +++ b/ddtrace/internal/openfeature/_flag_eval_evp_hook.py @@ -1,11 +1,11 @@ """ -FlagEvaluationHook — OpenFeature `finally_after` hook for EVP flagevaluation emission. +FlagEvalEVPHook — OpenFeature `finally_after` hook for EVP flagevaluation emission. Hook design: - Cheap capture only in finally_after (no aggregation, no serialization, no I/O). - Non-blocking enqueue to FlagEvaluationWriter. - The finally_after stage covers success, error, and default eval paths. -- Does NOT replace or modify the OTel FlagEvalHook in _flageval_metrics.py (the existing +- Does NOT replace or modify the OTel FlagEvalMetricsHook in _flageval_metrics.py (the existing feature_flag.evaluations OTel path is preserved unchanged). """ @@ -28,7 +28,7 @@ logger = get_logger(__name__) -class FlagEvaluationHook(Hook): +class FlagEvalEVPHook(Hook): """ OpenFeature Hook that enqueues cheap evaluation snapshots for EVP aggregation. @@ -115,6 +115,6 @@ def finally_after( except Exception: # Never propagate hook exceptions — best-effort telemetry. logger.debug( - "FlagEvaluationHook.finally_after: failed to enqueue eval snapshot", + "FlagEvalEVPHook.finally_after: failed to enqueue eval snapshot", exc_info=True, ) diff --git a/ddtrace/internal/openfeature/_flageval_metrics.py b/ddtrace/internal/openfeature/_flageval_metrics.py index c4cd18e8532..2c54eccf93c 100644 --- a/ddtrace/internal/openfeature/_flageval_metrics.py +++ b/ddtrace/internal/openfeature/_flageval_metrics.py @@ -136,7 +136,7 @@ def shutdown(self) -> None: log.debug("Flag evaluation metrics shutdown") -class FlagEvalHook(Hook): +class FlagEvalMetricsHook(Hook): """ OpenFeature Hook that tracks flag evaluation metrics. diff --git a/ddtrace/internal/openfeature/_flagevaluation_writer.py b/ddtrace/internal/openfeature/_flagevaluation_writer.py index 80060eaa347..2db1c2fdae3 100644 --- a/ddtrace/internal/openfeature/_flagevaluation_writer.py +++ b/ddtrace/internal/openfeature/_flagevaluation_writer.py @@ -34,6 +34,7 @@ from ddtrace import config as ddconfig from ddtrace.internal.evp_proxy.constants import DEFAULT_EVP_PAYLOAD_SIZE_LIMIT from ddtrace.internal.evp_proxy.constants import EVP_PROXY_AGENT_BASE_PATH +from ddtrace.internal.evp_proxy.constants import EVP_SUBDOMAIN_HEADER_EVENT_PLATFORM_VALUE from ddtrace.internal.evp_proxy.constants import EVP_SUBDOMAIN_HEADER_NAME from ddtrace.internal.logger import get_logger from ddtrace.internal.periodic import PeriodicService @@ -48,7 +49,7 @@ # EVP endpoint for flag evaluation events. FLAGEVALUATIONS_ENDPOINT = f"{EVP_PROXY_AGENT_BASE_PATH}/api/v2/flagevaluation" -EVP_SUBDOMAIN_VALUE = "event-platform-intake" +EVP_SUBDOMAIN_VALUE = EVP_SUBDOMAIN_HEADER_EVENT_PLATFORM_VALUE FLAGEVALUATIONS_PAYLOAD_SIZE_LIMIT = DEFAULT_EVP_PAYLOAD_SIZE_LIMIT _JSON_SEPARATORS = (",", ":") @@ -370,7 +371,7 @@ def __init__(self, interval: float = DEFAULT_FLUSH_INTERVAL, timeout: float = 2. self._drain_worker: typing.Optional[PeriodicThread] = None # ------------------------------------------------------------------ - # Public API used by FlagEvaluationHook + # Public API used by FlagEvalEVPHook # ------------------------------------------------------------------ def enqueue(self, event: _EvalEvent) -> None: diff --git a/ddtrace/internal/openfeature/_provider.py b/ddtrace/internal/openfeature/_provider.py index bfb45133857..6342795f8c3 100644 --- a/ddtrace/internal/openfeature/_provider.py +++ b/ddtrace/internal/openfeature/_provider.py @@ -25,10 +25,10 @@ from ddtrace.internal.native._native import ffe from ddtrace.internal.openfeature._config import _get_ffe_config from ddtrace.internal.openfeature._exposure import build_exposure_event +from ddtrace.internal.openfeature._flag_eval_evp_hook import FlagEvalEVPHook from ddtrace.internal.openfeature._flageval_metrics import METADATA_ALLOCATION_KEY -from ddtrace.internal.openfeature._flageval_metrics import FlagEvalHook from ddtrace.internal.openfeature._flageval_metrics import FlagEvalMetrics -from ddtrace.internal.openfeature._flagevaluation_hook import FlagEvaluationHook +from ddtrace.internal.openfeature._flageval_metrics import FlagEvalMetricsHook from ddtrace.internal.openfeature._flagevaluation_writer import EVAL_TIMESTAMP_METADATA_KEY from ddtrace.internal.openfeature._flagevaluation_writer import FlagEvaluationWriter from ddtrace.internal.openfeature._native import VariationType @@ -132,10 +132,10 @@ def __init__( # Initialize flag evaluation metrics tracking # Metrics are emitted via OTel when DD_METRICS_OTEL_ENABLED=true self._flag_eval_metrics: typing.Optional[FlagEvalMetrics] = None - self._flag_eval_hook: typing.Optional[FlagEvalHook] = None + self._flag_eval_metrics_hook: typing.Optional[FlagEvalMetricsHook] = None if self._enabled: self._flag_eval_metrics = FlagEvalMetrics() - self._flag_eval_hook = FlagEvalHook(self._flag_eval_metrics) + self._flag_eval_metrics_hook = FlagEvalMetricsHook(self._flag_eval_metrics) # EVP flagevaluation writer + hook — gated by DD_FLAGGING_EVALUATION_COUNTS_ENABLED # (default on). Gates ONLY the EVP path; the OTel path above is always registered @@ -147,13 +147,13 @@ def __init__( # environment at provider-construction time (the config var parses the live # environment via the DDConfig var system), which keeps the killswitch overridable # per-instance in tests. - self._flagevaluation_writer: typing.Optional[FlagEvaluationWriter] = None - self._flagevaluation_hook: typing.Optional[FlagEvaluationHook] = None + self._flag_eval_evp_writer: typing.Optional[FlagEvaluationWriter] = None + self._flag_eval_evp_hook: typing.Optional[FlagEvalEVPHook] = None evp_config = OpenFeatureConfig() evp_counts_enabled = evp_config.flagging_evaluation_counts_enabled if self._enabled and evp_counts_enabled: - self._flagevaluation_writer = FlagEvaluationWriter() - self._flagevaluation_hook = FlagEvaluationHook(self._flagevaluation_writer) + self._flag_eval_evp_writer = FlagEvaluationWriter() + self._flag_eval_evp_hook = FlagEvalEVPHook(self._flag_eval_evp_writer) def get_metadata(self) -> Metadata: """Returns provider metadata.""" @@ -169,21 +169,21 @@ def get_provider_hooks(self) -> list[typing.Any]: """ Returns provider-level hooks. - The flag evaluation hook is registered here to track metrics for + The OTel metrics hook is registered here to track metrics for every flag evaluation via the finally_after hook stage. Hook ordering: - 1. OTel FlagEvalHook (_flageval_metrics.py) — always registered when the provider + 1. OTel FlagEvalMetricsHook (_flageval_metrics.py) — always registered when the provider is enabled; emits the feature_flag.evaluations OTel counter (preserved unchanged). - 2. FlagEvaluationHook (_flagevaluation_hook.py) — registered only when + 2. FlagEvalEVPHook (_flag_eval_evp_hook.py) — registered only when DD_FLAGGING_EVALUATION_COUNTS_ENABLED is enabled (default on); enqueues cheap snapshots to FlagEvaluationWriter for EVP flagevaluation emission. """ hooks: list[typing.Any] = [] - if self._flag_eval_hook is not None: - hooks.append(self._flag_eval_hook) - if self._flagevaluation_hook is not None: - hooks.append(self._flagevaluation_hook) + if self._flag_eval_metrics_hook is not None: + hooks.append(self._flag_eval_metrics_hook) + if self._flag_eval_evp_hook is not None: + hooks.append(self._flag_eval_evp_hook) return hooks def initialize(self, evaluation_context: EvaluationContext) -> None: @@ -215,9 +215,9 @@ def initialize(self, evaluation_context: EvaluationContext) -> None: logger.debug("Exposure writer is already running", exc_info=True) # Start the EVP flagevaluation writer (if enabled via killswitch). - if self._flagevaluation_writer is not None: + if self._flag_eval_evp_writer is not None: try: - self._flagevaluation_writer.start() + self._flag_eval_evp_writer.start() logger.debug("FlagEvaluationWriter started") except ServiceStatusError: logger.debug("FlagEvaluationWriter is already running", exc_info=True) @@ -257,21 +257,21 @@ def shutdown(self) -> None: logger.debug("Exposure writer has already stopped", exc_info=True) # Stop the EVP flagevaluation writer (if it was started). - if self._flagevaluation_writer is not None: + if self._flag_eval_evp_writer is not None: try: - self._flagevaluation_writer.stop() - self._flagevaluation_writer.join() + self._flag_eval_evp_writer.stop() + self._flag_eval_evp_writer.join() logger.debug("FlagEvaluationWriter stopped") except ServiceStatusError: logger.debug("FlagEvaluationWriter has already stopped", exc_info=True) - self._flagevaluation_writer = None - self._flagevaluation_hook = None + self._flag_eval_evp_writer = None + self._flag_eval_evp_hook = None # Shutdown flag evaluation metrics if self._flag_eval_metrics is not None: self._flag_eval_metrics.shutdown() self._flag_eval_metrics = None - self._flag_eval_hook = None + self._flag_eval_metrics_hook = None # Clear exposure cache self.clear_exposure_cache() diff --git a/ddtrace/internal/openfeature/writer.py b/ddtrace/internal/openfeature/writer.py index 08a8f51bacf..8dca224d2af 100644 --- a/ddtrace/internal/openfeature/writer.py +++ b/ddtrace/internal/openfeature/writer.py @@ -9,6 +9,10 @@ from typing import TypedDict from ddtrace import config +from ddtrace.internal.evp_proxy.constants import DEFAULT_EVP_PAYLOAD_SIZE_LIMIT +from ddtrace.internal.evp_proxy.constants import EVP_PROXY_AGENT_BASE_PATH +from ddtrace.internal.evp_proxy.constants import EVP_SUBDOMAIN_HEADER_EVENT_PLATFORM_VALUE +from ddtrace.internal.evp_proxy.constants import EVP_SUBDOMAIN_HEADER_NAME from ddtrace.internal.logger import get_logger from ddtrace.internal.periodic import PeriodicService from ddtrace.internal.settings._agent import config as agent_config @@ -21,14 +25,12 @@ logger = get_logger(__name__) -EVP_PROXY_AGENT_BASE_PATH = "/evp_proxy/v2" EXPOSURE_ENDPOINT = "/api/v2/exposures" -EVP_SUBDOMAIN_HEADER_NAME = "X-Datadog-EVP-Subdomain" -EXPOSURE_SUBDOMAIN_NAME = "event-platform-intake" +EXPOSURE_SUBDOMAIN_NAME = EVP_SUBDOMAIN_HEADER_EVENT_PLATFORM_VALUE # Buffer and payload limits BUFFER_LIMIT = 1000 -PAYLOAD_SIZE_LIMIT = 5 << 20 # 5MB +PAYLOAD_SIZE_LIMIT = DEFAULT_EVP_PAYLOAD_SIZE_LIMIT # Default configuration DEFAULT_INTERVAL = 1.0 diff --git a/tests/openfeature/test_flagevaluation_hook.py b/tests/openfeature/test_flag_eval_evp_hook.py similarity index 90% rename from tests/openfeature/test_flagevaluation_hook.py rename to tests/openfeature/test_flag_eval_evp_hook.py index 2d6dd6aa009..1df95d0e414 100644 --- a/tests/openfeature/test_flagevaluation_hook.py +++ b/tests/openfeature/test_flag_eval_evp_hook.py @@ -1,5 +1,5 @@ """ -Tests for FlagEvaluationHook — finally_after cheap capture + non-blocking enqueue. +Tests for FlagEvalEVPHook — finally_after cheap capture + non-blocking enqueue. Validates the hook design: - finally_after does cheap capture only (no aggregation, no I/O) @@ -66,12 +66,12 @@ def writer(): @pytest.fixture def hook(writer): - from ddtrace.internal.openfeature._flagevaluation_hook import FlagEvaluationHook + from ddtrace.internal.openfeature._flag_eval_evp_hook import FlagEvalEVPHook - return FlagEvaluationHook(writer=writer) + return FlagEvalEVPHook(writer=writer) -class TestFlagEvaluationHook: +class TestFlagEvalEVPHook: def test_finally_after_calls_writer_enqueue_once(self, hook, writer): """finally_after must call writer.enqueue exactly once per evaluation.""" hc = _make_hook_context() @@ -207,11 +207,11 @@ class TestAsyncBoundary: def test_aggregate_not_called_on_hook_path(self): """Spy on the REAL writer's _aggregate — it must NOT run during finally_after.""" - from ddtrace.internal.openfeature._flagevaluation_hook import FlagEvaluationHook + from ddtrace.internal.openfeature._flag_eval_evp_hook import FlagEvalEVPHook from ddtrace.internal.openfeature._flagevaluation_writer import FlagEvaluationWriter real_writer = FlagEvaluationWriter(interval=10.0) - hook = FlagEvaluationHook(writer=real_writer) + hook = FlagEvalEVPHook(writer=real_writer) with mock.patch.object(real_writer, "_aggregate", wraps=real_writer._aggregate) as spy_aggregate: hc = _make_hook_context(attrs={"tier": "premium", "region": "us"}) @@ -236,12 +236,12 @@ def test_aggregate_not_called_on_hook_path(self): def test_canonical_key_not_computed_on_hook_path(self): """canonical_context_key (the keying cost) must not run during finally_after.""" - from ddtrace.internal.openfeature._flagevaluation_hook import FlagEvaluationHook + from ddtrace.internal.openfeature._flag_eval_evp_hook import FlagEvalEVPHook import ddtrace.internal.openfeature._flagevaluation_writer as writer_mod from ddtrace.internal.openfeature._flagevaluation_writer import FlagEvaluationWriter real_writer = FlagEvaluationWriter(interval=10.0) - hook = FlagEvaluationHook(writer=real_writer) + hook = FlagEvalEVPHook(writer=real_writer) with mock.patch.object(writer_mod, "canonical_context_key", wraps=writer_mod.canonical_context_key) as spy_key: hc = _make_hook_context(attrs={"a": "b"}) @@ -252,7 +252,7 @@ def test_canonical_key_not_computed_on_hook_path(self): class TestMetadataSourceMatchesOTelHook: """EVP hook reads allocation-key/eval metadata from the SAME source as the OTel hook. - The existing OTel FlagEvalHook reads allocation_key from + The existing OTel FlagEvalMetricsHook reads allocation_key from ``details.flag_metadata[METADATA_ALLOCATION_KEY]``. The EVP hook must read from the identical source so the two paths agree byte-for-byte on metadata. """ @@ -276,10 +276,10 @@ def test_evp_hook_reads_allocation_from_details_flag_metadata(self, hook, writer def test_otel_and_evp_hooks_extract_same_allocation_key(self): """Drive both hooks with identical details; both must surface the same allocation key.""" + from ddtrace.internal.openfeature._flag_eval_evp_hook import FlagEvalEVPHook from ddtrace.internal.openfeature._flageval_metrics import METADATA_ALLOCATION_KEY - from ddtrace.internal.openfeature._flageval_metrics import FlagEvalHook from ddtrace.internal.openfeature._flageval_metrics import FlagEvalMetrics - from ddtrace.internal.openfeature._flagevaluation_hook import FlagEvaluationHook + from ddtrace.internal.openfeature._flageval_metrics import FlagEvalMetricsHook from ddtrace.internal.openfeature._flagevaluation_writer import FlagEvaluationWriter details = _make_details(flag_metadata={METADATA_ALLOCATION_KEY: "shared-alloc"}) @@ -287,13 +287,13 @@ def test_otel_and_evp_hooks_extract_same_allocation_key(self): # OTel side: capture what FlagEvalMetrics.record received as allocation_key. metrics = mock.MagicMock(spec=FlagEvalMetrics) - otel_hook = FlagEvalHook(metrics) + otel_hook = FlagEvalMetricsHook(metrics) otel_hook.finally_after(hc, details, {}) otel_alloc = metrics.record.call_args.kwargs["allocation_key"] # EVP side: capture what the EVP hook enqueued as allocation_key. evp_writer = mock.MagicMock(spec=FlagEvaluationWriter) - evp_hook = FlagEvaluationHook(evp_writer) + evp_hook = FlagEvalEVPHook(evp_writer) evp_hook.finally_after(hc, details, {}) evp_alloc = evp_writer.enqueue.call_args[0][0].allocation_key @@ -303,7 +303,7 @@ def test_otel_and_evp_hooks_extract_same_allocation_key(self): class TestKillswitchGating: def test_default_enabled_registers_evp_hook(self): """Default (no env var set) must register the EVP hook + writer.""" - from ddtrace.internal.openfeature._flagevaluation_hook import FlagEvaluationHook + from ddtrace.internal.openfeature._flag_eval_evp_hook import FlagEvalEVPHook env = {k: v for k, v in os.environ.items() if k != "DD_FLAGGING_EVALUATION_COUNTS_ENABLED"} # No env var → enabled by default. @@ -314,9 +314,9 @@ def test_default_enabled_registers_evp_hook(self): from ddtrace.internal.openfeature._provider import DataDogProvider provider = DataDogProvider() - assert provider._flagevaluation_writer is not None - assert provider._flagevaluation_hook is not None - assert isinstance(provider._flagevaluation_hook, FlagEvaluationHook) + assert provider._flag_eval_evp_writer is not None + assert provider._flag_eval_evp_hook is not None + assert isinstance(provider._flag_eval_evp_hook, FlagEvalEVPHook) def test_killswitch_false_does_not_register_evp_hook(self): """DD_FLAGGING_EVALUATION_COUNTS_ENABLED=false must suppress EVP hook (killswitch).""" @@ -327,11 +327,11 @@ def test_killswitch_false_does_not_register_evp_hook(self): from ddtrace.internal.openfeature._provider import DataDogProvider provider = DataDogProvider() - assert provider._flagevaluation_writer is None - assert provider._flagevaluation_hook is None + assert provider._flag_eval_evp_writer is None + assert provider._flag_eval_evp_hook is None def test_killswitch_false_does_not_affect_otel_hook(self): - """Killswitch must not suppress the OTel FlagEvalHook (OTel non-regression).""" + """Killswitch must not suppress the OTel FlagEvalMetricsHook (OTel non-regression).""" with mock.patch.dict(os.environ, {"DD_FLAGGING_EVALUATION_COUNTS_ENABLED": "false"}): from tests.utils import override_global_config @@ -342,13 +342,13 @@ def test_killswitch_false_does_not_affect_otel_hook(self): provider = DataDogProvider() # OTel hook still present. - assert provider._flag_eval_hook is not None + assert provider._flag_eval_metrics_hook is not None # EVP hook absent. - assert provider._flagevaluation_hook is None + assert provider._flag_eval_evp_hook is None # get_provider_hooks still returns the OTel hook. hooks = provider.get_provider_hooks() assert len(hooks) == 1 - assert hooks[0] is provider._flag_eval_hook + assert hooks[0] is provider._flag_eval_metrics_hook def test_provider_shutdown_joins_evp_writer_final_flush(self): """Provider shutdown waits for FlagEvaluationWriter.on_shutdown final flush.""" @@ -377,5 +377,5 @@ def test_killswitch_enabled_true_registers_evp_hook(self): from ddtrace.internal.openfeature._provider import DataDogProvider provider = DataDogProvider() - assert provider._flagevaluation_writer is not None - assert provider._flagevaluation_hook is not None + assert provider._flag_eval_evp_writer is not None + assert provider._flag_eval_evp_hook is not None diff --git a/tests/openfeature/test_flag_eval_metrics.py b/tests/openfeature/test_flag_eval_metrics.py index 5e0d2184826..6e41fc163cf 100644 --- a/tests/openfeature/test_flag_eval_metrics.py +++ b/tests/openfeature/test_flag_eval_metrics.py @@ -1,7 +1,7 @@ """ Tests for flag evaluation metrics tracking. -Tests the FlagEvalMetrics class and FlagEvalHook that emit +Tests the FlagEvalMetrics class and FlagEvalMetricsHook that emit feature_flag.evaluations OTel metrics on every flag evaluation. """ @@ -21,8 +21,8 @@ from ddtrace.internal.openfeature._flageval_metrics import ATTR_REASON from ddtrace.internal.openfeature._flageval_metrics import ATTR_VARIANT from ddtrace.internal.openfeature._flageval_metrics import METADATA_ALLOCATION_KEY -from ddtrace.internal.openfeature._flageval_metrics import FlagEvalHook from ddtrace.internal.openfeature._flageval_metrics import FlagEvalMetrics +from ddtrace.internal.openfeature._flageval_metrics import FlagEvalMetricsHook from ddtrace.internal.openfeature._native import process_ffe_configuration from ddtrace.openfeature import DataDogProvider from tests.openfeature.config_helpers import create_boolean_flag @@ -260,14 +260,14 @@ def test_record_all_error_types(self): assert error_types == {"flag_not_found", "type_mismatch", "parse_error", "general"}, error_types -class TestFlagEvalHook: - """Test FlagEvalHook class.""" +class TestFlagEvalMetricsHook: + """Test FlagEvalMetricsHook class.""" def test_finally_after_calls_record(self): """finally_after should call metrics.record with correct arguments.""" mock_metrics = MagicMock(spec=FlagEvalMetrics) - hook = FlagEvalHook(mock_metrics) + hook = FlagEvalMetricsHook(mock_metrics) # Create mock hook context hook_context = MagicMock(spec=HookContext) @@ -296,7 +296,7 @@ def test_finally_after_with_error(self): """finally_after should pass error_code when present.""" mock_metrics = MagicMock(spec=FlagEvalMetrics) - hook = FlagEvalHook(mock_metrics) + hook = FlagEvalMetricsHook(mock_metrics) hook_context = MagicMock(spec=HookContext) hook_context.flag_key = "missing-flag" @@ -319,7 +319,7 @@ def test_finally_after_without_allocation_key(self): """finally_after should handle missing allocation_key.""" mock_metrics = MagicMock(spec=FlagEvalMetrics) - hook = FlagEvalHook(mock_metrics) + hook = FlagEvalMetricsHook(mock_metrics) hook_context = MagicMock(spec=HookContext) hook_context.flag_key = "test-flag" @@ -354,25 +354,25 @@ def clear_config(self): yield _set_ffe_config(None) - def test_provider_has_flag_eval_hook(self, provider): - """Provider should have flag evaluation hook when enabled.""" - assert provider._flag_eval_hook is not None + def test_provider_has_flag_eval_metrics_hook(self, provider): + """Provider should have the OTel flag evaluation metrics hook when enabled.""" + assert provider._flag_eval_metrics_hook is not None assert provider._flag_eval_metrics is not None - def test_get_provider_hooks_returns_flag_eval_hook(self, provider): - """get_provider_hooks should return the OTel flag eval hook (and optionally the EVP hook).""" + def test_get_provider_hooks_returns_flag_eval_metrics_hook(self, provider): + """get_provider_hooks should return the OTel metrics hook and optionally the EVP hook.""" hooks = provider.get_provider_hooks() - # The OTel FlagEvalHook is always first when the provider is enabled. - # The EVP FlagEvaluationHook is also registered by default (DD_FLAGGING_EVALUATION_COUNTS_ENABLED=true). + # The OTel FlagEvalMetricsHook is always first when the provider is enabled. + # The EVP FlagEvalEVPHook is also registered by default (DD_FLAGGING_EVALUATION_COUNTS_ENABLED=true). assert len(hooks) >= 1 - assert hooks[0] is provider._flag_eval_hook + assert hooks[0] is provider._flag_eval_metrics_hook def test_provider_disabled_has_no_hooks(self): """Provider should not have hooks when disabled.""" with override_global_config({"experimental_flagging_provider_enabled": False}): provider = DataDogProvider() - assert provider._flag_eval_hook is None + assert provider._flag_eval_metrics_hook is None assert provider._flag_eval_metrics is None assert provider.get_provider_hooks() == [] @@ -383,7 +383,7 @@ def test_shutdown_cleans_up_metrics(self, provider): provider.shutdown() assert provider._flag_eval_metrics is None - assert provider._flag_eval_hook is None + assert provider._flag_eval_metrics_hook is None def test_flag_metadata_includes_allocation_key(self, provider): """FlagResolutionDetails should include allocation_key in flag_metadata.""" diff --git a/tests/openfeature/test_flagevaluation_e2e.py b/tests/openfeature/test_flagevaluation_e2e.py index 0b78eaad1b4..68031703b1b 100644 --- a/tests/openfeature/test_flagevaluation_e2e.py +++ b/tests/openfeature/test_flagevaluation_e2e.py @@ -44,8 +44,8 @@ def provider_and_client(): with override_global_config({"experimental_flagging_provider_enabled": True}): provider = DataDogProvider() # Sanity: the EVP writer/hook are wired (killswitch default on). - assert provider._flagevaluation_writer is not None - assert provider._flagevaluation_hook is not None + assert provider._flag_eval_evp_writer is not None + assert provider._flag_eval_evp_hook is not None api.set_provider(provider) client = api.get_client() @@ -57,7 +57,7 @@ def provider_and_client(): def _drain(provider): """Aggregate everything the eval path enqueued, returning the emitted rows.""" - writer = provider._flagevaluation_writer + writer = provider._flag_eval_evp_writer with mock.patch.object(writer, "_send_payload") as mock_send: writer.periodic() if not mock_send.called: @@ -73,10 +73,10 @@ class TestEVPHookFiresOnRealEvalPath: def test_evp_hook_registered_in_provider_hooks(self, provider_and_client): provider, _ = provider_and_client - from ddtrace.internal.openfeature._flagevaluation_hook import FlagEvaluationHook + from ddtrace.internal.openfeature._flag_eval_evp_hook import FlagEvalEVPHook hooks = provider.get_provider_hooks() - assert any(isinstance(h, FlagEvaluationHook) for h in hooks) + assert any(isinstance(h, FlagEvalEVPHook) for h in hooks) def test_success_eval_enqueues_and_emits_row(self, provider_and_client): provider, client = provider_and_client @@ -185,12 +185,12 @@ class TestOTelNonRegressionAlongsideEVP: def test_both_otel_and_evp_hooks_registered(self, provider_and_client): provider, _ = provider_and_client - from ddtrace.internal.openfeature._flageval_metrics import FlagEvalHook - from ddtrace.internal.openfeature._flagevaluation_hook import FlagEvaluationHook + from ddtrace.internal.openfeature._flag_eval_evp_hook import FlagEvalEVPHook + from ddtrace.internal.openfeature._flageval_metrics import FlagEvalMetricsHook hooks = provider.get_provider_hooks() - assert any(isinstance(h, FlagEvalHook) for h in hooks), "OTel hook must remain registered" - assert any(isinstance(h, FlagEvaluationHook) for h in hooks), "EVP hook must be registered" + assert any(isinstance(h, FlagEvalMetricsHook) for h in hooks), "OTel hook must remain registered" + assert any(isinstance(h, FlagEvalEVPHook) for h in hooks), "EVP hook must be registered" def test_eval_drives_otel_record_and_evp_enqueue_together(self, provider_and_client): """A single eval feeds BOTH the OTel metric record and the EVP enqueue.""" @@ -201,7 +201,7 @@ def test_eval_drives_otel_record_and_evp_enqueue_together(self, provider_and_cli # Spy the OTel metrics record and the EVP writer enqueue. with ( mock.patch.object(provider._flag_eval_metrics, "record") as otel_record, - mock.patch.object(provider._flagevaluation_writer, "enqueue") as evp_enqueue, + mock.patch.object(provider._flag_eval_evp_writer, "enqueue") as evp_enqueue, ): client.get_boolean_value("dual-flag", False) From 623062cd0e940bca20ab35a6ae07f0cb4fc62e77 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 23 Jun 2026 16:01:56 -0400 Subject: [PATCH 24/37] refactor(openfeature): clarify flag eval logging hook --- .../openfeature_flagevaluation/scenario.py | 6 +-- ...evp_hook.py => _flag_eval_logging_hook.py} | 6 +-- .../openfeature/_flagevaluation_writer.py | 2 +- ddtrace/internal/openfeature/_provider.py | 14 +++--- ...hook.py => test_flag_eval_logging_hook.py} | 48 +++++++++---------- tests/openfeature/test_flag_eval_metrics.py | 4 +- tests/openfeature/test_flagevaluation_e2e.py | 14 +++--- 7 files changed, 47 insertions(+), 47 deletions(-) rename ddtrace/internal/openfeature/{_flag_eval_evp_hook.py => _flag_eval_logging_hook.py} (95%) rename tests/openfeature/{test_flag_eval_evp_hook.py => test_flag_eval_logging_hook.py} (90%) diff --git a/benchmarks/openfeature_flagevaluation/scenario.py b/benchmarks/openfeature_flagevaluation/scenario.py index 79f1f23ffb6..b9f90d768b5 100644 --- a/benchmarks/openfeature_flagevaluation/scenario.py +++ b/benchmarks/openfeature_flagevaluation/scenario.py @@ -13,7 +13,7 @@ * ``hook_plus_drain`` — end-to-end: N hook enqueues followed by a full queue drain + aggregate, the realistic per-flush shape. -The scenario builds its own ``FlagEvaluationWriter`` + ``FlagEvalEVPHook`` and drives +The scenario builds its own ``FlagEvaluationWriter`` + ``FlagEvalLoggingHook`` and drives them directly so the benchmark isolates the EVP pipeline (no network, no native eval). """ @@ -54,7 +54,7 @@ class OpenFeatureFlagEvaluation(bm.Scenario): num_context_fields: int def run(self): - from ddtrace.internal.openfeature._flag_eval_evp_hook import FlagEvalEVPHook + from ddtrace.internal.openfeature._flag_eval_logging_hook import FlagEvalLoggingHook from ddtrace.internal.openfeature._flagevaluation_writer import FlagEvaluationWriter mode = self.mode @@ -82,7 +82,7 @@ def run(self): ] writer = FlagEvaluationWriter(interval=3600.0) - hook = FlagEvalEVPHook(writer) + hook = FlagEvalLoggingHook(writer) if mode == "hook_enqueue": diff --git a/ddtrace/internal/openfeature/_flag_eval_evp_hook.py b/ddtrace/internal/openfeature/_flag_eval_logging_hook.py similarity index 95% rename from ddtrace/internal/openfeature/_flag_eval_evp_hook.py rename to ddtrace/internal/openfeature/_flag_eval_logging_hook.py index a2c3b54fb90..deee5f63dfa 100644 --- a/ddtrace/internal/openfeature/_flag_eval_evp_hook.py +++ b/ddtrace/internal/openfeature/_flag_eval_logging_hook.py @@ -1,5 +1,5 @@ """ -FlagEvalEVPHook — OpenFeature `finally_after` hook for EVP flagevaluation emission. +FlagEvalLoggingHook — OpenFeature `finally_after` hook for EVP flagevaluation emission. Hook design: - Cheap capture only in finally_after (no aggregation, no serialization, no I/O). @@ -28,7 +28,7 @@ logger = get_logger(__name__) -class FlagEvalEVPHook(Hook): +class FlagEvalLoggingHook(Hook): """ OpenFeature Hook that enqueues cheap evaluation snapshots for EVP aggregation. @@ -115,6 +115,6 @@ def finally_after( except Exception: # Never propagate hook exceptions — best-effort telemetry. logger.debug( - "FlagEvalEVPHook.finally_after: failed to enqueue eval snapshot", + "FlagEvalLoggingHook.finally_after: failed to enqueue eval snapshot", exc_info=True, ) diff --git a/ddtrace/internal/openfeature/_flagevaluation_writer.py b/ddtrace/internal/openfeature/_flagevaluation_writer.py index 2db1c2fdae3..9954bba140b 100644 --- a/ddtrace/internal/openfeature/_flagevaluation_writer.py +++ b/ddtrace/internal/openfeature/_flagevaluation_writer.py @@ -371,7 +371,7 @@ def __init__(self, interval: float = DEFAULT_FLUSH_INTERVAL, timeout: float = 2. self._drain_worker: typing.Optional[PeriodicThread] = None # ------------------------------------------------------------------ - # Public API used by FlagEvalEVPHook + # Public API used by FlagEvalLoggingHook # ------------------------------------------------------------------ def enqueue(self, event: _EvalEvent) -> None: diff --git a/ddtrace/internal/openfeature/_provider.py b/ddtrace/internal/openfeature/_provider.py index 6342795f8c3..52e3acd6f05 100644 --- a/ddtrace/internal/openfeature/_provider.py +++ b/ddtrace/internal/openfeature/_provider.py @@ -25,7 +25,7 @@ from ddtrace.internal.native._native import ffe from ddtrace.internal.openfeature._config import _get_ffe_config from ddtrace.internal.openfeature._exposure import build_exposure_event -from ddtrace.internal.openfeature._flag_eval_evp_hook import FlagEvalEVPHook +from ddtrace.internal.openfeature._flag_eval_logging_hook import FlagEvalLoggingHook from ddtrace.internal.openfeature._flageval_metrics import METADATA_ALLOCATION_KEY from ddtrace.internal.openfeature._flageval_metrics import FlagEvalMetrics from ddtrace.internal.openfeature._flageval_metrics import FlagEvalMetricsHook @@ -148,12 +148,12 @@ def __init__( # environment via the DDConfig var system), which keeps the killswitch overridable # per-instance in tests. self._flag_eval_evp_writer: typing.Optional[FlagEvaluationWriter] = None - self._flag_eval_evp_hook: typing.Optional[FlagEvalEVPHook] = None + self._flag_eval_logging_hook: typing.Optional[FlagEvalLoggingHook] = None evp_config = OpenFeatureConfig() evp_counts_enabled = evp_config.flagging_evaluation_counts_enabled if self._enabled and evp_counts_enabled: self._flag_eval_evp_writer = FlagEvaluationWriter() - self._flag_eval_evp_hook = FlagEvalEVPHook(self._flag_eval_evp_writer) + self._flag_eval_logging_hook = FlagEvalLoggingHook(self._flag_eval_evp_writer) def get_metadata(self) -> Metadata: """Returns provider metadata.""" @@ -175,15 +175,15 @@ def get_provider_hooks(self) -> list[typing.Any]: Hook ordering: 1. OTel FlagEvalMetricsHook (_flageval_metrics.py) — always registered when the provider is enabled; emits the feature_flag.evaluations OTel counter (preserved unchanged). - 2. FlagEvalEVPHook (_flag_eval_evp_hook.py) — registered only when + 2. FlagEvalLoggingHook (_flag_eval_logging_hook.py) — registered only when DD_FLAGGING_EVALUATION_COUNTS_ENABLED is enabled (default on); enqueues cheap snapshots to FlagEvaluationWriter for EVP flagevaluation emission. """ hooks: list[typing.Any] = [] if self._flag_eval_metrics_hook is not None: hooks.append(self._flag_eval_metrics_hook) - if self._flag_eval_evp_hook is not None: - hooks.append(self._flag_eval_evp_hook) + if self._flag_eval_logging_hook is not None: + hooks.append(self._flag_eval_logging_hook) return hooks def initialize(self, evaluation_context: EvaluationContext) -> None: @@ -265,7 +265,7 @@ def shutdown(self) -> None: except ServiceStatusError: logger.debug("FlagEvaluationWriter has already stopped", exc_info=True) self._flag_eval_evp_writer = None - self._flag_eval_evp_hook = None + self._flag_eval_logging_hook = None # Shutdown flag evaluation metrics if self._flag_eval_metrics is not None: diff --git a/tests/openfeature/test_flag_eval_evp_hook.py b/tests/openfeature/test_flag_eval_logging_hook.py similarity index 90% rename from tests/openfeature/test_flag_eval_evp_hook.py rename to tests/openfeature/test_flag_eval_logging_hook.py index 1df95d0e414..8a6ed2c2af5 100644 --- a/tests/openfeature/test_flag_eval_evp_hook.py +++ b/tests/openfeature/test_flag_eval_logging_hook.py @@ -1,5 +1,5 @@ """ -Tests for FlagEvalEVPHook — finally_after cheap capture + non-blocking enqueue. +Tests for FlagEvalLoggingHook — finally_after cheap capture + non-blocking enqueue. Validates the hook design: - finally_after does cheap capture only (no aggregation, no I/O) @@ -66,12 +66,12 @@ def writer(): @pytest.fixture def hook(writer): - from ddtrace.internal.openfeature._flag_eval_evp_hook import FlagEvalEVPHook + from ddtrace.internal.openfeature._flag_eval_logging_hook import FlagEvalLoggingHook - return FlagEvalEVPHook(writer=writer) + return FlagEvalLoggingHook(writer=writer) -class TestFlagEvalEVPHook: +class TestFlagEvalLoggingHook: def test_finally_after_calls_writer_enqueue_once(self, hook, writer): """finally_after must call writer.enqueue exactly once per evaluation.""" hc = _make_hook_context() @@ -207,11 +207,11 @@ class TestAsyncBoundary: def test_aggregate_not_called_on_hook_path(self): """Spy on the REAL writer's _aggregate — it must NOT run during finally_after.""" - from ddtrace.internal.openfeature._flag_eval_evp_hook import FlagEvalEVPHook + from ddtrace.internal.openfeature._flag_eval_logging_hook import FlagEvalLoggingHook from ddtrace.internal.openfeature._flagevaluation_writer import FlagEvaluationWriter real_writer = FlagEvaluationWriter(interval=10.0) - hook = FlagEvalEVPHook(writer=real_writer) + hook = FlagEvalLoggingHook(writer=real_writer) with mock.patch.object(real_writer, "_aggregate", wraps=real_writer._aggregate) as spy_aggregate: hc = _make_hook_context(attrs={"tier": "premium", "region": "us"}) @@ -236,12 +236,12 @@ def test_aggregate_not_called_on_hook_path(self): def test_canonical_key_not_computed_on_hook_path(self): """canonical_context_key (the keying cost) must not run during finally_after.""" - from ddtrace.internal.openfeature._flag_eval_evp_hook import FlagEvalEVPHook + from ddtrace.internal.openfeature._flag_eval_logging_hook import FlagEvalLoggingHook import ddtrace.internal.openfeature._flagevaluation_writer as writer_mod from ddtrace.internal.openfeature._flagevaluation_writer import FlagEvaluationWriter real_writer = FlagEvaluationWriter(interval=10.0) - hook = FlagEvalEVPHook(writer=real_writer) + hook = FlagEvalLoggingHook(writer=real_writer) with mock.patch.object(writer_mod, "canonical_context_key", wraps=writer_mod.canonical_context_key) as spy_key: hc = _make_hook_context(attrs={"a": "b"}) @@ -250,10 +250,10 @@ def test_canonical_key_not_computed_on_hook_path(self): class TestMetadataSourceMatchesOTelHook: - """EVP hook reads allocation-key/eval metadata from the SAME source as the OTel hook. + """logging hook reads allocation-key/eval metadata from the SAME source as the OTel hook. The existing OTel FlagEvalMetricsHook reads allocation_key from - ``details.flag_metadata[METADATA_ALLOCATION_KEY]``. The EVP hook must read from the + ``details.flag_metadata[METADATA_ALLOCATION_KEY]``. The logging hook must read from the identical source so the two paths agree byte-for-byte on metadata. """ @@ -265,7 +265,7 @@ def test_allocation_key_metadata_key_matches_otel_hook(self): assert _flagevaluation_writer.METADATA_ALLOCATION_KEY == _flageval_metrics.METADATA_ALLOCATION_KEY def test_evp_hook_reads_allocation_from_details_flag_metadata(self, hook, writer): - """EVP hook reads allocation_key from details.flag_metadata (not hook_context).""" + """logging hook reads allocation_key from details.flag_metadata (not hook_context).""" from ddtrace.internal.openfeature._flageval_metrics import METADATA_ALLOCATION_KEY hc = _make_hook_context() @@ -276,7 +276,7 @@ def test_evp_hook_reads_allocation_from_details_flag_metadata(self, hook, writer def test_otel_and_evp_hooks_extract_same_allocation_key(self): """Drive both hooks with identical details; both must surface the same allocation key.""" - from ddtrace.internal.openfeature._flag_eval_evp_hook import FlagEvalEVPHook + from ddtrace.internal.openfeature._flag_eval_logging_hook import FlagEvalLoggingHook from ddtrace.internal.openfeature._flageval_metrics import METADATA_ALLOCATION_KEY from ddtrace.internal.openfeature._flageval_metrics import FlagEvalMetrics from ddtrace.internal.openfeature._flageval_metrics import FlagEvalMetricsHook @@ -291,9 +291,9 @@ def test_otel_and_evp_hooks_extract_same_allocation_key(self): otel_hook.finally_after(hc, details, {}) otel_alloc = metrics.record.call_args.kwargs["allocation_key"] - # EVP side: capture what the EVP hook enqueued as allocation_key. + # EVP side: capture what the logging hook enqueued as allocation_key. evp_writer = mock.MagicMock(spec=FlagEvaluationWriter) - evp_hook = FlagEvalEVPHook(evp_writer) + evp_hook = FlagEvalLoggingHook(evp_writer) evp_hook.finally_after(hc, details, {}) evp_alloc = evp_writer.enqueue.call_args[0][0].allocation_key @@ -302,8 +302,8 @@ def test_otel_and_evp_hooks_extract_same_allocation_key(self): class TestKillswitchGating: def test_default_enabled_registers_evp_hook(self): - """Default (no env var set) must register the EVP hook + writer.""" - from ddtrace.internal.openfeature._flag_eval_evp_hook import FlagEvalEVPHook + """Default (no env var set) must register the logging hook + writer.""" + from ddtrace.internal.openfeature._flag_eval_logging_hook import FlagEvalLoggingHook env = {k: v for k, v in os.environ.items() if k != "DD_FLAGGING_EVALUATION_COUNTS_ENABLED"} # No env var → enabled by default. @@ -315,11 +315,11 @@ def test_default_enabled_registers_evp_hook(self): provider = DataDogProvider() assert provider._flag_eval_evp_writer is not None - assert provider._flag_eval_evp_hook is not None - assert isinstance(provider._flag_eval_evp_hook, FlagEvalEVPHook) + assert provider._flag_eval_logging_hook is not None + assert isinstance(provider._flag_eval_logging_hook, FlagEvalLoggingHook) def test_killswitch_false_does_not_register_evp_hook(self): - """DD_FLAGGING_EVALUATION_COUNTS_ENABLED=false must suppress EVP hook (killswitch).""" + """DD_FLAGGING_EVALUATION_COUNTS_ENABLED=false must suppress logging hook (killswitch).""" with mock.patch.dict(os.environ, {"DD_FLAGGING_EVALUATION_COUNTS_ENABLED": "false"}): from tests.utils import override_global_config @@ -328,7 +328,7 @@ def test_killswitch_false_does_not_register_evp_hook(self): provider = DataDogProvider() assert provider._flag_eval_evp_writer is None - assert provider._flag_eval_evp_hook is None + assert provider._flag_eval_logging_hook is None def test_killswitch_false_does_not_affect_otel_hook(self): """Killswitch must not suppress the OTel FlagEvalMetricsHook (OTel non-regression).""" @@ -343,8 +343,8 @@ def test_killswitch_false_does_not_affect_otel_hook(self): provider = DataDogProvider() # OTel hook still present. assert provider._flag_eval_metrics_hook is not None - # EVP hook absent. - assert provider._flag_eval_evp_hook is None + # logging hook absent. + assert provider._flag_eval_logging_hook is None # get_provider_hooks still returns the OTel hook. hooks = provider.get_provider_hooks() assert len(hooks) == 1 @@ -369,7 +369,7 @@ def test_provider_shutdown_joins_evp_writer_final_flush(self): assert writer.mock_calls.index(mock.call.stop()) < writer.mock_calls.index(mock.call.join()) def test_killswitch_enabled_true_registers_evp_hook(self): - """DD_FLAGGING_EVALUATION_COUNTS_ENABLED=true must register the EVP hook.""" + """DD_FLAGGING_EVALUATION_COUNTS_ENABLED=true must register the logging hook.""" with mock.patch.dict(os.environ, {"DD_FLAGGING_EVALUATION_COUNTS_ENABLED": "true"}): from tests.utils import override_global_config @@ -378,4 +378,4 @@ def test_killswitch_enabled_true_registers_evp_hook(self): provider = DataDogProvider() assert provider._flag_eval_evp_writer is not None - assert provider._flag_eval_evp_hook is not None + assert provider._flag_eval_logging_hook is not None diff --git a/tests/openfeature/test_flag_eval_metrics.py b/tests/openfeature/test_flag_eval_metrics.py index 6e41fc163cf..091ddf03261 100644 --- a/tests/openfeature/test_flag_eval_metrics.py +++ b/tests/openfeature/test_flag_eval_metrics.py @@ -360,10 +360,10 @@ def test_provider_has_flag_eval_metrics_hook(self, provider): assert provider._flag_eval_metrics is not None def test_get_provider_hooks_returns_flag_eval_metrics_hook(self, provider): - """get_provider_hooks should return the OTel metrics hook and optionally the EVP hook.""" + """get_provider_hooks should return the OTel metrics hook and optionally the logging hook.""" hooks = provider.get_provider_hooks() # The OTel FlagEvalMetricsHook is always first when the provider is enabled. - # The EVP FlagEvalEVPHook is also registered by default (DD_FLAGGING_EVALUATION_COUNTS_ENABLED=true). + # The FlagEvalLoggingHook is also registered by default (DD_FLAGGING_EVALUATION_COUNTS_ENABLED=true). assert len(hooks) >= 1 assert hooks[0] is provider._flag_eval_metrics_hook diff --git a/tests/openfeature/test_flagevaluation_e2e.py b/tests/openfeature/test_flagevaluation_e2e.py index 68031703b1b..8506a8379b8 100644 --- a/tests/openfeature/test_flagevaluation_e2e.py +++ b/tests/openfeature/test_flagevaluation_e2e.py @@ -4,7 +4,7 @@ These cover the lifecycle/exit paths that unit tests of the hook/writer in isolation cannot prove: -- The EVP hook actually fires on the provider's REAL OpenFeature evaluation entrypoint +- The logging hook actually fires on the provider's REAL OpenFeature evaluation entrypoint (driven through the OpenFeature client, not by calling the hook directly). - ALL evaluation exit paths are covered — success, native engine error, runtime default (flag-not-found / no-config), and disabled. @@ -45,7 +45,7 @@ def provider_and_client(): provider = DataDogProvider() # Sanity: the EVP writer/hook are wired (killswitch default on). assert provider._flag_eval_evp_writer is not None - assert provider._flag_eval_evp_hook is not None + assert provider._flag_eval_logging_hook is not None api.set_provider(provider) client = api.get_client() @@ -69,14 +69,14 @@ def _drain(provider): class TestEVPHookFiresOnRealEvalPath: - """The EVP hook fires on the provider's real evaluation entrypoint.""" + """The logging hook fires on the provider's real evaluation entrypoint.""" def test_evp_hook_registered_in_provider_hooks(self, provider_and_client): provider, _ = provider_and_client - from ddtrace.internal.openfeature._flag_eval_evp_hook import FlagEvalEVPHook + from ddtrace.internal.openfeature._flag_eval_logging_hook import FlagEvalLoggingHook hooks = provider.get_provider_hooks() - assert any(isinstance(h, FlagEvalEVPHook) for h in hooks) + assert any(isinstance(h, FlagEvalLoggingHook) for h in hooks) def test_success_eval_enqueues_and_emits_row(self, provider_and_client): provider, client = provider_and_client @@ -185,12 +185,12 @@ class TestOTelNonRegressionAlongsideEVP: def test_both_otel_and_evp_hooks_registered(self, provider_and_client): provider, _ = provider_and_client - from ddtrace.internal.openfeature._flag_eval_evp_hook import FlagEvalEVPHook + from ddtrace.internal.openfeature._flag_eval_logging_hook import FlagEvalLoggingHook from ddtrace.internal.openfeature._flageval_metrics import FlagEvalMetricsHook hooks = provider.get_provider_hooks() assert any(isinstance(h, FlagEvalMetricsHook) for h in hooks), "OTel hook must remain registered" - assert any(isinstance(h, FlagEvalEVPHook) for h in hooks), "EVP hook must be registered" + assert any(isinstance(h, FlagEvalLoggingHook) for h in hooks), "logging hook must be registered" def test_eval_drives_otel_record_and_evp_enqueue_together(self, provider_and_client): """A single eval feeds BOTH the OTel metric record and the EVP enqueue.""" From df99d97a14725c9943a2017e90026e7965ea26f0 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 23 Jun 2026 16:21:48 -0400 Subject: [PATCH 25/37] fix(debugging): remove stale uploader type ignore --- ddtrace/debugging/_uploader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/debugging/_uploader.py b/ddtrace/debugging/_uploader.py index 79abd44b809..3164d37af7d 100644 --- a/ddtrace/debugging/_uploader.py +++ b/ddtrace/debugging/_uploader.py @@ -231,7 +231,7 @@ def online(self) -> None: msg = "Debugger tracks not enabled" raise ValueError(msg) - def on_shutdown(self) -> None: # type: ignore[override] + def on_shutdown(self) -> None: self._flush() @classmethod From 61f63b91f2454303b3984ca161f342ed1fcc4f99 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 23 Jun 2026 18:03:58 -0400 Subject: [PATCH 26/37] test(openfeature): align flag eval logging hook naming --- .../test_flag_eval_logging_hook.py | 24 +++++++++---------- tests/openfeature/test_flagevaluation_e2e.py | 14 +++++------ 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/openfeature/test_flag_eval_logging_hook.py b/tests/openfeature/test_flag_eval_logging_hook.py index 8a6ed2c2af5..794aa88af61 100644 --- a/tests/openfeature/test_flag_eval_logging_hook.py +++ b/tests/openfeature/test_flag_eval_logging_hook.py @@ -264,7 +264,7 @@ def test_allocation_key_metadata_key_matches_otel_hook(self): # Same metadata key constant in both modules. assert _flagevaluation_writer.METADATA_ALLOCATION_KEY == _flageval_metrics.METADATA_ALLOCATION_KEY - def test_evp_hook_reads_allocation_from_details_flag_metadata(self, hook, writer): + def test_logging_hook_reads_allocation_from_details_flag_metadata(self, hook, writer): """logging hook reads allocation_key from details.flag_metadata (not hook_context).""" from ddtrace.internal.openfeature._flageval_metrics import METADATA_ALLOCATION_KEY @@ -274,7 +274,7 @@ def test_evp_hook_reads_allocation_from_details_flag_metadata(self, hook, writer event = writer.enqueue.call_args[0][0] assert event.allocation_key == "alloc-from-details" - def test_otel_and_evp_hooks_extract_same_allocation_key(self): + def test_metrics_and_logging_hooks_extract_same_allocation_key(self): """Drive both hooks with identical details; both must surface the same allocation key.""" from ddtrace.internal.openfeature._flag_eval_logging_hook import FlagEvalLoggingHook from ddtrace.internal.openfeature._flageval_metrics import METADATA_ALLOCATION_KEY @@ -291,17 +291,17 @@ def test_otel_and_evp_hooks_extract_same_allocation_key(self): otel_hook.finally_after(hc, details, {}) otel_alloc = metrics.record.call_args.kwargs["allocation_key"] - # EVP side: capture what the logging hook enqueued as allocation_key. - evp_writer = mock.MagicMock(spec=FlagEvaluationWriter) - evp_hook = FlagEvalLoggingHook(evp_writer) - evp_hook.finally_after(hc, details, {}) - evp_alloc = evp_writer.enqueue.call_args[0][0].allocation_key + # Logging side: capture what the logging hook enqueued as allocation_key. + logging_writer = mock.MagicMock(spec=FlagEvaluationWriter) + logging_hook = FlagEvalLoggingHook(logging_writer) + logging_hook.finally_after(hc, details, {}) + logging_alloc = logging_writer.enqueue.call_args[0][0].allocation_key - assert otel_alloc == evp_alloc == "shared-alloc" + assert otel_alloc == logging_alloc == "shared-alloc" class TestKillswitchGating: - def test_default_enabled_registers_evp_hook(self): + def test_default_enabled_registers_logging_hook(self): """Default (no env var set) must register the logging hook + writer.""" from ddtrace.internal.openfeature._flag_eval_logging_hook import FlagEvalLoggingHook @@ -318,7 +318,7 @@ def test_default_enabled_registers_evp_hook(self): assert provider._flag_eval_logging_hook is not None assert isinstance(provider._flag_eval_logging_hook, FlagEvalLoggingHook) - def test_killswitch_false_does_not_register_evp_hook(self): + def test_killswitch_false_does_not_register_logging_hook(self): """DD_FLAGGING_EVALUATION_COUNTS_ENABLED=false must suppress logging hook (killswitch).""" with mock.patch.dict(os.environ, {"DD_FLAGGING_EVALUATION_COUNTS_ENABLED": "false"}): from tests.utils import override_global_config @@ -350,7 +350,7 @@ def test_killswitch_false_does_not_affect_otel_hook(self): assert len(hooks) == 1 assert hooks[0] is provider._flag_eval_metrics_hook - def test_provider_shutdown_joins_evp_writer_final_flush(self): + def test_provider_shutdown_joins_logging_writer_final_flush(self): """Provider shutdown waits for FlagEvaluationWriter.on_shutdown final flush.""" from tests.utils import override_global_config @@ -368,7 +368,7 @@ def test_provider_shutdown_joins_evp_writer_final_flush(self): writer.join.assert_called_once() assert writer.mock_calls.index(mock.call.stop()) < writer.mock_calls.index(mock.call.join()) - def test_killswitch_enabled_true_registers_evp_hook(self): + def test_killswitch_enabled_true_registers_logging_hook(self): """DD_FLAGGING_EVALUATION_COUNTS_ENABLED=true must register the logging hook.""" with mock.patch.dict(os.environ, {"DD_FLAGGING_EVALUATION_COUNTS_ENABLED": "true"}): from tests.utils import override_global_config diff --git a/tests/openfeature/test_flagevaluation_e2e.py b/tests/openfeature/test_flagevaluation_e2e.py index 8506a8379b8..8b94c4eacba 100644 --- a/tests/openfeature/test_flagevaluation_e2e.py +++ b/tests/openfeature/test_flagevaluation_e2e.py @@ -68,10 +68,10 @@ def _drain(provider): return decoded.get("flagEvaluations", []) -class TestEVPHookFiresOnRealEvalPath: +class TestFlagEvalLoggingHookFiresOnRealEvalPath: """The logging hook fires on the provider's real evaluation entrypoint.""" - def test_evp_hook_registered_in_provider_hooks(self, provider_and_client): + def test_logging_hook_registered_in_provider_hooks(self, provider_and_client): provider, _ = provider_and_client from ddtrace.internal.openfeature._flag_eval_logging_hook import FlagEvalLoggingHook @@ -109,7 +109,7 @@ def test_variant_is_resolution_variant_not_value(self, provider_and_client): assert row["variant"]["key"] == details.variant -class TestEVPExitPathsCovered: +class TestFlagEvalLoggingExitPathsCovered: """success / engine-error / runtime-default / disabled exit paths are all captured.""" def test_flag_not_found_runtime_default_path(self, provider_and_client): @@ -180,10 +180,10 @@ def test_targeting_key_and_context_captured_on_success(self, provider_and_client assert row["context"]["evaluation"]["tier"] == "gold" -class TestOTelNonRegressionAlongsideEVP: - """The EVP path must NOT change which hooks the provider registers for OTel (non-regression).""" +class TestOTelNonRegressionAlongsideFlagEvalLogging: + """The logging path must NOT change which hooks the provider registers for OTel.""" - def test_both_otel_and_evp_hooks_registered(self, provider_and_client): + def test_both_metrics_and_logging_hooks_registered(self, provider_and_client): provider, _ = provider_and_client from ddtrace.internal.openfeature._flag_eval_logging_hook import FlagEvalLoggingHook from ddtrace.internal.openfeature._flageval_metrics import FlagEvalMetricsHook @@ -192,7 +192,7 @@ def test_both_otel_and_evp_hooks_registered(self, provider_and_client): assert any(isinstance(h, FlagEvalMetricsHook) for h in hooks), "OTel hook must remain registered" assert any(isinstance(h, FlagEvalLoggingHook) for h in hooks), "logging hook must be registered" - def test_eval_drives_otel_record_and_evp_enqueue_together(self, provider_and_client): + def test_eval_drives_otel_record_and_logging_enqueue_together(self, provider_and_client): """A single eval feeds BOTH the OTel metric record and the EVP enqueue.""" provider, client = provider_and_client config = create_config(create_boolean_flag("dual-flag", enabled=True, default_value=True)) From 4f1788bdb55377e5e3c834586b75e6d1fc90c334 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 23 Jun 2026 19:45:03 -0400 Subject: [PATCH 27/37] refactor(openfeature): name flag eval EVP hook explicitly --- ...logging_hook.py => _flag_eval_evp_hook.py} | 11 ++- .../openfeature/_flagevaluation_writer.py | 2 +- ddtrace/internal/openfeature/_provider.py | 14 ++-- ...ing_hook.py => test_flag_eval_evp_hook.py} | 74 +++++++++---------- tests/openfeature/test_flag_eval_metrics.py | 4 +- tests/openfeature/test_flagevaluation_e2e.py | 26 +++---- 6 files changed, 65 insertions(+), 66 deletions(-) rename ddtrace/internal/openfeature/{_flag_eval_logging_hook.py => _flag_eval_evp_hook.py} (91%) rename tests/openfeature/{test_flag_eval_logging_hook.py => test_flag_eval_evp_hook.py} (85%) diff --git a/ddtrace/internal/openfeature/_flag_eval_logging_hook.py b/ddtrace/internal/openfeature/_flag_eval_evp_hook.py similarity index 91% rename from ddtrace/internal/openfeature/_flag_eval_logging_hook.py rename to ddtrace/internal/openfeature/_flag_eval_evp_hook.py index deee5f63dfa..0d61a007f88 100644 --- a/ddtrace/internal/openfeature/_flag_eval_logging_hook.py +++ b/ddtrace/internal/openfeature/_flag_eval_evp_hook.py @@ -1,5 +1,5 @@ """ -FlagEvalLoggingHook — OpenFeature `finally_after` hook for EVP flagevaluation emission. +FlagEvalEVPHook — OpenFeature `finally_after` hook for EVP flagevaluation emission. Hook design: - Cheap capture only in finally_after (no aggregation, no serialization, no I/O). @@ -12,7 +12,6 @@ import time import typing -from openfeature.exception import ErrorCode from openfeature.flag_evaluation import FlagEvaluationDetails from openfeature.hook import Hook from openfeature.hook import HookContext @@ -28,7 +27,7 @@ logger = get_logger(__name__) -class FlagEvalLoggingHook(Hook): +class FlagEvalEVPHook(Hook): """ OpenFeature Hook that enqueues cheap evaluation snapshots for EVP aggregation. @@ -78,9 +77,9 @@ def finally_after( else: eval_time_ms = int(time.time() * 1000) - # Variant: absent or type mismatch signals a runtime default. + # Variant: absent variant signals a runtime default. variant = "" - if details.variant and details.error_code != ErrorCode.TYPE_MISMATCH: + if details.variant: variant = details.variant runtime_default = variant == "" @@ -115,6 +114,6 @@ def finally_after( except Exception: # Never propagate hook exceptions — best-effort telemetry. logger.debug( - "FlagEvalLoggingHook.finally_after: failed to enqueue eval snapshot", + "FlagEvalEVPHook.finally_after: failed to enqueue eval snapshot", exc_info=True, ) diff --git a/ddtrace/internal/openfeature/_flagevaluation_writer.py b/ddtrace/internal/openfeature/_flagevaluation_writer.py index 9954bba140b..2db1c2fdae3 100644 --- a/ddtrace/internal/openfeature/_flagevaluation_writer.py +++ b/ddtrace/internal/openfeature/_flagevaluation_writer.py @@ -371,7 +371,7 @@ def __init__(self, interval: float = DEFAULT_FLUSH_INTERVAL, timeout: float = 2. self._drain_worker: typing.Optional[PeriodicThread] = None # ------------------------------------------------------------------ - # Public API used by FlagEvalLoggingHook + # Public API used by FlagEvalEVPHook # ------------------------------------------------------------------ def enqueue(self, event: _EvalEvent) -> None: diff --git a/ddtrace/internal/openfeature/_provider.py b/ddtrace/internal/openfeature/_provider.py index 52e3acd6f05..6342795f8c3 100644 --- a/ddtrace/internal/openfeature/_provider.py +++ b/ddtrace/internal/openfeature/_provider.py @@ -25,7 +25,7 @@ from ddtrace.internal.native._native import ffe from ddtrace.internal.openfeature._config import _get_ffe_config from ddtrace.internal.openfeature._exposure import build_exposure_event -from ddtrace.internal.openfeature._flag_eval_logging_hook import FlagEvalLoggingHook +from ddtrace.internal.openfeature._flag_eval_evp_hook import FlagEvalEVPHook from ddtrace.internal.openfeature._flageval_metrics import METADATA_ALLOCATION_KEY from ddtrace.internal.openfeature._flageval_metrics import FlagEvalMetrics from ddtrace.internal.openfeature._flageval_metrics import FlagEvalMetricsHook @@ -148,12 +148,12 @@ def __init__( # environment via the DDConfig var system), which keeps the killswitch overridable # per-instance in tests. self._flag_eval_evp_writer: typing.Optional[FlagEvaluationWriter] = None - self._flag_eval_logging_hook: typing.Optional[FlagEvalLoggingHook] = None + self._flag_eval_evp_hook: typing.Optional[FlagEvalEVPHook] = None evp_config = OpenFeatureConfig() evp_counts_enabled = evp_config.flagging_evaluation_counts_enabled if self._enabled and evp_counts_enabled: self._flag_eval_evp_writer = FlagEvaluationWriter() - self._flag_eval_logging_hook = FlagEvalLoggingHook(self._flag_eval_evp_writer) + self._flag_eval_evp_hook = FlagEvalEVPHook(self._flag_eval_evp_writer) def get_metadata(self) -> Metadata: """Returns provider metadata.""" @@ -175,15 +175,15 @@ def get_provider_hooks(self) -> list[typing.Any]: Hook ordering: 1. OTel FlagEvalMetricsHook (_flageval_metrics.py) — always registered when the provider is enabled; emits the feature_flag.evaluations OTel counter (preserved unchanged). - 2. FlagEvalLoggingHook (_flag_eval_logging_hook.py) — registered only when + 2. FlagEvalEVPHook (_flag_eval_evp_hook.py) — registered only when DD_FLAGGING_EVALUATION_COUNTS_ENABLED is enabled (default on); enqueues cheap snapshots to FlagEvaluationWriter for EVP flagevaluation emission. """ hooks: list[typing.Any] = [] if self._flag_eval_metrics_hook is not None: hooks.append(self._flag_eval_metrics_hook) - if self._flag_eval_logging_hook is not None: - hooks.append(self._flag_eval_logging_hook) + if self._flag_eval_evp_hook is not None: + hooks.append(self._flag_eval_evp_hook) return hooks def initialize(self, evaluation_context: EvaluationContext) -> None: @@ -265,7 +265,7 @@ def shutdown(self) -> None: except ServiceStatusError: logger.debug("FlagEvaluationWriter has already stopped", exc_info=True) self._flag_eval_evp_writer = None - self._flag_eval_logging_hook = None + self._flag_eval_evp_hook = None # Shutdown flag evaluation metrics if self._flag_eval_metrics is not None: diff --git a/tests/openfeature/test_flag_eval_logging_hook.py b/tests/openfeature/test_flag_eval_evp_hook.py similarity index 85% rename from tests/openfeature/test_flag_eval_logging_hook.py rename to tests/openfeature/test_flag_eval_evp_hook.py index 794aa88af61..a63ff7eba0e 100644 --- a/tests/openfeature/test_flag_eval_logging_hook.py +++ b/tests/openfeature/test_flag_eval_evp_hook.py @@ -1,5 +1,5 @@ """ -Tests for FlagEvalLoggingHook — finally_after cheap capture + non-blocking enqueue. +Tests for FlagEvalEVPHook — finally_after cheap capture + non-blocking enqueue. Validates the hook design: - finally_after does cheap capture only (no aggregation, no I/O) @@ -66,12 +66,12 @@ def writer(): @pytest.fixture def hook(writer): - from ddtrace.internal.openfeature._flag_eval_logging_hook import FlagEvalLoggingHook + from ddtrace.internal.openfeature._flag_eval_evp_hook import FlagEvalEVPHook - return FlagEvalLoggingHook(writer=writer) + return FlagEvalEVPHook(writer=writer) -class TestFlagEvalLoggingHook: +class TestFlagEvalEVPHook: def test_finally_after_calls_writer_enqueue_once(self, hook, writer): """finally_after must call writer.enqueue exactly once per evaluation.""" hc = _make_hook_context() @@ -109,13 +109,13 @@ def test_finally_after_present_variant_not_runtime_default(self, hook, writer): event = writer.enqueue.call_args[0][0] assert event.runtime_default is False - def test_finally_after_type_mismatch_sets_runtime_default(self, hook, writer): + def test_finally_after_present_variant_with_type_mismatch_uses_variant(self, hook, writer): hc = _make_hook_context() details = _make_details(variant="wrong-type", error_code=ErrorCode.TYPE_MISMATCH) hook.finally_after(hc, details, {}) event = writer.enqueue.call_args[0][0] - assert event.runtime_default is True - assert event.variant == "" + assert event.runtime_default is False + assert event.variant == "wrong-type" def test_finally_after_does_not_enqueue_openfeature_reason(self, hook, writer): """OpenFeature reason is not an EVP field and must not enter the event snapshot.""" @@ -207,11 +207,11 @@ class TestAsyncBoundary: def test_aggregate_not_called_on_hook_path(self): """Spy on the REAL writer's _aggregate — it must NOT run during finally_after.""" - from ddtrace.internal.openfeature._flag_eval_logging_hook import FlagEvalLoggingHook + from ddtrace.internal.openfeature._flag_eval_evp_hook import FlagEvalEVPHook from ddtrace.internal.openfeature._flagevaluation_writer import FlagEvaluationWriter real_writer = FlagEvaluationWriter(interval=10.0) - hook = FlagEvalLoggingHook(writer=real_writer) + hook = FlagEvalEVPHook(writer=real_writer) with mock.patch.object(real_writer, "_aggregate", wraps=real_writer._aggregate) as spy_aggregate: hc = _make_hook_context(attrs={"tier": "premium", "region": "us"}) @@ -236,12 +236,12 @@ def test_aggregate_not_called_on_hook_path(self): def test_canonical_key_not_computed_on_hook_path(self): """canonical_context_key (the keying cost) must not run during finally_after.""" - from ddtrace.internal.openfeature._flag_eval_logging_hook import FlagEvalLoggingHook + from ddtrace.internal.openfeature._flag_eval_evp_hook import FlagEvalEVPHook import ddtrace.internal.openfeature._flagevaluation_writer as writer_mod from ddtrace.internal.openfeature._flagevaluation_writer import FlagEvaluationWriter real_writer = FlagEvaluationWriter(interval=10.0) - hook = FlagEvalLoggingHook(writer=real_writer) + hook = FlagEvalEVPHook(writer=real_writer) with mock.patch.object(writer_mod, "canonical_context_key", wraps=writer_mod.canonical_context_key) as spy_key: hc = _make_hook_context(attrs={"a": "b"}) @@ -250,10 +250,10 @@ def test_canonical_key_not_computed_on_hook_path(self): class TestMetadataSourceMatchesOTelHook: - """logging hook reads allocation-key/eval metadata from the SAME source as the OTel hook. + """EVP hook reads allocation-key/eval metadata from the SAME source as the OTel hook. The existing OTel FlagEvalMetricsHook reads allocation_key from - ``details.flag_metadata[METADATA_ALLOCATION_KEY]``. The logging hook must read from the + ``details.flag_metadata[METADATA_ALLOCATION_KEY]``. The EVP hook must read from the identical source so the two paths agree byte-for-byte on metadata. """ @@ -264,8 +264,8 @@ def test_allocation_key_metadata_key_matches_otel_hook(self): # Same metadata key constant in both modules. assert _flagevaluation_writer.METADATA_ALLOCATION_KEY == _flageval_metrics.METADATA_ALLOCATION_KEY - def test_logging_hook_reads_allocation_from_details_flag_metadata(self, hook, writer): - """logging hook reads allocation_key from details.flag_metadata (not hook_context).""" + def test_evp_hook_reads_allocation_from_details_flag_metadata(self, hook, writer): + """EVP hook reads allocation_key from details.flag_metadata (not hook_context).""" from ddtrace.internal.openfeature._flageval_metrics import METADATA_ALLOCATION_KEY hc = _make_hook_context() @@ -274,9 +274,9 @@ def test_logging_hook_reads_allocation_from_details_flag_metadata(self, hook, wr event = writer.enqueue.call_args[0][0] assert event.allocation_key == "alloc-from-details" - def test_metrics_and_logging_hooks_extract_same_allocation_key(self): + def test_metrics_and_evp_hooks_extract_same_allocation_key(self): """Drive both hooks with identical details; both must surface the same allocation key.""" - from ddtrace.internal.openfeature._flag_eval_logging_hook import FlagEvalLoggingHook + from ddtrace.internal.openfeature._flag_eval_evp_hook import FlagEvalEVPHook from ddtrace.internal.openfeature._flageval_metrics import METADATA_ALLOCATION_KEY from ddtrace.internal.openfeature._flageval_metrics import FlagEvalMetrics from ddtrace.internal.openfeature._flageval_metrics import FlagEvalMetricsHook @@ -291,19 +291,19 @@ def test_metrics_and_logging_hooks_extract_same_allocation_key(self): otel_hook.finally_after(hc, details, {}) otel_alloc = metrics.record.call_args.kwargs["allocation_key"] - # Logging side: capture what the logging hook enqueued as allocation_key. - logging_writer = mock.MagicMock(spec=FlagEvaluationWriter) - logging_hook = FlagEvalLoggingHook(logging_writer) - logging_hook.finally_after(hc, details, {}) - logging_alloc = logging_writer.enqueue.call_args[0][0].allocation_key + # EVP side: capture what the EVP hook enqueued as allocation_key. + evp_writer = mock.MagicMock(spec=FlagEvaluationWriter) + evp_hook = FlagEvalEVPHook(evp_writer) + evp_hook.finally_after(hc, details, {}) + evp_alloc = evp_writer.enqueue.call_args[0][0].allocation_key - assert otel_alloc == logging_alloc == "shared-alloc" + assert otel_alloc == evp_alloc == "shared-alloc" class TestKillswitchGating: - def test_default_enabled_registers_logging_hook(self): - """Default (no env var set) must register the logging hook + writer.""" - from ddtrace.internal.openfeature._flag_eval_logging_hook import FlagEvalLoggingHook + def test_default_enabled_registers_evp_hook(self): + """Default (no env var set) must register the EVP hook + writer.""" + from ddtrace.internal.openfeature._flag_eval_evp_hook import FlagEvalEVPHook env = {k: v for k, v in os.environ.items() if k != "DD_FLAGGING_EVALUATION_COUNTS_ENABLED"} # No env var → enabled by default. @@ -315,11 +315,11 @@ def test_default_enabled_registers_logging_hook(self): provider = DataDogProvider() assert provider._flag_eval_evp_writer is not None - assert provider._flag_eval_logging_hook is not None - assert isinstance(provider._flag_eval_logging_hook, FlagEvalLoggingHook) + assert provider._flag_eval_evp_hook is not None + assert isinstance(provider._flag_eval_evp_hook, FlagEvalEVPHook) - def test_killswitch_false_does_not_register_logging_hook(self): - """DD_FLAGGING_EVALUATION_COUNTS_ENABLED=false must suppress logging hook (killswitch).""" + def test_killswitch_false_does_not_register_evp_hook(self): + """DD_FLAGGING_EVALUATION_COUNTS_ENABLED=false must suppress EVP hook (killswitch).""" with mock.patch.dict(os.environ, {"DD_FLAGGING_EVALUATION_COUNTS_ENABLED": "false"}): from tests.utils import override_global_config @@ -328,7 +328,7 @@ def test_killswitch_false_does_not_register_logging_hook(self): provider = DataDogProvider() assert provider._flag_eval_evp_writer is None - assert provider._flag_eval_logging_hook is None + assert provider._flag_eval_evp_hook is None def test_killswitch_false_does_not_affect_otel_hook(self): """Killswitch must not suppress the OTel FlagEvalMetricsHook (OTel non-regression).""" @@ -343,14 +343,14 @@ def test_killswitch_false_does_not_affect_otel_hook(self): provider = DataDogProvider() # OTel hook still present. assert provider._flag_eval_metrics_hook is not None - # logging hook absent. - assert provider._flag_eval_logging_hook is None + # EVP hook absent. + assert provider._flag_eval_evp_hook is None # get_provider_hooks still returns the OTel hook. hooks = provider.get_provider_hooks() assert len(hooks) == 1 assert hooks[0] is provider._flag_eval_metrics_hook - def test_provider_shutdown_joins_logging_writer_final_flush(self): + def test_provider_shutdown_joins_evp_writer_final_flush(self): """Provider shutdown waits for FlagEvaluationWriter.on_shutdown final flush.""" from tests.utils import override_global_config @@ -368,8 +368,8 @@ def test_provider_shutdown_joins_logging_writer_final_flush(self): writer.join.assert_called_once() assert writer.mock_calls.index(mock.call.stop()) < writer.mock_calls.index(mock.call.join()) - def test_killswitch_enabled_true_registers_logging_hook(self): - """DD_FLAGGING_EVALUATION_COUNTS_ENABLED=true must register the logging hook.""" + def test_killswitch_enabled_true_registers_evp_hook(self): + """DD_FLAGGING_EVALUATION_COUNTS_ENABLED=true must register the EVP hook.""" with mock.patch.dict(os.environ, {"DD_FLAGGING_EVALUATION_COUNTS_ENABLED": "true"}): from tests.utils import override_global_config @@ -378,4 +378,4 @@ def test_killswitch_enabled_true_registers_logging_hook(self): provider = DataDogProvider() assert provider._flag_eval_evp_writer is not None - assert provider._flag_eval_logging_hook is not None + assert provider._flag_eval_evp_hook is not None diff --git a/tests/openfeature/test_flag_eval_metrics.py b/tests/openfeature/test_flag_eval_metrics.py index 091ddf03261..9e63f7ce480 100644 --- a/tests/openfeature/test_flag_eval_metrics.py +++ b/tests/openfeature/test_flag_eval_metrics.py @@ -360,10 +360,10 @@ def test_provider_has_flag_eval_metrics_hook(self, provider): assert provider._flag_eval_metrics is not None def test_get_provider_hooks_returns_flag_eval_metrics_hook(self, provider): - """get_provider_hooks should return the OTel metrics hook and optionally the logging hook.""" + """get_provider_hooks should return the OTel metrics hook and optionally the EVP hook.""" hooks = provider.get_provider_hooks() # The OTel FlagEvalMetricsHook is always first when the provider is enabled. - # The FlagEvalLoggingHook is also registered by default (DD_FLAGGING_EVALUATION_COUNTS_ENABLED=true). + # The FlagEvalEVPHook is also registered by default (DD_FLAGGING_EVALUATION_COUNTS_ENABLED=true). assert len(hooks) >= 1 assert hooks[0] is provider._flag_eval_metrics_hook diff --git a/tests/openfeature/test_flagevaluation_e2e.py b/tests/openfeature/test_flagevaluation_e2e.py index 8b94c4eacba..e04a1699c62 100644 --- a/tests/openfeature/test_flagevaluation_e2e.py +++ b/tests/openfeature/test_flagevaluation_e2e.py @@ -4,7 +4,7 @@ These cover the lifecycle/exit paths that unit tests of the hook/writer in isolation cannot prove: -- The logging hook actually fires on the provider's REAL OpenFeature evaluation entrypoint +- The EVP hook actually fires on the provider's REAL OpenFeature evaluation entrypoint (driven through the OpenFeature client, not by calling the hook directly). - ALL evaluation exit paths are covered — success, native engine error, runtime default (flag-not-found / no-config), and disabled. @@ -45,7 +45,7 @@ def provider_and_client(): provider = DataDogProvider() # Sanity: the EVP writer/hook are wired (killswitch default on). assert provider._flag_eval_evp_writer is not None - assert provider._flag_eval_logging_hook is not None + assert provider._flag_eval_evp_hook is not None api.set_provider(provider) client = api.get_client() @@ -68,15 +68,15 @@ def _drain(provider): return decoded.get("flagEvaluations", []) -class TestFlagEvalLoggingHookFiresOnRealEvalPath: - """The logging hook fires on the provider's real evaluation entrypoint.""" +class TestFlagEvalEVPHookFiresOnRealEvalPath: + """The EVP hook fires on the provider's real evaluation entrypoint.""" - def test_logging_hook_registered_in_provider_hooks(self, provider_and_client): + def test_evp_hook_registered_in_provider_hooks(self, provider_and_client): provider, _ = provider_and_client - from ddtrace.internal.openfeature._flag_eval_logging_hook import FlagEvalLoggingHook + from ddtrace.internal.openfeature._flag_eval_evp_hook import FlagEvalEVPHook hooks = provider.get_provider_hooks() - assert any(isinstance(h, FlagEvalLoggingHook) for h in hooks) + assert any(isinstance(h, FlagEvalEVPHook) for h in hooks) def test_success_eval_enqueues_and_emits_row(self, provider_and_client): provider, client = provider_and_client @@ -180,19 +180,19 @@ def test_targeting_key_and_context_captured_on_success(self, provider_and_client assert row["context"]["evaluation"]["tier"] == "gold" -class TestOTelNonRegressionAlongsideFlagEvalLogging: - """The logging path must NOT change which hooks the provider registers for OTel.""" +class TestOTelNonRegressionAlongsideFlagEvalEVP: + """The EVP path must NOT change which hooks the provider registers for OTel.""" - def test_both_metrics_and_logging_hooks_registered(self, provider_and_client): + def test_both_metrics_and_evp_hooks_registered(self, provider_and_client): provider, _ = provider_and_client - from ddtrace.internal.openfeature._flag_eval_logging_hook import FlagEvalLoggingHook + from ddtrace.internal.openfeature._flag_eval_evp_hook import FlagEvalEVPHook from ddtrace.internal.openfeature._flageval_metrics import FlagEvalMetricsHook hooks = provider.get_provider_hooks() assert any(isinstance(h, FlagEvalMetricsHook) for h in hooks), "OTel hook must remain registered" - assert any(isinstance(h, FlagEvalLoggingHook) for h in hooks), "logging hook must be registered" + assert any(isinstance(h, FlagEvalEVPHook) for h in hooks), "EVP hook must be registered" - def test_eval_drives_otel_record_and_logging_enqueue_together(self, provider_and_client): + def test_eval_drives_otel_record_and_evp_enqueue_together(self, provider_and_client): """A single eval feeds BOTH the OTel metric record and the EVP enqueue.""" provider, client = provider_and_client config = create_config(create_boolean_flag("dual-flag", enabled=True, default_value=True)) From 059705c4883f82b88972a396e7062534f349b9ed Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 23 Jun 2026 19:57:36 -0400 Subject: [PATCH 28/37] refactor(openfeature): update flag eval EVP benchmark --- benchmarks/openfeature_flagevaluation/scenario.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmarks/openfeature_flagevaluation/scenario.py b/benchmarks/openfeature_flagevaluation/scenario.py index b9f90d768b5..79f1f23ffb6 100644 --- a/benchmarks/openfeature_flagevaluation/scenario.py +++ b/benchmarks/openfeature_flagevaluation/scenario.py @@ -13,7 +13,7 @@ * ``hook_plus_drain`` — end-to-end: N hook enqueues followed by a full queue drain + aggregate, the realistic per-flush shape. -The scenario builds its own ``FlagEvaluationWriter`` + ``FlagEvalLoggingHook`` and drives +The scenario builds its own ``FlagEvaluationWriter`` + ``FlagEvalEVPHook`` and drives them directly so the benchmark isolates the EVP pipeline (no network, no native eval). """ @@ -54,7 +54,7 @@ class OpenFeatureFlagEvaluation(bm.Scenario): num_context_fields: int def run(self): - from ddtrace.internal.openfeature._flag_eval_logging_hook import FlagEvalLoggingHook + from ddtrace.internal.openfeature._flag_eval_evp_hook import FlagEvalEVPHook from ddtrace.internal.openfeature._flagevaluation_writer import FlagEvaluationWriter mode = self.mode @@ -82,7 +82,7 @@ def run(self): ] writer = FlagEvaluationWriter(interval=3600.0) - hook = FlagEvalLoggingHook(writer) + hook = FlagEvalEVPHook(writer) if mode == "hook_enqueue": From 0b49f06f5625536e31938331c35f749323adf45a Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 23 Jun 2026 20:40:40 -0400 Subject: [PATCH 29/37] chore(openfeature): drop candidate-only flageval benchmark --- .../openfeature_flagevaluation/config.yaml | 36 ----- .../requirements_scenario.txt | 6 - .../openfeature_flagevaluation/scenario.py | 153 ------------------ benchmarks/suitespec.yml | 13 -- 4 files changed, 208 deletions(-) delete mode 100644 benchmarks/openfeature_flagevaluation/config.yaml delete mode 100644 benchmarks/openfeature_flagevaluation/requirements_scenario.txt delete mode 100644 benchmarks/openfeature_flagevaluation/scenario.py diff --git a/benchmarks/openfeature_flagevaluation/config.yaml b/benchmarks/openfeature_flagevaluation/config.yaml deleted file mode 100644 index f9111dc909d..00000000000 --- a/benchmarks/openfeature_flagevaluation/config.yaml +++ /dev/null @@ -1,36 +0,0 @@ -defaults: &defaults - num_flags: 100 - num_context_fields: 8 - -# Eval hot path the caller pays: finally_after hook (cheap capture + non-blocking enqueue). -hook-enqueue: - <<: *defaults - mode: hook_enqueue - -# Eval hot path with a small context (typical server eval). -hook-enqueue-small-context: - <<: *defaults - mode: hook_enqueue - num_context_fields: 2 - -# Eval hot path with a large context (stresses the shallow copy on the hook path). -hook-enqueue-large-context: - <<: *defaults - mode: hook_enqueue - num_context_fields: 64 - -# Background worker hot path: flatten + deterministic prune + canonical key + two-tier insert. -aggregate: - <<: *defaults - mode: aggregate - -# Aggregation under high flag cardinality (more distinct full-tier buckets). -aggregate-high-cardinality: - <<: *defaults - mode: aggregate - num_flags: 2500 - -# End-to-end per-flush shape: N enqueues then a full drain + aggregate. -hook-plus-drain: - <<: *defaults - mode: hook_plus_drain diff --git a/benchmarks/openfeature_flagevaluation/requirements_scenario.txt b/benchmarks/openfeature_flagevaluation/requirements_scenario.txt deleted file mode 100644 index f6b50fc0b22..00000000000 --- a/benchmarks/openfeature_flagevaluation/requirements_scenario.txt +++ /dev/null @@ -1,6 +0,0 @@ -# openfeature-sdk provides EvaluationContext / FlagEvaluationDetails / HookContext, -# which the scenario imports directly. It is a test-time dependency (not a ddtrace -# runtime dependency), so the benchmark base venv does not install it otherwise. -# 0.8.0+ is required for the finally_after hook `details` parameter the scenario drives. -# Mirrors the `openfeature` venv pin in riotfile.py. -openfeature-sdk~=0.8.0 diff --git a/benchmarks/openfeature_flagevaluation/scenario.py b/benchmarks/openfeature_flagevaluation/scenario.py deleted file mode 100644 index 79f1f23ffb6..00000000000 --- a/benchmarks/openfeature_flagevaluation/scenario.py +++ /dev/null @@ -1,153 +0,0 @@ -"""Hot-path microbenchmark for the EVP ``flagevaluation`` aggregation pipeline. - -This measures the cost an OpenFeature evaluation actually pays for server-side EVP -flag-evaluation counting, mirroring the Go reference benchmark -(``dd-trace-go/openfeature/flagevaluation_test.go``): - -* ``hook_enqueue`` — the ``finally_after`` hook hot path: scalar capture + bounded - context snapshot + non-blocking bounded enqueue. This is what charges the caller's - evaluation goroutine/thread directly, so it must stay bounded. -* ``aggregate`` — the background worker hot path: canonical-context key + two-tier - aggregation over already-bounded snapshots. This runs OFF the eval path, but its - per-event cost still bounds throughput of the writer thread. -* ``hook_plus_drain`` — end-to-end: N hook enqueues followed by a full queue drain + - aggregate, the realistic per-flush shape. - -The scenario builds its own ``FlagEvaluationWriter`` + ``FlagEvalEVPHook`` and drives -them directly so the benchmark isolates the EVP pipeline (no network, no native eval). -""" - -import bm -from openfeature.evaluation_context import EvaluationContext -from openfeature.flag_evaluation import FlagEvaluationDetails -from openfeature.flag_evaluation import FlagType -from openfeature.flag_evaluation import Reason -from openfeature.hook import HookContext - - -def _make_hook_context(flag_key, targeting_key, attrs): - ctx = EvaluationContext(targeting_key=targeting_key, attributes=attrs) - return HookContext( - flag_key=flag_key, - flag_type=FlagType.BOOLEAN, - default_value=False, - evaluation_context=ctx, - ) - - -def _make_details(flag_key, variant, allocation_key): - return FlagEvaluationDetails( - flag_key=flag_key, - value=True, - variant=variant, - reason=Reason.TARGETING_MATCH, - flag_metadata={"allocation_key": allocation_key}, - ) - - -class OpenFeatureFlagEvaluation(bm.Scenario): - # Which segment of the pipeline to exercise. - mode: str - # Number of distinct flags cycled through (drives aggregation-map cardinality). - num_flags: int - # Number of evaluation-context attributes per evaluation (drives prune/key cost). - num_context_fields: int - - def run(self): - from ddtrace.internal.openfeature._flag_eval_evp_hook import FlagEvalEVPHook - from ddtrace.internal.openfeature._flagevaluation_writer import FlagEvaluationWriter - - mode = self.mode - num_flags = max(1, self.num_flags) - num_fields = max(0, self.num_context_fields) - - # Pre-build the per-evaluation inputs so the measured loop only exercises the - # hook/aggregation hot path, not fixture construction. - attrs = {"attr_{}".format(i): "value_{}".format(i) for i in range(num_fields)} - hook_contexts = [ - _make_hook_context( - flag_key="flag-{}".format(i % num_flags), - targeting_key="user-{}".format(i % num_flags), - attrs=attrs, - ) - for i in range(num_flags) - ] - details_list = [ - _make_details( - flag_key="flag-{}".format(i % num_flags), - variant="variant-{}".format(i % 4), - allocation_key="alloc-{}".format(i % num_flags), - ) - for i in range(num_flags) - ] - - writer = FlagEvaluationWriter(interval=3600.0) - hook = FlagEvalEVPHook(writer) - - if mode == "hook_enqueue": - - def _(loops): - n = num_flags - for i in range(loops): - idx = i % n - hook.finally_after(hook_contexts[idx], details_list[idx], {}) - # Keep the queue from saturating so we keep measuring the enqueue - # path rather than the drop-and-count path. - if writer._queue.full(): - writer._drain_queue() - - yield _ - - elif mode == "aggregate": - # Pre-capture snapshots once; the measured loop only runs _aggregate - # (canonical key + two-tier insert). - from ddtrace.internal.openfeature._flagevaluation_writer import _EvalEvent - from ddtrace.internal.openfeature._flagevaluation_writer import flatten_and_prune_context - - bounded_attrs = flatten_and_prune_context(attrs) - events = [ - _EvalEvent( - flag_key="flag-{}".format(i % num_flags), - variant="variant-{}".format(i % 4), - allocation_key="alloc-{}".format(i % num_flags), - targeting_key="user-{}".format(i % num_flags), - attrs=dict(bounded_attrs), - runtime_default=False, - error_message="", - eval_time_ms=1_760_000_000_000 + i, - ) - for i in range(num_flags) - ] - - def _(loops): - n = num_flags - for i in range(loops): - writer._aggregate(events[i % n]) - # Reset periodically so aggregation maps don't grow unbounded across - # a long benchmark loop (keeps the per-op cost representative). - if (i % 10000) == 0: - writer._full.clear() - writer._degraded.clear() - writer._per_flag_count.clear() - writer._global_count = 0 - - yield _ - - elif mode == "hook_plus_drain": - - def _(loops): - n = num_flags - for i in range(loops): - idx = i % n - hook.finally_after(hook_contexts[idx], details_list[idx], {}) - if writer._queue.full() or (i % n) == (n - 1): - writer._drain_queue() - writer._full.clear() - writer._degraded.clear() - writer._per_flag_count.clear() - writer._global_count = 0 - - yield _ - - else: - raise ValueError("unknown mode: {}".format(mode)) diff --git a/benchmarks/suitespec.yml b/benchmarks/suitespec.yml index 51f7241ab14..0802b8ed9ca 100644 --- a/benchmarks/suitespec.yml +++ b/benchmarks/suitespec.yml @@ -50,10 +50,6 @@ components: - ddtrace/appsec/iast/* - ddtrace/appsec/* - ddtrace/internal/settings/asm.py - openfeature_benchmark: - - ddtrace/openfeature/* - - ddtrace/internal/openfeature/* - - ddtrace/internal/settings/openfeature.py core: - ddtrace/internal/__init__.py - ddtrace/internal/_exceptions.py @@ -226,15 +222,6 @@ suites: - benchmarks/suitespec.yml cpus_per_run: 1 type: 'microbenchmark' - openfeature_flagevaluation: - paths: - - '@bootstrap' - - '@core' - - '@openfeature_benchmark' - - benchmarks/openfeature_flagevaluation/* - - benchmarks/suitespec.yml - cpus_per_run: 1 - type: 'microbenchmark' appsec_iast_aspects: paths: - '@bootstrap' From 54d100875e3df1283a2431b525a249f329cf066e Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 24 Jun 2026 07:00:17 -0400 Subject: [PATCH 30/37] fix(openfeature): serialize flagevaluation context values --- .../openfeature/_flagevaluation_writer.py | 22 +++++++-------- .../openfeature/test_flagevaluation_writer.py | 28 +++++++++++++++++++ 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/ddtrace/internal/openfeature/_flagevaluation_writer.py b/ddtrace/internal/openfeature/_flagevaluation_writer.py index 2db1c2fdae3..842038d5d12 100644 --- a/ddtrace/internal/openfeature/_flagevaluation_writer.py +++ b/ddtrace/internal/openfeature/_flagevaluation_writer.py @@ -22,7 +22,6 @@ """ from collections.abc import Mapping -from collections.abc import Sequence import http.client as httplib import json import queue @@ -105,7 +104,16 @@ def _json_dumps(obj: typing.Any) -> bytes: - return json.dumps(obj, separators=_JSON_SEPARATORS).encode("utf-8") + return json.dumps(obj, default=_json_default, separators=_JSON_SEPARATORS).encode("utf-8") + + +def _json_default(value: typing.Any) -> str: + if hasattr(value, "isoformat"): + try: + return value.isoformat() + except Exception: + pass + return str(value) def _count_metric(name: str, value: int, reason: typing.Optional[str] = None) -> None: @@ -211,15 +219,7 @@ def flatten_and_prune_context(attrs: dict[str, typing.Any]) -> dict[str, typing. def _json_safe_context_value(value: typing.Any) -> typing.Any: - if value is None or isinstance(value, (bool, int, float, str)): - return value - if isinstance(value, Mapping): - return {str(k): _json_safe_context_value(v) for k, v in value.items()} - if isinstance(value, Sequence) and not isinstance(value, (bytes, bytearray)): - return [_json_safe_context_value(v) for v in value] - if hasattr(value, "isoformat"): - return value.isoformat() - return str(value) + return json.loads(_json_dumps(value).decode("utf-8")) def _json_safe_context(attrs: dict[str, typing.Any]) -> dict[str, typing.Any]: diff --git a/tests/openfeature/test_flagevaluation_writer.py b/tests/openfeature/test_flagevaluation_writer.py index fcd37a2c703..e88e09064a3 100644 --- a/tests/openfeature/test_flagevaluation_writer.py +++ b/tests/openfeature/test_flagevaluation_writer.py @@ -13,6 +13,7 @@ from datetime import datetime from datetime import timezone +from decimal import Decimal import json import time from unittest import mock @@ -462,6 +463,33 @@ def test_openfeature_datetime_context_value_is_json_serialized(self, writer): assert ctx_eval["seen_at"] == timestamp.isoformat() assert ctx_eval["nested.created_at"] == timestamp.isoformat() + def test_openfeature_non_json_context_values_are_json_serialized(self, writer): + """Non-JSON-safe context values must not make payload encoding drop the batch.""" + + class CustomContextValue: + def __str__(self): + return "custom-value" + + writer.enqueue( + _make_event( + attrs={ + "amount": Decimal("12.34"), + "custom": CustomContextValue(), + "list": [Decimal("1.5"), CustomContextValue()], + } + ) + ) + + with mock.patch.object(writer, "_send_payload") as mock_send: + writer.periodic() + + payload_bytes = mock_send.call_args[0][0] + decoded = json.loads(payload_bytes) + ctx_eval = decoded["flagEvaluations"][0]["context"]["evaluation"] + assert ctx_eval["amount"] == "12.34" + assert ctx_eval["custom"] == "custom-value" + assert ctx_eval["list"] == ["1.5", "custom-value"] + def test_degraded_event_has_no_context_or_targeting_key(self, writer): """Degraded-tier events must not include targeting_key or context fields.""" # Force to degraded by saturating per-flag cap. From fb2f058333a0d59d704daa20c9c6b02e345d7fdb Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 24 Jun 2026 07:04:27 -0400 Subject: [PATCH 31/37] chore(openfeature): keep shutdown typing local --- ddtrace/debugging/_uploader.py | 2 +- ddtrace/internal/ci_visibility/encoder.py | 3 +-- ddtrace/internal/openfeature/_flagevaluation_writer.py | 4 ++-- ddtrace/internal/periodic.py | 3 ++- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ddtrace/debugging/_uploader.py b/ddtrace/debugging/_uploader.py index 3164d37af7d..79abd44b809 100644 --- a/ddtrace/debugging/_uploader.py +++ b/ddtrace/debugging/_uploader.py @@ -231,7 +231,7 @@ def online(self) -> None: msg = "Debugger tracks not enabled" raise ValueError(msg) - def on_shutdown(self) -> None: + def on_shutdown(self) -> None: # type: ignore[override] self._flush() @classmethod diff --git a/ddtrace/internal/ci_visibility/encoder.py b/ddtrace/internal/ci_visibility/encoder.py index 0ff1c6b023e..09533599149 100644 --- a/ddtrace/internal/ci_visibility/encoder.py +++ b/ddtrace/internal/ci_visibility/encoder.py @@ -23,7 +23,6 @@ from ddtrace.internal.ci_visibility.telemetry.payload import record_endpoint_payload_events_count from ddtrace.internal.ci_visibility.telemetry.payload import record_endpoint_payload_events_serialization_time from ddtrace.internal.encoding import JSONEncoderV2 -from ddtrace.internal.evp_proxy.constants import DEFAULT_EVP_PAYLOAD_SIZE_LIMIT from ddtrace.internal.logger import get_logger from ddtrace.internal.settings import env from ddtrace.internal.utils.time import StopWatch @@ -42,7 +41,7 @@ class CIVisibilityEncoderV01(BufferedEncoder): TEST_SUITE_EVENT_VERSION = 1 TEST_EVENT_VERSION = 2 ENDPOINT_TYPE = ENDPOINT.TEST_CYCLE - _MAX_PAYLOAD_SIZE = DEFAULT_EVP_PAYLOAD_SIZE_LIMIT + _MAX_PAYLOAD_SIZE = 5 * 1024 * 1024 # 5MB _MAX_META_TAG_VALUE_LENGTH = 5000 def __init__(self, *args: Any) -> None: diff --git a/ddtrace/internal/openfeature/_flagevaluation_writer.py b/ddtrace/internal/openfeature/_flagevaluation_writer.py index 842038d5d12..d697c235f77 100644 --- a/ddtrace/internal/openfeature/_flagevaluation_writer.py +++ b/ddtrace/internal/openfeature/_flagevaluation_writer.py @@ -110,7 +110,7 @@ def _json_dumps(obj: typing.Any) -> bytes: def _json_default(value: typing.Any) -> str: if hasattr(value, "isoformat"): try: - return value.isoformat() + return str(value.isoformat()) except Exception: pass return str(value) @@ -558,7 +558,7 @@ def periodic(self) -> None: for payload, num_events in result.payloads: self._send_payload(payload, num_events) - def on_shutdown(self) -> None: + def on_shutdown(self) -> None: # type: ignore[override] """Final flush on service shutdown — drains the queue and flushes before exit.""" self._stop_drain_worker() self.periodic() diff --git a/ddtrace/internal/periodic.py b/ddtrace/internal/periodic.py index 0902e9a7392..8481e505652 100644 --- a/ddtrace/internal/periodic.py +++ b/ddtrace/internal/periodic.py @@ -65,7 +65,8 @@ def join( if self._worker: self._worker.join(timeout) - def on_shutdown(self) -> None: + @staticmethod + def on_shutdown(): pass def periodic(self): From 41d10fa4f2bfb00e9cd0fb19bb91876c9ddc663a Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 24 Jun 2026 09:17:31 -0400 Subject: [PATCH 32/37] fix(openfeature): avoid bandit pass fallback --- ddtrace/internal/openfeature/_flagevaluation_writer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/internal/openfeature/_flagevaluation_writer.py b/ddtrace/internal/openfeature/_flagevaluation_writer.py index d697c235f77..bb7306a3afb 100644 --- a/ddtrace/internal/openfeature/_flagevaluation_writer.py +++ b/ddtrace/internal/openfeature/_flagevaluation_writer.py @@ -112,7 +112,7 @@ def _json_default(value: typing.Any) -> str: try: return str(value.isoformat()) except Exception: - pass + return str(value) return str(value) From 89ccdc376a61ed942a240d3c3568bace7360add6 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Sun, 28 Jun 2026 17:54:53 -0400 Subject: [PATCH 33/37] Add OpenFeature flagevaluation benchmarks --- .../openfeature_flagevaluation/config.yaml | 50 ++++++ .../requirements_scenario.txt | 5 + .../openfeature_flagevaluation/scenario.py | 146 ++++++++++++++++++ benchmarks/suitespec.yml | 14 ++ 4 files changed, 215 insertions(+) create mode 100644 benchmarks/openfeature_flagevaluation/config.yaml create mode 100644 benchmarks/openfeature_flagevaluation/requirements_scenario.txt create mode 100644 benchmarks/openfeature_flagevaluation/scenario.py diff --git a/benchmarks/openfeature_flagevaluation/config.yaml b/benchmarks/openfeature_flagevaluation/config.yaml new file mode 100644 index 00000000000..b70f81471ab --- /dev/null +++ b/benchmarks/openfeature_flagevaluation/config.yaml @@ -0,0 +1,50 @@ +defaults: &defaults + num_flags: 100 + num_users: 50 + num_context_fields: 10 + +# Eval hot path the caller pays: finally_after hook (cheap capture + non-blocking enqueue). +hook-enqueue-typical: + <<: *defaults + mode: hook_enqueue + +hook-enqueue-stress: + <<: *defaults + mode: hook_enqueue + num_flags: 10 + num_users: 1000 + num_context_fields: 250 + +hook-enqueue-scale: + <<: *defaults + mode: hook_enqueue + num_flags: 2500 + num_users: 500 + num_context_fields: 20 + +# Background worker hot path: canonical key + two-tier aggregation. +aggregate-typical: + <<: *defaults + mode: aggregate + +aggregate-stress: + <<: *defaults + mode: aggregate + num_flags: 10 + num_users: 1000 + num_context_fields: 250 + +aggregate-scale: + <<: *defaults + mode: aggregate + num_flags: 2500 + num_users: 500 + num_context_fields: 20 + +# End-to-end per-flush shape: N enqueues followed by drain + aggregate. +hook-plus-drain-scale: + <<: *defaults + mode: hook_plus_drain + num_flags: 2500 + num_users: 500 + num_context_fields: 20 diff --git a/benchmarks/openfeature_flagevaluation/requirements_scenario.txt b/benchmarks/openfeature_flagevaluation/requirements_scenario.txt new file mode 100644 index 00000000000..ade024d993d --- /dev/null +++ b/benchmarks/openfeature_flagevaluation/requirements_scenario.txt @@ -0,0 +1,5 @@ +# openfeature-sdk provides EvaluationContext / FlagEvaluationDetails / HookContext, +# which the scenario imports directly. It is a test-time dependency, not a ddtrace +# runtime dependency, so the benchmark base venv does not install it otherwise. +# Mirrors the openfeature venv pin in riotfile.py. +openfeature-sdk~=0.8.0 diff --git a/benchmarks/openfeature_flagevaluation/scenario.py b/benchmarks/openfeature_flagevaluation/scenario.py new file mode 100644 index 00000000000..4cd43eef071 --- /dev/null +++ b/benchmarks/openfeature_flagevaluation/scenario.py @@ -0,0 +1,146 @@ +"""Hot-path microbenchmark for the EVP ``flagevaluation`` aggregation pipeline. + +The profiles mirror the Go/Ruby OpenFeature EVP benchmark suite: + +* typical/100flags_50users_10fields +* stress/10flags_1000users_250fields +* scale/2500flags_500users_20fields + +The scenario drives ``FlagEvalEVPHook`` and ``FlagEvaluationWriter`` directly so the +benchmark isolates in-process EVP work: no network, no Remote Config backend, and no +native flag evaluation cost. +""" + +import bm +from openfeature.evaluation_context import EvaluationContext +from openfeature.flag_evaluation import FlagEvaluationDetails +from openfeature.flag_evaluation import FlagType +from openfeature.flag_evaluation import Reason +from openfeature.hook import HookContext + + +def _make_hook_context(flag_key, targeting_key, attrs): + ctx = EvaluationContext(targeting_key=targeting_key, attributes=attrs) + return HookContext( + flag_key=flag_key, + flag_type=FlagType.BOOLEAN, + default_value=False, + evaluation_context=ctx, + ) + + +def _make_details(flag_key, variant, allocation_key): + return FlagEvaluationDetails( + flag_key=flag_key, + value=True, + variant=variant, + reason=Reason.TARGETING_MATCH, + flag_metadata={"allocation_key": allocation_key}, + ) + + +class OpenFeatureFlagEvaluation(bm.Scenario): + # Which segment of the pipeline to exercise. + mode: str + # Number of distinct flags cycled through. + num_flags: int + # Number of distinct targeting keys cycled through. + num_users: int + # Number of evaluation-context attributes per evaluation. + num_context_fields: int + + def run(self): + from ddtrace.internal.openfeature._flag_eval_evp_hook import FlagEvalEVPHook + from ddtrace.internal.openfeature._flagevaluation_writer import FlagEvaluationWriter + + mode = self.mode + num_flags = max(1, self.num_flags) + num_users = max(1, self.num_users) + num_fields = max(0, self.num_context_fields) + cycle_count = max(num_flags, num_users) + + attrs = {"attr_{}".format(i): "value_{}".format(i) for i in range(num_fields)} + flag_keys = ["flag-{}".format(i) for i in range(num_flags)] + targeting_keys = ["user-{}".format(i) for i in range(num_users)] + hook_contexts = [ + _make_hook_context( + flag_key=flag_keys[i % num_flags], + targeting_key=targeting_keys[i % num_users], + attrs=attrs, + ) + for i in range(cycle_count) + ] + details_list = [ + _make_details( + flag_key=flag_keys[i % num_flags], + variant="variant-{}".format(i % 4), + allocation_key="alloc-{}".format(i % num_flags), + ) + for i in range(cycle_count) + ] + + writer = FlagEvaluationWriter(interval=3600.0) + hook = FlagEvalEVPHook(writer) + + if mode == "hook_enqueue": + + def _(loops): + for i in range(loops): + idx = i % cycle_count + hook.finally_after(hook_contexts[idx], details_list[idx], {}) + # Keep the queue from saturating so this measures steady-state + # enqueue, not queue overflow accounting. + if writer._queue.full(): + writer._drain_queue() + + yield _ + + elif mode == "aggregate": + from ddtrace.internal.openfeature._flagevaluation_writer import _EvalEvent + from ddtrace.internal.openfeature._flagevaluation_writer import flatten_and_prune_context + + bounded_attrs = flatten_and_prune_context(attrs) + events = [ + _EvalEvent( + flag_key=flag_keys[i % num_flags], + variant="variant-{}".format(i % 4), + allocation_key="alloc-{}".format(i % num_flags), + targeting_key=targeting_keys[i % num_users], + attrs=dict(bounded_attrs), + runtime_default=False, + error_message="", + eval_time_ms=1_760_000_000_000 + i, + ) + for i in range(cycle_count) + ] + + def _(loops): + for i in range(loops): + writer._aggregate(events[i % cycle_count]) + # Reset periodically so the measured loop is not dominated by + # ever-growing maps after enough unique buckets have been seen. + if i > 0 and (i % 10000) == 0: + writer._full.clear() + writer._degraded.clear() + writer._per_flag_count.clear() + writer._global_count = 0 + + yield _ + + elif mode == "hook_plus_drain": + + def _(loops): + for i in range(loops): + idx = i % cycle_count + hook.finally_after(hook_contexts[idx], details_list[idx], {}) + if writer._queue.full() or (i % cycle_count) == (cycle_count - 1): + writer._drain_queue() + writer._full.clear() + writer._degraded.clear() + writer._per_flag_count.clear() + writer._global_count = 0 + + yield _ + + else: + raise ValueError("unknown mode: {}".format(mode)) diff --git a/benchmarks/suitespec.yml b/benchmarks/suitespec.yml index 0802b8ed9ca..659e8f80662 100644 --- a/benchmarks/suitespec.yml +++ b/benchmarks/suitespec.yml @@ -50,6 +50,10 @@ components: - ddtrace/appsec/iast/* - ddtrace/appsec/* - ddtrace/internal/settings/asm.py + openfeature_benchmark: + - ddtrace/openfeature/* + - ddtrace/internal/openfeature/* + - ddtrace/internal/settings/openfeature.py core: - ddtrace/internal/__init__.py - ddtrace/internal/_exceptions.py @@ -222,6 +226,16 @@ suites: - benchmarks/suitespec.yml cpus_per_run: 1 type: 'microbenchmark' + openfeature_flagevaluation: + paths: + - '@bootstrap' + - '@core' + - '@openfeature_benchmark' + - '@vendor' + - benchmarks/openfeature_flagevaluation/* + - benchmarks/suitespec.yml + cpus_per_run: 1 + type: 'microbenchmark' appsec_iast_aspects: paths: - '@bootstrap' From d4f6195d3e217b5a626e18407e149041879f6084 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Mon, 6 Jul 2026 00:16:38 -0400 Subject: [PATCH 34/37] chore(openfeature): address flagevaluation review cleanup --- .../openfeature/_flagevaluation_writer.py | 13 +------------ ddtrace/internal/openfeature/_provider.py | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/ddtrace/internal/openfeature/_flagevaluation_writer.py b/ddtrace/internal/openfeature/_flagevaluation_writer.py index bb7306a3afb..a24c5e01036 100644 --- a/ddtrace/internal/openfeature/_flagevaluation_writer.py +++ b/ddtrace/internal/openfeature/_flagevaluation_writer.py @@ -36,6 +36,7 @@ from ddtrace.internal.evp_proxy.constants import EVP_SUBDOMAIN_HEADER_EVENT_PLATFORM_VALUE from ddtrace.internal.evp_proxy.constants import EVP_SUBDOMAIN_HEADER_NAME from ddtrace.internal.logger import get_logger +from ddtrace.internal.openfeature._flageval_metrics import METADATA_ALLOCATION_KEY as METADATA_ALLOCATION_KEY from ddtrace.internal.periodic import PeriodicService from ddtrace.internal.settings._agent import config as agent_config from ddtrace.internal.telemetry import telemetry_writer @@ -83,9 +84,6 @@ # Flag metadata key where the provider stamps the evaluation timestamp (ms). EVAL_TIMESTAMP_METADATA_KEY = "dd.eval.timestamp_ms" -# Metadata key for allocation_key (same as _flageval_metrics.py METADATA_ALLOCATION_KEY). -METADATA_ALLOCATION_KEY = "allocation_key" - # Type-tag bytes for the canonical context key encoding (mirrors Go's ctxTag* constants). _TAG_STR = b"s" _TAG_BOOL = b"b" @@ -752,15 +750,6 @@ def _encode_payload_event( return _PayloadEventResult(None, dropped_payload_limit=True) -def _build_payloads( - events: list[dict[str, typing.Any]], - context: dict[str, str], - payload_size_limit: int = FLAGEVALUATIONS_PAYLOAD_SIZE_LIMIT, -) -> typing.Iterator[tuple[bytes, int]]: - for payload in _build_payloads_with_stats(events, context, payload_size_limit).payloads: - yield payload - - def _build_payloads_with_stats( events: list[dict[str, typing.Any]], context: dict[str, str], diff --git a/ddtrace/internal/openfeature/_provider.py b/ddtrace/internal/openfeature/_provider.py index 6342795f8c3..f924b62a2e1 100644 --- a/ddtrace/internal/openfeature/_provider.py +++ b/ddtrace/internal/openfeature/_provider.py @@ -190,8 +190,11 @@ def initialize(self, evaluation_context: EvaluationContext) -> None: """ Initialize the provider. - Returns immediately; the provider transitions to READY asynchronously once + Returns immediately. This provider's internal status remains NOT_READY until Remote Config delivers the first FFE_FLAGS payload via on_configuration_received(). + openfeature-sdk 0.8.x still dispatches PROVIDER_READY after initialize() + returns, so flag resolution itself remains gated on the loaded config rather than + the SDK registry's ready event. If RC has already delivered config before initialize() runs (e.g. in the master process of a pre-fork server), the fast path sets READY synchronously so the SDK @@ -199,7 +202,7 @@ def initialize(self, evaluation_context: EvaluationContext) -> None: Provider lifecycle: NOT_READY -> initialize() returns -> RC delivers config -> on_configuration_received() - -> READY (PROVIDER_READY event emitted via emit_provider_ready) + -> READY """ if not self._enabled: return @@ -231,9 +234,12 @@ def initialize(self, evaluation_context: EvaluationContext) -> None: self._status = ProviderStatus.READY return # SDK will dispatch PROVIDER_READY - # Config not yet available — return without blocking. The provider stays - # NOT_READY; on_configuration_received() will flip it to READY and emit - # PROVIDER_READY when RC delivers the FFE_FLAGS payload. + # Config not yet available — return without blocking. This provider's + # internal status stays NOT_READY; on_configuration_received() will flip it + # to READY when RC delivers the FFE_FLAGS payload. Note that openfeature-sdk + # 0.8.x dispatches PROVIDER_READY unconditionally after initialize() + # returns, even while our internal status and evaluation path are still + # waiting for config. # AIDEV-NOTE: Do NOT block here with _config_received.wait(). Blocking # initialize() breaks gunicorn/uWSGI pre-fork workers: when the OpenFeature # SDK runs initialize() in a background thread, fork() kills that thread in @@ -557,6 +563,8 @@ def on_configuration_received(self) -> None: Updates status first, then signals the event for observers. Emits PROVIDER_READY for late arrivals after non-blocking initialize(). + Some openfeature-sdk versions also emit PROVIDER_READY immediately after + initialize() returns; this late event is the Datadog config-loaded signal. """ if not self._config_received.is_set(): self._status = ProviderStatus.READY From e914f4faf0d1607a643c945eb727ec5f3bd8486b Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Mon, 6 Jul 2026 00:16:50 -0400 Subject: [PATCH 35/37] fix(openfeature): harden flagevaluation context handling --- .../openfeature/_flagevaluation_writer.py | 57 ++++++++++++++----- .../openfeature/test_flagevaluation_writer.py | 26 ++++++++- 2 files changed, 68 insertions(+), 15 deletions(-) diff --git a/ddtrace/internal/openfeature/_flagevaluation_writer.py b/ddtrace/internal/openfeature/_flagevaluation_writer.py index a24c5e01036..8b9cdf3b379 100644 --- a/ddtrace/internal/openfeature/_flagevaluation_writer.py +++ b/ddtrace/internal/openfeature/_flagevaluation_writer.py @@ -22,6 +22,7 @@ """ from collections.abc import Mapping +from collections.abc import Sequence import http.client as httplib import json import queue @@ -96,6 +97,7 @@ FLAG_EVALUATION_SPLITS_METRIC = "flagevaluation.payload.splits" FLAG_EVALUATION_REASON_QUEUE_OVERFLOW = "queue_overflow" +FLAG_EVALUATION_REASON_CLOSED = "closed" FLAG_EVALUATION_REASON_DEGRADED_CAP = "degraded_cap" FLAG_EVALUATION_REASON_PAYLOAD_LIMIT = "payload_limit" FLAG_EVALUATION_REASON_CARDINALITY_CAP = "cardinality_cap" @@ -188,7 +190,7 @@ def flatten_and_prune_context(attrs: dict[str, typing.Any]) -> dict[str, typing. return {} flat: dict[str, typing.Any] = {} - _flatten_recursive("", attrs, flat) + _flatten_recursive("", attrs, flat, set()) if not flat: return {} @@ -224,20 +226,38 @@ def _json_safe_context(attrs: dict[str, typing.Any]) -> dict[str, typing.Any]: return {k: _json_safe_context_value(v) for k, v in attrs.items()} -def _flatten_recursive(prefix: str, attrs: typing.Any, out: dict[str, typing.Any]) -> None: - """Recursively flatten nested dicts into dot-notation keys.""" - if not isinstance(attrs, Mapping): - if prefix: - out[prefix] = attrs +def _flatten_recursive(prefix: str, attrs: typing.Any, out: dict[str, typing.Any], seen: set[int]) -> None: + """Recursively flatten nested maps and sequences into dot/bracket notation keys.""" + if isinstance(attrs, Mapping): + attrs_id = id(attrs) + if attrs_id in seen: + return + seen.add(attrs_id) + try: + for k, v in attrs.items(): + if not prefix and k in DEDICATED_TARGETING_KEY_CONTEXT_FIELDS: + continue + full_key = f"{prefix}.{k}" if prefix else str(k) + _flatten_recursive(full_key, v, out, seen) + finally: + seen.remove(attrs_id) + return + + if isinstance(attrs, Sequence) and not isinstance(attrs, (str, bytes, bytearray)): + attrs_id = id(attrs) + if attrs_id in seen: + return + seen.add(attrs_id) + try: + for idx, value in enumerate(attrs): + full_key = f"{prefix}[{idx}]" if prefix else f"[{idx}]" + _flatten_recursive(full_key, value, out, seen) + finally: + seen.remove(attrs_id) return - for k, v in attrs.items(): - if not prefix and k in DEDICATED_TARGETING_KEY_CONTEXT_FIELDS: - continue - full_key = f"{prefix}.{k}" if prefix else k - if isinstance(v, Mapping): - _flatten_recursive(full_key, v, out) - else: - out[full_key] = v + + if prefix: + out[prefix] = attrs # --------------------------------------------------------------------------- @@ -367,6 +387,7 @@ def __init__(self, interval: float = DEFAULT_FLUSH_INTERVAL, timeout: float = 2. self._dropped_degraded_overflow: int = 0 # degraded-cap overflow drops self._drain_worker: typing.Optional[PeriodicThread] = None + self._accepting_events = True # ------------------------------------------------------------------ # Public API used by FlagEvalEVPHook @@ -380,6 +401,11 @@ def enqueue(self, event: _EvalEvent) -> None: only memory bound. On queue.Full, increments _dropped_queue (observable) and returns immediately — never blocks the evaluation hot path. """ + if not self._accepting_events: + _count_metric(FLAG_EVALUATION_DROPPED_METRIC, 1, FLAG_EVALUATION_REASON_CLOSED) + logger.debug("FlagEvaluationWriter: dropped flag evaluation event after shutdown started") + return + bounded_event = _EvalEvent( flag_key=event.flag_key, variant=event.variant, @@ -406,6 +432,7 @@ def enqueue(self, event: _EvalEvent) -> None: # ------------------------------------------------------------------ def _start_service(self, *args: typing.Any, **kwargs: typing.Any) -> None: + self._accepting_events = True self._drain_worker = PeriodicThread( DRAIN_INTERVAL, target=self._drain_queue, @@ -420,6 +447,7 @@ def _start_service(self, *args: typing.Any, **kwargs: typing.Any) -> None: raise def _stop_service(self, *args: typing.Any, **kwargs: typing.Any) -> None: + self._accepting_events = False self._stop_drain_worker() super()._stop_service(*args, **kwargs) @@ -558,6 +586,7 @@ def periodic(self) -> None: def on_shutdown(self) -> None: # type: ignore[override] """Final flush on service shutdown — drains the queue and flushes before exit.""" + self._accepting_events = False self._stop_drain_worker() self.periodic() diff --git a/tests/openfeature/test_flagevaluation_writer.py b/tests/openfeature/test_flagevaluation_writer.py index e88e09064a3..5a63886cbdc 100644 --- a/tests/openfeature/test_flagevaluation_writer.py +++ b/tests/openfeature/test_flagevaluation_writer.py @@ -29,6 +29,7 @@ from ddtrace.internal.openfeature._flagevaluation_writer import FLAG_EVALUATION_DEGRADED_METRIC from ddtrace.internal.openfeature._flagevaluation_writer import FLAG_EVALUATION_DROPPED_METRIC from ddtrace.internal.openfeature._flagevaluation_writer import FLAG_EVALUATION_REASON_CARDINALITY_CAP +from ddtrace.internal.openfeature._flagevaluation_writer import FLAG_EVALUATION_REASON_CLOSED from ddtrace.internal.openfeature._flagevaluation_writer import FLAG_EVALUATION_REASON_DEGRADED_CAP from ddtrace.internal.openfeature._flagevaluation_writer import FLAG_EVALUATION_REASON_PAYLOAD_LIMIT from ddtrace.internal.openfeature._flagevaluation_writer import FLAG_EVALUATION_REASON_QUEUE_OVERFLOW @@ -190,6 +191,19 @@ def test_nested_attrs_flattened(self): assert "user.tier" in result assert result["user.id"] == "u1" + def test_nested_sequences_are_flattened_with_bracket_indexes(self): + attrs = {"cohorts": ["beta", {"name": "ga"}], "user": {"roles": ["admin"]}} + result = flatten_and_prune_context(attrs) + assert result["cohorts[0]"] == "beta" + assert result["cohorts[1].name"] == "ga" + assert result["user.roles[0]"] == "admin" + + def test_cycles_do_not_recurse_forever(self): + attrs = {} + attrs["self"] = attrs + result = flatten_and_prune_context(attrs) + assert result == {} + def test_dedicated_targeting_key_aliases_are_not_context_attrs(self): attrs = {"targetingKey": "user-1", "targeting_key": "user-1", "tier": "premium"} result = flatten_and_prune_context(attrs) @@ -340,6 +354,15 @@ def test_enqueue_flattens_nested_context_snapshot(self, writer): queued = writer._queue.get_nowait() assert queued.attrs == {"user.id": 123, "user.plan": "pro"} + def test_enqueue_after_shutdown_started_is_dropped_and_counted(self, writer): + writer.on_shutdown() + + with mock.patch(TELEMETRY_COUNT_PATCH) as mock_count: + writer.enqueue(_make_event(flag_key="late")) + + assert writer._queue.qsize() == 0 + _assert_count_metric(mock_count, FLAG_EVALUATION_DROPPED_METRIC, 1, FLAG_EVALUATION_REASON_CLOSED) + # --------------------------------------------------------------------------- # Periodic flush + EVP POST tests @@ -488,7 +511,8 @@ def __str__(self): ctx_eval = decoded["flagEvaluations"][0]["context"]["evaluation"] assert ctx_eval["amount"] == "12.34" assert ctx_eval["custom"] == "custom-value" - assert ctx_eval["list"] == ["1.5", "custom-value"] + assert ctx_eval["list[0]"] == "1.5" + assert ctx_eval["list[1]"] == "custom-value" def test_degraded_event_has_no_context_or_targeting_key(self, writer): """Degraded-tier events must not include targeting_key or context fields.""" From e6c25dcf11240b25326f584e834446a627b011e9 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Mon, 6 Jul 2026 00:30:20 -0400 Subject: [PATCH 36/37] test(openfeature): cover sdk ready before config --- tests/openfeature/test_provider_status.py | 25 +++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/openfeature/test_provider_status.py b/tests/openfeature/test_provider_status.py index fed6d4b15cb..d61f6c9f494 100644 --- a/tests/openfeature/test_provider_status.py +++ b/tests/openfeature/test_provider_status.py @@ -139,6 +139,31 @@ def on_provider_ready(event_details): api.remove_handler(ProviderEvent.PROVIDER_READY, on_provider_ready) api.clear_providers() + @pytest.mark.skipif(ProviderEvent is None, reason="ProviderEvent not available in SDK 0.6.0") + def test_sdk_ready_event_can_fire_before_datadog_config_ready(self): + """SDK-level PROVIDER_READY does not mean Datadog config has loaded.""" + ready_events = [] + ready_event = threading.Event() + + def on_provider_ready(event_details): + ready_events.append(event_details) + ready_event.set() + + api.add_handler(ProviderEvent.PROVIDER_READY, on_provider_ready) + + try: + with override_global_config({"experimental_flagging_provider_enabled": True}): + provider = DataDogProvider() + _set_provider(provider) + + assert ready_event.wait(timeout=1.0) + assert len(ready_events) >= 1 + assert provider._status == ProviderStatus.NOT_READY + assert not provider._config_received.is_set() + finally: + api.remove_handler(ProviderEvent.PROVIDER_READY, on_provider_ready) + api.clear_providers() + def test_provider_status_after_shutdown(self): """Test that provider returns to NOT_READY after shutdown.""" with override_global_config({"experimental_flagging_provider_enabled": True}): From 14e0293c71b7ad7643baef9d25ce58de9b74c662 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Mon, 6 Jul 2026 09:55:01 -0400 Subject: [PATCH 37/37] ci(openfeature): defer flagevaluation benchmark registration --- benchmarks/suitespec.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/benchmarks/suitespec.yml b/benchmarks/suitespec.yml index 659e8f80662..0802b8ed9ca 100644 --- a/benchmarks/suitespec.yml +++ b/benchmarks/suitespec.yml @@ -50,10 +50,6 @@ components: - ddtrace/appsec/iast/* - ddtrace/appsec/* - ddtrace/internal/settings/asm.py - openfeature_benchmark: - - ddtrace/openfeature/* - - ddtrace/internal/openfeature/* - - ddtrace/internal/settings/openfeature.py core: - ddtrace/internal/__init__.py - ddtrace/internal/_exceptions.py @@ -226,16 +222,6 @@ suites: - benchmarks/suitespec.yml cpus_per_run: 1 type: 'microbenchmark' - openfeature_flagevaluation: - paths: - - '@bootstrap' - - '@core' - - '@openfeature_benchmark' - - '@vendor' - - benchmarks/openfeature_flagevaluation/* - - benchmarks/suitespec.yml - cpus_per_run: 1 - type: 'microbenchmark' appsec_iast_aspects: paths: - '@bootstrap'