From 65a71aeb20d71f214511b9ad367ea8fd87b943f7 Mon Sep 17 00:00:00 2001 From: Aaron Personal Date: Tue, 7 Jul 2026 12:43:22 +0530 Subject: [PATCH 1/4] feat: add schema URL handling and alias-aware attribute resolution --- src/agentevals/api/otlp_processing.py | 14 ++- src/agentevals/extraction.py | 75 ++++++++++++++- src/agentevals/loader/otlp.py | 9 +- src/agentevals/streaming/processor.py | 3 + src/agentevals/trace_attrs.py | 6 ++ tests/test_extraction.py | 132 ++++++++++++++++++++++++++ 6 files changed, 230 insertions(+), 9 deletions(-) diff --git a/src/agentevals/api/otlp_processing.py b/src/agentevals/api/otlp_processing.py index 5dc6038..8892751 100644 --- a/src/agentevals/api/otlp_processing.py +++ b/src/agentevals/api/otlp_processing.py @@ -19,6 +19,7 @@ OTEL_GENAI_CONVERSATION_ID, OTEL_GENAI_INPUT_MESSAGES, OTEL_GENAI_OUTPUT_MESSAGES, + OTEL_SCHEMA_URL, OTEL_SCOPE, OTEL_SCOPE_VERSION, ) @@ -46,9 +47,10 @@ async def process_traces(body: dict, manager: StreamingTraceManager) -> None: scope = scope_span.get("scope", {}) scope_name = scope.get("name", "") scope_version = scope.get("version", "") + schema_url = scope_span.get("schemaUrl", "") for span_data in scope_span.get("spans", []): - span = _normalize_span(span_data, scope_name, scope_version) + span = _normalize_span(span_data, scope_name, scope_version, schema_url) trace_id = span.get("traceId", "") if not trace_id: @@ -200,12 +202,13 @@ def fix_protobuf_id_fields(data) -> None: _GENAI_EVENT_KEYS = {OTEL_GENAI_INPUT_MESSAGES, OTEL_GENAI_OUTPUT_MESSAGES} -def _normalize_span(span_data: dict, scope_name: str, scope_version: str) -> dict: +def _normalize_span(span_data: dict, scope_name: str, scope_version: str, schema_url: str = "") -> dict: """Normalize an OTLP span for the downstream pipeline. Performs two transformations: - 1. Injects otel.scope.name/version from the scopeSpans level into span - attributes (the pipeline expects them there). + 1. Injects otel.scope.name/version (and otel.schema_url) from the + scopeSpans level into span attributes (the pipeline expects them + there). 2. Promotes gen_ai.input.messages and gen_ai.output.messages from span events to span attributes. Some SDKs (e.g. Strands with OTEL_SEMCONV_STABILITY_OPT_IN=gen_ai_latest_experimental) store @@ -222,6 +225,9 @@ def _normalize_span(span_data: dict, scope_name: str, scope_version: str) -> dic if scope_version and OTEL_SCOPE_VERSION not in existing_keys: attrs.append({"key": OTEL_SCOPE_VERSION, "value": {"stringValue": scope_version}}) existing_keys.add(OTEL_SCOPE_VERSION) + if schema_url and OTEL_SCHEMA_URL not in existing_keys: + attrs.append({"key": OTEL_SCHEMA_URL, "value": {"stringValue": schema_url}}) + existing_keys.add(OTEL_SCHEMA_URL) for event in span.get("events", []): for attr in event.get("attributes", []): diff --git a/src/agentevals/extraction.py b/src/agentevals/extraction.py index 51b0348..96bf98e 100644 --- a/src/agentevals/extraction.py +++ b/src/agentevals/extraction.py @@ -13,6 +13,7 @@ import json import logging +import re from typing import Any, Protocol, TypedDict, TypeVar from .loader.base import Span, Trace @@ -22,6 +23,7 @@ ADK_SCOPE_VALUE, ADK_TOOL_CALL_ARGS, ADK_TOOL_RESPONSE, + GENAI_ATTRIBUTE_ALIASES, OTEL_ERROR_TYPE, OTEL_GENAI_INPUT_MESSAGES, OTEL_GENAI_OP, @@ -44,6 +46,7 @@ OTEL_GENAI_USAGE_CACHE_READ_TOKENS, OTEL_GENAI_USAGE_INPUT_TOKENS, OTEL_GENAI_USAGE_OUTPUT_TOKENS, + OTEL_SCHEMA_URL, OTEL_SCOPE, ) from .utils.genai_messages import ( @@ -58,6 +61,61 @@ FORMAT_DETECTION_SPAN_LIMIT = 10 +# --------------------------------------------------------------------------- +# Alias-aware attribute resolution +# --------------------------------------------------------------------------- + +# Matches a semver-like segment (e.g. "1.37.0") anywhere in a schema URL such +# as "https://opentelemetry.io/schemas/1.37.0". Deliberately permissive since +# schema_url values are free-form OTel-defined strings, not something we +# control the shape of. +_SEMCONV_VERSION_RE = re.compile(r"(\d+\.\d+\.\d+)") + + +def resolve_attr(attrs: dict[str, Any], canonical_key: str) -> Any | None: + """Look up *canonical_key* in *attrs*, falling back to older aliased names. + + Resolution order: + 1. The canonical (current/newest) key, if present with a truthy value. + 2. Each alias in ``GENAI_ATTRIBUTE_ALIASES[canonical_key]``, in order, + first truthy value wins. + 3. ``None`` if neither the canonical key nor any alias is present. + + The canonical value always takes precedence over aliases, even if both + are present on the same span. This resolves only against the flat attrs + dict at read time - it never mutates or rewrites stored/transmitted + attributes. + """ + value = attrs.get(canonical_key) + if value: + return value + + for alias in GENAI_ATTRIBUTE_ALIASES.get(canonical_key, []): + alias_value = attrs.get(alias) + if alias_value: + return alias_value + + return None + + +def resolve_semconv_version(schema_url: str | None) -> str: + """Resolve a schema_url into a semconv version string. + + Extracts a version-like segment (e.g. "1.37.0") from URLs such as + "https://opentelemetry.io/schemas/1.37.0". Degrades gracefully to + "unknown" for missing, empty, or malformed input - this function never + raises. + """ + if not schema_url or not isinstance(schema_url, str): + return "unknown" + + match = _SEMCONV_VERSION_RE.search(schema_url) + if match: + return match.group(1) + + return "unknown" + + # --------------------------------------------------------------------------- # Pure extraction functions (operate on flat attribute dicts) # --------------------------------------------------------------------------- @@ -189,18 +247,28 @@ class ExtendedModelInfo(TypedDict): cache_creation_tokens: int cache_read_tokens: int error_type: str | None + semconv_version: str def extract_extended_model_info_from_attrs(attrs: dict[str, Any]) -> ExtendedModelInfo: """Extract extended model and provider metadata from span attributes. - Uses gen_ai.system as fallback for provider when gen_ai.provider.name is - absent (backward compat with pre-v1.37.0 instrumentors). + Uses the alias-resolving lookup for attributes with known historical + renames (e.g. provider falls back to gen_ai.system when + gen_ai.provider.name - the canonical, current name - is absent, for + backward compat with pre-v1.37.0 instrumentors). + + ``semconv_version`` is resolved from the scope's ``schema_url`` (captured + at ingest as the ``otel.schema_url`` attribute) and degrades to + "unknown" when absent or malformed. This is purely informational + metadata about which semconv version emitted the span - it is never used + to detect whether a span is GenAI (that remains the sole responsibility + of the existing gen_ai.* presence checks). """ return { "request_model": attrs.get(OTEL_GENAI_REQUEST_MODEL), "response_model": attrs.get(OTEL_GENAI_RESPONSE_MODEL), - "provider": attrs.get(OTEL_GENAI_PROVIDER_NAME) or attrs.get(OTEL_GENAI_SYSTEM), + "provider": resolve_attr(attrs, OTEL_GENAI_PROVIDER_NAME), "finish_reasons": _parse_finish_reasons(attrs.get(OTEL_GENAI_RESPONSE_FINISH_REASONS)), "response_id": attrs.get(OTEL_GENAI_RESPONSE_ID), "temperature": _safe_cast(attrs.get(OTEL_GENAI_REQUEST_TEMPERATURE), float), @@ -208,6 +276,7 @@ def extract_extended_model_info_from_attrs(attrs: dict[str, Any]) -> ExtendedMod "cache_creation_tokens": _safe_cast(attrs.get(OTEL_GENAI_USAGE_CACHE_CREATION_TOKENS), int, 0), "cache_read_tokens": _safe_cast(attrs.get(OTEL_GENAI_USAGE_CACHE_READ_TOKENS), int, 0), "error_type": attrs.get(OTEL_ERROR_TYPE), + "semconv_version": resolve_semconv_version(attrs.get(OTEL_SCHEMA_URL)), } diff --git a/src/agentevals/loader/otlp.py b/src/agentevals/loader/otlp.py index 8155e02..7395df5 100644 --- a/src/agentevals/loader/otlp.py +++ b/src/agentevals/loader/otlp.py @@ -8,6 +8,7 @@ from ..trace_attrs import ( OTEL_GENAI_INPUT_MESSAGES, OTEL_GENAI_OUTPUT_MESSAGES, + OTEL_SCHEMA_URL, OTEL_SCOPE, OTEL_SCOPE_VERSION, ) @@ -98,16 +99,17 @@ def _parse_otlp_export(self, data: dict) -> list[Trace]: scope = scope_span.get("scope") or scope_span.get("instrumentationLibrary") or {} scope_name = scope.get("name", "") scope_version = scope.get("version", "") + schema_url = scope_span.get("schemaUrl", "") for span_data in scope_span.get("spans", []): - span = self._parse_span(span_data, resource_attrs, scope_name, scope_version) + span = self._parse_span(span_data, resource_attrs, scope_name, scope_version, schema_url) all_spans.append(span) return self._build_traces(all_spans) def _parse_otlp_spans(self, spans_data: list[dict]) -> list[Trace]: """Parse flat list of OTLP spans (JSONL format for streaming).""" - all_spans = [self._parse_span(span_data, {}, "", "") for span_data in spans_data] + all_spans = [self._parse_span(span_data, {}, "", "", "") for span_data in spans_data] return self._build_traces(all_spans) _GENAI_EVENT_KEYS = {OTEL_GENAI_INPUT_MESSAGES, OTEL_GENAI_OUTPUT_MESSAGES} @@ -118,6 +120,7 @@ def _parse_span( resource_attrs: dict, scope_name: str, scope_version: str, + schema_url: str = "", ) -> Span: """Convert OTLP span to normalized Span object.""" attributes = self._extract_attributes(span_data.get("attributes", [])) @@ -126,6 +129,8 @@ def _parse_span( attributes[OTEL_SCOPE] = scope_name if scope_version: attributes[OTEL_SCOPE_VERSION] = scope_version + if schema_url: + attributes[OTEL_SCHEMA_URL] = schema_url self._promote_genai_event_attributes(span_data, attributes) diff --git a/src/agentevals/streaming/processor.py b/src/agentevals/streaming/processor.py index 8f6367a..5116028 100644 --- a/src/agentevals/streaming/processor.py +++ b/src/agentevals/streaming/processor.py @@ -154,12 +154,15 @@ def force_flush(self, timeout_millis: int = 30000) -> bool: def _span_to_otlp(self, span: ReadableSpan) -> dict: scope_name = span.instrumentation_scope.name if span.instrumentation_scope else "" scope_version = span.instrumentation_scope.version if span.instrumentation_scope else "" + schema_url = span.instrumentation_scope.schema_url if span.instrumentation_scope else "" attributes = [] if scope_name: attributes.append({"key": "otel.scope.name", "value": {"stringValue": scope_name}}) if scope_version: attributes.append({"key": "otel.scope.version", "value": {"stringValue": scope_version}}) + if schema_url: + attributes.append({"key": "otel.schema_url", "value": {"stringValue": schema_url}}) if span.attributes: for key, value in span.attributes.items(): diff --git a/src/agentevals/trace_attrs.py b/src/agentevals/trace_attrs.py index 6bfc0df..5747bca 100644 --- a/src/agentevals/trace_attrs.py +++ b/src/agentevals/trace_attrs.py @@ -12,6 +12,7 @@ # OTel scope OTEL_SCOPE = "otel.scope.name" OTEL_SCOPE_VERSION = "otel.scope.version" +OTEL_SCHEMA_URL = "otel.schema_url" # Google ADK scope value ADK_SCOPE_VALUE = "gcp.vertex.agent" @@ -73,3 +74,8 @@ ADK_TOOL_CALL_ARGS = "gcp.vertex.agent.tool_call_args" ADK_TOOL_RESPONSE = "gcp.vertex.agent.tool_response" ADK_INVOCATION_ID = "gcp.vertex.agent.invocation_id" + +# GenAI attribute alias table +GENAI_ATTRIBUTE_ALIASES: dict[str, list[str]] = { + OTEL_GENAI_PROVIDER_NAME: [OTEL_GENAI_SYSTEM], +} diff --git a/tests/test_extraction.py b/tests/test_extraction.py index 52ec812..3cec9c6 100644 --- a/tests/test_extraction.py +++ b/tests/test_extraction.py @@ -21,6 +21,8 @@ is_invocation_span, is_llm_span, is_tool_span, + resolve_attr, + resolve_semconv_version, ) from agentevals.loader.base import Span, Trace from agentevals.trace_attrs import ( @@ -28,6 +30,7 @@ ADK_LLM_RESPONSE, ADK_SCOPE_VALUE, ADK_TOOL_CALL_ARGS, + GENAI_ATTRIBUTE_ALIASES, OTEL_ERROR_TYPE, OTEL_GENAI_AGENT_NAME, OTEL_GENAI_INPUT_MESSAGES, @@ -50,6 +53,7 @@ OTEL_GENAI_USAGE_CACHE_READ_TOKENS, OTEL_GENAI_USAGE_INPUT_TOKENS, OTEL_GENAI_USAGE_OUTPUT_TOKENS, + OTEL_SCHEMA_URL, OTEL_SCOPE, ) @@ -399,6 +403,16 @@ def test_mixed_types(self): def test_empty(self): assert flatten_otlp_attributes([]) == {} + def test_schema_url_flows_through(self): + # otel.schema_url is a plain string attribute; flatten_otlp_attributes + # has no allowlist, so it passes through like any other key. + result = flatten_otlp_attributes( + [ + {"key": OTEL_SCHEMA_URL, "value": {"stringValue": "https://opentelemetry.io/schemas/1.37.0"}}, + ] + ) + assert result == {OTEL_SCHEMA_URL: "https://opentelemetry.io/schemas/1.37.0"} + # --------------------------------------------------------------------------- # Span classification helpers @@ -452,6 +466,15 @@ def test_is_invocation_span_by_keyword(self): def test_is_invocation_span_false(self): assert not is_invocation_span(_span(op="chat gpt-4")) + def test_schema_url_is_never_used_for_genai_detection(self): + # schema_url is a generic OTel field, not a GenAI-specific signal. + # A span carrying only otel.schema_url (no gen_ai.* attributes) must + # not be classified as an LLM/tool span by its presence alone. + span = _span(tags={OTEL_SCHEMA_URL: "https://opentelemetry.io/schemas/1.37.0"}) + assert not is_llm_span(span) + assert not is_tool_span(span) + assert not is_adk_scope(span) + # --------------------------------------------------------------------------- # Extractor detection @@ -661,6 +684,70 @@ def test_find_tool_spans_in(self): assert [s.span_id for s in ext.find_tool_spans_in(root)] == ["tool1"] +# --------------------------------------------------------------------------- +# resolve_attr — alias-aware attribute lookup +# --------------------------------------------------------------------------- + + +class TestResolveAttr: + def test_canonical_present_alias_absent(self): + attrs = {OTEL_GENAI_PROVIDER_NAME: "openai"} + assert resolve_attr(attrs, OTEL_GENAI_PROVIDER_NAME) == "openai" + + def test_canonical_absent_alias_present(self): + attrs = {OTEL_GENAI_SYSTEM: "anthropic"} + assert resolve_attr(attrs, OTEL_GENAI_PROVIDER_NAME) == "anthropic" + + def test_both_present_canonical_wins(self): + attrs = { + OTEL_GENAI_PROVIDER_NAME: "openai", + OTEL_GENAI_SYSTEM: "old_value", + } + assert resolve_attr(attrs, OTEL_GENAI_PROVIDER_NAME) == "openai" + + def test_neither_present_returns_none(self): + assert resolve_attr({}, OTEL_GENAI_PROVIDER_NAME) is None + + def test_unknown_canonical_key_with_no_alias_entry_returns_none(self): + # A key with no entry in GENAI_ATTRIBUTE_ALIASES simply has no + # fallback list; absence should not raise. + assert resolve_attr({}, "gen_ai.some.unmapped.key") is None + + def test_alias_table_contains_expected_seed_entry(self): + # Sanity check that the table wiring itself is intact. + assert OTEL_GENAI_SYSTEM in GENAI_ATTRIBUTE_ALIASES.get(OTEL_GENAI_PROVIDER_NAME, []) + + +# --------------------------------------------------------------------------- +# resolve_semconv_version +# --------------------------------------------------------------------------- + + +class TestResolveSemconvVersion: + def test_valid_schema_url(self): + assert resolve_semconv_version("https://opentelemetry.io/schemas/1.37.0") == "1.37.0" + + def test_valid_schema_url_with_trailing_path(self): + assert resolve_semconv_version("https://opentelemetry.io/schemas/1.4.0/extra") == "1.4.0" + + def test_none_degrades_to_unknown(self): + assert resolve_semconv_version(None) == "unknown" + + def test_empty_string_degrades_to_unknown(self): + assert resolve_semconv_version("") == "unknown" + + def test_malformed_string_degrades_to_unknown(self): + assert resolve_semconv_version("not-a-schema-url") == "unknown" + + def test_non_string_input_degrades_to_unknown(self): + # Defensive: attrs values are technically Any; must not raise. + assert resolve_semconv_version(123) == "unknown" # type: ignore[arg-type] + + def test_never_raises_on_garbage_input(self): + for garbage in (object(), [], {}, 1.5): + assert resolve_semconv_version(garbage) == "unknown" # type: ignore[arg-type] + + # --------------------------------------------------------------------------- # extract_extended_model_info_from_attrs # --------------------------------------------------------------------------- @@ -793,6 +880,7 @@ def test_full_attribute_set(self): OTEL_GENAI_USAGE_CACHE_CREATION_TOKENS: 2000, OTEL_GENAI_USAGE_CACHE_READ_TOKENS: 5000, OTEL_ERROR_TYPE: None, + OTEL_SCHEMA_URL: "https://opentelemetry.io/schemas/1.37.0", } result = extract_extended_model_info_from_attrs(attrs) assert result["provider"] == "anthropic" @@ -805,6 +893,50 @@ def test_full_attribute_set(self): assert result["cache_creation_tokens"] == 2000 assert result["cache_read_tokens"] == 5000 assert result["error_type"] is None + assert result["semconv_version"] == "1.37.0" + + # ----------------------------------------------------------------- + # semconv_version / schema_url resolution + # ----------------------------------------------------------------- + + def test_semconv_version_newest_attribute_names_only(self): + # Fixture with only newest attribute names (gen_ai.provider.name, + # not gen_ai.system) plus a schema_url; all fields, including + # semconv_version, should populate normally. + attrs = { + OTEL_GENAI_PROVIDER_NAME: "openai", + OTEL_GENAI_REQUEST_MODEL: "gpt-4o", + OTEL_SCHEMA_URL: "https://opentelemetry.io/schemas/1.37.0", + } + result = extract_extended_model_info_from_attrs(attrs) + assert result["semconv_version"] == "1.37.0" + assert result["provider"] == "openai" + assert result["request_model"] == "gpt-4o" + + def test_semconv_version_with_aliased_attribute_names_only(self): + # Older instrumentor: gen_ai.system present, gen_ai.provider.name + # absent. provider should still populate via alias fallback, and + # semconv_version should resolve from an older schema_url. + attrs = { + OTEL_GENAI_SYSTEM: "anthropic", + OTEL_SCHEMA_URL: "https://opentelemetry.io/schemas/1.20.0", + } + result = extract_extended_model_info_from_attrs(attrs) + assert result["provider"] == "anthropic" + assert result["semconv_version"] == "1.20.0" + + def test_semconv_version_missing_schema_url_entirely(self): + # No schema_url at all -> "unknown", with no impact on other fields. + attrs = {OTEL_GENAI_PROVIDER_NAME: "openai", OTEL_GENAI_REQUEST_MODEL: "gpt-4o"} + result = extract_extended_model_info_from_attrs(attrs) + assert result["semconv_version"] == "unknown" + assert result["provider"] == "openai" + assert result["request_model"] == "gpt-4o" + + def test_semconv_version_malformed_schema_url(self): + attrs = {OTEL_SCHEMA_URL: "not-a-real-schema-url"} + result = extract_extended_model_info_from_attrs(attrs) + assert result["semconv_version"] == "unknown" # --------------------------------------------------------------------------- From d5a30ba96b65761b85a1538d89939c1e299dda40 Mon Sep 17 00:00:00 2001 From: Aaron Personal Date: Tue, 7 Jul 2026 14:45:29 +0530 Subject: [PATCH 2/4] fix: add UTF-8 encoding to file opening in loaders and runner --- src/agentevals/loader/auto.py | 2 +- src/agentevals/loader/jaeger.py | 2 +- src/agentevals/loader/otlp.py | 2 +- src/agentevals/runner.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/agentevals/loader/auto.py b/src/agentevals/loader/auto.py index 3ac3ddb..c50f3df 100644 --- a/src/agentevals/loader/auto.py +++ b/src/agentevals/loader/auto.py @@ -51,7 +51,7 @@ def detect_format(path: str) -> str | None: load can fall back to a default. """ try: - with open(path) as f: + with open(path, encoding="utf-8") as f: if path.lower().endswith(".jsonl"): return OTLP_JSON data = json.load(f) diff --git a/src/agentevals/loader/jaeger.py b/src/agentevals/loader/jaeger.py index cb02338..14a27fd 100644 --- a/src/agentevals/loader/jaeger.py +++ b/src/agentevals/loader/jaeger.py @@ -43,7 +43,7 @@ def format_name(self) -> str: return "jaeger-json" def load(self, source: str) -> list[Trace]: - with open(source) as f: + with open(source, encoding="utf-8") as f: raw = json.load(f) if not isinstance(raw, dict) or "data" not in raw: diff --git a/src/agentevals/loader/otlp.py b/src/agentevals/loader/otlp.py index 7395df5..b322d58 100644 --- a/src/agentevals/loader/otlp.py +++ b/src/agentevals/loader/otlp.py @@ -36,7 +36,7 @@ def format_name(self) -> str: def load(self, source: str) -> list[Trace]: """Load OTLP JSON file or JSONL (one span per line).""" - with open(source) as f: + with open(source, encoding="utf-8") as f: content = f.read().strip() if not content: diff --git a/src/agentevals/runner.py b/src/agentevals/runner.py index ebca577..eae4066 100644 --- a/src/agentevals/runner.py +++ b/src/agentevals/runner.py @@ -64,7 +64,7 @@ class RunResult(BaseModel): def load_eval_set(path: str) -> EvalSet: - with open(path) as f: + with open(path, encoding="utf-8") as f: data = json.load(f) return EvalSet.model_validate(data) From c6a92ba9db7b9ed78bf223f27a081dcd25ef4ed9 Mon Sep 17 00:00:00 2001 From: Aaron Personal Date: Tue, 7 Jul 2026 14:45:39 +0530 Subject: [PATCH 3/4] feat: add semconv_version to TraceConversionMetadata and related metadata extraction --- src/agentevals/api/models.py | 1 + src/agentevals/api/routes.py | 1 + src/agentevals/trace_metrics.py | 2 ++ tests/test_api.py | 21 +++++++++++++++++++++ tests/test_runner.py | 17 +++++++++++++++++ 5 files changed, 42 insertions(+) diff --git a/src/agentevals/api/models.py b/src/agentevals/api/models.py index a1cf74e..d5d0aac 100644 --- a/src/agentevals/api/models.py +++ b/src/agentevals/api/models.py @@ -119,6 +119,7 @@ class TraceConversionMetadata(CamelModel): model: str | None = None response_model: str | None = None provider: str | None = None + semconv_version: str | None = None start_time: int | None = None user_input_preview: str | None = None final_output_preview: str | None = None diff --git a/src/agentevals/api/routes.py b/src/agentevals/api/routes.py index 29de1d2..fac096a 100644 --- a/src/agentevals/api/routes.py +++ b/src/agentevals/api/routes.py @@ -487,6 +487,7 @@ async def convert_trace_files( meta = TraceConversionMetadata( agent_name=meta_dict.get("agent_name"), model=meta_dict.get("model"), + semconv_version=meta_dict.get("semconv_version"), start_time=meta_dict.get("start_time"), user_input_preview=meta_dict.get("user_input_preview"), final_output_preview=meta_dict.get("final_output_preview"), diff --git a/src/agentevals/trace_metrics.py b/src/agentevals/trace_metrics.py index 8009957..6e8b792 100644 --- a/src/agentevals/trace_metrics.py +++ b/src/agentevals/trace_metrics.py @@ -185,6 +185,7 @@ def extract_trace_metadata(trace, extractor=None) -> dict[str, Any]: "model": None, "response_model": None, "provider": None, + "semconv_version": "unknown", "start_time": None, "user_input_preview": None, "final_output_preview": None, @@ -209,6 +210,7 @@ def extract_trace_metadata(trace, extractor=None) -> dict[str, Any]: metadata["response_model"] = ext["response_model"] if ext["provider"]: metadata["provider"] = ext["provider"] + metadata["semconv_version"] = ext["semconv_version"] user_text = extract_user_text_from_attrs(llm_spans[0].tags) if user_text: diff --git a/tests/test_api.py b/tests/test_api.py index 627707d..8953b27 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1598,6 +1598,27 @@ def test_tempo_json_upload_auto_detected(self): assert traces[0]["traceId"] == "dd547580319ab0312cee07f1def50dad" assert len(traces[0]["invocations"]) == 1 + @patch("agentevals.api.routes.extract_trace_metadata") + def test_convert_metadata_includes_semconv_version(self, mock_extract_trace_metadata): + mock_extract_trace_metadata.return_value = { + "agent_name": "agent", + "model": "gpt-4o", + "semconv_version": "1.39.0", + "start_time": 123, + "user_input_preview": "hello", + "final_output_preview": "world", + } + + body = _assert_envelope( + self.client.post( + "/api/convert", + files={"trace_files": ("trace.json", io.BytesIO(self._tempo_fixture_bytes()))}, + ) + ) + + metadata = body["data"]["traces"][0]["metadata"] + assert metadata["semconvVersion"] == "1.39.0" + def test_unknown_shape_returns_load_warning(self): unknown = json.dumps({"some_other_shape": []}).encode() resp = self.client.post( diff --git a/tests/test_runner.py b/tests/test_runner.py index e537d18..f91bcff 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -261,6 +261,23 @@ def testextract_trace_metadata_adk(self): assert metadata["final_output_preview"] is not None assert len(metadata["final_output_preview"]) > 0 + def test_extract_trace_metadata_semconv_version_unknown_when_schema_missing(self): + trace = _make_tool_trace(["tool_a"]) + metadata = extract_trace_metadata(trace) + assert metadata["semconv_version"] == "unknown" + + def test_extract_trace_metadata_semconv_version_unknown_when_schema_malformed(self): + trace = _make_tool_trace(["tool_a"]) + trace.all_spans[1].tags["otel.schema_url"] = "not-a-semconv-version" + metadata = extract_trace_metadata(trace) + assert metadata["semconv_version"] == "unknown" + + def test_extract_trace_metadata_semconv_version_from_valid_schema_url(self): + trace = _make_tool_trace(["tool_a"]) + trace.all_spans[1].tags["otel.schema_url"] = "https://opentelemetry.io/schemas/1.39.0" + metadata = extract_trace_metadata(trace) + assert metadata["semconv_version"] == "1.39.0" + class TestTrajectoryMatchType: """Verify trajectory_match_type produces different scores on the same trace. From 5256b4433114aad4fb03c28590600f7c5b60459e Mon Sep 17 00:00:00 2001 From: Aaron Personal Date: Tue, 7 Jul 2026 17:17:04 +0530 Subject: [PATCH 4/4] revert: undo utf-8 changes --- src/agentevals/loader/auto.py | 2 +- src/agentevals/loader/jaeger.py | 2 +- src/agentevals/loader/otlp.py | 2 +- src/agentevals/runner.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/agentevals/loader/auto.py b/src/agentevals/loader/auto.py index c50f3df..3ac3ddb 100644 --- a/src/agentevals/loader/auto.py +++ b/src/agentevals/loader/auto.py @@ -51,7 +51,7 @@ def detect_format(path: str) -> str | None: load can fall back to a default. """ try: - with open(path, encoding="utf-8") as f: + with open(path) as f: if path.lower().endswith(".jsonl"): return OTLP_JSON data = json.load(f) diff --git a/src/agentevals/loader/jaeger.py b/src/agentevals/loader/jaeger.py index 14a27fd..cb02338 100644 --- a/src/agentevals/loader/jaeger.py +++ b/src/agentevals/loader/jaeger.py @@ -43,7 +43,7 @@ def format_name(self) -> str: return "jaeger-json" def load(self, source: str) -> list[Trace]: - with open(source, encoding="utf-8") as f: + with open(source) as f: raw = json.load(f) if not isinstance(raw, dict) or "data" not in raw: diff --git a/src/agentevals/loader/otlp.py b/src/agentevals/loader/otlp.py index b322d58..7395df5 100644 --- a/src/agentevals/loader/otlp.py +++ b/src/agentevals/loader/otlp.py @@ -36,7 +36,7 @@ def format_name(self) -> str: def load(self, source: str) -> list[Trace]: """Load OTLP JSON file or JSONL (one span per line).""" - with open(source, encoding="utf-8") as f: + with open(source) as f: content = f.read().strip() if not content: diff --git a/src/agentevals/runner.py b/src/agentevals/runner.py index eae4066..ebca577 100644 --- a/src/agentevals/runner.py +++ b/src/agentevals/runner.py @@ -64,7 +64,7 @@ class RunResult(BaseModel): def load_eval_set(path: str) -> EvalSet: - with open(path, encoding="utf-8") as f: + with open(path) as f: data = json.load(f) return EvalSet.model_validate(data)