Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/agentevals/api/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,7 @@ class TraceConversionMetadata(CamelModel):
model: str | None = None
response_model: str | None = None
provider: str | None = None
semconv_version: str | None = None

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Per the schema spec, the version number is scoped to a Schema Family (the URL prefix before the version). So basically two URLs with the same version but different hosts are different schemas, and "OpenTelemetry schema version numbers match OpenTelemetry Semantic Conventions version numbers" is only true for the OTel official family. So a bare "1.40.0" from a custom family (e.g. https://randomcompany.internal/schemas/1.40.0) would be recorded as if it were OTel semconv 1.40.0.

Since this field is informational and never drives logic, I wouldn't hard-gate on the host (mirrors exist, and the full otel.schema_url attribute is already preserved for anyone who needs the family). But please make this clear and add note in the field description / the resolve_semconv_version docstring that this is the version segment of the emitting scope's schema URL, comparable only within a schema family and equal to the OTel semconv version for the OTel family.

We could even call it schema_version, since "semconv" asserts the OTel family already.

start_time: int | None = None
user_input_preview: str | None = None
final_output_preview: str | None = None
Expand Down
14 changes: 10 additions & 4 deletions src/agentevals/api/otlp_processing.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
OTEL_GENAI_CONVERSATION_ID,
OTEL_GENAI_INPUT_MESSAGES,
OTEL_GENAI_OUTPUT_MESSAGES,
OTEL_SCHEMA_URL,
OTEL_SCOPE,
OTEL_SCOPE_VERSION,
)
Expand Down Expand Up @@ -46,9 +47,10 @@ async def process_traces(body: dict, manager: StreamingTraceManager) -> None:
scope = scope_span.get("scope", {})
scope_name = scope.get("name", "")
scope_version = scope.get("version", "")
schema_url = scope_span.get("schemaUrl", "")

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the same as the loader path so both ingest routes should behave the same. See my other comment in otlp.py.


for span_data in scope_span.get("spans", []):
span = _normalize_span(span_data, scope_name, scope_version)
span = _normalize_span(span_data, scope_name, scope_version, schema_url)
trace_id = span.get("traceId", "")

if not trace_id:
Expand Down Expand Up @@ -200,12 +202,13 @@ def fix_protobuf_id_fields(data) -> None:
_GENAI_EVENT_KEYS = {OTEL_GENAI_INPUT_MESSAGES, OTEL_GENAI_OUTPUT_MESSAGES}


def _normalize_span(span_data: dict, scope_name: str, scope_version: str) -> dict:
def _normalize_span(span_data: dict, scope_name: str, scope_version: str, schema_url: str = "") -> dict:
"""Normalize an OTLP span for the downstream pipeline.

Performs two transformations:
1. Injects otel.scope.name/version from the scopeSpans level into span
attributes (the pipeline expects them there).
1. Injects otel.scope.name/version (and otel.schema_url) from the
scopeSpans level into span attributes (the pipeline expects them
there).
2. Promotes gen_ai.input.messages and gen_ai.output.messages from span
events to span attributes. Some SDKs (e.g. Strands with
OTEL_SEMCONV_STABILITY_OPT_IN=gen_ai_latest_experimental) store
Expand All @@ -222,6 +225,9 @@ def _normalize_span(span_data: dict, scope_name: str, scope_version: str) -> dic
if scope_version and OTEL_SCOPE_VERSION not in existing_keys:
attrs.append({"key": OTEL_SCOPE_VERSION, "value": {"stringValue": scope_version}})
existing_keys.add(OTEL_SCOPE_VERSION)
if schema_url and OTEL_SCHEMA_URL not in existing_keys:
attrs.append({"key": OTEL_SCHEMA_URL, "value": {"stringValue": schema_url}})
existing_keys.add(OTEL_SCHEMA_URL)

for event in span.get("events", []):
for attr in event.get("attributes", []):
Expand Down
1 change: 1 addition & 0 deletions src/agentevals/api/routes.py
Original file line number Diff line number Diff line change
Expand Up @@ -487,6 +487,7 @@ async def convert_trace_files(
meta = TraceConversionMetadata(
agent_name=meta_dict.get("agent_name"),
model=meta_dict.get("model"),
semconv_version=meta_dict.get("semconv_version"),
start_time=meta_dict.get("start_time"),
user_input_preview=meta_dict.get("user_input_preview"),
final_output_preview=meta_dict.get("final_output_preview"),
Expand Down
75 changes: 72 additions & 3 deletions src/agentevals/extraction.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@

import json
import logging
import re
from typing import Any, Protocol, TypedDict, TypeVar

from .loader.base import Span, Trace
Expand All @@ -22,6 +23,7 @@
ADK_SCOPE_VALUE,
ADK_TOOL_CALL_ARGS,
ADK_TOOL_RESPONSE,
GENAI_ATTRIBUTE_ALIASES,
OTEL_ERROR_TYPE,
OTEL_GENAI_INPUT_MESSAGES,
OTEL_GENAI_OP,
Expand All @@ -44,6 +46,7 @@
OTEL_GENAI_USAGE_CACHE_READ_TOKENS,
OTEL_GENAI_USAGE_INPUT_TOKENS,
OTEL_GENAI_USAGE_OUTPUT_TOKENS,
OTEL_SCHEMA_URL,
OTEL_SCOPE,
)
from .utils.genai_messages import (
Expand All @@ -58,6 +61,61 @@

FORMAT_DETECTION_SPAN_LIMIT = 10

# ---------------------------------------------------------------------------
# Alias-aware attribute resolution
# ---------------------------------------------------------------------------

# Matches a semver-like segment (e.g. "1.37.0") anywhere in a schema URL such
# as "https://opentelemetry.io/schemas/1.37.0". Deliberately permissive since
# schema_url values are free-form OTel-defined strings, not something we
# control the shape of.
_SEMCONV_VERSION_RE = re.compile(r"(\d+\.\d+\.\d+)")
Comment on lines +68 to +72

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is incorrect. Can you please follow the definition described here: https://opentelemetry.io/docs/specs/otel/schemas/#schema-url

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We'd also want a test for this.



def resolve_attr(attrs: dict[str, Any], canonical_key: str) -> Any | None:
"""Look up *canonical_key* in *attrs*, falling back to older aliased names.
Resolution order:
1. The canonical (current/newest) key, if present with a truthy value.
2. Each alias in ``GENAI_ATTRIBUTE_ALIASES[canonical_key]``, in order,
first truthy value wins.
3. ``None`` if neither the canonical key nor any alias is present.
The canonical value always takes precedence over aliases, even if both
are present on the same span. This resolves only against the flat attrs
dict at read time - it never mutates or rewrites stored/transmitted
attributes.
"""
value = attrs.get(canonical_key)
if value:
return value

for alias in GENAI_ATTRIBUTE_ALIASES.get(canonical_key, []):
alias_value = attrs.get(alias)
if alias_value:
return alias_value

return None


def resolve_semconv_version(schema_url: str | None) -> str:
"""Resolve a schema_url into a semconv version string.
Extracts a version-like segment (e.g. "1.37.0") from URLs such as
"https://opentelemetry.io/schemas/1.37.0". Degrades gracefully to
"unknown" for missing, empty, or malformed input - this function never
raises.
"""
if not schema_url or not isinstance(schema_url, str):
return "unknown"

match = _SEMCONV_VERSION_RE.search(schema_url)
if match:
return match.group(1)

return "unknown"

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since TraceConversionMetadata.semconv_version is typed, this can never be null, but "unknown". Let's return None and serialise to null.



# ---------------------------------------------------------------------------
# Pure extraction functions (operate on flat attribute dicts)
# ---------------------------------------------------------------------------
Expand Down Expand Up @@ -189,25 +247,36 @@ class ExtendedModelInfo(TypedDict):
cache_creation_tokens: int
cache_read_tokens: int
error_type: str | None
semconv_version: str


def extract_extended_model_info_from_attrs(attrs: dict[str, Any]) -> ExtendedModelInfo:
"""Extract extended model and provider metadata from span attributes.
Uses gen_ai.system as fallback for provider when gen_ai.provider.name is
absent (backward compat with pre-v1.37.0 instrumentors).
Uses the alias-resolving lookup for attributes with known historical
renames (e.g. provider falls back to gen_ai.system when
gen_ai.provider.name - the canonical, current name - is absent, for
backward compat with pre-v1.37.0 instrumentors).
``semconv_version`` is resolved from the scope's ``schema_url`` (captured
at ingest as the ``otel.schema_url`` attribute) and degrades to
"unknown" when absent or malformed. This is purely informational
metadata about which semconv version emitted the span - it is never used
to detect whether a span is GenAI (that remains the sole responsibility
of the existing gen_ai.* presence checks).
"""
return {
"request_model": attrs.get(OTEL_GENAI_REQUEST_MODEL),
"response_model": attrs.get(OTEL_GENAI_RESPONSE_MODEL),
"provider": attrs.get(OTEL_GENAI_PROVIDER_NAME) or attrs.get(OTEL_GENAI_SYSTEM),
"provider": resolve_attr(attrs, OTEL_GENAI_PROVIDER_NAME),
"finish_reasons": _parse_finish_reasons(attrs.get(OTEL_GENAI_RESPONSE_FINISH_REASONS)),
"response_id": attrs.get(OTEL_GENAI_RESPONSE_ID),
"temperature": _safe_cast(attrs.get(OTEL_GENAI_REQUEST_TEMPERATURE), float),
"max_tokens": _safe_cast(attrs.get(OTEL_GENAI_REQUEST_MAX_TOKENS), int),
"cache_creation_tokens": _safe_cast(attrs.get(OTEL_GENAI_USAGE_CACHE_CREATION_TOKENS), int, 0),
"cache_read_tokens": _safe_cast(attrs.get(OTEL_GENAI_USAGE_CACHE_READ_TOKENS), int, 0),
"error_type": attrs.get(OTEL_ERROR_TYPE),
"semconv_version": resolve_semconv_version(attrs.get(OTEL_SCHEMA_URL)),
}


Expand Down
9 changes: 7 additions & 2 deletions src/agentevals/loader/otlp.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from ..trace_attrs import (
OTEL_GENAI_INPUT_MESSAGES,
OTEL_GENAI_OUTPUT_MESSAGES,
OTEL_SCHEMA_URL,
OTEL_SCOPE,
OTEL_SCOPE_VERSION,
)
Expand Down Expand Up @@ -98,16 +99,17 @@ def _parse_otlp_export(self, data: dict) -> list[Trace]:
scope = scope_span.get("scope") or scope_span.get("instrumentationLibrary") or {}
scope_name = scope.get("name", "")
scope_version = scope.get("version", "")
schema_url = scope_span.get("schemaUrl", "")

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OTLP defines schemaUrl at both ScopeSpans and ResourceSpans. This reads only the scope level, so a trace setting it only at the resource level reports "unknown". resource_span is already in scope from the loop above. Scope stays first as the more specific signal.

Suggested change
schema_url = scope_span.get("schemaUrl", "")
schema_url = scope_span.get("schemaUrl", "") or resource_span.get("schemaUrl", "")


for span_data in scope_span.get("spans", []):
span = self._parse_span(span_data, resource_attrs, scope_name, scope_version)
span = self._parse_span(span_data, resource_attrs, scope_name, scope_version, schema_url)
all_spans.append(span)

return self._build_traces(all_spans)

def _parse_otlp_spans(self, spans_data: list[dict]) -> list[Trace]:
"""Parse flat list of OTLP spans (JSONL format for streaming)."""
all_spans = [self._parse_span(span_data, {}, "", "") for span_data in spans_data]
all_spans = [self._parse_span(span_data, {}, "", "", "") for span_data in spans_data]
return self._build_traces(all_spans)

_GENAI_EVENT_KEYS = {OTEL_GENAI_INPUT_MESSAGES, OTEL_GENAI_OUTPUT_MESSAGES}
Expand All @@ -118,6 +120,7 @@ def _parse_span(
resource_attrs: dict,
scope_name: str,
scope_version: str,
schema_url: str = "",
) -> Span:
"""Convert OTLP span to normalized Span object."""
attributes = self._extract_attributes(span_data.get("attributes", []))
Expand All @@ -126,6 +129,8 @@ def _parse_span(
attributes[OTEL_SCOPE] = scope_name
if scope_version:
attributes[OTEL_SCOPE_VERSION] = scope_version
if schema_url:
attributes[OTEL_SCHEMA_URL] = schema_url

self._promote_genai_event_attributes(span_data, attributes)

Expand Down
3 changes: 3 additions & 0 deletions src/agentevals/streaming/processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -154,12 +154,15 @@ def force_flush(self, timeout_millis: int = 30000) -> bool:
def _span_to_otlp(self, span: ReadableSpan) -> dict:
scope_name = span.instrumentation_scope.name if span.instrumentation_scope else ""
scope_version = span.instrumentation_scope.version if span.instrumentation_scope else ""
schema_url = span.instrumentation_scope.schema_url if span.instrumentation_scope else ""

attributes = []
if scope_name:
attributes.append({"key": "otel.scope.name", "value": {"stringValue": scope_name}})
if scope_version:
attributes.append({"key": "otel.scope.version", "value": {"stringValue": scope_version}})
if schema_url:
attributes.append({"key": "otel.schema_url", "value": {"stringValue": schema_url}})

if span.attributes:
for key, value in span.attributes.items():
Expand Down
6 changes: 6 additions & 0 deletions src/agentevals/trace_attrs.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
# OTel scope
OTEL_SCOPE = "otel.scope.name"
OTEL_SCOPE_VERSION = "otel.scope.version"
OTEL_SCHEMA_URL = "otel.schema_url"

# Google ADK scope value
ADK_SCOPE_VALUE = "gcp.vertex.agent"
Expand Down Expand Up @@ -73,3 +74,8 @@
ADK_TOOL_CALL_ARGS = "gcp.vertex.agent.tool_call_args"
ADK_TOOL_RESPONSE = "gcp.vertex.agent.tool_response"
ADK_INVOCATION_ID = "gcp.vertex.agent.invocation_id"

# GenAI attribute alias table
GENAI_ATTRIBUTE_ALIASES: dict[str, list[str]] = {
OTEL_GENAI_PROVIDER_NAME: [OTEL_GENAI_SYSTEM],
}
2 changes: 2 additions & 0 deletions src/agentevals/trace_metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,7 @@ def extract_trace_metadata(trace, extractor=None) -> dict[str, Any]:
"model": None,
"response_model": None,
"provider": None,
"semconv_version": "unknown",
"start_time": None,
"user_input_preview": None,
"final_output_preview": None,
Expand All @@ -209,6 +210,7 @@ def extract_trace_metadata(trace, extractor=None) -> dict[str, Any]:
metadata["response_model"] = ext["response_model"]
if ext["provider"]:
metadata["provider"] = ext["provider"]
metadata["semconv_version"] = ext["semconv_version"]

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Here, ext comes from llm_spans[0] only, so a trace mixing instrumentors at different versions will only record the first. It's fine, but please add a comment calling this out.


user_text = extract_user_text_from_attrs(llm_spans[0].tags)
if user_text:
Expand Down
21 changes: 21 additions & 0 deletions tests/test_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -1598,6 +1598,27 @@ def test_tempo_json_upload_auto_detected(self):
assert traces[0]["traceId"] == "dd547580319ab0312cee07f1def50dad"
assert len(traces[0]["invocations"]) == 1

@patch("agentevals.api.routes.extract_trace_metadata")

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Mocking extract_trace_metadata like this verifies the wiring and camelCase serialization, which is good, but can we please also cover OTLP with scopeSpans.schemaUrl to /api/convert to semconvVersion, so we have something e2e as well?

def test_convert_metadata_includes_semconv_version(self, mock_extract_trace_metadata):
mock_extract_trace_metadata.return_value = {
"agent_name": "agent",
"model": "gpt-4o",
"semconv_version": "1.39.0",
"start_time": 123,
"user_input_preview": "hello",
"final_output_preview": "world",
}

body = _assert_envelope(
self.client.post(
"/api/convert",
files={"trace_files": ("trace.json", io.BytesIO(self._tempo_fixture_bytes()))},
)
)

metadata = body["data"]["traces"][0]["metadata"]
assert metadata["semconvVersion"] == "1.39.0"

def test_unknown_shape_returns_load_warning(self):
unknown = json.dumps({"some_other_shape": []}).encode()
resp = self.client.post(
Expand Down
Loading