Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ Current scenarios:
- `server-initiated-controller` (Server initiates connection and client wants Controller): start the server first, let the client advertise a listener, let the server connect in, observe controller state, send a control command, and verify the server recorded it
- `server-initiated-flac` (Server initiates connection and client wants FLAC): start the server first with PCM audio decoded from `almost_silent.flac`, let the client advertise a listener and FLAC as its only supported audio format, let the server connect in, encode the PCM to FLAC using the SDK, stream it to the client, and compare the transported FLAC bytes
- `server-initiated-opus` (Server initiates connection and client wants OPUS): start the server first with PCM audio decoded from `almost_silent.flac`, let the client advertise a listener and OPUS as its only supported audio format, let the server connect in, encode the PCM to OPUS using the SDK, stream it to the client, and compare the transported OPUS bytes
- `server-initiated-drift-injection` (Server injects timestamp drift mid-stream): start the server first, then the client. The server deliberately skews audio chunk timestamps partway through the stream; a conformant client recovers the original audio within the spec's soft/hard correction thresholds. No SDK in the matrix declares the `stream-sync-drift-correction` capability yet, so every case currently fails fast and the matrix surfaces the gap

## Current coverage

Expand Down
1 change: 1 addition & 0 deletions src/conformance/implementations.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@
supports_opus=True,
supports_discovery=True,
supported_role_families=("player", "metadata", "controller", "artwork"),
supported_capabilities=("stream-sync-drift-correction",),
),
),
"sendspin-dotnet": ImplementationSpec(
Expand Down
32 changes: 28 additions & 4 deletions src/conformance/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,14 @@
InitiatorRole = Literal["server", "client"]
RoleName = Literal["server", "client"]
RoleFamily = Literal["player", "metadata", "controller", "artwork"]
VerificationMode = Literal["audio-pcm", "audio-encoded-bytes", "metadata", "controller", "artwork"]
VerificationMode = Literal[
"audio-pcm",
"audio-encoded-bytes",
"metadata",
"controller",
"artwork",
"capability-only",
]


@dataclass(frozen=True)
Expand All @@ -28,6 +35,7 @@ class RoleSpec:
supports_opus: bool = False
supports_discovery: bool = False
supported_role_families: tuple[RoleFamily, ...] = ()
supported_capabilities: tuple[str, ...] = ()
reason: str | None = None

def supports_initiator(self, initiator_role: InitiatorRole) -> bool:
Expand All @@ -49,6 +57,12 @@ def supports_role_families(self, role_families: tuple[RoleFamily, ...]) -> bool:
supported = set(self.supported_role_families)
return all(role_family in supported for role_family in role_families)

def supports_capability(self, capability: str | None) -> bool:
"""Return whether this role exposes the SDK surface a scenario requires."""
if capability is None:
return True
return capability in self.supported_capabilities

def unsupported_reason(
self,
*,
Expand All @@ -57,9 +71,12 @@ def unsupported_reason(
scenario: "ScenarioSpec",
) -> str | None:
"""Explain why this role cannot execute the scenario."""
if self.supports_initiator(scenario.initiator_role) and self.supports_codec(
scenario.preferred_codec
) and self.supports_role_families(scenario.required_role_families):
if (
self.supports_initiator(scenario.initiator_role)
and self.supports_codec(scenario.preferred_codec)
and self.supports_role_families(scenario.required_role_families)
and self.supports_capability(scenario.required_capability)
):
return None

if scenario.initiator_role == "server":
Expand All @@ -85,6 +102,12 @@ def unsupported_reason(
f"{missing_display} required by {scenario.id}."
)

if not self.supports_capability(scenario.required_capability):
return (
f"{implementation} {role} adapter does not expose the "
f"{scenario.required_capability!r} capability required by {scenario.id}."
)

return (
f"{implementation} {role} adapter does not support "
f"{scenario.preferred_codec.upper()} transport required by {scenario.id}."
Expand Down Expand Up @@ -115,6 +138,7 @@ class ScenarioSpec:
required_role_families: tuple[RoleFamily, ...]
verification_mode: VerificationMode
extra_cli_args: tuple[tuple[str, str], ...] = field(default_factory=tuple)
required_capability: str | None = None

def cli_args(self) -> dict[str, str]:
"""Return scenario-wide CLI arguments passed to both roles."""
Expand Down
5 changes: 5 additions & 0 deletions src/conformance/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -742,6 +742,11 @@ def _compare_summaries(
return _compare_controller_summaries(server_summary, client_summary)
if scenario.verification_mode == "artwork":
return _compare_artwork_summaries(server_summary, client_summary)
if scenario.verification_mode == "capability-only":
# Reserved for scenarios whose only purpose today is to surface SDK capability
# gaps. No SDK in the matrix supports them yet, so every case fails fast via
# the capability check before reaching this comparison.
return False, "No SDK in the matrix supports this scenario yet"
raise ValueError(f"Unsupported verification mode: {scenario.verification_mode}")


Expand Down
19 changes: 19 additions & 0 deletions src/conformance/scenarios.py
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,24 @@
)


SERVER_INITIATED_DRIFT_INJECTION = ScenarioSpec(
id="server-initiated-drift-injection",
display_name="Server injects timestamp drift mid-stream",
description=(
"Start the server first, then the client. The server deliberately skews its "
"audio chunk timestamps by a known amount partway through the stream. A "
"conformant client recovers the original audio within the spec's soft/hard "
"correction thresholds; a non-conformant client either drops audio or "
"produces a hash that diverges from the fixture by more than tolerance."
),
initiator_role="server",
preferred_codec="pcm",
required_role_families=("player",),
verification_mode="capability-only",
required_capability="stream-sync-drift-correction",
)


SCENARIO_LIST: tuple[ScenarioSpec, ...] = (
CLIENT_INITIATED_PCM,
SERVER_INITIATED_PCM,
Expand All @@ -149,6 +167,7 @@
SERVER_INITIATED_CONTROLLER,
SERVER_INITIATED_FLAC,
SERVER_INITIATED_OPUS,
SERVER_INITIATED_DRIFT_INJECTION,
)

SCENARIOS: dict[str, ScenarioSpec] = {scenario.id: scenario for scenario in SCENARIO_LIST}
Expand Down
Loading