diff --git a/README.md b/README.md index e246170..6aaa7ea 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ Current scenarios: - `server-initiated-controller` (Server initiates connection and client wants Controller): start the server first, let the client advertise a listener, let the server connect in, observe controller state, send a control command, and verify the server recorded it - `server-initiated-flac` (Server initiates connection and client wants FLAC): start the server first with PCM audio decoded from `almost_silent.flac`, let the client advertise a listener and FLAC as its only supported audio format, let the server connect in, encode the PCM to FLAC using the SDK, stream it to the client, and compare the transported FLAC bytes - `server-initiated-opus` (Server initiates connection and client wants OPUS): start the server first with PCM audio decoded from `almost_silent.flac`, let the client advertise a listener and OPUS as its only supported audio format, let the server connect in, encode the PCM to OPUS using the SDK, stream it to the client, and compare the transported OPUS bytes +- `server-initiated-drift-injection` (Server injects timestamp drift mid-stream): start the server first, then the client. The server deliberately skews audio chunk timestamps partway through the stream; a conformant client recovers the original audio within the spec's soft/hard correction thresholds. No SDK in the matrix declares the `stream-sync-drift-correction` capability yet, so every case currently fails fast and the matrix surfaces the gap ## Current coverage diff --git a/src/conformance/implementations.py b/src/conformance/implementations.py index 42f07e5..8e937ea 100644 --- a/src/conformance/implementations.py +++ b/src/conformance/implementations.py @@ -61,6 +61,7 @@ supports_opus=True, supports_discovery=True, supported_role_families=("player", "metadata", "controller", "artwork"), + supported_capabilities=("stream-sync-drift-correction",), ), ), "sendspin-dotnet": ImplementationSpec( diff --git a/src/conformance/models.py b/src/conformance/models.py index 9eab1d5..b16349d 100644 --- a/src/conformance/models.py +++ b/src/conformance/models.py @@ -11,7 +11,14 @@ InitiatorRole = Literal["server", "client"] RoleName = Literal["server", "client"] RoleFamily = Literal["player", "metadata", "controller", "artwork"] -VerificationMode = Literal["audio-pcm", "audio-encoded-bytes", "metadata", "controller", "artwork"] +VerificationMode = Literal[ + "audio-pcm", + "audio-encoded-bytes", + "metadata", + "controller", + "artwork", + "capability-only", +] @dataclass(frozen=True) @@ -28,6 +35,7 @@ class RoleSpec: supports_opus: bool = False supports_discovery: bool = False supported_role_families: tuple[RoleFamily, ...] = () + supported_capabilities: tuple[str, ...] = () reason: str | None = None def supports_initiator(self, initiator_role: InitiatorRole) -> bool: @@ -49,6 +57,12 @@ def supports_role_families(self, role_families: tuple[RoleFamily, ...]) -> bool: supported = set(self.supported_role_families) return all(role_family in supported for role_family in role_families) + def supports_capability(self, capability: str | None) -> bool: + """Return whether this role exposes the SDK surface a scenario requires.""" + if capability is None: + return True + return capability in self.supported_capabilities + def unsupported_reason( self, *, @@ -57,9 +71,12 @@ def unsupported_reason( scenario: "ScenarioSpec", ) -> str | None: """Explain why this role cannot execute the scenario.""" - if self.supports_initiator(scenario.initiator_role) and self.supports_codec( - scenario.preferred_codec - ) and self.supports_role_families(scenario.required_role_families): + if ( + self.supports_initiator(scenario.initiator_role) + and self.supports_codec(scenario.preferred_codec) + and self.supports_role_families(scenario.required_role_families) + and self.supports_capability(scenario.required_capability) + ): return None if scenario.initiator_role == "server": @@ -85,6 +102,12 @@ def unsupported_reason( f"{missing_display} required by {scenario.id}." ) + if not self.supports_capability(scenario.required_capability): + return ( + f"{implementation} {role} adapter does not expose the " + f"{scenario.required_capability!r} capability required by {scenario.id}." + ) + return ( f"{implementation} {role} adapter does not support " f"{scenario.preferred_codec.upper()} transport required by {scenario.id}." @@ -115,6 +138,7 @@ class ScenarioSpec: required_role_families: tuple[RoleFamily, ...] verification_mode: VerificationMode extra_cli_args: tuple[tuple[str, str], ...] = field(default_factory=tuple) + required_capability: str | None = None def cli_args(self) -> dict[str, str]: """Return scenario-wide CLI arguments passed to both roles.""" diff --git a/src/conformance/runner.py b/src/conformance/runner.py index 38eeff6..f3d8b2e 100644 --- a/src/conformance/runner.py +++ b/src/conformance/runner.py @@ -742,6 +742,11 @@ def _compare_summaries( return _compare_controller_summaries(server_summary, client_summary) if scenario.verification_mode == "artwork": return _compare_artwork_summaries(server_summary, client_summary) + if scenario.verification_mode == "capability-only": + # Reserved for scenarios whose only purpose today is to surface SDK capability + # gaps. No SDK in the matrix supports them yet, so every case fails fast via + # the capability check before reaching this comparison. + return False, "No SDK in the matrix supports this scenario yet" raise ValueError(f"Unsupported verification mode: {scenario.verification_mode}") diff --git a/src/conformance/scenarios.py b/src/conformance/scenarios.py index e8a4a4f..0fd0b22 100644 --- a/src/conformance/scenarios.py +++ b/src/conformance/scenarios.py @@ -141,6 +141,24 @@ ) +SERVER_INITIATED_DRIFT_INJECTION = ScenarioSpec( + id="server-initiated-drift-injection", + display_name="Server injects timestamp drift mid-stream", + description=( + "Start the server first, then the client. The server deliberately skews its " + "audio chunk timestamps by a known amount partway through the stream. A " + "conformant client recovers the original audio within the spec's soft/hard " + "correction thresholds; a non-conformant client either drops audio or " + "produces a hash that diverges from the fixture by more than tolerance." + ), + initiator_role="server", + preferred_codec="pcm", + required_role_families=("player",), + verification_mode="capability-only", + required_capability="stream-sync-drift-correction", +) + + SCENARIO_LIST: tuple[ScenarioSpec, ...] = ( CLIENT_INITIATED_PCM, SERVER_INITIATED_PCM, @@ -149,6 +167,7 @@ SERVER_INITIATED_CONTROLLER, SERVER_INITIATED_FLAC, SERVER_INITIATED_OPUS, + SERVER_INITIATED_DRIFT_INJECTION, ) SCENARIOS: dict[str, ScenarioSpec] = {scenario.id: scenario for scenario in SCENARIO_LIST}