From aed711d8885b7138346ff009766caee748d57e93 Mon Sep 17 00:00:00 2001 From: Paulus Schoutsen Date: Wed, 27 May 2026 14:11:05 +0200 Subject: [PATCH] Add server-initiated-drift-injection scenario (#57) Adds the drift-injection scenario from the audit (Sendspin/conformance#57) and a `required_capability` mechanism on `ScenarioSpec` so scenarios can gate matrix participation on SDK-side surface area. The matching `supported_capabilities` tuple on `RoleSpec` lets each adapter declare which named capabilities it implements. - `stream-sync-drift-correction` is declared on the aiosendspin server (it can host the harness-level timestamp skew) and on no client. All client cases fail fast with a clear "does not expose the capability" reason, which is exactly the conformance signal the audit wants surfaced. The verification mode is `capability-only` until SDK implementations land and the test can compare recovered audio against a tolerance derived from the spec thresholds. This is the first of eight stacked PRs that wire the audit's recommended scenarios into the matrix. --- README.md | 1 + src/conformance/implementations.py | 1 + src/conformance/models.py | 32 ++++++++++++++++++++++++++---- src/conformance/runner.py | 5 +++++ src/conformance/scenarios.py | 19 ++++++++++++++++++ 5 files changed, 54 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e246170..6aaa7ea 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ Current scenarios: - `server-initiated-controller` (Server initiates connection and client wants Controller): start the server first, let the client advertise a listener, let the server connect in, observe controller state, send a control command, and verify the server recorded it - `server-initiated-flac` (Server initiates connection and client wants FLAC): start the server first with PCM audio decoded from `almost_silent.flac`, let the client advertise a listener and FLAC as its only supported audio format, let the server connect in, encode the PCM to FLAC using the SDK, stream it to the client, and compare the transported FLAC bytes - `server-initiated-opus` (Server initiates connection and client wants OPUS): start the server first with PCM audio decoded from `almost_silent.flac`, let the client advertise a listener and OPUS as its only supported audio format, let the server connect in, encode the PCM to OPUS using the SDK, stream it to the client, and compare the transported OPUS bytes +- `server-initiated-drift-injection` (Server injects timestamp drift mid-stream): start the server first, then the client. The server deliberately skews audio chunk timestamps partway through the stream; a conformant client recovers the original audio within the spec's soft/hard correction thresholds. No SDK in the matrix declares the `stream-sync-drift-correction` capability yet, so every case currently fails fast and the matrix surfaces the gap ## Current coverage diff --git a/src/conformance/implementations.py b/src/conformance/implementations.py index 42f07e5..8e937ea 100644 --- a/src/conformance/implementations.py +++ b/src/conformance/implementations.py @@ -61,6 +61,7 @@ supports_opus=True, supports_discovery=True, supported_role_families=("player", "metadata", "controller", "artwork"), + supported_capabilities=("stream-sync-drift-correction",), ), ), "sendspin-dotnet": ImplementationSpec( diff --git a/src/conformance/models.py b/src/conformance/models.py index 9eab1d5..b16349d 100644 --- a/src/conformance/models.py +++ b/src/conformance/models.py @@ -11,7 +11,14 @@ InitiatorRole = Literal["server", "client"] RoleName = Literal["server", "client"] RoleFamily = Literal["player", "metadata", "controller", "artwork"] -VerificationMode = Literal["audio-pcm", "audio-encoded-bytes", "metadata", "controller", "artwork"] +VerificationMode = Literal[ + "audio-pcm", + "audio-encoded-bytes", + "metadata", + "controller", + "artwork", + "capability-only", +] @dataclass(frozen=True) @@ -28,6 +35,7 @@ class RoleSpec: supports_opus: bool = False supports_discovery: bool = False supported_role_families: tuple[RoleFamily, ...] = () + supported_capabilities: tuple[str, ...] = () reason: str | None = None def supports_initiator(self, initiator_role: InitiatorRole) -> bool: @@ -49,6 +57,12 @@ def supports_role_families(self, role_families: tuple[RoleFamily, ...]) -> bool: supported = set(self.supported_role_families) return all(role_family in supported for role_family in role_families) + def supports_capability(self, capability: str | None) -> bool: + """Return whether this role exposes the SDK surface a scenario requires.""" + if capability is None: + return True + return capability in self.supported_capabilities + def unsupported_reason( self, *, @@ -57,9 +71,12 @@ def unsupported_reason( scenario: "ScenarioSpec", ) -> str | None: """Explain why this role cannot execute the scenario.""" - if self.supports_initiator(scenario.initiator_role) and self.supports_codec( - scenario.preferred_codec - ) and self.supports_role_families(scenario.required_role_families): + if ( + self.supports_initiator(scenario.initiator_role) + and self.supports_codec(scenario.preferred_codec) + and self.supports_role_families(scenario.required_role_families) + and self.supports_capability(scenario.required_capability) + ): return None if scenario.initiator_role == "server": @@ -85,6 +102,12 @@ def unsupported_reason( f"{missing_display} required by {scenario.id}." ) + if not self.supports_capability(scenario.required_capability): + return ( + f"{implementation} {role} adapter does not expose the " + f"{scenario.required_capability!r} capability required by {scenario.id}." + ) + return ( f"{implementation} {role} adapter does not support " f"{scenario.preferred_codec.upper()} transport required by {scenario.id}." @@ -115,6 +138,7 @@ class ScenarioSpec: required_role_families: tuple[RoleFamily, ...] verification_mode: VerificationMode extra_cli_args: tuple[tuple[str, str], ...] = field(default_factory=tuple) + required_capability: str | None = None def cli_args(self) -> dict[str, str]: """Return scenario-wide CLI arguments passed to both roles.""" diff --git a/src/conformance/runner.py b/src/conformance/runner.py index 38eeff6..f3d8b2e 100644 --- a/src/conformance/runner.py +++ b/src/conformance/runner.py @@ -742,6 +742,11 @@ def _compare_summaries( return _compare_controller_summaries(server_summary, client_summary) if scenario.verification_mode == "artwork": return _compare_artwork_summaries(server_summary, client_summary) + if scenario.verification_mode == "capability-only": + # Reserved for scenarios whose only purpose today is to surface SDK capability + # gaps. No SDK in the matrix supports them yet, so every case fails fast via + # the capability check before reaching this comparison. + return False, "No SDK in the matrix supports this scenario yet" raise ValueError(f"Unsupported verification mode: {scenario.verification_mode}") diff --git a/src/conformance/scenarios.py b/src/conformance/scenarios.py index e8a4a4f..0fd0b22 100644 --- a/src/conformance/scenarios.py +++ b/src/conformance/scenarios.py @@ -141,6 +141,24 @@ ) +SERVER_INITIATED_DRIFT_INJECTION = ScenarioSpec( + id="server-initiated-drift-injection", + display_name="Server injects timestamp drift mid-stream", + description=( + "Start the server first, then the client. The server deliberately skews its " + "audio chunk timestamps by a known amount partway through the stream. A " + "conformant client recovers the original audio within the spec's soft/hard " + "correction thresholds; a non-conformant client either drops audio or " + "produces a hash that diverges from the fixture by more than tolerance." + ), + initiator_role="server", + preferred_codec="pcm", + required_role_families=("player",), + verification_mode="capability-only", + required_capability="stream-sync-drift-correction", +) + + SCENARIO_LIST: tuple[ScenarioSpec, ...] = ( CLIENT_INITIATED_PCM, SERVER_INITIATED_PCM, @@ -149,6 +167,7 @@ SERVER_INITIATED_CONTROLLER, SERVER_INITIATED_FLAC, SERVER_INITIATED_OPUS, + SERVER_INITIATED_DRIFT_INJECTION, ) SCENARIOS: dict[str, ScenarioSpec] = {scenario.id: scenario for scenario in SCENARIO_LIST}