From af3baaef7ee117d5cc9b1f49d07e224efe2632c9 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 14 Mar 2026 01:12:12 +0000 Subject: [PATCH 01/82] Increment Version to 0.13.1 --- ovoscope/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovoscope/version.py b/ovoscope/version.py index 3b482a3..d18db45 100644 --- a/ovoscope/version.py +++ b/ovoscope/version.py @@ -2,7 +2,7 @@ VERSION_MAJOR = 0 VERSION_MINOR = 13 VERSION_BUILD = 1 -VERSION_ALPHA = 1 +VERSION_ALPHA = 0 # END_VERSION_BLOCK __version__ = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}" + ( From d0b58b686043197de1d0e832ddd68731cd706695 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Thu, 14 May 2026 22:52:08 +0100 Subject: [PATCH 02/82] Merge pull request #54 from TigreGotico/feat/nebulento-palavreado-pipelines feat: add NEBULENTO_PIPELINE and PALAVREADO_PIPELINE stage groups --- ovoscope/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ovoscope/__init__.py b/ovoscope/__init__.py index 3d88df6..d346738 100644 --- a/ovoscope/__init__.py +++ b/ovoscope/__init__.py @@ -75,6 +75,11 @@ "ovos-m2v-pipeline-medium", "ovos-m2v-pipeline-low", ] +# Nebulento โ€” fuzzy intent matching (ConfidenceMatcherPipeline). Single OPM +# entry point; the pipeline manager handles confidence-tier routing. +NEBULENTO_PIPELINE = ["ovos-nebulento-pipeline-plugin"] +# Palavreado โ€” keyword/slot intent parser (ConfidenceMatcherPipeline). +PALAVREADO_PIPELINE = ["palavreado"] # Standard test pipeline โ€” all standard built-in stages. # This requires ovos-adapt-pipeline-plugin and ovos-padatious-pipeline-plugin. From ad74906ed9b02af34ef18d94ed481e5a8a5b59c8 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Thu, 14 May 2026 21:52:18 +0000 Subject: [PATCH 03/82] Increment Version to 0.14.0a1 --- ovoscope/version.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ovoscope/version.py b/ovoscope/version.py index d18db45..ed3d0b8 100644 --- a/ovoscope/version.py +++ b/ovoscope/version.py @@ -1,8 +1,8 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 -VERSION_MINOR = 13 -VERSION_BUILD = 1 -VERSION_ALPHA = 0 +VERSION_MINOR = 14 +VERSION_BUILD = 0 +VERSION_ALPHA = 1 # END_VERSION_BLOCK __version__ = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}" + ( From 654ef16d25abefc8712db0084e3fd98f559031f7 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Thu, 14 May 2026 21:52:53 +0000 Subject: [PATCH 04/82] Update Changelog --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a2ec47..fcc89ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,12 @@ # Changelog -## [0.13.1a1](https://github.com/TigreGotico/ovoscope/tree/0.13.1a1) (2026-03-14) +## [0.14.0a1](https://github.com/TigreGotico/ovoscope/tree/0.14.0a1) (2026-05-14) -[Full Changelog](https://github.com/TigreGotico/ovoscope/compare/0.13.0...0.13.1a1) +[Full Changelog](https://github.com/TigreGotico/ovoscope/compare/0.13.1...0.14.0a1) **Merged pull requests:** -- fix: thread names in bus coverage report [\#52](https://github.com/TigreGotico/ovoscope/pull/52) ([JarbasAl](https://github.com/JarbasAl)) +- feat: add NEBULENTO\_PIPELINE and PALAVREADO\_PIPELINE stage groups [\#54](https://github.com/TigreGotico/ovoscope/pull/54) ([JarbasAl](https://github.com/JarbasAl)) From a01c5c9466738a28f44af0d617430d5c01ebc507 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Thu, 14 May 2026 22:56:50 +0100 Subject: [PATCH 05/82] feat(e2e): reusable harness, bus helpers, and intent-registration shims (#55) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ConfidenceMatcherPipeline plugins (nebulento, palavreado, padacioso, padatious, adapt, โ€ฆ) all reproduce roughly the same end-to-end test boilerplate: spin up MiniCroft pinned to one pipeline, mutate Configuration()["intents"][config_key], emit utterances, capture either the dispatched intent Message or complete_intent_failure, then restore. This change extracts that shape into ovoscope so a plugin author can focus on engine-specific behaviour. New module ovoscope/e2e.py: - E2EPipelineHarness: unittest.TestCase base. Subclasses declare PIPELINE_ID, CONFIG_KEY, PLUGIN_CONFIG, SKILL_ID and inherit send_and_capture / expect_no_match / make_utterance, plus Configuration save+restore and per-test skill detach. - Standalone bus helpers (no MiniCroft required): make_session, make_utterance_message, wait_for_match, wait_for_failure. - Engine-family registration shims: register_padatious_intent / register_padatious_entity (padatious, padacioso, nebulento, ...) register_adapt_vocab / register_adapt_intent (adapt, palavreado, ...) detach_intent / detach_skill (generic). All names are re-exported from the top-level `ovoscope` package. A focused unit-test module exercises every helper against a FakeBus in well under a second; no MiniCroft startup is required. ๐Ÿค– Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 4.7 --- ovoscope/__init__.py | 18 ++ ovoscope/e2e.py | 362 +++++++++++++++++++++++++++++ test/unittests/test_e2e_helpers.py | 171 ++++++++++++++ 3 files changed, 551 insertions(+) create mode 100644 ovoscope/e2e.py create mode 100644 test/unittests/test_e2e_helpers.py diff --git a/ovoscope/__init__.py b/ovoscope/__init__.py index d346738..08d8542 100644 --- a/ovoscope/__init__.py +++ b/ovoscope/__init__.py @@ -1281,3 +1281,21 @@ def assert_namespace_cleared(self, namespace: str) -> None: f"Expected namespace {namespace!r} to be cleared, " f"but no matching message was captured." ) + + +# --------------------------------------------------------------------------- +# Public re-exports โ€” see ovoscope/e2e.py for full docs +# --------------------------------------------------------------------------- +from ovoscope.e2e import ( # noqa: E402,F401 + E2EPipelineHarness, + detach_intent, + detach_skill, + make_session, + make_utterance_message, + register_adapt_intent, + register_adapt_vocab, + register_padatious_entity, + register_padatious_intent, + wait_for_failure, + wait_for_match, +) diff --git a/ovoscope/e2e.py b/ovoscope/e2e.py new file mode 100644 index 0000000..04e1672 --- /dev/null +++ b/ovoscope/e2e.py @@ -0,0 +1,362 @@ +"""End-to-end test scaffolding for ConfidenceMatcherPipeline plugins. + +Most pipeline plugins (Adapt, Padatious, Padacioso, Nebulento, Palavreado, โ€ฆ) +need the same end-to-end shape: + +1. Mutate ``Configuration()["intents"][]`` with a per-test config. +2. Spin up a ``MiniCroft`` pinned to that one pipeline. +3. Drive the bus with utterances and capture the dispatched intent message + (or the ``complete_intent_failure`` signal). +4. Tear everything down without leaking state into the next test class. + +This module factors out that shape so a plugin only has to subclass +``E2EPipelineHarness`` and declare a handful of class attributes. It also +exposes the standalone bus helpers (``wait_for_match``, +``make_utterance_message``, โ€ฆ) and engine-family registration shims +(``register_padatious_intent``, ``register_adapt_intent``, โ€ฆ) for the cases +where pytest-style tests are preferred over ``unittest``. +""" +from __future__ import annotations + +import threading +import time +import unittest +from typing import Any, ClassVar, Dict, List, Optional + +from ovos_bus_client.message import Message +from ovos_bus_client.session import Session +from ovos_config.config import Configuration + + +# --------------------------------------------------------------------------- +# Standalone bus helpers (work with any FakeBus / MessageBusClient) +# --------------------------------------------------------------------------- + +def make_session( + session_id: str = "ovoscope-test", + *, + pipeline: Optional[List[str]] = None, + blacklisted_intents: Optional[List[str]] = None, + blacklisted_skills: Optional[List[str]] = None, + lang: str = "en-US", +) -> Session: + """Build a ``Session`` with the most common overrides preset.""" + kwargs: Dict[str, Any] = {"session_id": session_id, "lang": lang} + if pipeline is not None: + kwargs["pipeline"] = pipeline + if blacklisted_intents is not None: + kwargs["blacklisted_intents"] = blacklisted_intents + if blacklisted_skills is not None: + kwargs["blacklisted_skills"] = blacklisted_skills + return Session(**kwargs) + + +def make_utterance_message( + utterance: str, + *, + lang: str = "en-US", + session: Optional[Session] = None, +) -> Message: + """Build a ``recognizer_loop:utterance`` Message, optional Session override.""" + ctx: Dict[str, Any] = {} + if session is not None: + ctx["session"] = session.serialize() + return Message( + "recognizer_loop:utterance", + data={"utterances": [utterance], "lang": lang}, + context=ctx, + ) + + +def wait_for_match( + bus, + expected_types: List[str], + *, + timeout: float = 5.0, +) -> Optional[Message]: + """Subscribe to ``expected_types`` and ``complete_intent_failure``; return + the first match Message, or ``None`` on failure / timeout. + + The caller is responsible for emitting the utterance *after* calling this + helper if used in a pytest style โ€” for the ``unittest`` style use + :meth:`E2EPipelineHarness.send_and_capture` which emits internally. + """ + got: List[Message] = [] + done = threading.Event() + failed = threading.Event() + + def _on_match(msg: Message) -> None: + got.append(msg) + done.set() + + def _on_fail(_msg: Message) -> None: + failed.set() + done.set() + + for t in expected_types: + bus.on(t, _on_match) + bus.on("complete_intent_failure", _on_fail) + try: + done.wait(timeout=timeout) + finally: + for t in expected_types: + bus.remove(t, _on_match) + bus.remove("complete_intent_failure", _on_fail) + if failed.is_set() and not got: + return None + return got[0] if got else None + + +def wait_for_failure(bus, *, timeout: float = 2.0) -> bool: + """Wait for a ``complete_intent_failure`` Message; return whether one fired.""" + failed = threading.Event() + + def _on_fail(_msg: Message) -> None: + failed.set() + + bus.on("complete_intent_failure", _on_fail) + try: + failed.wait(timeout=timeout) + finally: + bus.remove("complete_intent_failure", _on_fail) + return failed.is_set() + + +# --------------------------------------------------------------------------- +# Intent-registration shims โ€” emit the bus event a given engine family expects +# --------------------------------------------------------------------------- +# +# Padatious family (padatious, padacioso, nebulento, โ€ฆ) โ€” registers intents by +# emitting ``padatious:register_intent`` with inline ``samples``. +# Adapt family (adapt, palavreado, โ€ฆ) โ€” registers vocab + IntentBuilder. + +def register_padatious_intent( + bus, name: str, samples: List[str], *, lang: str = "en-US", + settle: float = 0.1, +) -> None: + bus.emit(Message("padatious:register_intent", { + "name": name, "samples": samples, "lang": lang, + })) + if settle: + time.sleep(settle) + + +def register_padatious_entity( + bus, name: str, samples: List[str], *, lang: str = "en-US", + settle: float = 0.1, +) -> None: + bus.emit(Message("padatious:register_entity", { + "name": name, "samples": samples, "lang": lang, + })) + if settle: + time.sleep(settle) + + +def register_adapt_vocab( + bus, entity_type: str, words: List[str], *, lang: str = "en-US", + settle: float = 0.1, +) -> None: + for word in words: + bus.emit(Message("register_vocab", { + "entity_value": word, "entity_type": entity_type, "lang": lang, + })) + if settle: + time.sleep(settle) + + +def register_adapt_intent(bus, builder, *, lang: str = "en-US", + settle: float = 0.1) -> None: + """Register an Adapt intent. + + ``builder`` may be an ``IntentBuilder`` (will be ``.build()``-ed) or an + already-built intent with a ``__dict__`` payload. + """ + intent = builder.build() if hasattr(builder, "build") else builder + msg = Message("register_intent", intent.__dict__) + msg.context["lang"] = lang + bus.emit(msg) + if settle: + time.sleep(settle) + + +def detach_intent(bus, intent_name: str, *, settle: float = 0.1) -> None: + bus.emit(Message("detach_intent", {"intent_name": intent_name})) + if settle: + time.sleep(settle) + + +def detach_skill(bus, skill_id: str, *, settle: float = 0.1) -> None: + bus.emit(Message("detach_skill", {"skill_id": skill_id})) + if settle: + time.sleep(settle) + + +# --------------------------------------------------------------------------- +# unittest.TestCase harness +# --------------------------------------------------------------------------- + +class E2EPipelineHarness(unittest.TestCase): + """Base class for end-to-end tests of a single pipeline plugin. + + Subclass and set the four class attributes: + + ``PIPELINE_ID`` + OPM ``opm.pipeline`` entry-point name to pin MiniCroft to + (e.g. ``"ovos-nebulento-pipeline-plugin"``). + ``CONFIG_KEY`` + Key under ``Configuration()["intents"]`` for plugin config. + ``PLUGIN_CONFIG`` + Dict merged into ``Configuration()["intents"][CONFIG_KEY]`` before + MiniCroft starts. Restored on teardown. + ``SKILL_ID`` + Skill id used by helpers when registering intents. Detached in + ``setUp`` to keep tests isolated. + + The harness then exposes: + + - ``self.mc`` โ€” the running ``MiniCroft`` + - ``self.bus`` โ€” shortcut to ``self.mc.bus`` + - ``self.pipeline`` โ€” the loaded pipeline plugin instance + - ``self.send_and_capture(utterance, expected_types, โ€ฆ)`` + - ``self.expect_no_match(utterance, โ€ฆ)`` + - ``self.make_utterance(utterance, session=โ€ฆ)`` + """ + + PIPELINE_ID: ClassVar[str] = "" + CONFIG_KEY: ClassVar[str] = "" + PLUGIN_CONFIG: ClassVar[Dict[str, Any]] = {} + SKILL_ID: ClassVar[str] = "test_skill_ovoscope" + DEFAULT_LANG: ClassVar[str] = "en-US" + STARTUP_MAX_WAIT: ClassVar[float] = 60.0 + + mc: ClassVar[Any] + pipeline: ClassVar[Any] + _orig_intents_cfg: ClassVar[Any] = None + + @classmethod + def setUpClass(cls) -> None: + if not cls.PIPELINE_ID or not cls.CONFIG_KEY: + raise unittest.SkipTest( + f"{cls.__name__} must set PIPELINE_ID and CONFIG_KEY" + ) + # Import here so importing this module does not pull in MiniCroft. + from ovoscope import get_minicroft + + cfg = Configuration() + intents_cfg = cfg.setdefault("intents", {}) + cls._orig_intents_cfg = intents_cfg.get(cls.CONFIG_KEY) + intents_cfg[cls.CONFIG_KEY] = dict(cls.PLUGIN_CONFIG or {}) + + cls.mc = get_minicroft( + skill_ids=[], + lang=cls.DEFAULT_LANG, + default_pipeline=[cls.PIPELINE_ID], + max_wait=cls.STARTUP_MAX_WAIT, + ) + cls.pipeline = cls.mc.intents.pipeline_plugins[cls.PIPELINE_ID] + + @classmethod + def tearDownClass(cls) -> None: + try: + cls.mc.stop() + finally: + cfg = Configuration() + intents_cfg = cfg.get("intents", {}) + if cls._orig_intents_cfg is None: + intents_cfg.pop(cls.CONFIG_KEY, None) + else: + intents_cfg[cls.CONFIG_KEY] = cls._orig_intents_cfg + + @property + def bus(self): + return self.mc.bus + + def setUp(self) -> None: + # Isolate tests by detaching this skill_id's registrations. + detach_skill(self.bus, self.SKILL_ID) + + # -- helpers -------------------------------------------------------- + + def make_utterance(self, utterance: str, *, + session: Optional[Session] = None) -> Message: + return make_utterance_message( + utterance, lang=self.DEFAULT_LANG, session=session + ) + + def send_and_capture( + self, + utterance: str, + expected_types: List[str], + *, + timeout: float = 5.0, + session: Optional[Session] = None, + ) -> Optional[Message]: + """Emit ``utterance`` and return the first match Message (or None).""" + got: List[Message] = [] + done = threading.Event() + failed = threading.Event() + + def _on_match(msg: Message) -> None: + got.append(msg) + done.set() + + def _on_fail(_msg: Message) -> None: + failed.set() + done.set() + + for t in expected_types: + self.bus.on(t, _on_match) + self.bus.on("complete_intent_failure", _on_fail) + try: + self.bus.emit(self.make_utterance(utterance, session=session)) + done.wait(timeout=timeout) + finally: + for t in expected_types: + self.bus.remove(t, _on_match) + self.bus.remove("complete_intent_failure", _on_fail) + if failed.is_set() and not got: + return None + return got[0] if got else None + + def expect_no_match( + self, + utterance: str, + *, + timeout: float = 2.0, + session: Optional[Session] = None, + ) -> None: + """Assert that emitting ``utterance`` produces a ``complete_intent_failure``.""" + failed = threading.Event() + + def _on_fail(_msg: Message) -> None: + failed.set() + + self.bus.on("complete_intent_failure", _on_fail) + try: + self.bus.emit(self.make_utterance(utterance, session=session)) + failed.wait(timeout=timeout) + finally: + self.bus.remove("complete_intent_failure", _on_fail) + self.assertTrue( + failed.is_set(), + f"Expected no match for {utterance!r} but no " + f"complete_intent_failure was emitted.", + ) + + +__all__ = [ + # bus helpers + "make_session", + "make_utterance_message", + "wait_for_match", + "wait_for_failure", + # registration shims + "register_padatious_intent", + "register_padatious_entity", + "register_adapt_vocab", + "register_adapt_intent", + "detach_intent", + "detach_skill", + # harness + "E2EPipelineHarness", +] diff --git a/test/unittests/test_e2e_helpers.py b/test/unittests/test_e2e_helpers.py new file mode 100644 index 0000000..776bd57 --- /dev/null +++ b/test/unittests/test_e2e_helpers.py @@ -0,0 +1,171 @@ +"""Fast unit tests for the engine-agnostic helpers in ``ovoscope.e2e``. + +These do not spin up MiniCroft โ€” they exercise the standalone helpers +against a plain ``FakeBus`` so they run in well under a second and can +catch regressions in the helper logic itself. +""" +import threading +import unittest +from unittest.mock import patch + +from ovos_bus_client.message import Message +from ovos_utils.fakebus import FakeBus + +from ovoscope.e2e import ( + detach_intent, + detach_skill, + make_session, + make_utterance_message, + register_adapt_vocab, + register_padatious_entity, + register_padatious_intent, + wait_for_failure, + wait_for_match, +) + + +class TestMakeSession(unittest.TestCase): + def test_only_session_id(self): + s = make_session("abc") + d = s.serialize() + self.assertEqual(d["session_id"], "abc") + + def test_pipeline_override(self): + s = make_session("x", pipeline=["foo", "bar"]) + self.assertEqual(s.serialize()["pipeline"], ["foo", "bar"]) + + def test_blacklists_round_trip(self): + s = make_session( + "y", + blacklisted_intents=["sk:hi"], + blacklisted_skills=["sk"], + ) + data = s.serialize() + self.assertEqual(data["blacklisted_intents"], ["sk:hi"]) + self.assertEqual(data["blacklisted_skills"], ["sk"]) + + +class TestMakeUtteranceMessage(unittest.TestCase): + def test_no_session(self): + m = make_utterance_message("hello there") + self.assertEqual(m.msg_type, "recognizer_loop:utterance") + self.assertEqual(m.data["utterances"], ["hello there"]) + self.assertEqual(m.data["lang"], "en-US") + self.assertNotIn("session", m.context) + + def test_with_session(self): + s = make_session("sid", pipeline=["pipeline-x"]) + m = make_utterance_message("hi", session=s) + self.assertIn("session", m.context) + self.assertEqual(m.context["session"]["pipeline"], ["pipeline-x"]) + + def test_lang_propagates(self): + m = make_utterance_message("oi", lang="pt-PT") + self.assertEqual(m.data["lang"], "pt-PT") + + +class TestRegistrationShims(unittest.TestCase): + def setUp(self): + self.bus = FakeBus() + self.captured = [] + # Wildcard-ish: subscribe to each known type and append. + for t in ( + "padatious:register_intent", + "padatious:register_entity", + "register_vocab", + "detach_intent", + "detach_skill", + ): + self.bus.on(t, self._append) + + def _append(self, msg): + self.captured.append(msg) + + def _types(self): + return [m.msg_type for m in self.captured] + + def test_register_padatious_intent_emits_event(self): + register_padatious_intent( + self.bus, "sk:hi", ["hi", "hello"], settle=0.0 + ) + self.assertEqual(self._types(), ["padatious:register_intent"]) + d = self.captured[0].data + self.assertEqual(d["name"], "sk:hi") + self.assertEqual(d["samples"], ["hi", "hello"]) + self.assertEqual(d["lang"], "en-US") + + def test_register_padatious_entity_emits_event(self): + register_padatious_entity( + self.bus, "item", ["milk", "bread"], settle=0.0 + ) + self.assertEqual(self._types(), ["padatious:register_entity"]) + self.assertEqual(self.captured[0].data["samples"], ["milk", "bread"]) + + def test_register_adapt_vocab_emits_one_event_per_word(self): + register_adapt_vocab( + self.bus, "sk:Light", ["light", "lamp", "bulb"], settle=0.0 + ) + self.assertEqual(self._types(), ["register_vocab"] * 3) + values = [m.data["entity_value"] for m in self.captured] + self.assertEqual(values, ["light", "lamp", "bulb"]) + for m in self.captured: + self.assertEqual(m.data["entity_type"], "sk:Light") + + def test_detach_helpers(self): + detach_intent(self.bus, "sk:hi", settle=0.0) + detach_skill(self.bus, "sk", settle=0.0) + self.assertEqual(self._types(), ["detach_intent", "detach_skill"]) + self.assertEqual(self.captured[0].data["intent_name"], "sk:hi") + self.assertEqual(self.captured[1].data["skill_id"], "sk") + + +class TestWaitHelpers(unittest.TestCase): + def setUp(self): + self.bus = FakeBus() + + def _emit_after(self, delay, msg): + timer = threading.Timer(delay, lambda: self.bus.emit(msg)) + timer.daemon = True + timer.start() + return timer + + def test_wait_for_match_returns_message(self): + self._emit_after(0.05, Message("sk:hello", {"x": 1})) + msg = wait_for_match(self.bus, ["sk:hello"], timeout=1.0) + self.assertIsNotNone(msg) + self.assertEqual(msg.msg_type, "sk:hello") + + def test_wait_for_match_returns_none_on_failure(self): + self._emit_after(0.05, Message("complete_intent_failure", {})) + msg = wait_for_match(self.bus, ["sk:hello"], timeout=1.0) + self.assertIsNone(msg) + + def test_wait_for_match_returns_none_on_timeout(self): + msg = wait_for_match(self.bus, ["sk:nope"], timeout=0.1) + self.assertIsNone(msg) + + def test_wait_for_failure_true_when_event_fires(self): + self._emit_after(0.05, Message("complete_intent_failure", {})) + self.assertTrue(wait_for_failure(self.bus, timeout=1.0)) + + def test_wait_for_failure_false_on_timeout(self): + self.assertFalse(wait_for_failure(self.bus, timeout=0.1)) + + +class TestHarnessClassValidation(unittest.TestCase): + """Without spinning MiniCroft, ensure missing PIPELINE_ID/CONFIG_KEY skips + rather than crashing. This protects against accidental abstract instantiation. + """ + + def test_missing_ids_raises_skip(self): + from ovoscope.e2e import E2EPipelineHarness + + class _Bad(E2EPipelineHarness): + pass + + with self.assertRaises(unittest.SkipTest): + _Bad.setUpClass() + + +if __name__ == "__main__": + unittest.main() From 1945fd28547c4f5307010a4ae4a880b072674046 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Thu, 14 May 2026 21:57:06 +0000 Subject: [PATCH 06/82] Increment Version to 0.15.0a1 --- ovoscope/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovoscope/version.py b/ovoscope/version.py index ed3d0b8..f8773cc 100644 --- a/ovoscope/version.py +++ b/ovoscope/version.py @@ -1,6 +1,6 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 -VERSION_MINOR = 14 +VERSION_MINOR = 15 VERSION_BUILD = 0 VERSION_ALPHA = 1 # END_VERSION_BLOCK From 26e208860b92a021d2471b91109f0ef8ad87c0e2 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Thu, 14 May 2026 21:57:29 +0000 Subject: [PATCH 07/82] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fcc89ff..753275b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.15.0a1](https://github.com/TigreGotico/ovoscope/tree/0.15.0a1) (2026-05-14) + +[Full Changelog](https://github.com/TigreGotico/ovoscope/compare/0.14.0a1...0.15.0a1) + +**Merged pull requests:** + +- feat\(e2e\): reusable harness, bus helpers, and intent-registration shims [\#55](https://github.com/TigreGotico/ovoscope/pull/55) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.14.0a1](https://github.com/TigreGotico/ovoscope/tree/0.14.0a1) (2026-05-14) [Full Changelog](https://github.com/TigreGotico/ovoscope/compare/0.13.1...0.14.0a1) From 1115a80dd92f11e6748141150128b7f88a9c98d4 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Thu, 14 May 2026 23:32:06 +0100 Subject: [PATCH 08/82] feat(intent-cases): file-based intent test layout + accuracy reporter (#58) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds ovoscope.intent_cases: skill authors describe expected intent routing in plain-text files under test/end2end/cases//.intent.test (positive) test/end2end/cases//no_match.test (negative) One utterance per line, '#' comments / blank lines ignored. Adding a phrase, intent, or whole new language is a pure text edit; no Python. API: - load_intent_cases(cases_dir) -> [IntentCase] - assert_intent_case(minicroft, skill_id, handlers, case, pipeline) - register_intent_case_tests(globals(), skill_id=..., handlers=..., cases_dir=...) โ€” one call in a test module generates TestCase classes for Padatious / Padacioso / M2V / DefaultPipeline, each with one method per (lang, utterance). A test passes if any tier of the pipeline family routes the utterance correctly, matching production cascade behaviour. Pytest plugin adds an intent-case accuracy reporter: --ovoscope-accuracy-report=PATH write JSON pivot --ovoscope-accuracy-min=RATIO fail session if overall < --ovoscope-accuracy-baseline=PATH fail session if accuracy drops vs a previous report --ovoscope-accuracy-tolerant downgrade individual case fails to xfail; only the aggregate gate can block the run. The terminal summary prints a per-(pipeline, lang, intent) pivot โ€” easy to wire into CI as a regression gate that blocks PRs lowering routing accuracy. Co-authored-by: Claude Opus 4.7 --- ovoscope/__init__.py | 8 + ovoscope/intent_cases.py | 382 ++++++++++++++++++++++++++++++++++++++ ovoscope/pytest_plugin.py | 236 ++++++++++++++++++++++- 3 files changed, 620 insertions(+), 6 deletions(-) create mode 100644 ovoscope/intent_cases.py diff --git a/ovoscope/__init__.py b/ovoscope/__init__.py index 08d8542..5d83f94 100644 --- a/ovoscope/__init__.py +++ b/ovoscope/__init__.py @@ -1286,6 +1286,14 @@ def assert_namespace_cleared(self, namespace: str) -> None: # --------------------------------------------------------------------------- # Public re-exports โ€” see ovoscope/e2e.py for full docs # --------------------------------------------------------------------------- +from ovoscope.intent_cases import ( # noqa: E402,F401 + DEFAULT_IGNORE_MESSAGES, + DEFAULT_PIPELINE_FAMILIES, + IntentCase, + assert_intent_case, + load_intent_cases, + register_intent_case_tests, +) from ovoscope.e2e import ( # noqa: E402,F401 E2EPipelineHarness, detach_intent, diff --git a/ovoscope/intent_cases.py b/ovoscope/intent_cases.py new file mode 100644 index 0000000..9e1721e --- /dev/null +++ b/ovoscope/intent_cases.py @@ -0,0 +1,382 @@ +"""File-based intent test cases for OVOS skills. + +Lets skill authors describe expected intent routing as plain-text files +under ``test/end2end/cases//`` โ€” adding a phrase, intent, or whole +new language is a pure text edit; no Python required. + +Layout +------ + +:: + + test/end2end/cases/ + / + .intent.test # one utterance per line, expected + # to match + no_match.test # utterances expected to match + # NO intent of this skill + +``#`` comments and blank lines are ignored. + +Usage (one call, in a test module owned by the skill) +----------------------------------------------------- + +:: + + # test/end2end/test_intents.py + from pathlib import Path + from ovoscope.intent_cases import register_intent_case_tests + + register_intent_case_tests( + globals(), + skill_id="ovos-skill-personal.openvoiceos", + handlers={ + "WhoAreYou.intent": "PersonalSkill.handle_who_are_you_intent", + "WhatAreYou.intent": "PersonalSkill.handle_what_are_you_intent", + # ... + }, + cases_dir=Path(__file__).parent / "cases", + ) + +The call creates four ``unittest.TestCase`` classes in the caller's +module โ€” one per pipeline family (Padatious, Padacioso, Model2Vec) plus +one for the full default OVOS stack โ€” each containing one ``test_*`` +method per (lang, utterance) pair. A test passes if any tier of its +pipeline family routes the utterance to the expected intent, which +matches realistic production cascade behaviour. + +Override the generated set with ``pipelines={"name": [stage, ...]}`` if +you only want a subset, or to test against custom pipeline stages. +""" +from __future__ import annotations + +import dataclasses +import time +from copy import deepcopy +from pathlib import Path +from typing import Dict, Iterable, Iterator, List, Optional, Union +from unittest import TestCase + +from ovos_bus_client.message import Message +from ovos_bus_client.session import Session +from ovos_utils.log import LOG + +# Imports from the top-level package โ€” guarded to avoid a circular import at +# module load. The runtime calls happen well after package init. +from ovoscope import (DEFAULT_TEST_PIPELINE, End2EndTest, M2V_PIPELINE, + PADACIOSO_PIPELINE, PADATIOUS_PIPELINE, get_minicroft) + +__all__ = [ + "IntentCase", + "load_intent_cases", + "assert_intent_case", + "register_intent_case_tests", + "DEFAULT_IGNORE_MESSAGES", + "DEFAULT_PIPELINE_FAMILIES", +] + +DEFAULT_IGNORE_MESSAGES: List[str] = [ + "speak", + "mycroft.audio.play_sound", + "ovos.common_play.stop.response", +] + +# Named pipeline families generated by default. +DEFAULT_PIPELINE_FAMILIES: Dict[str, List[str]] = { + "Padatious": PADATIOUS_PIPELINE, + "Padacioso": PADACIOSO_PIPELINE, + "M2V": M2V_PIPELINE, + "DefaultPipeline": DEFAULT_TEST_PIPELINE, +} + + +@dataclasses.dataclass(frozen=True) +class IntentCase: + """A single expectation: ``utterance`` in ``lang`` should match ``intent``. + + ``intent`` is ``None`` to assert no intent of the skill under test + fires (the utterance falls through to ``complete_intent_failure``). + """ + lang: str + utterance: str + intent: Optional[str] + source: Path + + +# --------------------------------------------------------------------------- +# Case-file discovery +# --------------------------------------------------------------------------- +def _read_lines(path: Path) -> List[str]: + out: List[str] = [] + for raw in path.read_text(encoding="utf-8").splitlines(): + line = raw.strip() + if line and not line.startswith("#"): + out.append(line) + return out + + +def load_intent_cases(cases_dir: Union[str, Path], + known_intents: Optional[Iterable[str]] = None + ) -> List[IntentCase]: + """Discover every ``IntentCase`` under ``cases_dir``. + + Args: + cases_dir: directory containing ``/.intent.test`` and + optionally ``/no_match.test`` files. + known_intents: if given, every ``.intent`` filename found + is validated against this set โ€” a typo will raise + ``AssertionError`` instead of silently being skipped. + + Returns: + List of cases in stable (lang, file, line) order. + """ + base = Path(cases_dir) + if not base.is_dir(): + return [] + known = set(known_intents) if known_intents is not None else None + cases: List[IntentCase] = [] + for lang_dir in sorted(base.iterdir()): + if not lang_dir.is_dir() or lang_dir.name.startswith("_"): + continue + lang = lang_dir.name + for case_file in sorted(lang_dir.glob("*.test")): + if case_file.name == "no_match.test": + expected: Optional[str] = None + elif case_file.stem.endswith(".intent"): + expected = case_file.stem # ".intent" + if known is not None and expected not in known: + raise AssertionError( + f"{case_file} targets unknown intent " + f"{expected!r}; expected one of {sorted(known)}") + else: + continue + for utt in _read_lines(case_file): + cases.append(IntentCase(lang=lang, utterance=utt, + intent=expected, source=case_file)) + return cases + + +# --------------------------------------------------------------------------- +# Single-case assertion +# --------------------------------------------------------------------------- +def assert_intent_case(minicroft, skill_id: str, handlers: Dict[str, str], + case: IntentCase, pipeline: List[str], + *, + ignore_messages: Optional[List[str]] = None, + timeout: float = 30) -> None: + """Fire ``case.utterance`` through ``pipeline`` and assert routing. + + Asserts that an ``IntentCase`` with ``intent=None`` falls through to + ``complete_intent_failure``; otherwise asserts the expected intent + and handler-lifecycle messages fire in order. + + Args: + minicroft: a running ``MiniCroft`` instance with ``skill_id`` loaded. + skill_id: the full skill id under test (e.g. ``"my-skill.author"``). + handlers: ``{intent_name: handler_method_name}`` for the skill. + case: the case to run. + pipeline: list of pipeline stage ids to populate + ``session.pipeline`` with. + ignore_messages: extra non-deterministic / noisy message types to + filter out of the comparison. + timeout: per-case execution timeout, in seconds. + """ + ignored = list(ignore_messages or DEFAULT_IGNORE_MESSAGES) + + session = Session(f"intent-case-{case.lang}-{case.utterance[:24]}") + session.lang = case.lang + session.pipeline = list(pipeline) + + source = Message( + "recognizer_loop:utterance", + {"utterances": [case.utterance], "lang": case.lang}, + {"session": session.serialize()}, + ) + + if case.intent is None: + final_session = deepcopy(session) + expected_messages = [ + source, + Message("complete_intent_failure", + {"utterances": [case.utterance], "lang": case.lang}, {}), + Message("ovos.utterance.handled", {}, {}), + ] + test = End2EndTest( + minicroft=minicroft, + skill_ids=[skill_id], + eof_msgs=["ovos.utterance.handled"], + flip_points=["recognizer_loop:utterance"], + ignore_messages=ignored, + source_message=source, + final_session=final_session, + expected_messages=expected_messages, + test_msg_data=False, + test_msg_context=False, + ) + else: + if case.intent not in handlers: + raise AssertionError( + f"No handler mapping for intent {case.intent!r} " + f"(case from {case.source}). Add it to ``handlers``.") + handler = handlers[case.intent] + final_session = deepcopy(session) + final_session.active_skills = [(skill_id, 0.0)] + expected_messages = [ + source, + Message(f"{skill_id}.activate", {}, {"skill_id": skill_id}), + Message(f"{skill_id}:{case.intent}", + {"utterance": case.utterance, "lang": case.lang}, + {"skill_id": skill_id}), + Message("mycroft.skill.handler.start", + {"name": handler}, {"skill_id": skill_id}), + Message("mycroft.skill.handler.complete", + {"name": handler}, {"skill_id": skill_id}), + Message("ovos.utterance.handled", {}, {"skill_id": skill_id}), + ] + test = End2EndTest( + minicroft=minicroft, + skill_ids=[skill_id], + eof_msgs=["ovos.utterance.handled"], + flip_points=["recognizer_loop:utterance"], + ignore_messages=ignored, + source_message=source, + final_session=final_session, + activation_points=[f"{skill_id}:{case.intent}"], + expected_messages=expected_messages, + ) + test.execute(timeout=timeout) + + +# --------------------------------------------------------------------------- +# Shared minicroft helper +# --------------------------------------------------------------------------- +_SHARED_MINICROFT_KEY = "_ovoscope_shared_minicroft" + + +def _shared_minicroft(skill_id: str, langs: List[str], m2v_warmup: float): + """Lazily create one MiniCroft and warm m2v's label index. + + Caches the instance on a process-global so every generated class + shares the same boot. + """ + cache = globals().setdefault(_SHARED_MINICROFT_KEY, {}) + key = (skill_id, tuple(langs)) + if key not in cache: + LOG.set_level("CRITICAL") + secondary = [l for l in langs if l != "en-US"] + mc = get_minicroft([skill_id], secondary_langs=secondary or None) + # m2v consumes ``padatious:register_intent`` to (re)build its label + # index; ``mycroft.ready`` triggers ``handle_sync_intents``. Without + # this nudge the first m2v call logs "No model classes match + # registered intents" and falls through. + mc.bus.emit(Message("mycroft.ready", {}, {})) + if m2v_warmup > 0: + time.sleep(m2v_warmup) + cache[key] = mc + return cache[key] + + +# --------------------------------------------------------------------------- +# Test-class generator +# --------------------------------------------------------------------------- +def _slug(s: str) -> str: + out = [] + for ch in s: + if ch.isalnum(): + out.append(ch) + elif ch in " -_/": + out.append("_") + return "".join(out).strip("_") or "x" + + +def _build_test_class(name: str, pipeline: List[str], cases: List[IntentCase], + skill_id: str, handlers: Dict[str, str], + langs: List[str], + ignore_messages: Optional[List[str]], + timeout: float, + m2v_warmup: float, + doc: str) -> type: + def _make(case: IntentCase): + def _test(self): + mc = _shared_minicroft(skill_id, langs, m2v_warmup) + assert_intent_case(mc, skill_id, handlers, case, pipeline, + ignore_messages=ignore_messages, + timeout=timeout) + label = case.intent.split(".")[0] if case.intent else "no_match" + _test.__doc__ = (f"[{case.lang}] {case.utterance!r} -> " + f"{case.intent if case.intent else 'no match'} ({name})") + # Attach metadata for the pytest accuracy/coverage reporter. + _test._intent_case = case # type: ignore[attr-defined] + _test._intent_case_pipeline = name # type: ignore[attr-defined] + _test._intent_case_skill_id = skill_id # type: ignore[attr-defined] + return _test, label + + body: Dict[str, object] = {"__doc__": doc} + for case in cases: + method, label = _make(case) + slug = _slug(case.utterance) + attr = f"test_{case.lang.replace('-', '_')}__{label}__{slug}" + # avoid clobbers if two utterances slug-collide + base = attr + n = 2 + while attr in body: + attr = f"{base}_{n}" + n += 1 + body[attr] = method + return type(name, (TestCase,), body) + + +def register_intent_case_tests( + target_globals: dict, + *, + skill_id: str, + handlers: Dict[str, str], + cases_dir: Union[str, Path], + pipelines: Optional[Dict[str, List[str]]] = None, + ignore_messages: Optional[List[str]] = None, + timeout: float = 30, + m2v_warmup: float = 10.0, + ) -> Dict[str, type]: + """Create per-pipeline ``TestCase`` classes in ``target_globals``. + + Args: + target_globals: pass ``globals()`` from the caller test module โ€” + generated classes are inserted here so pytest collects them. + skill_id: full skill plugin id (e.g. ``"my-skill.author"``). + handlers: mapping ``{".intent": ""}`` + covering every intent referenced by case files. + cases_dir: directory containing ``/.intent.test`` and + optional ``/no_match.test`` files. + pipelines: ``{class_suffix: pipeline_stage_list}`` to override the + default per-family classes. Each entry becomes a + ``Test`` class. Defaults to one class per family + in :data:`DEFAULT_PIPELINE_FAMILIES`. + ignore_messages: extra message types to filter out of comparison. + timeout: per-case execution timeout, in seconds. + m2v_warmup: seconds to sleep after booting the MiniCroft so the + m2v pipeline finishes syncing its label index. Set to 0 if + you're not running m2v cases. + + Returns: + ``{class_name: class_object}`` for every class created. + """ + cases = load_intent_cases(cases_dir, known_intents=handlers.keys()) + if not cases: + # Empty cases dir: skip silently so a freshly-copied template + # doesn't fail collection before any .test files are added. + return {} + + langs = sorted({c.lang for c in cases}) + families = pipelines if pipelines is not None else DEFAULT_PIPELINE_FAMILIES + + created: Dict[str, type] = {} + for suffix, pipeline in families.items(): + cls_name = f"Test{suffix}" + doc = (f"ovoscope intent-case tests for {skill_id} on pipeline " + f"{pipeline!r}.") + cls = _build_test_class(cls_name, pipeline, cases, skill_id, handlers, + langs, ignore_messages, timeout, m2v_warmup, + doc) + target_globals[cls_name] = cls + created[cls_name] = cls + return created diff --git a/ovoscope/pytest_plugin.py b/ovoscope/pytest_plugin.py index 5745d87..03688ed 100644 --- a/ovoscope/pytest_plugin.py +++ b/ovoscope/pytest_plugin.py @@ -96,6 +96,41 @@ def pytest_addoption(parser): metavar="PATTERN", help="Exclude skills/components matching this regex from the coverage report.", ) + group.addoption( + "--ovoscope-accuracy-report", + action="store", + default=None, + metavar="PATH", + help="Write per-(pipeline, lang, intent) accuracy report as JSON.", + ) + group.addoption( + "--ovoscope-accuracy-min", + action="store", + type=float, + default=None, + metavar="RATIO", + help=("Minimum overall intent-case pass rate (0.0-1.0). If set and " + "the rate falls below, the session exits non-zero โ€” useful " + "as a CI gate on regression in intent routing accuracy."), + ) + group.addoption( + "--ovoscope-accuracy-baseline", + action="store", + default=None, + metavar="PATH", + help=("Path to a previous --ovoscope-accuracy-report JSON. The " + "session fails if overall accuracy is lower than the " + "baseline (helpful for blocking PRs that lower accuracy)."), + ) + group.addoption( + "--ovoscope-accuracy-tolerant", + action="store_true", + default=False, + help=("Downgrade individual intent-case failures to xfail so they " + "don't fail the build; only the aggregate accuracy gate " + "(--ovoscope-accuracy-min / --ovoscope-accuracy-baseline) " + "can block the session."), + ) @pytest.fixture(scope="class") @@ -307,12 +342,9 @@ def patched_execute(self, *args, **kwargs): print(f"\nERROR: Failed to save bus coverage report to {cov_file}: {exc}") -def pytest_terminal_summary(terminalreporter, exitstatus, config): # noqa: ARG001 - """Print the merged bus coverage report at the end of the pytest session. - - Only runs if at least one test used the ``bus_coverage_session`` fixture - (or ``--ovoscope-bus-cov`` was used) and reports were collected. - """ +def _bus_coverage_summary(terminalreporter, config): + """Print the merged bus coverage report (factored out so the combined + ``pytest_terminal_summary`` below can call both reporters).""" reports = getattr(config, "_bus_coverage_reports", None) if not reports: return @@ -321,3 +353,195 @@ def pytest_terminal_summary(terminalreporter, exitstatus, config): # noqa: ARG0 terminalreporter.write_sep("=", "Bus Coverage Report") report.print_report(verbose=verbose) terminalreporter.write_line("") + + +# --------------------------------------------------------------------------- +# Intent-case accuracy reporter +# +# Aggregates pass/fail per (pipeline, lang, intent) for tests generated by +# :func:`ovoscope.intent_cases.register_intent_case_tests`. Each generated +# method carries an ``_intent_case`` attribute; we read it during the +# ``pytest_runtest_logreport`` hook, accumulate, then emit a report and an +# optional pass/fail gate during ``pytest_terminal_summary``. +# --------------------------------------------------------------------------- +def _resolve_intent_case_meta(item): + """Return (IntentCase, pipeline_name, skill_id) or None if not generated.""" + func = getattr(item, "function", None) + if func is None: + return None + case = getattr(func, "_intent_case", None) + if case is None: + return None + return (case, + getattr(func, "_intent_case_pipeline", "Unknown"), + getattr(func, "_intent_case_skill_id", "")) + + +def pytest_collection_modifyitems(config, items): + """In tolerant mode, mark every intent-case test as ``xfail(strict=False)``. + + That lets the suite measure per-case routing accuracy without forcing + every CI run to be green only when 100% of cases match. The aggregate + gate (``--ovoscope-accuracy-min`` / ``--ovoscope-accuracy-baseline``) + is the real blocker. + """ + if not config.getoption("--ovoscope-accuracy-tolerant"): + return + for item in items: + if _resolve_intent_case_meta(item) is not None: + item.add_marker(pytest.mark.xfail(reason="intent-case (tolerant)", + strict=False, run=True)) + + +def pytest_runtest_logreport(report): + """Record intent-case outcomes on the session config.""" + if report.when != "call": + return + item_func = getattr(report, "_intent_case_func", None) + # We can't get the item directly from a logreport; fall back to nodeid. + # The accumulator is keyed by nodeid -> (case, pipeline) which we set in + # ``pytest_runtest_setup``. + accum = getattr(pytest_runtest_logreport, "_accum", None) + if accum is None: + return + meta = accum["meta"].get(report.nodeid) + if meta is None: + return + case, pipeline, skill_id = meta + passed = report.outcome == "passed" or ( + report.outcome == "skipped" + and isinstance(report.longrepr, tuple) + and "XPASS" in str(report.longrepr)) + # xfail/xpass handling: in tolerant mode a "failure" surfaces as xfail. + if report.outcome == "skipped" and hasattr(report, "wasxfail"): + passed = False + accum["results"].append({ + "nodeid": report.nodeid, + "skill_id": skill_id, + "pipeline": pipeline, + "lang": case.lang, + "intent": case.intent or "no_match", + "utterance": case.utterance, + "source": str(case.source), + "passed": passed, + }) + + +def pytest_runtest_setup(item): + """Index intent-case metadata by nodeid for the logreport hook.""" + meta = _resolve_intent_case_meta(item) + if meta is None: + return + accum = getattr(pytest_runtest_logreport, "_accum", None) + if accum is None: + accum = {"results": [], "meta": {}} + pytest_runtest_logreport._accum = accum + accum["meta"][item.nodeid] = meta + + +def _accuracy_summary(results): + """Aggregate results into pivot tables for reporting.""" + by_pipeline: Dict[str, Dict[str, int]] = {} + by_pipeline_lang: Dict[tuple, Dict[str, int]] = {} + by_pipeline_intent: Dict[tuple, Dict[str, int]] = {} + total_pass = total = 0 + for r in results: + total += 1 + if r["passed"]: + total_pass += 1 + for bucket, key in ( + (by_pipeline, r["pipeline"]), + (by_pipeline_lang, (r["pipeline"], r["lang"])), + (by_pipeline_intent, (r["pipeline"], r["intent"])), + ): + d = bucket.setdefault(key, {"pass": 0, "total": 0}) + d["total"] += 1 + if r["passed"]: + d["pass"] += 1 + overall = (total_pass / total) if total else 0.0 + return { + "overall_accuracy": overall, + "passed": total_pass, + "total": total, + "by_pipeline": by_pipeline, + "by_pipeline_lang": {f"{p}|{l}": v for (p, l), v in by_pipeline_lang.items()}, + "by_pipeline_intent": {f"{p}|{i}": v for (p, i), v in by_pipeline_intent.items()}, + } + + +def pytest_terminal_summary(terminalreporter, exitstatus, config): # noqa: ARG001 + """Combined session summary: bus coverage + intent-case accuracy.""" + _bus_coverage_summary(terminalreporter, config) + accum = getattr(pytest_runtest_logreport, "_accum", None) + if not accum or not accum["results"]: + return + summary = _accuracy_summary(accum["results"]) + + tr = terminalreporter + tr.write_sep("=", "ovoscope Intent-Case Accuracy") + tr.write_line( + f"Overall: {summary['passed']}/{summary['total']} " + f"= {summary['overall_accuracy']:.1%}") + tr.write_line("") + tr.write_line("By pipeline:") + for pipe, d in sorted(summary["by_pipeline"].items()): + ratio = d["pass"] / d["total"] if d["total"] else 0.0 + tr.write_line(f" {pipe:24s} {d['pass']:4d}/{d['total']:<4d} {ratio:>6.1%}") + tr.write_line("") + tr.write_line("By pipeline x lang:") + for key in sorted(summary["by_pipeline_lang"]): + d = summary["by_pipeline_lang"][key] + ratio = d["pass"] / d["total"] if d["total"] else 0.0 + tr.write_line(f" {key:36s} {d['pass']:4d}/{d['total']:<4d} {ratio:>6.1%}") + tr.write_line("") + tr.write_line("By pipeline x intent:") + for key in sorted(summary["by_pipeline_intent"]): + d = summary["by_pipeline_intent"][key] + ratio = d["pass"] / d["total"] if d["total"] else 0.0 + tr.write_line(f" {key:48s} {d['pass']:4d}/{d['total']:<4d} {ratio:>6.1%}") + + # Persist to JSON if requested. + report_path = config.getoption("--ovoscope-accuracy-report") + if report_path: + import json + import os + os.makedirs(os.path.dirname(os.path.abspath(report_path)) or ".", + exist_ok=True) + with open(report_path, "w", encoding="utf-8") as fh: + json.dump({"summary": summary, "results": accum["results"]}, + fh, indent=2) + tr.write_line(f"\nWrote accuracy report -> {report_path}") + + # Gate the session on minimum / baseline accuracy. + min_acc = config.getoption("--ovoscope-accuracy-min") + baseline_path = config.getoption("--ovoscope-accuracy-baseline") + failures = [] + if min_acc is not None and summary["overall_accuracy"] < min_acc: + failures.append( + f"overall accuracy {summary['overall_accuracy']:.1%} < " + f"required {min_acc:.1%}") + if baseline_path: + try: + import json as _json + with open(baseline_path, "r", encoding="utf-8") as fh: + base = _json.load(fh)["summary"]["overall_accuracy"] + if summary["overall_accuracy"] < base: + failures.append( + f"overall accuracy {summary['overall_accuracy']:.1%} < " + f"baseline {base:.1%} ({baseline_path})") + except Exception as exc: + tr.write_line(f"\nWARNING: could not read baseline " + f"{baseline_path}: {exc}") + if failures: + tr.write_sep("!", "ovoscope accuracy gate FAILED") + for f in failures: + tr.write_line(f" - {f}") + # Mark the session as failed. + config._ovoscope_accuracy_gate_failed = True + + +def pytest_sessionfinish(session, exitstatus): # noqa: ARG001 + """Propagate accuracy-gate failure as a non-zero exit status.""" + if getattr(session.config, "_ovoscope_accuracy_gate_failed", False): + if session.exitstatus == 0: + session.exitstatus = 1 From c08f29e7738786660e631c39a428c94e43ab1aea Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Thu, 14 May 2026 22:32:15 +0000 Subject: [PATCH 09/82] Increment Version to 0.16.0a1 --- ovoscope/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovoscope/version.py b/ovoscope/version.py index f8773cc..09a6f32 100644 --- a/ovoscope/version.py +++ b/ovoscope/version.py @@ -1,6 +1,6 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 -VERSION_MINOR = 15 +VERSION_MINOR = 16 VERSION_BUILD = 0 VERSION_ALPHA = 1 # END_VERSION_BLOCK From a4494d44e99ce1eca8659d211bb0de5dcbdcda3e Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Thu, 14 May 2026 22:32:36 +0000 Subject: [PATCH 10/82] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 753275b..bf01a87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.16.0a1](https://github.com/TigreGotico/ovoscope/tree/0.16.0a1) (2026-05-14) + +[Full Changelog](https://github.com/TigreGotico/ovoscope/compare/0.15.0a1...0.16.0a1) + +**Merged pull requests:** + +- feat\(intent-cases\): file-based intent test layout + pytest accuracy gate [\#58](https://github.com/TigreGotico/ovoscope/pull/58) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.15.0a1](https://github.com/TigreGotico/ovoscope/tree/0.15.0a1) (2026-05-14) [Full Changelog](https://github.com/TigreGotico/ovoscope/compare/0.14.0a1...0.15.0a1) From e02b62d40cf5698fd574b17707c05a7db83963a2 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Fri, 15 May 2026 00:35:14 +0100 Subject: [PATCH 11/82] feat(intent-cases): markdown reporter, baseline diff, auto-discovery, deterministic m2v warmup (#60) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(intent-cases): markdown reporter, baseline diff, auto-discovery Three follow-up improvements to the intent-case test framework: 1. **Markdown report (--ovoscope-accuracy-md=PATH)** Render the per-(pipeline, lang, intent) pivot as Markdown with collapsible sections. Drops into the gh-automations PR-comment workflow as a new '๐ŸŽฏ Intent-Case Accuracy' section alongside the existing skill-tests and bus-coverage panels. Also surfaces a 'Hardest utterances' table (top-N by cross-pipeline pass rate) so reviewers can see which phrasings need locale tuning. 2. **Structural baseline diff** Replace the scalar pass-rate baseline gate with a full diff: identifies which (pipeline, lang, intent, utterance) cases regressed (was-pass -> now-fail) vs recovered. The PR comment now lists the regressed cases verbatim, and the session fails if any regression is detected. JSON output includes a baseline_diff block for downstream tooling. 3. **Auto-discovery via conftest** Skills can now opt in to intent-case tests by declaring a single dict in test/end2end/conftest.py: ovoscope_intent_cases = dict( skill_id='my-skill.author', handlers={...}, ) The pytest plugin walks loaded conftest modules at configure time and calls register_intent_case_tests() automatically. The explicit API stays supported and unchanged. Plus a deterministic m2v warm-up: instead of time.sleep(10), wait for the burst of padatious:register_intent events to settle (quiet-window heuristic) then pad for the 3 s in-plugin debounce. Falls back to a sleep if the bus introspection fails for any reason. 7 new unit tests cover the loader, summary, baseline diff, and markdown emitter โ€” no live MiniCroft required, runs in <1 s. Co-Authored-By: Claude Opus 4.7 * fix(intent-cases): auto-discovery via pytest_pycollect_makemodule The previous auto-discovery used pytest_configure to walk sys.modules for conftest.py files โ€” but pytest doesn't collect tests from conftest.py, and conftests aren't loaded yet at pytest_configure time either. Switched to a pytest_pycollect_makemodule hookwrapper that fires once per candidate test module: if the module declares a top-level 'ovoscope_intent_cases' dict, the helper injects the generated TestCase classes into its namespace before pytest's standard Python-class collector walks it. Result: a skill's complete intent-case wiring is now this 3-line file: # test/end2end/test_intents.py ovoscope_intent_cases = dict( skill_id='my-skill.author', handlers={'WhoAreYou.intent': 'MySkill.handle_who', ...}, ) Verified end-to-end: - 12 tests collected from a 3-line shim in 0.14s - TestM2V slice ran live (1 XPASS, 2 XFAIL matching the known m2v-misroutes-'who are you' divergence canary) in 36 s - Markdown / JSON / accuracy gate all fired correctly Explicit register_intent_case_tests(globals(), ...) continues to work unchanged. Co-Authored-By: Claude Opus 4.7 --------- Co-authored-by: Claude Opus 4.7 --- ovoscope/intent_cases.py | 139 +++++++++++- ovoscope/pytest_plugin.py | 335 ++++++++++++++++++++++++++-- test/unittests/test_intent_cases.py | 142 ++++++++++++ 3 files changed, 590 insertions(+), 26 deletions(-) create mode 100644 test/unittests/test_intent_cases.py diff --git a/ovoscope/intent_cases.py b/ovoscope/intent_cases.py index 9e1721e..a6fa0b5 100644 --- a/ovoscope/intent_cases.py +++ b/ovoscope/intent_cases.py @@ -253,11 +253,72 @@ def assert_intent_case(minicroft, skill_id: str, handlers: Dict[str, str], _SHARED_MINICROFT_KEY = "_ovoscope_shared_minicroft" +def _wait_for_m2v_sync(mc, max_wait: float = 15.0, + quiet_window: float = 0.5) -> float: + """Wait deterministically for the m2v pipeline to finish its + ``handle_sync_intents`` debounce after ``mycroft.ready``. + + The m2v plugin (`ovos_m2v_pipeline/__init__.py:handle_sync_intents`) + sleeps 3 s then re-queries the adapt + padatious intent manifests. + Until that returns, the pipeline matches against an empty label set + and every utterance falls through. We avoid relying on a fixed + ``time.sleep`` by: + + 1. Subscribing to every ``padatious:register_intent`` / + ``register_intent`` event on the bus. + 2. Emitting ``mycroft.ready`` (which triggers + ``handle_sync_intents``). + 3. Waiting until no new register events have arrived for + ``quiet_window`` seconds AND at least one register event has + actually been observed (or ``max_wait`` is exhausted). + 4. Adding a small constant grace period for the in-plugin + ``time.sleep(3)`` debounce to actually return. + + Returns the seconds slept (for diagnostics). + """ + seen = {"count": 0, "last_t": 0.0} + + def on_register(_serialized): + seen["count"] += 1 + seen["last_t"] = time.monotonic() + + mc.bus.ee.on("padatious:register_intent", + lambda _: on_register(None)) + mc.bus.ee.on("register_intent", lambda _: on_register(None)) + + # Wildcard "message" listener catches the serialized form on FakeBus. + def on_any(serialized): + try: + t = serialized.get("type") if isinstance(serialized, dict) else None + except Exception: + return + if not t: + return + if t == "padatious:register_intent" or t == "register_intent": + on_register(None) + + mc.bus.ee.on("message", on_any) + + t0 = time.monotonic() + mc.bus.emit(Message("mycroft.ready", {}, {})) + + # Wait for the burst of register events to settle. + while time.monotonic() - t0 < max_wait: + time.sleep(0.1) + if seen["count"] > 0 and (time.monotonic() - seen["last_t"]) > quiet_window: + break + # m2v's handle_sync_intents does an internal time.sleep(3) before the + # actual intent set update. Pad for that ceiling. + time.sleep(3.5) + return time.monotonic() - t0 + + def _shared_minicroft(skill_id: str, langs: List[str], m2v_warmup: float): """Lazily create one MiniCroft and warm m2v's label index. Caches the instance on a process-global so every generated class - shares the same boot. + shares the same boot. ``m2v_warmup`` is now an *upper bound*: if the + deterministic event-based wait finishes faster, we return early. """ cache = globals().setdefault(_SHARED_MINICROFT_KEY, {}) key = (skill_id, tuple(langs)) @@ -265,13 +326,15 @@ def _shared_minicroft(skill_id: str, langs: List[str], m2v_warmup: float): LOG.set_level("CRITICAL") secondary = [l for l in langs if l != "en-US"] mc = get_minicroft([skill_id], secondary_langs=secondary or None) - # m2v consumes ``padatious:register_intent`` to (re)build its label - # index; ``mycroft.ready`` triggers ``handle_sync_intents``. Without - # this nudge the first m2v call logs "No model classes match - # registered intents" and falls through. - mc.bus.emit(Message("mycroft.ready", {}, {})) if m2v_warmup > 0: - time.sleep(m2v_warmup) + try: + _wait_for_m2v_sync(mc, max_wait=max(m2v_warmup, 5.0)) + except Exception: + # If the deterministic wait fails for any reason + # (e.g. FakeBus internals change), fall back to a sleep + # so the suite still runs. + mc.bus.emit(Message("mycroft.ready", {}, {})) + time.sleep(m2v_warmup) cache[key] = mc return cache[key] @@ -379,4 +442,66 @@ def register_intent_case_tests( doc) target_globals[cls_name] = cls created[cls_name] = cls + # Mark generated classes so the pytest auto-discovery hook can skip a + # cases_dir whose tests are already registered by an explicit call. + target_globals["_ovoscope_intent_cases_registered"] = True + return created + + +# --------------------------------------------------------------------------- +# Pytest auto-discovery (no Python boilerplate required in the skill). +# +# A skill can opt in by adding a ``conftest.py`` next to its +# ``cases/`` directory containing: +# +# ovoscope_intent_cases = dict( +# skill_id="my-skill.author", +# handlers={"DoX.intent": "MySkill.handle_do_x", ...}, +# ) +# +# The ``pytest_collect_directory`` hook on the ovoscope pytest plugin +# discovers the conftest, walks ``/cases/`` and generates the same +# TestCase classes ``register_intent_case_tests`` would have created โ€” +# zero boilerplate in the test module. +# --------------------------------------------------------------------------- +def autodiscover_from_conftest(conftest_dir: Union[str, Path], + target_globals: dict) -> Dict[str, type]: + """Look for ``ovoscope_intent_cases`` config in a conftest namespace + and call :func:`register_intent_case_tests` accordingly. + + The conftest must expose a dict like:: + + ovoscope_intent_cases = { + "skill_id": "my-skill.author", + "handlers": {"WhoAreYou.intent": "MySkill.handle_who", ...}, + # optional overrides: + "cases_dir": "cases", + "pipelines": {"Custom": [...]}, + "ignore_messages": [...], + "timeout": 30, + "m2v_warmup": 10.0, + } + + Returns ``{}`` if the conftest has no ``ovoscope_intent_cases`` or + the cases directory does not exist. + """ + cfg = target_globals.get("ovoscope_intent_cases") + if not cfg: + return {} + base = Path(conftest_dir) + cases_dir = Path(cfg.get("cases_dir", "cases")) + if not cases_dir.is_absolute(): + cases_dir = base / cases_dir + if not cases_dir.is_dir(): + return {} + return register_intent_case_tests( + target_globals, + skill_id=cfg["skill_id"], + handlers=cfg["handlers"], + cases_dir=cases_dir, + pipelines=cfg.get("pipelines"), + ignore_messages=cfg.get("ignore_messages"), + timeout=cfg.get("timeout", 30), + m2v_warmup=cfg.get("m2v_warmup", 10.0), + ) return created diff --git a/ovoscope/pytest_plugin.py b/ovoscope/pytest_plugin.py index 03688ed..389444d 100644 --- a/ovoscope/pytest_plugin.py +++ b/ovoscope/pytest_plugin.py @@ -52,6 +52,8 @@ def test_something(self, minicroft, bus_coverage_session): if TYPE_CHECKING: from ovoscope.bus_coverage import BusCoverageReport +from pathlib import Path + import pytest from ovoscope import MiniCroft, get_minicroft, End2EndTest @@ -131,6 +133,24 @@ def pytest_addoption(parser): "(--ovoscope-accuracy-min / --ovoscope-accuracy-baseline) " "can block the session."), ) + group.addoption( + "--ovoscope-accuracy-md", + action="store", + default=None, + metavar="PATH", + help=("Write a Markdown intent-case accuracy report (the format " + "consumed by the OpenVoiceOS gh-automations PR-comment " + "workflow). Pairs naturally with --ovoscope-accuracy-report."), + ) + group.addoption( + "--ovoscope-accuracy-top-n", + action="store", + type=int, + default=10, + metavar="N", + help=("Show the N hardest utterances (lowest cross-pipeline pass " + "rate) in the Markdown report. Default: 10."), + ) @pytest.fixture(scope="class") @@ -377,6 +397,59 @@ def _resolve_intent_case_meta(item): getattr(func, "_intent_case_skill_id", "")) +def _autodiscover_intent_cases(config): + """Walk pytest's loaded test modules and trigger auto-discovery. + + Pytest collects tests from files matching ``test_*.py`` (or + ``*_test.py``) โ€” *not* from ``conftest.py``. So the auto-discovery + target is a thin shim module like ``test_intent_cases.py`` that + declares:: + + ovoscope_intent_cases = dict(skill_id=..., handlers=...) + + On the very first ``pytest_pycollect_makemodule`` we resolve the + module, scan it for that declaration, and inject the generated + TestCase classes into its namespace before pytest collects items + from it. The user writes one variable assignment, no + ``register_intent_case_tests`` call, no ``globals()`` argument. + + Skills already calling :func:`register_intent_case_tests` explicitly + are skipped via the ``_ovoscope_intent_cases_registered`` marker. + """ + # Tracked in pytest_pycollect_makemodule; this helper kept for symmetry + # / future use (e.g. a CLI hook). + return None + + +@pytest.hookimpl(hookwrapper=True) +def pytest_pycollect_makemodule(module_path, path, parent): + """Auto-register intent-case tests on shim modules that declare + ``ovoscope_intent_cases = {...}``. + + The hookwrapper imports the module first, lets pytest build the + collector, then injects the generated TestCase classes into the + module's namespace so the standard Python-class collector finds them. + """ + from ovoscope.intent_cases import autodiscover_from_conftest + + outcome = yield + collector = outcome.get_result() + if collector is None: + return + try: + mod = collector.obj # imports the module if not already loaded + except Exception: + return + if not hasattr(mod, "ovoscope_intent_cases"): + return + if getattr(mod, "_ovoscope_intent_cases_registered", False): + return + try: + autodiscover_from_conftest(Path(mod.__file__).parent, mod.__dict__) + except Exception as exc: # noqa: BLE001 + print(f"ovoscope auto-discovery skipped for {mod.__file__}: {exc}") + + def pytest_collection_modifyitems(config, items): """In tolerant mode, mark every intent-case test as ``xfail(strict=False)``. @@ -440,10 +513,18 @@ def pytest_runtest_setup(item): def _accuracy_summary(results): - """Aggregate results into pivot tables for reporting.""" + """Aggregate results into pivot tables for reporting. + + Adds, on top of the previous pivots, a per-utterance roll-up that + feeds the "hardest utterances" section of the Markdown report. + """ by_pipeline: Dict[str, Dict[str, int]] = {} by_pipeline_lang: Dict[tuple, Dict[str, int]] = {} by_pipeline_intent: Dict[tuple, Dict[str, int]] = {} + # Cross-pipeline rollup: (lang, intent, utterance) -> {pass, total, + # failing_pipelines}. Lets us surface "which exact phrasing routes + # poorly across the whole stack" in one place. + by_utterance: Dict[tuple, Dict[str, object]] = {} total_pass = total = 0 for r in results: total += 1 @@ -458,6 +539,16 @@ def _accuracy_summary(results): d["total"] += 1 if r["passed"]: d["pass"] += 1 + utt_key = (r["lang"], r["intent"], r["utterance"]) + u = by_utterance.setdefault(utt_key, { + "lang": r["lang"], "intent": r["intent"], + "utterance": r["utterance"], + "pass": 0, "total": 0, "failing_pipelines": []}) + u["total"] += 1 + if r["passed"]: + u["pass"] += 1 + else: + u["failing_pipelines"].append(r["pipeline"]) overall = (total_pass / total) if total else 0.0 return { "overall_accuracy": overall, @@ -466,9 +557,179 @@ def _accuracy_summary(results): "by_pipeline": by_pipeline, "by_pipeline_lang": {f"{p}|{l}": v for (p, l), v in by_pipeline_lang.items()}, "by_pipeline_intent": {f"{p}|{i}": v for (p, i), v in by_pipeline_intent.items()}, + "by_utterance": list(by_utterance.values()), } +# --------------------------------------------------------------------------- +# Baseline diff +# --------------------------------------------------------------------------- +def _result_key(r: dict) -> tuple: + """Stable identity for a single (pipeline, lang, intent, utterance) case.""" + return (r.get("pipeline", ""), r.get("lang", ""), + r.get("intent", ""), r.get("utterance", "")) + + +def _baseline_diff(baseline_results, current_results): + """Compare two result lists by case key, returning a structural diff. + + Returns ``{"regressed": [...], "recovered": [...], "added": [...], + "removed": [...], "baseline_accuracy": float}``. Each item in + ``regressed`` / ``recovered`` is the matching *current* result dict + so callers can quote the offending utterance verbatim. + """ + base_by_key = {_result_key(r): r for r in baseline_results or []} + cur_by_key = {_result_key(r): r for r in current_results or []} + + regressed, recovered, added, removed = [], [], [], [] + for k, cur in cur_by_key.items(): + if k not in base_by_key: + added.append(cur) + continue + base = base_by_key[k] + if base.get("passed") and not cur.get("passed"): + regressed.append(cur) + elif (not base.get("passed")) and cur.get("passed"): + recovered.append(cur) + for k, base in base_by_key.items(): + if k not in cur_by_key: + removed.append(base) + + base_total = len(baseline_results or []) + base_pass = sum(1 for r in (baseline_results or []) if r.get("passed")) + base_acc = (base_pass / base_total) if base_total else 0.0 + + return { + "regressed": regressed, "recovered": recovered, + "added": added, "removed": removed, + "baseline_accuracy": base_acc, "baseline_passed": base_pass, + "baseline_total": base_total, + } + + +# --------------------------------------------------------------------------- +# Markdown report +# --------------------------------------------------------------------------- +def _accuracy_markdown(summary, results, baseline_diff=None, top_n=10): + """Render the intent-case accuracy summary as Markdown. + + Mirrors the table-and-collapsible-section style used by the existing + bus-coverage section in the gh-automations PR-comment workflow: + headline line, summary table, per-(pipeline,lang) and + per-(pipeline,intent) breakdowns in ``
`` blocks, then a + "hardest utterances" call-out and, when present, a baseline diff. + """ + overall = summary["overall_accuracy"] + icon = "โœ…" if overall >= 0.9 else ("โš ๏ธ" if overall >= 0.7 else "โŒ") + lines = [ + f"{icon} **{summary['passed']}/{summary['total']}** intent-case " + f"checks passed โ€” overall accuracy **{overall:.1%}**.", + "", + ] + + # --- Per-pipeline summary table (always visible) --- + lines.append("| Pipeline | Pass / Total | Accuracy |") + lines.append("|---|---:|---:|") + for pipe, d in sorted(summary["by_pipeline"].items()): + ratio = d["pass"] / d["total"] if d["total"] else 0.0 + lines.append(f"| `{pipe}` | {d['pass']} / {d['total']} | {ratio:.1%} |") + lines.append("") + + # --- Per-(pipeline, lang) --- + lines.append("
Per-pipeline ร— language") + lines.append("") + lines.append("| Pipeline | Lang | Pass / Total | Accuracy |") + lines.append("|---|---|---:|---:|") + for key in sorted(summary["by_pipeline_lang"]): + pipe, lang = key.split("|", 1) + d = summary["by_pipeline_lang"][key] + ratio = d["pass"] / d["total"] if d["total"] else 0.0 + lines.append(f"| `{pipe}` | `{lang}` | {d['pass']} / {d['total']} | {ratio:.1%} |") + lines.append("") + lines.append("
") + lines.append("") + + # --- Per-(pipeline, intent) --- + lines.append("
Per-pipeline ร— intent") + lines.append("") + lines.append("| Pipeline | Intent | Pass / Total | Accuracy |") + lines.append("|---|---|---:|---:|") + for key in sorted(summary["by_pipeline_intent"]): + pipe, intent = key.split("|", 1) + d = summary["by_pipeline_intent"][key] + ratio = d["pass"] / d["total"] if d["total"] else 0.0 + lines.append(f"| `{pipe}` | `{intent}` | {d['pass']} / {d['total']} | {ratio:.1%} |") + lines.append("") + lines.append("
") + lines.append("") + + # --- Hardest utterances (lowest cross-pipeline pass rate) --- + utts = list(summary.get("by_utterance", [])) + # Only utterances that failed at least once and aren't no_match + hard = [u for u in utts + if u["total"] > 0 and u["pass"] < u["total"] + and u["intent"] != "no_match"] + hard.sort(key=lambda u: (u["pass"] / u["total"], -u["total"])) + if hard: + lines.append(f"
Hardest utterances " + f"(top {min(top_n, len(hard))})") + lines.append("") + lines.append("| Lang | Intent | Utterance | Pass / Total | Failing pipelines |") + lines.append("|---|---|---|---:|---|") + for u in hard[:top_n]: + fail_pipes = ", ".join(f"`{p}`" for p in sorted(set(u["failing_pipelines"]))) + lines.append(f"| `{u['lang']}` | `{u['intent']}` " + f"| {u['utterance']} " + f"| {u['pass']} / {u['total']} | {fail_pipes} |") + lines.append("") + lines.append("
") + lines.append("") + + # --- Baseline diff (when supplied) --- + if baseline_diff is not None: + regressed = baseline_diff["regressed"] + recovered = baseline_diff["recovered"] + delta = overall - baseline_diff["baseline_accuracy"] + delta_str = f"{delta:+.1%}" + head = (f"vs baseline ({baseline_diff['baseline_passed']}/" + f"{baseline_diff['baseline_total']} = " + f"{baseline_diff['baseline_accuracy']:.1%}): " + f"{len(regressed)} regressed, {len(recovered)} recovered, " + f"ฮ” {delta_str}") + lines.append(f"**Baseline diff** โ€” {head}") + lines.append("") + if regressed: + lines.append("
โŒ Regressed " + f"({len(regressed)})") + lines.append("") + lines.append("| Pipeline | Lang | Intent | Utterance |") + lines.append("|---|---|---|---|") + for r in regressed[:50]: + lines.append(f"| `{r['pipeline']}` | `{r['lang']}` | " + f"`{r['intent']}` | {r['utterance']} |") + if len(regressed) > 50: + lines.append(f"| โ€ฆ | โ€ฆ | โ€ฆ | _+{len(regressed)-50} more_ |") + lines.append("") + lines.append("
") + lines.append("") + if recovered: + lines.append("
โœ… Recovered " + f"({len(recovered)})") + lines.append("") + lines.append("| Pipeline | Lang | Intent | Utterance |") + lines.append("|---|---|---|---|") + for r in recovered[:50]: + lines.append(f"| `{r['pipeline']}` | `{r['lang']}` | " + f"`{r['intent']}` | {r['utterance']} |") + if len(recovered) > 50: + lines.append(f"| โ€ฆ | โ€ฆ | โ€ฆ | _+{len(recovered)-50} more_ |") + lines.append("") + lines.append("
") + lines.append("") + + return "\n".join(lines).rstrip() + "\n" + + def pytest_terminal_summary(terminalreporter, exitstatus, config): # noqa: ARG001 """Combined session summary: bus coverage + intent-case accuracy.""" _bus_coverage_summary(terminalreporter, config) @@ -500,43 +761,79 @@ def pytest_terminal_summary(terminalreporter, exitstatus, config): # noqa: ARG0 ratio = d["pass"] / d["total"] if d["total"] else 0.0 tr.write_line(f" {key:48s} {d['pass']:4d}/{d['total']:<4d} {ratio:>6.1%}") - # Persist to JSON if requested. + # Load baseline (if any) and compute structural diff up-front so both + # the JSON / Markdown outputs and the gate can reuse it. + baseline_path = config.getoption("--ovoscope-accuracy-baseline") + baseline_diff = None + baseline_warning = None + if baseline_path: + try: + import json as _json + with open(baseline_path, "r", encoding="utf-8") as fh: + baseline_doc = _json.load(fh) + baseline_diff = _baseline_diff( + baseline_doc.get("results") or [], + accum["results"]) + except Exception as exc: + baseline_warning = (f"could not read baseline " + f"{baseline_path}: {exc}") + tr.write_line(f"\nWARNING: {baseline_warning}") + + # Persist JSON. report_path = config.getoption("--ovoscope-accuracy-report") if report_path: import json import os os.makedirs(os.path.dirname(os.path.abspath(report_path)) or ".", exist_ok=True) + doc = {"summary": summary, "results": accum["results"]} + if baseline_diff is not None: + doc["baseline_diff"] = { + "regressed": baseline_diff["regressed"], + "recovered": baseline_diff["recovered"], + "added": baseline_diff["added"], + "removed": baseline_diff["removed"], + "baseline_accuracy": baseline_diff["baseline_accuracy"], + "baseline_passed": baseline_diff["baseline_passed"], + "baseline_total": baseline_diff["baseline_total"], + } with open(report_path, "w", encoding="utf-8") as fh: - json.dump({"summary": summary, "results": accum["results"]}, - fh, indent=2) + json.dump(doc, fh, indent=2) tr.write_line(f"\nWrote accuracy report -> {report_path}") - # Gate the session on minimum / baseline accuracy. + # Persist Markdown (for PR-comment ingestion). + md_path = config.getoption("--ovoscope-accuracy-md") + if md_path: + import os + os.makedirs(os.path.dirname(os.path.abspath(md_path)) or ".", + exist_ok=True) + top_n = config.getoption("--ovoscope-accuracy-top-n") + md = _accuracy_markdown(summary, accum["results"], + baseline_diff=baseline_diff, top_n=top_n) + with open(md_path, "w", encoding="utf-8") as fh: + fh.write(md) + tr.write_line(f"Wrote accuracy markdown -> {md_path}") + + # Gate the session. min_acc = config.getoption("--ovoscope-accuracy-min") - baseline_path = config.getoption("--ovoscope-accuracy-baseline") failures = [] if min_acc is not None and summary["overall_accuracy"] < min_acc: failures.append( f"overall accuracy {summary['overall_accuracy']:.1%} < " f"required {min_acc:.1%}") - if baseline_path: - try: - import json as _json - with open(baseline_path, "r", encoding="utf-8") as fh: - base = _json.load(fh)["summary"]["overall_accuracy"] - if summary["overall_accuracy"] < base: - failures.append( - f"overall accuracy {summary['overall_accuracy']:.1%} < " - f"baseline {base:.1%} ({baseline_path})") - except Exception as exc: - tr.write_line(f"\nWARNING: could not read baseline " - f"{baseline_path}: {exc}") + if baseline_diff is not None: + if baseline_diff["regressed"]: + top_regression = baseline_diff["regressed"][0] + failures.append( + f"{len(baseline_diff['regressed'])} cases regressed vs " + f"baseline (first: `{top_regression['pipeline']}` / " + f"`{top_regression['lang']}` / " + f"`{top_regression['intent']}` / " + f"{top_regression['utterance']!r})") if failures: tr.write_sep("!", "ovoscope accuracy gate FAILED") for f in failures: tr.write_line(f" - {f}") - # Mark the session as failed. config._ovoscope_accuracy_gate_failed = True diff --git a/test/unittests/test_intent_cases.py b/test/unittests/test_intent_cases.py new file mode 100644 index 0000000..af15cc8 --- /dev/null +++ b/test/unittests/test_intent_cases.py @@ -0,0 +1,142 @@ +# Copyright 2024 OpenVoiceOS +# Licensed under the Apache License, Version 2.0 +"""Unit tests for ovoscope.intent_cases helpers and the accuracy reporter. + +Tests focus on the parts that don't need a live MiniCroft: the file-layout +loader, the JSON summary roll-ups, the structural baseline diff, and the +Markdown report renderer. The end-to-end execution path is covered by +the consumer skill's e2e suite. +""" +from pathlib import Path +from unittest import TestCase + +from ovoscope.intent_cases import (IntentCase, _read_lines, load_intent_cases) +from ovoscope.pytest_plugin import (_accuracy_markdown, _accuracy_summary, + _baseline_diff) + + +class TestLoadIntentCases(TestCase): + """Discovery of /.intent.test and no_match.test files.""" + + def _write(self, tmp, lang, name, body): + d = tmp / "cases" / lang + d.mkdir(parents=True, exist_ok=True) + (d / name).write_text(body, encoding="utf-8") + + def test_loads_intent_and_no_match_files(self): + import tempfile + with tempfile.TemporaryDirectory() as td: + tmp = Path(td) + self._write(tmp, "en-US", "WhoAreYou.intent.test", + "# header\nwho are you\nwhat is your name\n") + self._write(tmp, "en-US", "no_match.test", "what time is it\n") + cases = load_intent_cases(tmp / "cases", + known_intents=["WhoAreYou.intent"]) + self.assertEqual(len(cases), 3) + self.assertEqual(sum(1 for c in cases if c.intent is None), 1) + self.assertTrue(all(c.lang == "en-US" for c in cases)) + + def test_unknown_intent_raises(self): + import tempfile + with tempfile.TemporaryDirectory() as td: + tmp = Path(td) + self._write(tmp, "en-US", "Mystery.intent.test", "x\n") + with self.assertRaises(AssertionError): + load_intent_cases(tmp / "cases", + known_intents=["WhoAreYou.intent"]) + + def test_skips_comments_and_blank_lines(self): + import tempfile + with tempfile.TemporaryDirectory() as td: + tmp = Path(td) + self._write(tmp, "en-US", "WhoAreYou.intent.test", + "# a\n\n \nfoo\n#bar\n") + cases = load_intent_cases(tmp / "cases", + known_intents=["WhoAreYou.intent"]) + self.assertEqual([c.utterance for c in cases], ["foo"]) + + +class TestAccuracySummary(TestCase): + """Aggregate roll-ups feeding the markdown / terminal pivots.""" + + def _r(self, **kw): + base = {"pipeline": "TestX", "lang": "en-US", + "intent": "WhoAreYou.intent", "utterance": "u", + "passed": True, "source": ""} + base.update(kw) + return base + + def test_overall_and_pivots(self): + results = [ + self._r(), + self._r(passed=False), + self._r(pipeline="TestY"), + ] + s = _accuracy_summary(results) + self.assertEqual(s["total"], 3) + self.assertEqual(s["passed"], 2) + self.assertAlmostEqual(s["overall_accuracy"], 2 / 3) + # Per-pipeline bucket exists for both pipelines. + self.assertEqual(s["by_pipeline"]["TestX"]["total"], 2) + self.assertEqual(s["by_pipeline"]["TestY"]["pass"], 1) + # by_utterance carries failing_pipelines for diagnosis. + by_utt = {(u["lang"], u["intent"], u["utterance"]): u + for u in s["by_utterance"]} + u = by_utt[("en-US", "WhoAreYou.intent", "u")] + self.assertIn("TestX", u["failing_pipelines"]) + + +class TestBaselineDiff(TestCase): + """Structural baseline diff: regressed vs recovered.""" + + def _r(self, pipeline, passed, utterance="u"): + return {"pipeline": pipeline, "lang": "en-US", + "intent": "WhoAreYou.intent", "utterance": utterance, + "passed": passed} + + def test_diff_classifies_flips(self): + baseline = [self._r("TestX", True), self._r("TestY", False)] + current = [self._r("TestX", False), self._r("TestY", True), + self._r("TestZ", True)] + d = _baseline_diff(baseline, current) + self.assertEqual(len(d["regressed"]), 1) + self.assertEqual(d["regressed"][0]["pipeline"], "TestX") + self.assertEqual(len(d["recovered"]), 1) + self.assertEqual(d["recovered"][0]["pipeline"], "TestY") + self.assertEqual(len(d["added"]), 1) + self.assertEqual(d["added"][0]["pipeline"], "TestZ") + self.assertEqual(d["baseline_total"], 2) + + +class TestAccuracyMarkdown(TestCase): + """Markdown report includes every section we need for PR comments.""" + + def test_markdown_contains_expected_sections(self): + results = [ + {"pipeline": "TestA", "lang": "en-US", "intent": "I.intent", + "utterance": "ok phrase", "passed": True}, + {"pipeline": "TestA", "lang": "en-US", "intent": "I.intent", + "utterance": "bad phrase", "passed": False}, + ] + s = _accuracy_summary(results) + md = _accuracy_markdown(s, results) + self.assertIn("intent-case checks passed", md) + self.assertIn("| Pipeline | Pass / Total | Accuracy |", md) + self.assertIn("Per-pipeline ร— language", md) + self.assertIn("Per-pipeline ร— intent", md) + self.assertIn("Hardest utterances", md) + # The failing utterance must be quoted in the hardest list. + self.assertIn("bad phrase", md) + + def test_baseline_diff_section_when_supplied(self): + baseline = [{"pipeline": "TestA", "lang": "en-US", + "intent": "I.intent", "utterance": "phrase", + "passed": True}] + current = [{"pipeline": "TestA", "lang": "en-US", + "intent": "I.intent", "utterance": "phrase", + "passed": False}] + s = _accuracy_summary(current) + d = _baseline_diff(baseline, current) + md = _accuracy_markdown(s, current, baseline_diff=d) + self.assertIn("Baseline diff", md) + self.assertIn("Regressed (1)", md) From 74abf09f63dd35826982f5f00491a69c3723ef9c Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Thu, 14 May 2026 23:35:27 +0000 Subject: [PATCH 12/82] Increment Version to 0.17.0a1 --- ovoscope/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovoscope/version.py b/ovoscope/version.py index 09a6f32..d1f8d5d 100644 --- a/ovoscope/version.py +++ b/ovoscope/version.py @@ -1,6 +1,6 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 -VERSION_MINOR = 16 +VERSION_MINOR = 17 VERSION_BUILD = 0 VERSION_ALPHA = 1 # END_VERSION_BLOCK From fbd523def9af14abe3fb71862e7afb8127fec62f Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Thu, 14 May 2026 23:35:54 +0000 Subject: [PATCH 13/82] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf01a87..37e9470 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.17.0a1](https://github.com/TigreGotico/ovoscope/tree/0.17.0a1) (2026-05-14) + +[Full Changelog](https://github.com/TigreGotico/ovoscope/compare/0.16.0a1...0.17.0a1) + +**Merged pull requests:** + +- feat\(intent-cases\): markdown reporter, baseline diff, auto-discovery, deterministic m2v warmup [\#60](https://github.com/TigreGotico/ovoscope/pull/60) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.16.0a1](https://github.com/TigreGotico/ovoscope/tree/0.16.0a1) (2026-05-14) [Full Changelog](https://github.com/TigreGotico/ovoscope/compare/0.15.0a1...0.16.0a1) From 09546b5ebb24d76958de60384390eea15581dd8c Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Wed, 20 May 2026 21:14:53 +0100 Subject: [PATCH 14/82] fix(pipeline-harness): default _SinkSkill bus to FakeBus (#62) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(pipeline-harness): defer _SinkSkill bus subscription via property PipelineHarness.__enter__ constructs _SinkSkill(bus=None) and assigns the real bus only after MiniCroft is created. _SinkSkill.__init__ was unconditionally calling bus.on(...) on the None bus, crashing with AttributeError before MiniCroft could be built โ€” so PipelineHarness was unusable in any context. Move the subscription into a bus property setter so: - _SinkSkill(bus=None) is safe - assigning a real bus after construction registers handlers - rebinding to a new bus detaches the old subscriptions first Adds regression tests covering all four paths. * refactor(pipeline-harness): default _SinkSkill bus to FakeBus, forbid None Per review feedback: rather than special-casing bus=None, always have a real bus. _SinkSkill now constructs a FakeBus by default when no bus is supplied; setting bus=None after construction raises ValueError. PipelineHarness drops the explicit bus=None / late-rebind dance โ€” it constructs _SinkSkill() with the default FakeBus and rebinds to MiniCroft's real bus in __enter__. --- ovoscope/pipeline.py | 33 +++++++++++--- test/unittests/test_pipeline_harness.py | 58 +++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 6 deletions(-) create mode 100644 test/unittests/test_pipeline_harness.py diff --git a/ovoscope/pipeline.py b/ovoscope/pipeline.py index e8ccbd2..4ad0ba1 100644 --- a/ovoscope/pipeline.py +++ b/ovoscope/pipeline.py @@ -42,12 +42,32 @@ class _SinkSkill: return it from :meth:`match`. """ - def __init__(self, bus: Any, skill_id: str = "__ovoscope_sink__") -> None: - self.bus = bus + def __init__(self, bus: Optional[Any] = None, skill_id: str = "__ovoscope_sink__") -> None: + from ovos_utils.fakebus import FakeBus + self.skill_id = skill_id self._last_match: Optional[Message] = None - bus.on("intent.service.skills.activated", self._handle) - bus.on("intent_failure", self._handle_failure) + self._bus: Any = bus if bus is not None else FakeBus() + self._bus.on("intent.service.skills.activated", self._handle) + self._bus.on("intent_failure", self._handle_failure) + + @property + def bus(self) -> Any: + return self._bus + + @bus.setter + def bus(self, new_bus: Any) -> None: + if new_bus is None: + raise ValueError("_SinkSkill.bus cannot be None; pass a real bus or omit to default to FakeBus.") + # Detach handlers from the previous bus before rebinding. + try: + self._bus.remove("intent.service.skills.activated", self._handle) + self._bus.remove("intent_failure", self._handle_failure) + except Exception: + pass + self._bus = new_bus + new_bus.on("intent.service.skills.activated", self._handle) + new_bus.on("intent_failure", self._handle_failure) def _handle(self, message: Any) -> None: """Capture matched intent messages.""" @@ -105,8 +125,9 @@ def __enter__(self) -> "PipelineHarness": """Start MiniCroft with the specified pipeline and no skills.""" from ovoscope import get_minicroft - # Inject internal sink skill to capture matched intents - sink_skill = _SinkSkill(bus=None) # bus set after MiniCroft creation + # Inject internal sink skill to capture matched intents. + # Constructed with a default FakeBus; rebound to MiniCroft's real bus below. + sink_skill = _SinkSkill() self._mc = get_minicroft( skill_ids=[], diff --git a/test/unittests/test_pipeline_harness.py b/test/unittests/test_pipeline_harness.py new file mode 100644 index 0000000..24ee782 --- /dev/null +++ b/test/unittests/test_pipeline_harness.py @@ -0,0 +1,58 @@ +"""Regression tests for ovoscope.pipeline._SinkSkill.""" + +import pytest + +from ovos_utils.fakebus import FakeBus + +from ovoscope.pipeline import _SinkSkill + + +class _RecordingBus: + def __init__(self): + self.handlers = [] + self.removed = [] + + def on(self, event, handler): + self.handlers.append((event, handler)) + + def remove(self, event, handler): + self.removed.append((event, handler)) + + +class TestSinkSkillBusHandling: + def test_default_constructs_with_fakebus(self): + # Regression: previously _SinkSkill(bus=None) crashed and + # PipelineHarness relied on passing None then rebinding. Now bus + # defaults to a FakeBus so construction is always safe and the + # skill is immediately usable. + sink = _SinkSkill() + assert isinstance(sink.bus, FakeBus) + assert sink._last_match is None + + def test_explicit_none_falls_back_to_fakebus(self): + sink = _SinkSkill(bus=None) + assert isinstance(sink.bus, FakeBus) + + def test_constructs_with_supplied_bus(self): + bus = _RecordingBus() + sink = _SinkSkill(bus=bus) + events = [e for e, _ in bus.handlers] + assert "intent.service.skills.activated" in events + assert "intent_failure" in events + + def test_rebinding_bus_detaches_previous(self): + old = _RecordingBus() + new = _RecordingBus() + sink = _SinkSkill(bus=old) + sink.bus = new + old_removed = [e for e, _ in old.removed] + assert "intent.service.skills.activated" in old_removed + assert "intent_failure" in old_removed + new_events = [e for e, _ in new.handlers] + assert "intent.service.skills.activated" in new_events + assert "intent_failure" in new_events + + def test_setting_bus_to_none_raises(self): + sink = _SinkSkill() + with pytest.raises(ValueError): + sink.bus = None From 581759188a66fe1e6e2f1446b9eb03bfec100608 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Wed, 20 May 2026 20:15:05 +0000 Subject: [PATCH 15/82] Increment Version to 0.17.1a1 --- ovoscope/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovoscope/version.py b/ovoscope/version.py index d1f8d5d..b095899 100644 --- a/ovoscope/version.py +++ b/ovoscope/version.py @@ -1,7 +1,7 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 VERSION_MINOR = 17 -VERSION_BUILD = 0 +VERSION_BUILD = 1 VERSION_ALPHA = 1 # END_VERSION_BLOCK From 0d68cd3408b6d830cda1901adcc39a752446ac14 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Wed, 20 May 2026 20:15:28 +0000 Subject: [PATCH 16/82] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 37e9470..1d62389 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.17.1a1](https://github.com/TigreGotico/ovoscope/tree/0.17.1a1) (2026-05-20) + +[Full Changelog](https://github.com/TigreGotico/ovoscope/compare/0.17.0a1...0.17.1a1) + +**Merged pull requests:** + +- fix\(pipeline-harness\): default \_SinkSkill bus to FakeBus [\#62](https://github.com/TigreGotico/ovoscope/pull/62) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.17.0a1](https://github.com/TigreGotico/ovoscope/tree/0.17.0a1) (2026-05-14) [Full Changelog](https://github.com/TigreGotico/ovoscope/compare/0.16.0a1...0.17.0a1) From 2958e2e23dbcbd2e2051b5cc99fd914e7477811b Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Wed, 10 Jun 2026 23:18:01 +0100 Subject: [PATCH 17/82] feat(phal): plugin_factories for MiniPHAL and PHALTest (#65) * feat(phal): add plugin_factories to MiniPHAL and PHALTest Plugins that register bus handlers in __init__ must be constructed with the harness FakeBus, not with a pre-existing bus. The new plugin_factories parameter accepts callables (bus) -> plugin that are invoked during __enter__, ensuring the plugin is always wired to the MiniPHAL bus. Also fixes the deprecated ovos_utils.messagebus import to use ovos_bus_client.message.Message directly. Co-Authored-By: Claude Fable 5 * docs: NLnet/NGI0 funding attribution --------- Co-authored-by: Claude Fable 5 --- README.md | 6 +++ ovoscope/phal.py | 42 +++++++++++++++-- test/unittests/test_phal.py | 94 +++++++++++++++++++++++++++++++++++++ 3 files changed, 138 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c781d9b..435eaa2 100644 --- a/README.md +++ b/README.md @@ -141,3 +141,9 @@ In the interest of transparency, two files are maintained as a public record of significant AI-assisted session. These files are intentionally published so that contributors and users can understand how the project evolves and where AI assistance has been applied. + +## Credits + +Funded by [NGI0 Commons Fund](https://nlnet.nl/project/OpenVoiceOS) / [NLnet](https://nlnet.nl) +under grant agreement No [101135429](https://cordis.europa.eu/project/id/101135429), +through the European Commission's [Next Generation Internet](https://ngi.eu) programme. diff --git a/ovoscope/phal.py b/ovoscope/phal.py index 6349574..7ef0901 100644 --- a/ovoscope/phal.py +++ b/ovoscope/phal.py @@ -34,10 +34,10 @@ import threading import time from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional from ovos_utils.fakebus import FakeBus -from ovos_utils.messagebus import Message +from ovos_bus_client.message import Message class MiniPHAL: @@ -50,6 +50,13 @@ class MiniPHAL: plugin_ids: OPM entry-point IDs of the PHAL plugins to load. plugin_instances: Pre-built or mocked plugin instances keyed by plugin_id. When provided the corresponding entry in *plugin_ids* is skipped. + Note: instances must have been constructed with the same ``FakeBus`` + that ``MiniPHAL`` provides โ€” use *plugin_factories* instead when + the plugin must be built inside the harness context. + plugin_factories: Callables ``(bus: FakeBus) -> plugin`` keyed by + plugin_id. The factory is called during ``__enter__`` so the plugin + is always wired to the harness ``FakeBus``. Takes precedence over + *plugin_instances* for the same plugin_id. config: Per-plugin configuration overrides keyed by plugin_id. Example:: @@ -60,16 +67,29 @@ class MiniPHAL: ) as phal: phal.emit(Message("system.reboot")) phal.assert_emitted("system.reboot.confirmed") + + Factory example (plugin must be built with the harness bus):: + + from ovos_phal_plugin_tools import OVOSToolsPHALPlugin + + with MiniPHAL( + plugin_ids=["ovos-phal-plugin-tools"], + plugin_factories={"ovos-phal-plugin-tools": lambda bus: OVOSToolsPHALPlugin(bus=bus)}, + ) as phal: + phal.emit(Message("ovos.tools.list", {})) + phal.assert_emitted("ovos.tools.list.response") """ def __init__( self, plugin_ids: Optional[List[str]] = None, plugin_instances: Optional[Dict[str, Any]] = None, + plugin_factories: Optional[Dict[str, Callable[[FakeBus], Any]]] = None, config: Optional[Dict[str, Dict[str, Any]]] = None, ) -> None: self.plugin_ids: List[str] = plugin_ids or [] self.plugin_instances: Dict[str, Any] = plugin_instances or {} + self.plugin_factories: Dict[str, Callable[[FakeBus], Any]] = plugin_factories or {} self.config: Dict[str, Dict[str, Any]] = config or {} self._bus: FakeBus = FakeBus() self._captured: List[Message] = [] @@ -108,9 +128,19 @@ def _capture(self, message: Any) -> None: self._captured.append(message) def _load_plugins(self) -> None: - """Load PHAL plugins via OPM or use pre-built instances.""" + """Load PHAL plugins via factories, pre-built instances, or OPM.""" for plugin_id in self.plugin_ids: - if plugin_id in self.plugin_instances: + if plugin_id in self.plugin_factories: + try: + instance = self.plugin_factories[plugin_id](self._bus) + except Exception as exc: + import warnings + warnings.warn( + f"Factory for PHAL plugin {plugin_id!r} raised: {exc}", + stacklevel=2, + ) + instance = None + elif plugin_id in self.plugin_instances: instance = self.plugin_instances[plugin_id] else: instance = self._instantiate_plugin(plugin_id) @@ -215,6 +245,8 @@ class PHALTest: expected_types: Message types that MUST appear in the capture. forbidden_types: Message types that MUST NOT appear. plugin_instances: Pre-built plugin instances (keyed by plugin_id). + plugin_factories: Callables ``(bus) -> plugin`` keyed by plugin_id. + Use this when the plugin must be constructed with the harness bus. config: Per-plugin config overrides. timeout: Maximum seconds to wait for expected messages (default 5.0). @@ -236,6 +268,7 @@ class PHALTest: expected_types: List[str] = field(default_factory=list) forbidden_types: List[str] = field(default_factory=list) plugin_instances: Dict[str, Any] = field(default_factory=dict) + plugin_factories: Dict[str, Callable[[FakeBus], Any]] = field(default_factory=dict) config: Dict[str, Dict[str, Any]] = field(default_factory=dict) timeout: float = 5.0 @@ -251,6 +284,7 @@ def execute(self) -> List[Message]: with MiniPHAL( plugin_ids=self.plugin_ids, plugin_instances=self.plugin_instances, + plugin_factories=self.plugin_factories, config=self.config, ) as phal: phal.emit(self.trigger_message, wait=0.1) diff --git a/test/unittests/test_phal.py b/test/unittests/test_phal.py index bccc368..e43168e 100644 --- a/test/unittests/test_phal.py +++ b/test/unittests/test_phal.py @@ -141,3 +141,97 @@ def test_phal_test_execute_with_instance(self): ) result = t.execute() assert isinstance(result, list) + + +# --------------------------------------------------------------------------- +# plugin_factories +# --------------------------------------------------------------------------- + + +class TestPluginFactories: + """Tests for the plugin_factories parameter added to MiniPHAL and PHALTest.""" + + def _make_responder_factory(self, trigger_type: str, response_type: str): + """Return a factory callable that wires a triggerโ†’response handler on the bus.""" + + def factory(bus: FakeBus): + plugin = MagicMock() + plugin.shutdown = MagicMock() + bus.on(trigger_type, lambda m: bus.emit(Message(response_type))) + return plugin + + return factory + + def test_factory_called_with_harness_bus(self): + """Factory receives the MiniPHAL internal bus.""" + received_buses = [] + + def factory(bus: FakeBus): + received_buses.append(bus) + return MagicMock() + + with MiniPHAL( + plugin_ids=["test-plugin"], + plugin_factories={"test-plugin": factory}, + ) as phal: + assert len(received_buses) == 1 + assert received_buses[0] is phal._bus + + def test_factory_plugin_receives_emitted_message(self): + """Plugin built by factory can handle messages emitted via phal.emit().""" + factory = self._make_responder_factory("req.type", "resp.type") + with MiniPHAL( + plugin_ids=["echo-plugin"], + plugin_factories={"echo-plugin": factory}, + ) as phal: + phal.emit(Message("req.type"), wait=0.1) + msg = phal.assert_emitted("resp.type", timeout=2.0) + assert msg.msg_type == "resp.type" + + def test_factory_takes_precedence_over_plugin_instances(self): + """When both factory and instance are provided, factory wins.""" + factory_calls = [] + + def factory(bus: FakeBus): + factory_calls.append(True) + return MagicMock() + + stale_instance = MagicMock() + with MiniPHAL( + plugin_ids=["dual-plugin"], + plugin_factories={"dual-plugin": factory}, + plugin_instances={"dual-plugin": stale_instance}, + ) as phal: + assert len(factory_calls) == 1 + assert "dual-plugin" in phal._loaded + # stale_instance was NOT loaded + assert phal._loaded["dual-plugin"] is not stale_instance + + def test_factory_raising_warns_and_skips_plugin(self): + """A factory that raises issues a warning and the plugin is not loaded.""" + import warnings + + def bad_factory(bus: FakeBus): + raise RuntimeError("factory error") + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + with MiniPHAL( + plugin_ids=["bad-plugin"], + plugin_factories={"bad-plugin": bad_factory}, + ) as phal: + assert "bad-plugin" not in phal._loaded + assert any("bad-plugin" in str(warning.message) for warning in w) + + def test_phal_test_plugin_factories_field(self): + """PHALTest.plugin_factories is forwarded to MiniPHAL.""" + factory = self._make_responder_factory("ovos.req", "ovos.resp") + t = PHALTest( + plugin_ids=["my-phal"], + trigger_message=Message("ovos.req"), + expected_types=["ovos.resp"], + plugin_factories={"my-phal": factory}, + timeout=2.0, + ) + captured = t.execute() + assert any(m.msg_type == "ovos.resp" for m in captured) From 5d9f4ec3320aaf2590dcb01f49b6bef19c6b1615 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Wed, 10 Jun 2026 22:18:11 +0000 Subject: [PATCH 18/82] Increment Version to 0.18.0a1 --- ovoscope/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ovoscope/version.py b/ovoscope/version.py index b095899..46c184e 100644 --- a/ovoscope/version.py +++ b/ovoscope/version.py @@ -1,7 +1,7 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 -VERSION_MINOR = 17 -VERSION_BUILD = 1 +VERSION_MINOR = 18 +VERSION_BUILD = 0 VERSION_ALPHA = 1 # END_VERSION_BLOCK From fd03c0e4f28f7399905c68e00dd227aa9c2aec42 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Wed, 10 Jun 2026 22:18:42 +0000 Subject: [PATCH 19/82] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d62389..f97d5d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.18.0a1](https://github.com/TigreGotico/ovoscope/tree/0.18.0a1) (2026-06-10) + +[Full Changelog](https://github.com/TigreGotico/ovoscope/compare/0.17.1a1...0.18.0a1) + +**Merged pull requests:** + +- feat\(phal\): plugin\_factories for MiniPHAL and PHALTest [\#65](https://github.com/TigreGotico/ovoscope/pull/65) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.17.1a1](https://github.com/TigreGotico/ovoscope/tree/0.17.1a1) (2026-05-20) [Full Changelog](https://github.com/TigreGotico/ovoscope/compare/0.17.0a1...0.17.1a1) From cefb075b1d455925a712b253a6dbe29bfc19a252 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Fri, 12 Jun 2026 23:37:02 +0100 Subject: [PATCH 20/82] feat: MiniVoiceLoop + simple/classic listener bus-sequence harnesses (#67) Add file-driven, in-process harnesses for the three OVOS listener services, each wiring the real service to a FakeBus with mock mic/VAD/STT/wake-word plugins and capturing the recognizer_loop:* bus sequence: - MiniVoiceLoop (ovos-dinkum-listener): feed_chunks drives _detect_ww for the wake-word / verifier-chain gate (closes #64); feed_file runs the full DinkumVoiceLoop.run() state machine over an audio file. - MiniSimpleListener (ovos-simple-listener): drives the SimpleListener loop over an audio file with the canonical bus callbacks. - MiniClassicListener (mycroft-classic-listener): RecognizerLoop event->FakeBus bridge plus a best-effort file-driven harness. Shared ListenerHarness base provides bus capture and the assert_record_begin / assert_wakeword_detected / assert_wakeword_suppressed / assert_utterance helpers. MockFileMicrophone and MockStreamingSTT are shared across backends. Adds tests (gated on each optional listener dependency) and docs/voice-loop.md. Co-authored-by: Claude Opus 4.8 (1M context) --- docs/index.md | 39 +- docs/voice-loop.md | 203 +++++ ovoscope/__init__.py | 19 + ovoscope/classic_listener.py | 393 +++++++++ ovoscope/simple_listener.py | 223 +++++ ovoscope/voice_loop.py | 1056 +++++++++++++++++++++++ test/unittests/test_classic_listener.py | 110 +++ test/unittests/test_simple_listener.py | 78 ++ test/unittests/test_voice_loop.py | 302 +++++++ 9 files changed, 2422 insertions(+), 1 deletion(-) create mode 100644 docs/voice-loop.md create mode 100644 ovoscope/classic_listener.py create mode 100644 ovoscope/simple_listener.py create mode 100644 ovoscope/voice_loop.py create mode 100644 test/unittests/test_classic_listener.py create mode 100644 test/unittests/test_simple_listener.py create mode 100644 test/unittests/test_voice_loop.py diff --git a/docs/index.md b/docs/index.md index 0c113f5..b9833ed 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,6 +11,7 @@ | [pydantic-integration.md](pydantic-integration.md) | Using `ovos-pydantic-models` with OvoScope | | [audio-testing.md](audio-testing.md) | `AudioServiceHarness`, `PlaybackServiceHarness` โ€” testing audio services | | [listener.md](listener.md) | `MiniListener`, `get_mini_listener`, `ListenerTest`, `MockVADEngine`, `MockHotWordEngine`, `VADTest`, `WakeWordTest` โ€” testing audio transformer plugins, STT pipeline, VAD, and wake-word | +| [voice-loop.md](voice-loop.md) | `MiniVoiceLoop` / `MiniSimpleListener` / `MiniClassicListener` โ€” file-driven bus-sequence testing for the ovos-dinkum, ovos-simple, and mycroft-classic listener services (wake-word โ†’ record-begin โ†’ utterance), with verifier-chain gating | | [gui-testing.md](gui-testing.md) | `GUICaptureSession` โ€” asserting GUI page navigation and namespace values | | [bus-coverage.md](bus-coverage.md) | `BusCoverageTracker`, `BusCoverageReport` โ€” measuring handler and emitter coverage per skill | ## Conceptual Model @@ -84,6 +85,17 @@ from ovoscope.listener import ( VADTest, # declarative VAD test runner WakeWordTest, # declarative WakeWord test runner ) +# Listener-service bus-sequence testing (dinkum / simple / classic) +from ovoscope import ( + MiniVoiceLoop, # ovos-dinkum-listener: feed PCM chunks or an audio file + MiniSimpleListener, # ovos-simple-listener: drive the loop over an audio file + MiniClassicListener, # mycroft-classic-listener: best-effort file drive + bridge + get_mini_voice_loop, # factory: create MiniVoiceLoop + VoiceLoopTest, # declarative wake-word โ†’ bus-sequence test runner + MiniHotwordContainer, # controllable hotword container with a verifier chain + MockFileMicrophone, # file-backed mic plugin shared across listener harnesses + MockStreamingSTT, # configurable transcript STT mock +) ``` Type aliases also exported: ```python @@ -115,12 +127,37 @@ msgs = listener.feed_audio(b"\x00" * 1024) listener.shutdown() ``` +## Listener-Service Bus-Sequence Testing + +OVOS has several listener **services** โ€” ovos-dinkum-listener, ovos-simple-listener, +and mycroft-classic-listener โ€” each emitting the same `recognizer_loop:*` bus +events. `MiniVoiceLoop`, `MiniSimpleListener`, and `MiniClassicListener` each +wire their real service to a `FakeBus` with mock mic/VAD/STT/wake-word plugins, +drive it over an arbitrary audio file (or PCM frames), and capture the emitted +sequence โ€” sharing one set of assertion helpers. `MiniVoiceLoop` also exercises +the dinkum verifier-chain gate that decides whether a detection survives. + +See [voice-loop.md](voice-loop.md) for full API reference and usage patterns. + +```python +from unittest.mock import Mock +from ovoscope.voice_loop import MiniVoiceLoop, MockHotWordEngine + +ww = MockHotWordEngine("hey_mycroft", trigger_after=3) +accepting = Mock(); accepting.verify.return_value = True + +with MiniVoiceLoop(ww_instances={"hey_mycroft": ww}, + verifiers=[accepting]) as vl: + msgs = vl.feed_chunks([b"\x00" * 512] * 5) + vl.assert_record_begin_emitted(msgs) +``` + ## What OvoScope Does NOT Do - Does not start a real WebSocket MessageBus server โ€” uses `FakeBus` (in-process pub/sub). - Does not load PHAL plugins or the audio service โ€” only skills and the intent pipeline. - Does not test GUI rendering โ€” GUI namespace messages are ignored by default (`ignore_gui=True`). - Does not test TTS โ€” operates at the `recognizer_loop:utterance` level (see [audio-testing.md](audio-testing.md) for TTS lifecycle testing). -- `MiniListener` covers `AudioTransformersService`, the STT pipeline, and mock VAD/WakeWord engines โ€” not the full `DinkumVoiceLoop` state machine. +- `MiniListener` covers `AudioTransformersService`, the STT pipeline, and mock VAD/WakeWord engines. `MiniVoiceLoop` / `MiniSimpleListener` / `MiniClassicListener` drive the dinkum, simple, and classic listener **services** from an audio file and capture the `recognizer_loop:*` bus sequence; the classic file drive is best-effort (energy-based pipeline). ## Quick Links | Resource | Path | |---|---| diff --git a/docs/voice-loop.md b/docs/voice-loop.md new file mode 100644 index 0000000..aee94df --- /dev/null +++ b/docs/voice-loop.md @@ -0,0 +1,203 @@ +# Listener Service Bus-Sequence Testing + +OVOS ships more than one listener **service**, and each emits the same +``recognizer_loop:*`` bus events as audio flows through it. ovoscope provides an +in-process harness for each, sharing one capture bus and one set of assertion +helpers (:class:`ovoscope.voice_loop.ListenerHarness`): + +| Service | Harness | Drive | +|---|---|---| +| ovos-dinkum-listener | `MiniVoiceLoop` | `feed_chunks` (wake-word / verifier gate) + `feed_file` (full `DinkumVoiceLoop.run()`) | +| ovos-simple-listener | `MiniSimpleListener` | `feed_file` (full `SimpleListener` loop) | +| mycroft-classic-listener | `MiniClassicListener` | `feed_file` (best-effort) + `bridge_recognizer_loop_to_bus` | + +Each harness wires its real listener to a `FakeBus` with mock mic / VAD / STT / +wake-word plugins, runs it over an arbitrary audio file (or PCM frames), and +captures the emitted bus sequence. The mocks live in +`ovoscope.voice_loop`: `MockFileMicrophone`, `MockStreamingSTT`, +`MockVADEngine`, `MockHotWordEngine`. + +## Shared assertion helpers + +Every harness inherits these (each takes an optional message list, defaulting to +the last feed result, and returns the checked list): + +- `assert_record_begin_emitted()` โ€” `recognizer_loop:record_begin` present. +- `assert_wakeword_detected()` โ€” both `recognizer_loop:wakeword` and `โ€ฆ:record_begin`. +- `assert_wakeword_suppressed()` โ€” neither wake-word nor record-begin present. +- `assert_utterance_emitted(utterance=None)` โ€” a `recognizer_loop:utterance` (optionally with the given text). + +--- + +## ovos-dinkum-listener โ€” `MiniVoiceLoop` + +### Wake-word / verifier gate (`feed_chunks`) + +Feeds PCM frames straight through `DinkumVoiceLoop._detect_ww` to assert the +wake-word detection and the verifier chain in isolation. + +```python +from unittest.mock import Mock +from ovoscope.voice_loop import MiniVoiceLoop, MockHotWordEngine + +SILENT = b"\x00" * 512 + +def loop(verifiers): + ww = MockHotWordEngine("hey_mycroft", trigger_after=3) + return MiniVoiceLoop(ww_instances={"hey_mycroft": ww}, verifiers=verifiers) + +acc = Mock(); acc.verify.return_value = True +with loop([acc]) as vl: + vl.assert_wakeword_detected(vl.feed_chunks([SILENT] * 5)) + +rej = Mock(); rej.verify.return_value = False +with loop([rej]) as vl: + vl.assert_wakeword_suppressed(vl.feed_chunks([SILENT] * 5)) + +boom = Mock(); boom.verify.side_effect = RuntimeError("boom") +with loop([boom]) as vl: # fail-open + vl.assert_record_begin_emitted(vl.feed_chunks([SILENT] * 5)) +``` + +| Sequence | Expected bus events | +|---|---| +| WW detected + all verifiers accept | `recognizer_loop:wakeword` + `โ€ฆ:record_begin` | +| WW detected + a verifier rejects | suppressed โ€” no `recognizer_loop:*` | +| WW detected + a verifier raises (fail-open) | `โ€ฆ:record_begin` emitted | +| No WW detected | no `recognizer_loop:*` | + +The verifier gate lives inside `DinkumVoiceLoop._detect_ww` and is only present +in ovos-dinkum-listener builds that ship the hotword-verifier feature +(`HotwordContainer.verify`). On a build without it the gate is absent and a +detection is never suppressed โ€” assert accordingly for the version under test. + +### Full loop from an audio file (`feed_file`) + +Runs the whole `DinkumVoiceLoop.run()` state machine over an audio file through a +`MockFileMicrophone`, emitting the full record-begin โ†’ record-end โ†’ utterance +sequence. + +```python +from ovoscope.voice_loop import MiniVoiceLoop, MockStreamingSTT + +stt = MockStreamingSTT(transcript="what time is it") +with MiniVoiceLoop(stt_instance=stt) as vl: + msgs = vl.feed_file("command.wav") + vl.assert_record_begin_emitted(msgs) + vl.assert_utterance_emitted("what time is it", msgs) +``` + +An empty `MockStreamingSTT` transcript yields +`recognizer_loop:speech.recognition.unknown` instead of an utterance. + +`MiniVoiceLoop` builds a real `DinkumVoiceLoop`; it raises `RuntimeError` when +ovos-dinkum-listener is not installed. + +--- + +## ovos-simple-listener โ€” `MiniSimpleListener` + +Drives the real `SimpleListener` thread with the canonical bus callbacks (a +per-instance mirror of `OVOSCallbacks`). + +```python +from ovoscope.simple_listener import MiniSimpleListener +from ovoscope.voice_loop import MockHotWordEngine, MockStreamingSTT + +with MiniSimpleListener( + wakeword=MockHotWordEngine("hey_mycroft", trigger_after=2), + stt_instance=MockStreamingSTT(transcript="turn on the lights"), +) as sl: + msgs = sl.feed_file("command.wav") + sl.assert_record_begin_emitted(msgs) + sl.assert_utterance_emitted("turn on the lights", msgs) +``` + +The default timing is tightened (`min_speech_seconds=0`, +`max_silence_seconds=0.1`) so a file-driven command ends promptly; raise them to +mimic production. Raises `RuntimeError` when ovos-simple-listener is absent. + +--- + +## mycroft-classic-listener โ€” `MiniClassicListener` + +The classic listener is a threaded, energy-based pipeline. Two entry points: + +### Event bridge (always available) + +`bridge_recognizer_loop_to_bus(loop, bus)` forwards a `RecognizerLoop`'s internal +EventEmitter events onto a `FakeBus`, exactly as the classic listener's +`service.py` does. Drive the loop with real audio and assert on the bus: + +```python +from ovoscope.classic_listener import bridge_recognizer_loop_to_bus +from ovos_utils.fakebus import FakeBus + +bus = FakeBus() +bridge_recognizer_loop_to_bus(loop, bus) # loop = a RecognizerLoop +``` + +### Best-effort file drive + +Injects a file-backed audio source and mock wake-word / STT into a fresh +`RecognizerLoop` and runs it to completion: + +```python +from ovoscope.classic_listener import MiniClassicListener +from ovoscope.voice_loop import MockHotWordEngine, MockStreamingSTT + +with MiniClassicListener( + wakeword=MockHotWordEngine("hey_mycroft", trigger_after=1), + stt_instance=MockStreamingSTT(transcript="hello world"), +) as cl: + msgs = cl.feed_file("command.wav", tail_silence_seconds=3.0) + cl.assert_record_begin_emitted(msgs) + cl.assert_utterance_emitted("hello world", msgs) +``` + +The file drive depends on the energy-based recogniser, so assertions are +presence-based (a busy pipeline may emit more than one cycle before it is +stopped). Use `classic_listener_available()` to gate tests on the environment; +`MiniClassicListener(...)` (built mode) raises `RuntimeError` when the package is +absent. + +--- + +## Declarative helper โ€” `VoiceLoopTest` + +For the dinkum backend, `VoiceLoopTest` runs a scenario and asserts in one call โ€” +via `feed_chunks` by default, or `feed_file` when `audio_file` is set: + +```python +from unittest.mock import Mock +from ovoscope.voice_loop import VoiceLoopTest, MockHotWordEngine, MockStreamingSTT + +# verifier gate +accepting = Mock(); accepting.verify.return_value = True +VoiceLoopTest( + ww_instances={"hey_mycroft": MockHotWordEngine(trigger_after=3)}, + verifiers=[accepting], + audio_chunks=[b"\x00" * 512] * 5, + expect_record_begin=True, +).execute() + +# full loop from a file +VoiceLoopTest( + audio_file="command.wav", + stt_instance=MockStreamingSTT(transcript="what time is it"), + expect_utterance="what time is it", +).execute() +``` + +## API surface + +| Symbol | Description | +|---|---| +| `ListenerHarness` | Base: FakeBus capture + assertion helpers + file-mic. | +| `MiniVoiceLoop` / `get_mini_voice_loop` | ovos-dinkum-listener harness + factory. | +| `MiniHotwordContainer` | Controllable hotword container with a fail-open verifier chain. | +| `MiniSimpleListener` / `get_mini_simple_listener` | ovos-simple-listener harness + factory. | +| `MiniClassicListener` | mycroft-classic-listener harness (best-effort). | +| `bridge_recognizer_loop_to_bus` / `classic_listener_available` | Classic event-bridge + capability probe. | +| `MockFileMicrophone`, `MockStreamingSTT`, `MockVADEngine`, `MockHotWordEngine` | Mock plugins shared across backends. | +| `VoiceLoopTest` | Declarative dinkum scenario runner. | diff --git a/ovoscope/__init__.py b/ovoscope/__init__.py index 5d83f94..8816468 100644 --- a/ovoscope/__init__.py +++ b/ovoscope/__init__.py @@ -1107,6 +1107,25 @@ def assert_spoke(self, text: str, lang: str = "en-US", timeout: int = 30) -> Non else: raise +from ovoscope.voice_loop import ( # noqa: F401 + ListenerHarness, + MiniVoiceLoop, + MiniHotwordContainer, + MockFileMicrophone, + MockStreamingSTT, + get_mini_voice_loop, + VoiceLoopTest, +) +from ovoscope.simple_listener import ( # noqa: F401 + MiniSimpleListener, + get_mini_simple_listener, +) +from ovoscope.classic_listener import ( # noqa: F401 + MiniClassicListener, + bridge_recognizer_loop_to_bus, + classic_listener_available, +) + @dataclasses.dataclass class GUICaptureSession: diff --git a/ovoscope/classic_listener.py b/ovoscope/classic_listener.py new file mode 100644 index 0000000..ecf9a8a --- /dev/null +++ b/ovoscope/classic_listener.py @@ -0,0 +1,393 @@ +# Copyright 2024 OpenVoiceOS +# Licensed under the Apache License, Version 2.0 +"""MiniClassicListener โ€” drive mycroft-classic-listener for bus-sequence testing. + +``mycroft-classic-listener`` is an alternative OVOS listener service built around +a ``RecognizerLoop`` (a ``pyee`` ``EventEmitter``) running an energy-based +``ResponsiveRecognizer`` across producer/consumer threads. Its service layer +bridges the loop's internal events (``recognizer_loop:record_begin``, +``โ€ฆ:wakeword``, ``โ€ฆ:record_end``, ``โ€ฆ:utterance``, +``โ€ฆ:speech.recognition.unknown``) onto the OVOS messagebus. + +Two pieces are provided: + +* :func:`bridge_recognizer_loop_to_bus` โ€” the reusable eventโ†’bus bridge (mirrors + the classic listener's ``service.py``). Wire any ``RecognizerLoop`` to a + ``FakeBus`` and assert on the captured sequence. +* :class:`MiniClassicListener` โ€” a best-effort, file-driven harness that injects + a :class:`FileAudioSource` and mock wake-word/STT into a ``RecognizerLoop`` and + runs it to completion, sharing the assertion helpers of + :class:`ovoscope.voice_loop.ListenerHarness`. + +The classic pipeline is energy-threshold based; the file drive is best-effort. +Use :func:`classic_listener_available` to gate tests on the environment. + +Example โ€” event bridge:: + + from ovoscope.classic_listener import bridge_recognizer_loop_to_bus + from ovos_utils.fakebus import FakeBus + + bus = FakeBus() + bridge_recognizer_loop_to_bus(loop, bus) # loop = a RecognizerLoop + # ... drive `loop` with real audio; assert on `bus` +""" + +from __future__ import annotations + +import time +from pathlib import Path +from typing import Any, List, Optional, Union + +from ovos_bus_client.message import Message +from ovos_utils.fakebus import FakeBus + +from ovoscope.voice_loop import ( + ListenerHarness, + MockHotWordEngine, + MockStreamingSTT, + _read_audio, +) + + +# --------------------------------------------------------------------------- +# Event โ†’ bus bridge (mirrors mycroft_classic_listener/service.py) +# --------------------------------------------------------------------------- + +def bridge_recognizer_loop_to_bus(loop: Any, bus: FakeBus) -> Any: + """Forward a ``RecognizerLoop``'s internal events onto a ``FakeBus``. + + Registers handlers on *loop* (any object with an ``on(event, handler)`` + EventEmitter API) that re-emit the listener events as bus :class:`Message` + objects, exactly as the classic listener's service layer does. + + Args: + loop: A ``RecognizerLoop`` (or compatible ``EventEmitter``). + bus: The :class:`FakeBus` to emit translated messages on. + + Returns: + *loop*, for chaining. + """ + ctx = {"client_name": "mycroft_listener", "source": "audio"} + + loop.on( + "recognizer_loop:record_begin", + lambda *a: bus.emit(Message("recognizer_loop:record_begin", context=ctx)), + ) + loop.on( + "recognizer_loop:record_end", + lambda *a: bus.emit(Message("recognizer_loop:record_end", context=ctx)), + ) + loop.on( + "recognizer_loop:wakeword", + lambda event=None, *a: bus.emit( + Message("recognizer_loop:wakeword", event or {}) + ), + ) + loop.on( + "recognizer_loop:utterance", + lambda event=None, *a: bus.emit(Message( + "recognizer_loop:utterance", + event or {}, + {**ctx, "destination": ["skills"]}, + )), + ) + loop.on( + "recognizer_loop:speech.recognition.unknown", + lambda *a: bus.emit( + Message("recognizer_loop:speech.recognition.unknown", context=ctx) + ), + ) + loop.on( + "recognizer_loop:awoken", + lambda *a: bus.emit(Message("mycroft.awoken", context=ctx)), + ) + return loop + + +# --------------------------------------------------------------------------- +# File-backed audio source (classic Microphone subclass) +# --------------------------------------------------------------------------- + +class _FileStream: + """Minimal stream serving file PCM then silence, frame by frame. + + Mirrors the read contract the classic ``ResponsiveRecognizer`` expects from + a microphone stream: ``read(num_frames, of_exc)`` returns + ``num_frames * sample_width`` bytes. + + Args: + pcm: Raw PCM bytes of the audio. + sample_width: Bytes per sample. + tail_silence_bytes: Trailing silence appended after the audio. + """ + + def __init__(self, pcm: bytes, sample_width: int, tail_silence_bytes: int) -> None: + self._data: bytes = pcm + (b"\x00" * tail_silence_bytes) + self._sample_width: int = sample_width + self._pos: int = 0 + + def read(self, num_frames: int, of_exc: bool = False) -> bytes: + """Return ``num_frames`` of audio, padding with silence past EOF.""" + nbytes = num_frames * self._sample_width + chunk = self._data[self._pos:self._pos + nbytes] + self._pos += len(chunk) + if len(chunk) < nbytes: + chunk = chunk + b"\x00" * (nbytes - len(chunk)) + return chunk + + def close(self) -> None: + """No-op close.""" + + def stop_stream(self) -> None: + """No-op stop.""" + + def is_stopped(self) -> bool: + return False + + +def _make_file_audio_source( + audio: Union[bytes, str, Path], + chunk_size: int, + tail_silence_seconds: float, +) -> Any: + """Build a classic ``Microphone`` whose stream reads from *audio*. + + Bypasses ``Microphone.__init__`` (which opens PyAudio and needs a real input + device) while still satisfying the ``isinstance(source, Microphone)`` check + in ``ResponsiveRecognizer.listen``. + """ + from mycroft_classic_listener.mic import Microphone + + pcm, sr, sw, _ch = _read_audio(audio, 16000, 2, 1) + + class FileAudioSource(Microphone): + """File-backed classic microphone (no PyAudio).""" + + def __init__(self) -> None: + self.device_index = None + self.format = 8 # pyaudio.paInt16 + self.SAMPLE_WIDTH = sw + self.SAMPLE_RATE = sr + self.CHUNK = chunk_size + self.muted = False + self.audio = None + self.stream = None + self._pcm = pcm + self._tail = int(tail_silence_seconds * sr * sw) + + def __enter__(self): + self.stream = _FileStream(self._pcm, self.SAMPLE_WIDTH, self._tail) + return self + + def __exit__(self, *_): + self.stream = None + + def restart(self): + self.stream = _FileStream(self._pcm, self.SAMPLE_WIDTH, self._tail) + + def duration_to_bytes(self, sec): + return int(sec * self.SAMPLE_RATE * self.SAMPLE_WIDTH) + + def mute(self): + self.muted = True + + def unmute(self): + self.muted = False + + def is_muted(self): + return self.muted + + return FileAudioSource() + + +def _build_recognizer_loop(file_source: Any, wakeword: Any, stt: Any) -> Any: + """Build a ``RecognizerLoop`` with mocks injected, bypassing plugin/hardware. + + Subclasses ``RecognizerLoop`` to replace ``_load_config`` (which would open + PyAudio and load real wake-word plugins) and ``start_async`` (which would + create a real STT) with the injected file source, wake-word engine, and STT. + """ + from mycroft_classic_listener.listener import ( + AudioConsumer, + AudioProducer, + RecognizerLoop, + RecognizerLoopState, + ResponsiveRecognizer, + ) + try: + from queue import Queue + except ImportError: # pragma: no cover + from Queue import Queue # type: ignore + + class _InjectedRecognizerLoop(RecognizerLoop): + def __init__(self) -> None: + super(RecognizerLoop, self).__init__() # EventEmitter.__init__ + self._watchdog = lambda: None + self.mute_calls = 0 + self.lang = "en-us" + self.config = {} + self.microphone = file_source + self.wakeword_recognizer = wakeword + self.wakeup_recognizer = MockHotWordEngine("wake_up", trigger_after=10**9) + self.responsive_recognizer = ResponsiveRecognizer( + self.wakeword_recognizer, self._watchdog + ) + self.state = RecognizerLoopState() + self._config_hash = None + + def start_async(self) -> None: + self.state.running = True + self.producer = AudioProducer( + self.state, Queue(), self.microphone, + self.responsive_recognizer, self, None, + ) + # share one queue between producer and consumer + queue = self.producer.queue + self.producer.start() + self.consumer = AudioConsumer( + self.state, queue, self, stt, + self.wakeup_recognizer, self.wakeword_recognizer, + ) + self.consumer.start() + + def stop(self) -> None: + self.state.running = False + try: + self.producer.stop() + self.producer.join(timeout=2.0) + self.consumer.join(timeout=2.0) + except Exception: + pass + + return _InjectedRecognizerLoop() + + +def classic_listener_available() -> bool: + """Return ``True`` if mycroft-classic-listener can be imported here.""" + import importlib.util + + try: + return all( + importlib.util.find_spec(mod) is not None + for mod in ( + "mycroft_classic_listener.listener", + "mycroft_classic_listener.mic", + ) + ) + except ModuleNotFoundError: + return False + + +# --------------------------------------------------------------------------- +# MiniClassicListener +# --------------------------------------------------------------------------- + +class MiniClassicListener(ListenerHarness): + """Best-effort in-process mycroft-classic-listener harness. + + Wires a ``RecognizerLoop`` (built with a file audio source + mock wake-word / + STT, or one supplied by the caller) to a ``FakeBus`` via + :func:`bridge_recognizer_loop_to_bus`, then drives it over an audio file. + + Args: + recognizer_loop: A pre-built ``RecognizerLoop`` (or compatible + EventEmitter). When provided, only the bus bridge is wired and the + caller drives the loop; :meth:`feed_file` is unavailable. + wakeword: Wake-word engine for the built loop (defaults to a + :class:`MockHotWordEngine`). + stt_instance: STT engine for the built loop (defaults to + :class:`MockStreamingSTT`). + bus: Optional :class:`FakeBus` to capture on. + + Raises: + RuntimeError: If *recognizer_loop* is ``None`` and + mycroft-classic-listener is not importable. + """ + + def __init__( + self, + recognizer_loop: Optional[Any] = None, + *, + wakeword: Optional[Any] = None, + stt_instance: Optional[Any] = None, + bus: Optional[FakeBus] = None, + ) -> None: + super().__init__(bus) + self._built = recognizer_loop is None + self._wakeword = wakeword + self._stt = stt_instance + + if recognizer_loop is not None: + self.loop: Any = recognizer_loop + bridge_recognizer_loop_to_bus(self.loop, self.bus) + else: + if not classic_listener_available(): + raise RuntimeError( + "mycroft-classic-listener is required to build a " + "MiniClassicListener. Install it with: " + "pip install mycroft-classic-listener" + ) + self.loop = None # built per-run in feed_file (needs the audio) + + def feed_file( + self, + audio: Union[bytes, str, Path], + *, + tail_silence_seconds: float = 2.0, + chunk_size: int = 1024, + timeout: float = 15.0, + ) -> List[Message]: + """Run the classic loop over an audio file and capture bus events. + + Builds a fresh ``RecognizerLoop`` with a :class:`FileAudioSource`, bridges + it to the bus, runs it until a command finishes + (``recognizer_loop:record_end``) or *timeout* elapses, then stops it. + + Args: + audio: Path to a ``.wav`` file, WAV bytes, or raw PCM bytes. + tail_silence_seconds: Trailing silence appended after the audio so + the energy recogniser can end the command. + chunk_size: Frames per microphone read. + timeout: Maximum seconds to wait for the command to finish. + + Returns: + The list of :class:`Message` objects emitted during the run. + + Raises: + RuntimeError: If this harness was constructed with an external loop. + """ + if not self._built: + raise RuntimeError( + "feed_file is only available when MiniClassicListener builds the " + "RecognizerLoop. Drive the supplied loop yourself and assert on " + ".bus instead." + ) + + wakeword = self._wakeword or MockHotWordEngine("hey_mycroft", trigger_after=1) + stt = self._stt or MockStreamingSTT() + source = _make_file_audio_source(audio, chunk_size, tail_silence_seconds) + + self.loop = _build_recognizer_loop(source, wakeword, stt) + bridge_recognizer_loop_to_bus(self.loop, self.bus) + + self._messages.clear() + self.loop.start_async() + try: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if self._has(self._messages, "recognizer_loop:record_end"): + break + time.sleep(0.02) + finally: + self.loop.stop() + + self._last_messages = list(self._messages) + return list(self._messages) + + def shutdown(self) -> None: + """Stop the loop if the harness owns it.""" + if self._built and self.loop is not None: + try: + self.loop.stop() + except Exception: + pass diff --git a/ovoscope/simple_listener.py b/ovoscope/simple_listener.py new file mode 100644 index 0000000..2317cb5 --- /dev/null +++ b/ovoscope/simple_listener.py @@ -0,0 +1,223 @@ +# Copyright 2024 OpenVoiceOS +# Licensed under the Apache License, Version 2.0 +"""MiniSimpleListener โ€” drive ovos-simple-listener for bus-sequence testing. + +``ovos-simple-listener`` is an alternative OVOS listener service: a single +``SimpleListener`` thread that reads microphone chunks, detects a wake word (or +VAD activation), records the command, and dispatches a small set of callbacks. +Its canonical bus integration (``OVOSCallbacks`` in +``ovos_simple_listener.__main__``) emits ``recognizer_loop:wakeword`` / +``โ€ฆ:record_begin`` on activation, ``โ€ฆ:utterance`` / +``โ€ฆ:speech.recognition.unknown`` after STT, and ``โ€ฆ:record_end`` when the command +finishes. + +:class:`MiniSimpleListener` wires a ``SimpleListener`` to a ``FakeBus`` with mock +wake-word / VAD / STT plugins and a :class:`MockFileMicrophone`, runs the loop +over an arbitrary audio file, and captures the emitted bus sequence โ€” sharing the +assertion helpers of :class:`ovoscope.voice_loop.ListenerHarness`. + +Example:: + + from ovoscope.simple_listener import MiniSimpleListener + from ovoscope.voice_loop import MockHotWordEngine, MockStreamingSTT + + with MiniSimpleListener( + wakeword=MockHotWordEngine("hey_mycroft", trigger_after=2), + stt_instance=MockStreamingSTT(transcript="turn on the lights"), + ) as sl: + msgs = sl.feed_file("command.wav") + sl.assert_record_begin_emitted(msgs) + sl.assert_utterance_emitted("turn on the lights", msgs) +""" + +from __future__ import annotations + +import time +from pathlib import Path +from typing import Any, List, Optional, Union + +from ovos_bus_client.message import Message +from ovos_utils.fakebus import FakeBus + +from ovoscope.voice_loop import ( + ListenerHarness, + MockHotWordEngine, + MockStreamingSTT, + MockVADEngine, +) + + +class _SimpleBusCallbacks: + """Per-instance ``ListenerCallbacks`` that emit on a ``FakeBus``. + + Mirrors the canonical ``OVOSCallbacks`` from + ``ovos_simple_listener.__main__`` (minus the listen-sound playback), but + binds the bus per instance instead of on the class so concurrent harnesses + do not clobber one another. + + Args: + bus: The :class:`FakeBus` to emit listener events on. + """ + + def __init__(self, bus: FakeBus) -> None: + self.bus: FakeBus = bus + + def listen_callback(self) -> None: + """Activation: emit wakeword + record-begin.""" + self.bus.emit(Message("recognizer_loop:wakeword")) + self.bus.emit(Message("recognizer_loop:record_begin")) + + def end_listen_callback(self) -> None: + """Command finished: emit record-end.""" + self.bus.emit(Message("recognizer_loop:record_end")) + + def audio_callback(self, audio: Any) -> None: + """Recorded command audio is available (no-op).""" + + def error_callback(self, audio: Any) -> None: + """Empty transcription: emit the recognition-unknown event.""" + self.bus.emit(Message("recognizer_loop:speech.recognition.unknown")) + + def text_callback(self, utterance: str, lang: str) -> None: + """Transcription succeeded: emit the utterance.""" + self.bus.emit(Message( + "recognizer_loop:utterance", + {"utterances": [utterance], "lang": lang}, + )) + + +class MiniSimpleListener(ListenerHarness): + """In-process ovos-simple-listener harness for bus-sequence testing. + + Args: + wakeword: Wake-word engine (``update`` + ``found_wake_word``). Defaults + to a :class:`MockHotWordEngine` keyed ``"hey_mycroft"``. Pass + ``None`` to use VAD-only activation. + vad_instance: VAD engine (defaults to :class:`MockVADEngine`). + stt_instance: Streaming STT engine (defaults to + :class:`MockStreamingSTT` returning no transcript). + min_speech_seconds: Minimum command speech before a silence can end it. + max_silence_seconds: Trailing silence that ends a command. + max_speech_seconds: Hard cap on command length. + bus: Optional :class:`FakeBus` to capture on. + + Raises: + RuntimeError: If ovos-simple-listener is not installed. + + The default timing is tightened (``min_speech_seconds=0`` / + ``max_silence_seconds=0.1``) so a file-driven command ends promptly and + deterministically; raise them to mimic production behaviour. + """ + + def __init__( + self, + *, + wakeword: Optional[Any] = "__default__", + vad_instance: Optional[Any] = None, + stt_instance: Optional[Any] = None, + min_speech_seconds: float = 0.0, + max_silence_seconds: float = 0.1, + max_speech_seconds: float = 8.0, + bus: Optional[FakeBus] = None, + ) -> None: + super().__init__(bus) + + try: + from ovos_simple_listener import SimpleListener + except ImportError as e: + raise RuntimeError( + "ovos-simple-listener is required to use MiniSimpleListener. " + "Install it with: pip install ovos-simple-listener" + ) from e + + if wakeword == "__default__": + wakeword = MockHotWordEngine("hey_mycroft", trigger_after=2) + + self.callbacks = _SimpleBusCallbacks(self.bus) + self.listener = SimpleListener( + wakeword=wakeword, + mic=None, # supplied per-run by feed_file + vad=vad_instance if vad_instance is not None else MockVADEngine(), + stt=stt_instance if stt_instance is not None else MockStreamingSTT(), + min_speech_seconds=min_speech_seconds, + max_silence_seconds=max_silence_seconds, + max_speech_seconds=max_speech_seconds, + callbacks=self.callbacks, + ) + + def feed_file( + self, + audio: Union[bytes, str, Path], + *, + silence_tail_chunks: int = 25, + chunk_size: int = 2048, + timeout: float = 10.0, + ) -> List[Message]: + """Run the simple listener over an audio file and capture bus events. + + Streams *audio* through a :class:`MockFileMicrophone`, runs the listener + thread until the command completes (``recognizer_loop:record_end``) or + *timeout* elapses, then stops it. + + Args: + audio: Path to a ``.wav`` file, WAV bytes, or raw PCM bytes. + silence_tail_chunks: Trailing silent frames appended after the audio + so the command can end. + chunk_size: Bytes per microphone read. + timeout: Maximum seconds to wait for the command to finish. + + Returns: + The list of :class:`Message` objects emitted during the run. + """ + mic = self._build_file_mic(audio, silence_tail_chunks, chunk_size) + self.listener.mic = mic + + self._messages.clear() + self.listener.start() # Thread.start โ†’ runs the loop in the background + try: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if self._has(self._messages, "recognizer_loop:record_end"): + break + time.sleep(0.02) + finally: + self.listener.stop() + self.listener.join(timeout=2.0) + + self._last_messages = list(self._messages) + return list(self._messages) + + def shutdown(self) -> None: + """Stop the listener thread if still running.""" + try: + self.listener.stop() + if self.listener.is_alive(): + self.listener.join(timeout=2.0) + except Exception: + pass + + +def get_mini_simple_listener( + wakeword: Optional[Any] = "__default__", + vad_instance: Optional[Any] = None, + stt_instance: Optional[Any] = None, + bus: Optional[FakeBus] = None, +) -> MiniSimpleListener: + """Factory: create a ready-to-feed :class:`MiniSimpleListener`. + + Args: + wakeword: Wake-word engine (defaults to a :class:`MockHotWordEngine`). + vad_instance: VAD engine (defaults to :class:`MockVADEngine`). + stt_instance: Streaming STT engine (defaults to + :class:`MockStreamingSTT`). + bus: Optional :class:`FakeBus` to capture on. + + Returns: + A fully initialised :class:`MiniSimpleListener`. + """ + return MiniSimpleListener( + wakeword=wakeword, + vad_instance=vad_instance, + stt_instance=stt_instance, + bus=bus, + ) diff --git a/ovoscope/voice_loop.py b/ovoscope/voice_loop.py new file mode 100644 index 0000000..ec208bb --- /dev/null +++ b/ovoscope/voice_loop.py @@ -0,0 +1,1056 @@ +# Copyright 2024 OpenVoiceOS +# Licensed under the Apache License, Version 2.0 +"""MiniVoiceLoop โ€” drive ``DinkumVoiceLoop`` bus sequences for ovoscope. + +Where :class:`ovoscope.listener.MiniListener` covers the +``AudioTransformersService``, STT, and mock VAD/WakeWord engines, it does **not** +exercise the full ``DinkumVoiceLoop`` state machine. The bus events that matter +for wake-word handling (``recognizer_loop:wakeword``, +``recognizer_loop:record_begin``, ``recognizer_loop:record_end``, +``recognizer_loop:utterance``) are emitted as side-effects of the voice loop as +PCM chunks flow through it. + +``MiniVoiceLoop`` wires a real ``DinkumVoiceLoop`` to a ``FakeBus`` with no-op +mic/STT/transformer plugins, a controllable hotword container, and an optional +verifier chain. It supports two drive modes: + +* :meth:`MiniVoiceLoop.feed_chunks` โ€” feed PCM frames directly through + ``_detect_ww`` to assert the wake-word detection / verifier gate in isolation. +* :meth:`MiniVoiceLoop.feed_file` โ€” run the **whole** ``DinkumVoiceLoop.run()`` + state machine, reading an arbitrary audio file through a file-backed + microphone plugin, so the full record-begin โ†’ wakeword โ†’ command โ†’ record-end + โ†’ utterance sequence is captured. + +Example โ€” verifier gate (``_detect_ww`` only):: + + from unittest.mock import Mock + from ovoscope.voice_loop import MiniVoiceLoop, MockHotWordEngine + + SILENT_CHUNK = b"\\x00" * 512 + ww = MockHotWordEngine(key_phrase="hey_mycroft", trigger_after=3) + accepting = Mock(); accepting.verify.return_value = True + + with MiniVoiceLoop(ww_instances={"hey_mycroft": ww}, + verifiers=[accepting]) as vl: + msgs = vl.feed_chunks([SILENT_CHUNK] * 5) + vl.assert_record_begin_emitted(msgs) + +Example โ€” full loop driven from an audio file:: + + from ovoscope.voice_loop import MiniVoiceLoop, MockStreamingSTT + + stt = MockStreamingSTT(transcript="what time is it") + with MiniVoiceLoop(stt_instance=stt) as vl: + msgs = vl.feed_file("command.wav") + assert any(m.msg_type == "recognizer_loop:utterance" for m in msgs) + +The verifier gate lives inside ``DinkumVoiceLoop._detect_ww`` and is only present +in ovos-dinkum-listener builds that ship the hotword-verifier feature +(``HotwordContainer.verify``). On a build without it the gate is absent and the +detection is never suppressed โ€” assert accordingly for the version under test. +""" + +from __future__ import annotations + +import io +import wave +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple, Union + +from ovos_bus_client.message import Message +from ovos_utils.fakebus import FakeBus + +# Re-export the engine mocks so callers have a single import site for the +# voice-loop harness. +from ovoscope.listener import MockHotWordEngine, MockVADEngine # noqa: F401 + + +# --------------------------------------------------------------------------- +# Internal helpers +# --------------------------------------------------------------------------- + +def _read_audio( + audio: Union[bytes, str, Path], + sample_rate: int, + sample_width: int, + sample_channels: int, +) -> Tuple[bytes, int, int, int]: + """Read raw PCM from a WAV file/bytes (or raw PCM) for the file mic. + + Args: + audio: Path to a ``.wav`` file, WAV bytes, or raw PCM bytes. + sample_rate: Fallback sample rate when no WAV header is present. + sample_width: Fallback sample width (bytes). + sample_channels: Fallback channel count. + + Returns: + ``(pcm_bytes, sample_rate, sample_width, sample_channels)``. + """ + if isinstance(audio, (str, Path)): + with open(audio, "rb") as fh: + raw = fh.read() + else: + raw = audio + + try: + with wave.open(io.BytesIO(raw)) as wf: + sample_rate = wf.getframerate() + sample_width = wf.getsampwidth() + sample_channels = wf.getnchannels() + pcm = wf.readframes(wf.getnframes()) + return pcm, sample_rate, sample_width, sample_channels + except (wave.Error, EOFError): + # not a WAV container โ€” treat input as raw PCM + return raw, sample_rate, sample_width, sample_channels + + +# --------------------------------------------------------------------------- +# Mock plugin stand-ins for the DinkumVoiceLoop dependencies +# --------------------------------------------------------------------------- + +class _MockMicrophone: + """Silent stand-in for the listener ``Microphone`` plugin. + + Used when no audio file is driven; ``_detect_ww`` does not read from the + mic, but the dataclass requires one and a few derived values + (``sample_rate``, ``seconds_per_chunk``) are referenced by adjacent loop + stages. + + Args: + sample_rate: Audio sample rate in Hz. + sample_width: Sample width in bytes. + sample_channels: Channel count. + chunk_size: Bytes per read. + """ + + def __init__( + self, + sample_rate: int = 16000, + sample_width: int = 2, + sample_channels: int = 1, + chunk_size: int = 2048, + ) -> None: + self.sample_rate: int = sample_rate + self.sample_width: int = sample_width + self.sample_channels: int = sample_channels + self.chunk_size: int = chunk_size + + @property + def seconds_per_chunk(self) -> float: + """Duration in seconds of one ``chunk_size`` read.""" + frames = self.chunk_size / max(self.sample_width, 1) + return frames / max(self.sample_rate, 1) + + def read_chunk(self) -> bytes: + """Return a silent chunk (the harness feeds audio explicitly).""" + return b"\x00" * self.chunk_size + + def start(self) -> None: + """No-op start hook.""" + + def stop(self) -> None: + """No-op stop hook.""" + + +class MockFileMicrophone: + """File-backed ``Microphone`` plugin for the voice loop. + + Streams an arbitrary audio file (or raw PCM) into ``DinkumVoiceLoop.run()`` + one ``chunk_size`` frame at a time, then appends a tail of silent frames so + a silence-based VAD can detect the end of the command. When every frame has + been read, :meth:`read_chunk` invokes :attr:`on_exhausted` (used by + :class:`MiniVoiceLoop` to stop the loop) and returns ``None``. + + Args: + audio: Path to a ``.wav`` file, WAV bytes, or raw PCM bytes. + chunk_size: Bytes per :meth:`read_chunk`. + sample_rate: Fallback sample rate when the input has no WAV header. + sample_width: Fallback sample width (bytes). + sample_channels: Fallback channel count. + silence_chunks: Number of trailing silent frames appended after the + file so the command can end and the loop can return to wake-word + detection. + + Attributes: + on_exhausted: Optional zero-arg callback invoked once the audio is fully + consumed (set by :class:`MiniVoiceLoop` to stop the loop). + """ + + def __init__( + self, + audio: Union[bytes, str, Path], + chunk_size: int = 2048, + sample_rate: int = 16000, + sample_width: int = 2, + sample_channels: int = 1, + silence_chunks: int = 25, + ) -> None: + pcm, sr, sw, ch = _read_audio( + audio, sample_rate, sample_width, sample_channels + ) + self.sample_rate: int = sr + self.sample_width: int = sw + self.sample_channels: int = ch + self.chunk_size: int = chunk_size + + self._chunks: List[bytes] = [ + pcm[i:i + chunk_size] for i in range(0, len(pcm), chunk_size) + ] + self._chunks.extend(b"\x00" * chunk_size for _ in range(silence_chunks)) + self._idx: int = 0 + self.on_exhausted: Optional[Any] = None + + @property + def seconds_per_chunk(self) -> float: + """Duration in seconds of one ``chunk_size`` read.""" + frames = self.chunk_size / max(self.sample_width, 1) + return frames / max(self.sample_rate, 1) + + def read_chunk(self) -> Optional[bytes]: + """Return the next audio frame, or ``None`` when exhausted. + + On exhaustion the :attr:`on_exhausted` callback is invoked so the loop + can stop. + """ + if self._idx >= len(self._chunks): + if self.on_exhausted is not None: + self.on_exhausted() + return None + chunk = self._chunks[self._idx] + self._idx += 1 + return chunk + + def start(self) -> None: + """No-op start hook.""" + + def stop(self) -> None: + """No-op stop hook.""" + + +class MockStreamingSTT: + """Configurable streaming STT engine for the voice loop. + + Returns a fixed transcript when the recorded command is finalized. An empty + transcript yields ``recognizer_loop:speech.recognition.unknown`` (matching + the real listener), which is useful for asserting the silent-recording path. + + Args: + transcript: Text returned by :meth:`transcribe`. Empty string means "no + transcription". + confidence: Confidence score paired with the transcript. + lang: Language tag reported by the ``lang`` property. + """ + + can_stream: bool = False + + def __init__( + self, + transcript: str = "", + confidence: float = 1.0, + lang: str = "en-us", + ) -> None: + self.transcript: str = transcript + self.confidence: float = confidence + self._lang: str = lang + self.fed_chunks: int = 0 + + @property + def lang(self) -> str: + """Language tag for transcription.""" + return self._lang + + def stream_start(self, *args: Any, **kwargs: Any) -> None: + """Begin a streaming transcription session (no-op).""" + + def stream_data(self, chunk: bytes) -> None: + """Feed audio to the streaming session.""" + self.fed_chunks += 1 + + def stream_stop(self) -> str: + """End the session and return the transcript text.""" + return self.transcript + + def execute(self, audio: Any = None, language: Optional[str] = None) -> Optional[str]: + """Non-streaming transcription (used by mycroft-classic-listener). + + Args: + audio: Recorded audio clip (ignored). + language: Language override (ignored). + + Returns: + The configured transcript, or ``None`` when empty (so the classic + consumer emits ``recognizer_loop:speech.recognition.unknown``). + """ + return self.transcript or None + + def transcribe( + self, audio: Any = None, lang: Optional[str] = None + ) -> List[Tuple[str, float]]: + """Return the configured transcript as ``[(text, confidence)]``. + + Always returns a single ``(text, confidence)`` pair โ€” the text is the + empty string when no transcript is configured. This matches the + ``transcribe(audio)[0][0]`` access pattern used by ovos-simple-listener + while still letting the dinkum loop's confidence filter run; harness + callbacks treat an empty transcript as "no utterance". + + Args: + audio: Ignored (the loop streams audio via :meth:`stream_data`). + lang: Ignored language override. + + Returns: + ``[(transcript, confidence)]``. + """ + return [(self.transcript, self.confidence)] + + def shutdown(self) -> None: + """Graceful shutdown (no-op).""" + + +class _MockTransformers: + """No-op ``AudioTransformersService`` stand-in. + + Args: + bus: The :class:`FakeBus` the loop is wired to. + """ + + def __init__(self, bus: FakeBus) -> None: + self.bus: FakeBus = bus + self.hotword_chunks: List[bytes] = [] + + def feed_hotword(self, chunk: bytes) -> None: + """Record a chunk forwarded after wake-word detection.""" + self.hotword_chunks.append(chunk) + + def feed_audio(self, chunk: bytes) -> None: + """Consume a non-speech chunk (no-op).""" + + def feed_speech(self, chunk: bytes) -> None: + """Consume a speech chunk (no-op).""" + + def transform(self, chunk: bytes) -> Tuple[bytes, Dict[str, Any]]: + """Return the chunk unchanged with empty context.""" + return chunk, {} + + def shutdown(self) -> None: + """Graceful shutdown (no-op).""" + + +class MiniHotwordContainer: + """Controllable hotword container for :class:`MiniVoiceLoop`. + + Implements the subset of ``ovos_dinkum_listener.voice_loop.HotwordContainer`` + that the voice loop relies on, without requiring the wrapped engines to be + real ``HotWordEngine`` subclasses โ€” so ovoscope's :class:`MockHotWordEngine` + can drive the loop directly. + + Every registered engine is treated as a **listen word** (it triggers the STT + stage). :meth:`update` and :meth:`found` are state-aware so the loop's + separate hotword-detection branch (``_detect_hot``) does not double-count or + mis-fire the listen engines. + + The :meth:`verify` chain mirrors the real container's **fail-open** + semantics: a verifier that returns ``False`` suppresses the detection; a + verifier that raises is skipped (the detection survives). + + Args: + ww_instances: Mapping of wake-word name โ†’ engine instance (each engine + implements ``update(chunk)``, ``found_wake_word() -> bool`` and + ``reset()``). + verifiers: Optional list of verifier objects with a + ``verify(ww_audio: bytes) -> bool`` method. + + Attributes: + state: Listening state, assigned by the voice loop during detection. + reload_on_failure: Always ``False`` (no engine reloading in tests). + """ + + def __init__( + self, + ww_instances: Dict[str, Any], + verifiers: Optional[List[Any]] = None, + ) -> None: + self._engines: Dict[str, Any] = dict(ww_instances) + self.verifiers: List[Any] = list(verifiers) if verifiers else [] + self.state: Any = None + self.reload_on_failure: bool = False + + def _active_engines(self) -> Dict[str, Any]: + """Return the engines relevant to the current listening state. + + All registered engines are listen words, so they are active in the + ``LISTEN`` state (and when no state has been set yet, as used by direct + ``_detect_ww`` feeding). In every other state โ€” ``HOTWORD``, + ``WAKEUP``, ``RECORDING`` โ€” no engines are active. + """ + name = getattr(self.state, "name", None) + if name in (None, "LISTEN"): + return self._engines + return {} + + def update(self, chunk: bytes) -> None: + """Feed *chunk* to the engines active in the current state. + + Args: + chunk: Raw PCM audio bytes. + """ + for engine in self._active_engines().values(): + engine.update(chunk) + + def found(self) -> Optional[str]: + """Return the name of the first active engine reporting a detection. + + A detected engine is reset so a stale ``update_count`` does not re-fire + the wake word on subsequent chunks (e.g. during the silence tail of a + file-driven run). + + Returns: + The wake-word name, or ``None`` if no active engine fired. + """ + for name, engine in self._active_engines().items(): + if engine.found_wake_word(): + engine.reset() + return name + return None + + def get_ww(self, ww: str) -> Dict[str, Any]: + """Return metadata for wake word *ww*. + + Mirrors the keys ``DinkumVoiceLoop`` and the listener's hotword callback + read. The wake word is treated as a listen word (it triggers the STT + stage), with no confirmation sound. + + Args: + ww: Wake-word name. + + Returns: + Metadata dict for the wake word. + + Raises: + ValueError: If *ww* is not registered. + """ + if ww not in self._engines: + raise ValueError(f"Requested ww not defined: {ww}") + engine = self._engines[ww] + return { + "key_phrase": ww, + "module": getattr(engine, "key_phrase", ww), + "engine": engine.__class__.__name__, + "sound": None, + "listen": True, + "utterance": None, + "stt_lang": "en-us", + "bus_event": None, + "wakeup": False, + "stopword": False, + } + + def verify(self, ww_audio: bytes) -> bool: + """Run the verifier chain against the wake-word audio. + + Fail-open: only an explicit ``False`` return suppresses the detection; + a verifier that raises is skipped. + + Args: + ww_audio: Raw PCM bytes of the audio that triggered the engine. + + Returns: + ``True`` if every verifier accepts (or none are configured), + ``False`` if any verifier rejects the audio. + """ + for verifier in self.verifiers: + try: + if not verifier.verify(ww_audio): + return False + except Exception: + # fail-open: a raising verifier does not discard the detection + pass + return True + + def reset(self) -> None: + """Reset all wrapped engines (called by the loop after a command).""" + for engine in self._engines.values(): + try: + engine.reset() + except Exception: + pass + + def shutdown(self) -> None: + """Shut down all wrapped engines gracefully.""" + for engine in self._engines.values(): + try: + engine.shutdown() + except Exception: + pass + + +# --------------------------------------------------------------------------- +# Shared harness base +# --------------------------------------------------------------------------- + +class ListenerHarness: + """Base for in-process listener-service harnesses. + + Owns a :class:`FakeBus`, captures every ``Message`` emitted on it, and + provides the common ``recognizer_loop:*`` assertion helpers. Concrete + backends (:class:`MiniVoiceLoop` for ovos-dinkum-listener, + ``MiniSimpleListener`` for ovos-simple-listener, ``MiniClassicListener`` for + mycroft-classic-listener) wire their specific listener to this bus and add a + drive method (``feed_file`` / ``feed_chunks``). + + Args: + bus: Optional :class:`FakeBus` to capture on. Defaults to a fresh bus. + """ + + def __init__(self, bus: Optional[FakeBus] = None) -> None: + self.bus: FakeBus = bus if bus is not None else FakeBus() + self._messages: List[Message] = [] + self._last_messages: List[Message] = [] + self.bus.on("message", self._capture) + + def _capture(self, msg: Any) -> None: + if isinstance(msg, str): + try: + msg = Message.deserialize(msg) + except Exception: + return + self._messages.append(msg) + + # ------------------------------------------------------------------ + # File microphone + # ------------------------------------------------------------------ + + @staticmethod + def _build_file_mic( + audio: Union[bytes, str, Path], + silence_chunks: int, + chunk_size: int, + ) -> MockFileMicrophone: + """Build a :class:`MockFileMicrophone` for *audio*.""" + return MockFileMicrophone( + audio, chunk_size=chunk_size, silence_chunks=silence_chunks + ) + + # ------------------------------------------------------------------ + # Assertion helpers + # ------------------------------------------------------------------ + + @staticmethod + def _has(messages: List[Message], msg_type: str) -> bool: + return any(m.msg_type == msg_type for m in messages) + + def _resolve(self, messages: Optional[List[Message]]) -> List[Message]: + return messages if messages is not None else self._last_messages + + def assert_record_begin_emitted( + self, messages: Optional[List[Message]] = None + ) -> List[Message]: + """Assert ``recognizer_loop:record_begin`` was emitted. + + Args: + messages: Messages to check; defaults to the last feed result. + + Returns: + The checked message list. + + Raises: + AssertionError: If no record-begin event is present. + """ + msgs = self._resolve(messages) + assert self._has(msgs, "recognizer_loop:record_begin"), ( + "Expected 'recognizer_loop:record_begin' but it was not emitted. " + f"Captured: {[m.msg_type for m in msgs]}" + ) + return msgs + + def assert_wakeword_detected( + self, messages: Optional[List[Message]] = None + ) -> List[Message]: + """Assert a wake word was detected and recording began. + + Checks for both ``recognizer_loop:wakeword`` and + ``recognizer_loop:record_begin``. + + Args: + messages: Messages to check; defaults to the last feed result. + + Returns: + The checked message list. + + Raises: + AssertionError: If either expected event is missing. + """ + msgs = self._resolve(messages) + captured = [m.msg_type for m in msgs] + assert self._has(msgs, "recognizer_loop:wakeword"), ( + "Expected 'recognizer_loop:wakeword' but it was not emitted. " + f"Captured: {captured}" + ) + assert self._has(msgs, "recognizer_loop:record_begin"), ( + "Expected 'recognizer_loop:record_begin' but it was not emitted. " + f"Captured: {captured}" + ) + return msgs + + def assert_wakeword_suppressed( + self, messages: Optional[List[Message]] = None + ) -> List[Message]: + """Assert no wake-word recording was triggered. + + Verifies that neither ``recognizer_loop:wakeword`` nor + ``recognizer_loop:record_begin`` was emitted. + + Args: + messages: Messages to check; defaults to the last feed result. + + Returns: + The checked message list. + + Raises: + AssertionError: If a wake-word or record-begin event is present. + """ + msgs = self._resolve(messages) + captured = [m.msg_type for m in msgs] + assert not self._has(msgs, "recognizer_loop:record_begin"), ( + "Expected wake word to be suppressed, but " + f"'recognizer_loop:record_begin' was emitted. Captured: {captured}" + ) + assert not self._has(msgs, "recognizer_loop:wakeword"), ( + "Expected wake word to be suppressed, but " + f"'recognizer_loop:wakeword' was emitted. Captured: {captured}" + ) + return msgs + + def assert_utterance_emitted( + self, + utterance: Optional[str] = None, + messages: Optional[List[Message]] = None, + ) -> List[Message]: + """Assert a ``recognizer_loop:utterance`` was emitted. + + Args: + utterance: When given, also assert this exact text is among the + emitted utterances. + messages: Messages to check; defaults to the last feed result. + + Returns: + The checked message list. + + Raises: + AssertionError: If no utterance (or the named one) was emitted. + """ + msgs = self._resolve(messages) + utts: List[str] = [] + for m in msgs: + if m.msg_type == "recognizer_loop:utterance": + utts.extend(m.data.get("utterances", [])) + assert utts, ( + "Expected 'recognizer_loop:utterance' but it was not emitted. " + f"Captured: {[m.msg_type for m in msgs]}" + ) + if utterance is not None: + assert utterance in utts, ( + f"Expected utterance {utterance!r} but got: {utts}" + ) + return msgs + + # ------------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------------ + + def shutdown(self) -> None: + """Release backend resources (overridden by subclasses).""" + + def __enter__(self) -> "ListenerHarness": + return self + + def __exit__(self, *_: Any) -> None: + self.shutdown() + + +# --------------------------------------------------------------------------- +# MiniVoiceLoop +# --------------------------------------------------------------------------- + +class MiniVoiceLoop(ListenerHarness): + """In-process ``DinkumVoiceLoop`` harness for bus-sequence testing. + + Captures every ``Message`` emitted on the ``FakeBus`` as audio flows through + the loop. The voice-loop callbacks are wired to emit the same bus events as + the real listener service (``recognizer_loop:wakeword``, + ``recognizer_loop:record_begin``, ``recognizer_loop:record_end``, + ``recognizer_loop:utterance`` / ``โ€ฆ:speech.recognition.unknown``). + + Args: + voice_loop: A pre-built ``DinkumVoiceLoop`` instance. When provided, the + caller owns its plugins/callbacks, which should emit on this + harness's *bus* to be captured. When ``None``, a loop is built from + the mock arguments below. + ww_instances: Mapping of wake-word name โ†’ engine instance. Defaults to a + single :class:`MockHotWordEngine` keyed ``"hey_mycroft"``. + verifiers: Optional verifier objects (``verify(audio) -> bool``) gating + detection. + vad_instance: Optional VAD engine (defaults to :class:`MockVADEngine`). + stt_instance: Optional streaming STT engine (defaults to a + :class:`MockStreamingSTT` returning no transcript). Used by + :meth:`feed_file`. + bus: Optional :class:`FakeBus` to capture on. Defaults to a fresh bus. + + Raises: + RuntimeError: If *voice_loop* is ``None`` and ovos-dinkum-listener is not + installed. + + Example:: + + from ovoscope.voice_loop import MiniVoiceLoop, MockHotWordEngine + + ww = MockHotWordEngine("hey_mycroft", trigger_after=3) + with MiniVoiceLoop(ww_instances={"hey_mycroft": ww}) as vl: + msgs = vl.feed_chunks([b"\\x00" * 512] * 5) + vl.assert_record_begin_emitted(msgs) + """ + + def __init__( + self, + voice_loop: Optional[Any] = None, + *, + ww_instances: Optional[Dict[str, Any]] = None, + verifiers: Optional[List[Any]] = None, + vad_instance: Optional[Any] = None, + stt_instance: Optional[Any] = None, + bus: Optional[FakeBus] = None, + ) -> None: + super().__init__(bus) + + self.hotwords: Optional[MiniHotwordContainer] = None + if voice_loop is not None: + self.voice_loop: Any = voice_loop + hw = getattr(voice_loop, "hotwords", None) + if isinstance(hw, MiniHotwordContainer): + self.hotwords = hw + else: + self.voice_loop = self._build_voice_loop( + ww_instances=ww_instances, + verifiers=verifiers, + vad_instance=vad_instance, + stt_instance=stt_instance, + ) + + # ------------------------------------------------------------------ + # Construction + # ------------------------------------------------------------------ + + def _build_voice_loop( + self, + ww_instances: Optional[Dict[str, Any]], + verifiers: Optional[List[Any]], + vad_instance: Optional[Any], + stt_instance: Optional[Any], + ) -> Any: + """Build a ``DinkumVoiceLoop`` wired to this harness's bus. + + Returns: + A ready-to-feed ``DinkumVoiceLoop`` instance. + + Raises: + RuntimeError: If ovos-dinkum-listener is not installed. + """ + try: + from ovos_dinkum_listener.voice_loop.voice_loop import DinkumVoiceLoop + except ImportError as e: + raise RuntimeError( + "ovos-dinkum-listener is required to build a MiniVoiceLoop. " + "Install it with: pip install ovos-dinkum-listener" + ) from e + + if ww_instances is None: + ww_instances = {"hey_mycroft": MockHotWordEngine("hey_mycroft")} + + self.hotwords = MiniHotwordContainer(ww_instances, verifiers=verifiers) + + return DinkumVoiceLoop( + mic=_MockMicrophone(), + hotwords=self.hotwords, + stt=stt_instance if stt_instance is not None else MockStreamingSTT(), + fallback_stt=None, + vad=vad_instance if vad_instance is not None else MockVADEngine(), + transformers=_MockTransformers(self.bus), + wake_callback=self._emit_record_begin, + record_end_callback=self._emit_record_end, + text_callback=self._emit_stt_text, + listenword_audio_callback=self._emit_wakeword, + hotword_audio_callback=self._emit_wakeword, + ) + + # ------------------------------------------------------------------ + # Bus-emitting callbacks (mirror ovos-dinkum-listener service.py) + # ------------------------------------------------------------------ + + def _emit_record_begin(self) -> None: + """Emit ``recognizer_loop:record_begin`` (the real wake callback).""" + self.bus.emit(Message("recognizer_loop:record_begin")) + + def _emit_record_end(self) -> None: + """Emit ``recognizer_loop:record_end`` (the real record-end callback).""" + self.bus.emit(Message("recognizer_loop:record_end")) + + def _emit_wakeword(self, audio_bytes: bytes, ww_context: Dict[str, Any]) -> None: + """Emit ``recognizer_loop:wakeword`` for a detected listen word. + + Mirrors the listen-word branch of the listener's ``_hotword_audio`` + callback so the captured bus sequence matches the real service. + + Args: + audio_bytes: Raw hotword audio collected by the loop. + ww_context: Wake-word metadata from :meth:`MiniHotwordContainer.get_ww`. + """ + payload = dict(ww_context) + key_phrase = ww_context.get("key_phrase", "") + payload["utterance"] = key_phrase.replace("_", " ").replace("-", " ") + context = { + "client_name": "ovos_dinkum_listener", + "source": "audio", + "destination": ["skills"], + } + self.bus.emit(Message("recognizer_loop:wakeword", payload, context)) + + def _emit_stt_text( + self, transcripts: List[Tuple[str, float]], stt_context: Dict[str, Any] + ) -> None: + """Emit the STT result (mirrors the listener's ``_stt_text`` callback). + + Emits ``recognizer_loop:utterance`` for a non-empty transcript, or + ``recognizer_loop:speech.recognition.unknown`` when transcription is + empty. + + Args: + transcripts: List of ``(text, confidence)`` from the STT engine. + stt_context: Context dict accumulated by the loop. + """ + utts = [t[0] for t in transcripts if t[0]] if transcripts else [] + if utts: + lang = stt_context.get("lang") or "en-us" + self.bus.emit(Message( + "recognizer_loop:utterance", + {"utterances": utts, "lang": lang}, + stt_context, + )) + else: + self.bus.emit(Message( + "recognizer_loop:speech.recognition.unknown", + context=stt_context, + )) + + # ------------------------------------------------------------------ + # Feeding + # ------------------------------------------------------------------ + + def feed_chunks(self, chunks: List[bytes]) -> List[Message]: + """Feed PCM *chunks* through the voice loop's wake-word detection. + + Each chunk is passed to ``DinkumVoiceLoop._detect_ww``; all bus messages + emitted as side-effects are collected and returned. This drives only + the wake-word / verifier stage โ€” use :meth:`feed_file` to run the full + state machine. + + Args: + chunks: Ordered list of raw PCM audio frames. + + Returns: + The list of :class:`Message` objects emitted during this call. + """ + self._messages.clear() + for chunk in chunks: + self.voice_loop._detect_ww(chunk) + self._last_messages = list(self._messages) + return list(self._messages) + + def feed_file( + self, + audio: Union[bytes, str, Path], + *, + silence_tail_chunks: int = 25, + chunk_size: int = 2048, + ) -> List[Message]: + """Run the full ``DinkumVoiceLoop`` state machine over an audio file. + + Swaps in a :class:`MockFileMicrophone` that streams *audio* through the + loop, drives ``start()`` + ``run()`` to completion, and returns every + bus message emitted along the way. A tail of silent frames is appended + so a silence-based VAD can end the command and the loop can finalize the + utterance. + + Args: + audio: Path to a ``.wav`` file, WAV bytes, or raw PCM bytes. + silence_tail_chunks: Trailing silent frames appended after the audio + (gives the VAD a chance to detect end-of-command). + chunk_size: Bytes per microphone read. + + Returns: + The list of :class:`Message` objects emitted during the run. + """ + mic = MockFileMicrophone( + audio, + chunk_size=chunk_size, + silence_chunks=silence_tail_chunks, + ) + mic.on_exhausted = self.voice_loop.stop + self.voice_loop.mic = mic + + self._messages.clear() + self.voice_loop.start() + self.voice_loop.run() + self._last_messages = list(self._messages) + return list(self._messages) + + # ------------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------------ + + def shutdown(self) -> None: + """Shut down the hotword container and wrapped engines.""" + if self.hotwords is not None: + self.hotwords.shutdown() + + +# --------------------------------------------------------------------------- +# Factory +# --------------------------------------------------------------------------- + +def get_mini_voice_loop( + ww_instances: Optional[Dict[str, Any]] = None, + verifiers: Optional[List[Any]] = None, + vad_instance: Optional[Any] = None, + stt_instance: Optional[Any] = None, + bus: Optional[FakeBus] = None, +) -> MiniVoiceLoop: + """Factory: create a ready-to-feed :class:`MiniVoiceLoop`. + + Args: + ww_instances: Mapping of wake-word name โ†’ engine instance. Defaults to a + single :class:`MockHotWordEngine` keyed ``"hey_mycroft"``. + verifiers: Optional verifier objects gating detection. + vad_instance: Optional VAD engine (defaults to :class:`MockVADEngine`). + stt_instance: Optional streaming STT engine (defaults to + :class:`MockStreamingSTT`). + bus: Optional :class:`FakeBus` to capture on. + + Returns: + A fully initialised :class:`MiniVoiceLoop`. + + Example:: + + from ovoscope.voice_loop import get_mini_voice_loop, MockHotWordEngine + + vl = get_mini_voice_loop( + ww_instances={"hey_mycroft": MockHotWordEngine(trigger_after=3)}, + ) + try: + vl.assert_record_begin_emitted(vl.feed_chunks([b"\\x00" * 512] * 3)) + finally: + vl.shutdown() + """ + return MiniVoiceLoop( + ww_instances=ww_instances, + verifiers=verifiers, + vad_instance=vad_instance, + stt_instance=stt_instance, + bus=bus, + ) + + +# --------------------------------------------------------------------------- +# Declarative test helper +# --------------------------------------------------------------------------- + +@dataclass +class VoiceLoopTest: + """Declarative wake-word โ†’ bus-sequence test for the voice loop. + + Drives a :class:`MiniVoiceLoop` and asserts whether the wake-word recording + sequence was triggered. Feeds PCM frames via ``feed_chunks`` by default, or + an audio file via ``feed_file`` when *audio_file* is set. + + Example โ€” verifier gate:: + + from unittest.mock import Mock + from ovoscope.voice_loop import VoiceLoopTest, MockHotWordEngine + + accepting = Mock(); accepting.verify.return_value = True + VoiceLoopTest( + ww_instances={"hey_mycroft": MockHotWordEngine(trigger_after=3)}, + verifiers=[accepting], + audio_chunks=[b"\\x00" * 512] * 5, + expect_record_begin=True, + ).execute() + + Example โ€” full loop from a file:: + + from ovoscope.voice_loop import VoiceLoopTest, MockStreamingSTT + + VoiceLoopTest( + audio_file="command.wav", + stt_instance=MockStreamingSTT(transcript="what time is it"), + expect_utterance="what time is it", + ).execute() + """ + + ww_instances: Optional[Dict[str, Any]] = None + """Mapping of wake-word name โ†’ engine instance.""" + + verifiers: Optional[List[Any]] = None + """Verifier objects (``verify(audio) -> bool``) gating detection.""" + + vad_instance: Optional[Any] = None + """Optional VAD engine (defaults to :class:`MockVADEngine`).""" + + stt_instance: Optional[Any] = None + """Optional streaming STT engine (defaults to :class:`MockStreamingSTT`).""" + + audio_chunks: List[bytes] = field( + default_factory=lambda: [b"\x00" * 512] * 5 + ) + """PCM frames fed via ``feed_chunks`` (ignored when *audio_file* is set).""" + + audio_file: Optional[Union[bytes, str, Path]] = None + """When set, run the full loop over this audio via ``feed_file``.""" + + expect_record_begin: bool = True + """Assert ``recognizer_loop:record_begin`` was emitted (``True``) or + suppressed (``False``).""" + + expect_utterance: Optional[str] = None + """When set, assert this utterance text was emitted (implies full-loop).""" + + def execute(self) -> List[Message]: + """Run the test and assert the configured expectations. + + Returns: + The captured :class:`Message` list. + + Raises: + AssertionError: If an expectation is not met. + """ + vl = MiniVoiceLoop( + ww_instances=self.ww_instances, + verifiers=self.verifiers, + vad_instance=self.vad_instance, + stt_instance=self.stt_instance, + ) + try: + if self.audio_file is not None: + messages = vl.feed_file(self.audio_file) + else: + messages = vl.feed_chunks(self.audio_chunks) + + if self.expect_record_begin: + vl.assert_record_begin_emitted(messages) + else: + vl.assert_wakeword_suppressed(messages) + + if self.expect_utterance is not None: + vl.assert_utterance_emitted(self.expect_utterance, messages) + return messages + finally: + vl.shutdown() diff --git a/test/unittests/test_classic_listener.py b/test/unittests/test_classic_listener.py new file mode 100644 index 0000000..bda322e --- /dev/null +++ b/test/unittests/test_classic_listener.py @@ -0,0 +1,110 @@ +# Copyright 2024 Jarbas AI +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for the MiniClassicListener harness (mycroft-classic-listener). + +The event-bridge test does not need the classic listener installed (it drives a +plain EventEmitter). The file-drive test exercises the real RecognizerLoop and +is gated on the package being importable. +""" +import io +import unittest +import wave + +from ovos_utils.fakebus import FakeBus + +from ovoscope.classic_listener import ( + MiniClassicListener, + bridge_recognizer_loop_to_bus, + classic_listener_available, +) +from ovoscope.voice_loop import MockHotWordEngine, MockStreamingSTT + +try: + from pyee import EventEmitter + HAS_PYEE = True +except ImportError: + HAS_PYEE = False + +HAS_CLASSIC = classic_listener_available() + + +def _wav(speech_seconds=2.0, sample_rate=16000, sample_width=2): + buf = io.BytesIO() + with wave.open(buf, "wb") as w: + w.setframerate(sample_rate) + w.setsampwidth(sample_width) + w.setnchannels(1) + w.writeframes(b"\x10\x20" * int(sample_rate * speech_seconds)) + return buf.getvalue() + + +@unittest.skipUnless(HAS_PYEE, "pyee not installed") +class TestEventBridge(unittest.TestCase): + """The RecognizerLoop event โ†’ FakeBus bridge (no classic listener needed).""" + + def test_bridge_translates_events(self): + """Internal loop events become recognizer_loop:* bus messages.""" + loop = EventEmitter() + harness = MiniClassicListener(recognizer_loop=loop) + + loop.emit("recognizer_loop:record_begin") + loop.emit("recognizer_loop:wakeword", {"utterance": "hey mycroft"}) + loop.emit("recognizer_loop:utterance", + {"utterances": ["hello world"], "lang": "en-us"}) + loop.emit("recognizer_loop:record_end") + harness._last_messages = list(harness._messages) + + harness.assert_record_begin_emitted() + harness.assert_wakeword_detected() + harness.assert_utterance_emitted("hello world") + + def test_bridge_unknown_event(self): + """The unknown event is forwarded to the bus.""" + bus = FakeBus() + seen = [] + bus.on("message", lambda m: seen.append( + m if isinstance(m, str) else m.serialize())) + loop = EventEmitter() + bridge_recognizer_loop_to_bus(loop, bus) + loop.emit("recognizer_loop:speech.recognition.unknown") + self.assertTrue( + any("speech.recognition.unknown" in s for s in seen) + ) + + def test_feed_file_requires_built_loop(self): + """feed_file is unavailable when an external loop was supplied.""" + harness = MiniClassicListener(recognizer_loop=EventEmitter()) + with self.assertRaises(RuntimeError): + harness.feed_file(b"\x00" * 1024) + + +@unittest.skipUnless(HAS_CLASSIC, "mycroft-classic-listener not installed") +class TestMiniClassicListenerFileDrive(unittest.TestCase): + """Best-effort file-driven test against a real RecognizerLoop.""" + + def test_full_sequence_with_utterance(self): + """Driving an audio file yields record-begin and the utterance.""" + with MiniClassicListener( + wakeword=MockHotWordEngine("hey_mycroft", trigger_after=1), + stt_instance=MockStreamingSTT(transcript="hello world"), + ) as cl: + msgs = cl.feed_file(_wav(2.0), tail_silence_seconds=3.0, timeout=20) + types = [m.msg_type for m in msgs] + self.assertIn("recognizer_loop:record_begin", types) + self.assertIn("recognizer_loop:record_end", types) + cl.assert_utterance_emitted("hello world", msgs) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/unittests/test_simple_listener.py b/test/unittests/test_simple_listener.py new file mode 100644 index 0000000..481c5f2 --- /dev/null +++ b/test/unittests/test_simple_listener.py @@ -0,0 +1,78 @@ +# Copyright 2024 Jarbas AI +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for the MiniSimpleListener harness (ovos-simple-listener).""" +import io +import unittest +import wave + +from ovoscope.simple_listener import MiniSimpleListener +from ovoscope.voice_loop import MockHotWordEngine, MockStreamingSTT + +try: + import ovos_simple_listener # noqa: F401 + HAS_SIMPLE = True +except ImportError: + HAS_SIMPLE = False + + +def _wav(speech_seconds=0.6, sample_rate=16000, sample_width=2): + buf = io.BytesIO() + with wave.open(buf, "wb") as w: + w.setframerate(sample_rate) + w.setsampwidth(sample_width) + w.setnchannels(1) + w.writeframes(b"\x10\x20" * int(sample_rate * speech_seconds)) + return buf.getvalue() + + +@unittest.skipUnless(HAS_SIMPLE, "ovos-simple-listener not installed") +class TestMiniSimpleListener(unittest.TestCase): + """MiniSimpleListener integration tests against a real SimpleListener.""" + + def test_full_sequence_with_utterance(self): + """A wake word + command yields the full bus sequence + utterance.""" + with MiniSimpleListener( + wakeword=MockHotWordEngine("hey_mycroft", trigger_after=2), + stt_instance=MockStreamingSTT(transcript="turn on the lights"), + ) as sl: + msgs = sl.feed_file(_wav(), silence_tail_chunks=20) + types = [m.msg_type for m in msgs] + self.assertIn("recognizer_loop:wakeword", types) + self.assertIn("recognizer_loop:record_begin", types) + self.assertIn("recognizer_loop:record_end", types) + sl.assert_utterance_emitted("turn on the lights", msgs) + + def test_empty_transcript_unknown(self): + """An empty transcript emits the recognition-unknown event.""" + with MiniSimpleListener( + wakeword=MockHotWordEngine("hey_mycroft", trigger_after=2), + stt_instance=MockStreamingSTT(transcript=""), + ) as sl: + msgs = sl.feed_file(_wav(), silence_tail_chunks=20) + types = [m.msg_type for m in msgs] + self.assertIn("recognizer_loop:speech.recognition.unknown", types) + self.assertNotIn("recognizer_loop:utterance", types) + + def test_record_begin_helper(self): + """The shared assert_record_begin_emitted helper works here too.""" + with MiniSimpleListener( + wakeword=MockHotWordEngine("hey_mycroft", trigger_after=1), + stt_instance=MockStreamingSTT(transcript="hello"), + ) as sl: + sl.feed_file(_wav(), silence_tail_chunks=20) + sl.assert_record_begin_emitted() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/unittests/test_voice_loop.py b/test/unittests/test_voice_loop.py new file mode 100644 index 0000000..473c8c1 --- /dev/null +++ b/test/unittests/test_voice_loop.py @@ -0,0 +1,302 @@ +# Copyright 2024 Jarbas AI +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for the MiniVoiceLoop harness. + +TestMiniHotwordContainer โ€” container update/found/get_ww/verify (no dinkum) +TestMiniVoiceLoop โ€” feed_chunks + helpers (needs ovos-dinkum-listener) +TestVerifierGate โ€” verifier suppression (needs the hotword-verifier gate) +TestVoiceLoopTest โ€” declarative helper +""" +import inspect +import io +import unittest +import wave +from unittest.mock import Mock + +from ovoscope.voice_loop import ( + MiniHotwordContainer, + MiniVoiceLoop, + MockHotWordEngine, + MockStreamingSTT, + VoiceLoopTest, + get_mini_voice_loop, +) + +_SILENCE = b"\x00" * 512 + + +def _wav(speech_seconds=0.6, sample_rate=16000, sample_width=2): + """Build in-memory WAV bytes of non-zero 'speech'.""" + buf = io.BytesIO() + with wave.open(buf, "wb") as w: + w.setframerate(sample_rate) + w.setsampwidth(sample_width) + w.setnchannels(1) + w.writeframes(b"\x10\x20" * int(sample_rate * speech_seconds)) + return buf.getvalue() + +# ovos-dinkum-listener is an optional test dependency โ€” the harness can build a +# real DinkumVoiceLoop only when it is installed. +try: + from ovos_dinkum_listener.voice_loop.voice_loop import DinkumVoiceLoop + HAS_DINKUM = True + # The verifier gate is only present in builds shipping the hotword-verifier + # feature; suppression assertions require it. + HAS_VERIFY_GATE = "self.hotwords.verify" in inspect.getsource( + DinkumVoiceLoop._detect_ww + ) +except ImportError: + HAS_DINKUM = False + HAS_VERIFY_GATE = False + + +def _accepting(): + v = Mock() + v.verify.return_value = True + return v + + +def _rejecting(): + v = Mock() + v.verify.return_value = False + return v + + +def _raising(): + v = Mock() + v.verify.side_effect = RuntimeError("verifier boom") + return v + + +# --------------------------------------------------------------------------- +# MiniHotwordContainer (no dinkum required) +# --------------------------------------------------------------------------- + +class TestMiniHotwordContainer(unittest.TestCase): + """MiniHotwordContainer unit tests.""" + + def test_found_after_threshold(self): + """found() returns the ww name once the engine fires.""" + c = MiniHotwordContainer( + {"hey_mycroft": MockHotWordEngine(trigger_after=2)} + ) + c.update(_SILENCE) + self.assertIsNone(c.found()) + c.update(_SILENCE) + self.assertEqual(c.found(), "hey_mycroft") + + def test_found_none_when_not_triggered(self): + """found() returns None before the threshold.""" + c = MiniHotwordContainer( + {"hey_mycroft": MockHotWordEngine(trigger_after=10)} + ) + c.update(_SILENCE) + self.assertIsNone(c.found()) + + def test_update_feeds_all_engines(self): + """update() forwards chunks to every registered engine.""" + a = MockHotWordEngine("hey_mycroft", trigger_after=1) + b = MockHotWordEngine("hey_jarbas", trigger_after=1) + c = MiniHotwordContainer({"hey_mycroft": a, "hey_jarbas": b}) + c.update(_SILENCE) + self.assertEqual(a.update_count, 1) + self.assertEqual(b.update_count, 1) + + def test_get_ww_metadata(self): + """get_ww() returns listen-word metadata for a known ww.""" + c = MiniHotwordContainer({"hey_mycroft": MockHotWordEngine()}) + meta = c.get_ww("hey_mycroft") + self.assertEqual(meta["key_phrase"], "hey_mycroft") + self.assertTrue(meta["listen"]) + self.assertIsNone(meta["sound"]) + + def test_get_ww_unknown_raises(self): + """get_ww() raises ValueError for an unregistered ww.""" + c = MiniHotwordContainer({"hey_mycroft": MockHotWordEngine()}) + with self.assertRaises(ValueError): + c.get_ww("unknown") + + def test_verify_accept(self): + """verify() returns True when all verifiers accept.""" + c = MiniHotwordContainer({}, verifiers=[_accepting(), _accepting()]) + self.assertTrue(c.verify(b"audio")) + + def test_verify_reject(self): + """verify() returns False when any verifier rejects.""" + c = MiniHotwordContainer({}, verifiers=[_accepting(), _rejecting()]) + self.assertFalse(c.verify(b"audio")) + + def test_verify_fail_open(self): + """verify() ignores a raising verifier (fail-open).""" + c = MiniHotwordContainer({}, verifiers=[_raising(), _accepting()]) + self.assertTrue(c.verify(b"audio")) + + def test_verify_no_verifiers(self): + """verify() returns True when no verifiers are configured.""" + c = MiniHotwordContainer({}) + self.assertTrue(c.verify(b"audio")) + + +# --------------------------------------------------------------------------- +# MiniVoiceLoop (requires ovos-dinkum-listener) +# --------------------------------------------------------------------------- + +@unittest.skipUnless(HAS_DINKUM, "ovos-dinkum-listener not installed") +class TestMiniVoiceLoop(unittest.TestCase): + """MiniVoiceLoop integration tests against a real DinkumVoiceLoop.""" + + def _loop(self, trigger_after=3, verifiers=None): + ww = MockHotWordEngine("hey_mycroft", trigger_after=trigger_after) + return MiniVoiceLoop( + ww_instances={"hey_mycroft": ww}, verifiers=verifiers + ) + + def test_accept_emits_record_begin(self): + """WW detected + accepting verifier emits the record-begin sequence.""" + with self._loop(verifiers=[_accepting()]) as vl: + msgs = vl.feed_chunks([_SILENCE] * 5) + types = {m.msg_type for m in msgs} + self.assertIn("recognizer_loop:wakeword", types) + self.assertIn("recognizer_loop:record_begin", types) + + def test_no_wakeword_no_events(self): + """No detection emits no recognizer_loop events.""" + with self._loop(trigger_after=100, verifiers=[_accepting()]) as vl: + msgs = vl.feed_chunks([_SILENCE] * 5) + self.assertFalse( + any(m.msg_type.startswith("recognizer_loop:") for m in msgs) + ) + + def test_raising_verifier_fails_open(self): + """A raising verifier does not suppress detection (fail-open).""" + with self._loop(verifiers=[_raising()]) as vl: + vl.assert_record_begin_emitted(vl.feed_chunks([_SILENCE] * 5)) + + def test_default_ww_instances(self): + """Omitting ww_instances uses a default hey_mycroft engine.""" + with MiniVoiceLoop() as vl: + vl.assert_record_begin_emitted(vl.feed_chunks([_SILENCE] * 3)) + + def test_assert_wakeword_detected_helper(self): + """assert_wakeword_detected passes on a real detection.""" + with self._loop() as vl: + vl.feed_chunks([_SILENCE] * 5) + vl.assert_wakeword_detected() # operates on last feed result + + def test_assert_wakeword_detected_fails_without_detection(self): + """assert_wakeword_detected raises when nothing fired.""" + with self._loop(trigger_after=100) as vl: + vl.feed_chunks([_SILENCE] * 5) + with self.assertRaises(AssertionError): + vl.assert_wakeword_detected() + + def test_factory(self): + """get_mini_voice_loop wires a usable harness.""" + vl = get_mini_voice_loop( + ww_instances={"hey_mycroft": MockHotWordEngine(trigger_after=2)}, + ) + try: + vl.assert_record_begin_emitted(vl.feed_chunks([_SILENCE] * 3)) + finally: + vl.shutdown() + + def test_feed_file_full_sequence(self): + """feed_file drives the whole loop to a transcribed utterance.""" + ww = MockHotWordEngine("hey_mycroft", trigger_after=2) + stt = MockStreamingSTT(transcript="what time is it") + with MiniVoiceLoop(ww_instances={"hey_mycroft": ww}, stt_instance=stt) as vl: + msgs = vl.feed_file(_wav(), silence_tail_chunks=30) + types = [m.msg_type for m in msgs] + self.assertIn("recognizer_loop:record_begin", types) + self.assertIn("recognizer_loop:record_end", types) + vl.assert_utterance_emitted("what time is it", msgs) + + def test_feed_file_empty_transcript_unknown(self): + """feed_file with no transcript emits the recognition-unknown event.""" + ww = MockHotWordEngine("hey_mycroft", trigger_after=2) + with MiniVoiceLoop(ww_instances={"hey_mycroft": ww}, + stt_instance=MockStreamingSTT(transcript="")) as vl: + msgs = vl.feed_file(_wav(), silence_tail_chunks=30) + types = [m.msg_type for m in msgs] + self.assertIn("recognizer_loop:speech.recognition.unknown", types) + self.assertNotIn("recognizer_loop:utterance", types) + + +# --------------------------------------------------------------------------- +# Verifier suppression gate (requires the hotword-verifier feature) +# --------------------------------------------------------------------------- + +@unittest.skipUnless( + HAS_VERIFY_GATE, "ovos-dinkum-listener build lacks the hotword-verifier gate" +) +class TestVerifierGate(unittest.TestCase): + """Tests that depend on DinkumVoiceLoop gating on hotwords.verify().""" + + def test_reject_suppresses(self): + """A rejecting verifier suppresses the whole record sequence.""" + ww = MockHotWordEngine("hey_mycroft", trigger_after=3) + with MiniVoiceLoop( + ww_instances={"hey_mycroft": ww}, verifiers=[_rejecting()] + ) as vl: + msgs = vl.feed_chunks([_SILENCE] * 5) + vl.assert_wakeword_suppressed(msgs) + + def test_voice_loop_test_expect_suppressed(self): + """VoiceLoopTest(expect_record_begin=False) passes on rejection.""" + VoiceLoopTest( + ww_instances={"hey_mycroft": MockHotWordEngine(trigger_after=3)}, + verifiers=[_rejecting()], + audio_chunks=[_SILENCE] * 5, + expect_record_begin=False, + ).execute() + + +# --------------------------------------------------------------------------- +# VoiceLoopTest declarative helper (requires ovos-dinkum-listener) +# --------------------------------------------------------------------------- + +@unittest.skipUnless(HAS_DINKUM, "ovos-dinkum-listener not installed") +class TestVoiceLoopTest(unittest.TestCase): + """VoiceLoopTest declarative helper tests.""" + + def test_expect_record_begin_passes(self): + """Passes when record_begin is expected and emitted.""" + VoiceLoopTest( + ww_instances={"hey_mycroft": MockHotWordEngine(trigger_after=3)}, + verifiers=[_accepting()], + audio_chunks=[_SILENCE] * 5, + expect_record_begin=True, + ).execute() + + def test_expect_record_begin_fails_without_detection(self): + """Raises AssertionError when record_begin expected but absent.""" + with self.assertRaises(AssertionError): + VoiceLoopTest( + ww_instances={"hey_mycroft": MockHotWordEngine(trigger_after=100)}, + audio_chunks=[_SILENCE] * 5, + expect_record_begin=True, + ).execute() + + def test_audio_file_with_expected_utterance(self): + """audio_file path drives the full loop and asserts the utterance.""" + VoiceLoopTest( + ww_instances={"hey_mycroft": MockHotWordEngine(trigger_after=2)}, + stt_instance=MockStreamingSTT(transcript="hello world"), + audio_file=_wav(), + expect_utterance="hello world", + ).execute() + + +if __name__ == "__main__": + unittest.main() From 5a34567c8734974b7a1284228121a6a01adb1199 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Fri, 12 Jun 2026 22:37:13 +0000 Subject: [PATCH 21/82] Increment Version to 0.19.0a1 --- ovoscope/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovoscope/version.py b/ovoscope/version.py index 46c184e..d880880 100644 --- a/ovoscope/version.py +++ b/ovoscope/version.py @@ -1,6 +1,6 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 -VERSION_MINOR = 18 +VERSION_MINOR = 19 VERSION_BUILD = 0 VERSION_ALPHA = 1 # END_VERSION_BLOCK From 64745b4df29b2ecaa5c4eeb06a3642df74fea93c Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Fri, 12 Jun 2026 22:37:42 +0000 Subject: [PATCH 22/82] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f97d5d4..ac96aa8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.19.0a1](https://github.com/TigreGotico/ovoscope/tree/0.19.0a1) (2026-06-12) + +[Full Changelog](https://github.com/TigreGotico/ovoscope/compare/0.18.0a1...0.19.0a1) + +**Merged pull requests:** + +- feat: MiniVoiceLoop + simple/classic listener bus-sequence harnesses [\#67](https://github.com/TigreGotico/ovoscope/pull/67) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.18.0a1](https://github.com/TigreGotico/ovoscope/tree/0.18.0a1) (2026-06-10) [Full Changelog](https://github.com/TigreGotico/ovoscope/compare/0.17.1a1...0.18.0a1) From 8afb4a05c3a35387b4116495db9d41b0a74bb0ec Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 13 Jun 2026 03:18:30 +0100 Subject: [PATCH 23/82] docs: standardize NGI0 Commons Fund attribution (#69) Use the canonical funding block (developer + funder + correct NGI0 Commons Fund banner) across the repo, replacing divergent ad-hoc credit notes. --- README.md | 24 ++++++++++++++++++------ ngi.png | Bin 0 -> 18699 bytes 2 files changed, 18 insertions(+), 6 deletions(-) create mode 100644 ngi.png diff --git a/README.md b/README.md index 435eaa2..4c16bb6 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,24 @@ stages and deliberately excludes persona, Ollama, OCP, and m2v plugins. | [docs/pydantic-integration.md](docs/pydantic-integration.md) | Typed message models with `ovos-pydantic-models` | | [FAQ.md](FAQ.md) | Common questions and gotchas | --- + +--- + +## Credits + +Developed by [TigreGรณtico](https://tigregotico.pt) for +[OpenVoiceOS](https://openvoiceos.org). + +[![NGI0 Commons Fund](./ngi.png)](https://nlnet.nl/project/OpenVoiceOS) + +This project was funded through the [NGI0 Commons Fund](https://nlnet.nl/commonsfund), +a fund established by [NLnet](https://nlnet.nl) with financial support from the +European Commission's [Next Generation Internet](https://ngi.eu) programme, under +the aegis of [DG Communications Networks, Content and Technology](https://commission.europa.eu/about-european-commission/departments-and-executive-agencies/communications-networks-content-and-technology_en) +under grant agreement No [101135429](https://cordis.europa.eu/project/id/101135429). + +--- + ## License [Apache 2.0](LICENSE) --- @@ -141,9 +159,3 @@ In the interest of transparency, two files are maintained as a public record of significant AI-assisted session. These files are intentionally published so that contributors and users can understand how the project evolves and where AI assistance has been applied. - -## Credits - -Funded by [NGI0 Commons Fund](https://nlnet.nl/project/OpenVoiceOS) / [NLnet](https://nlnet.nl) -under grant agreement No [101135429](https://cordis.europa.eu/project/id/101135429), -through the European Commission's [Next Generation Internet](https://ngi.eu) programme. diff --git a/ngi.png b/ngi.png new file mode 100644 index 0000000000000000000000000000000000000000..bfd401afd74bd0e512c332577c152264fe231b88 GIT binary patch literal 18699 zcmZ^K1yCG8yY1qOFYfNa-Q9x|EVu-BcXvw&?oJ2@5V(;|-_Tdp=!oyIP_DJ0)@3W5XL1P#mc+R*b zietq@E(*b5ixg^mE$!t7>r9f&zYDM&zo@NcU|@&gLznt^HahN-cM6#Qtdli;GWg?L z!l*YGu{v>e4@H*4{Bf`2%o{;Gks6WoKP}ftC}b)Zg#Mp4u^5Dh<-bS?q)-+Mtn zRlEEC(_sGhIQ&L9^naa#^nZ_Yf>^Q7|Eq!i=XfrZRFQD-X8ycq;dg6~W@|xh1 zHbTMYu~kZK=BALQDM_G7`UH z!g18foD4D6Kd1alEgXJ9p&;XnDD+T!ii&m=_NXY~r;0}DKSOZ`4$;|0#{5;h6;1lED1yupoWYjgZ}f5FB2*^|YG9!ar>%;^sw0CSUv6b% z_6R?XBEk8$dHn38!x3J#ShnSm2RSGP;w2T$x6N8=`m%KL`;#sEj#yWF$M|3TO@zK5 zE|l>-5E7#5-Q0RoTWK9Bk^FjMZ1&EzxyhJMH^^2=&$vsLLWC0o&uiN%u-L0ge9-Q^ z@DE<#1U-blxWLP;hdm7r8We>*I#|6&CJh+wvMr$`UZ=3@n-yzTq@~pUF{7CM-UPXL z?<;>$gZ^~iE+6%z(L_@=NNl;0?E8?eh3z{wRSVZ}Q9gKfKAAvG7u6q@t#!@JFs>OT zl{`r{9@{@|_X-yg7V4rNELKg>d!#KQ=^6fA-A>aKv`PiBEe(VY$!C3Y zIyfro@FOK{o|)6Odl5})GNwmJwb!E^>z*rT&OiD3BLpL~38dc$N9}1JtSh@ve0h$N z#!iQwJ_@h@{V7w6lq@+kZn2%Dsq*{x?k4+{^Cio?f|{C1lRqupcAo~<6d1c^&#BrP zm5Mr0g@haij86;5VtAHA3d)-qLc)j|;F2#Hzcbe)*Pc!}Rjb(z6C)Y1fMsawcai*k zyaMFo_zUvOh4E~JU#jd*@8fykl{E;7h-ys-kf>>Cx0+TQ`r6!&5>itmDk_-t+uWkv znZAjP6(izQ0?vZS%WD-Wi6mIB0Zy3<26zORBx0ci#9KOV?co+1WH#O95EEn>A)!DyRxw*L;7L;>^qL=C|w-;;8()pYzPi`P*(NULq z%CTmuOCI14MUG@Al}?PM%DMfvZ+t9M;;H`9o>*Mgqo+q*crpNORhVh5cPaw z3;L?mYEK{s(L(lX_tt}9Qaf5|DuI1hXOO3q;r_*L*9;>Lycjp8Izb7mwrl@EMpc9G z*i2`gOm8f}WN~$V9OR^K-A=>yEHJTlffRG-TNf zHM1$GFL>uHCB}HbO-?#^l0a~B-j|d_7!Ur@^-B{7#$6hkp!a!Kj21DnXEvU8j(s16 z#9;x~>wO5MI2}!N36{Q?{X9`dT#QW?%$5F_Kh>Nss|Q!o1|D9Eoy=&9puxuogFs!% zC&>2#QE#e*Awp)omkFPL3itfHB})wlNp2b?UK}xBlpF(Oh4M~Uc*OQLIXPK%cJnhm zJ-snDQZc2YE-wG!w zMgTHY0^kVZEaot{eq4(6{!MJ^9-+cy*d*FuKr9%oE_uR#r`z14n3-)A0-8Ii~?bIsCqdWH{fkzt+|X4@66?mQH9beArT8#6r41FA!S5 zO?|TTHRI#4hsqKIflNC2)3dHA(6B3E5MfS!&t)`q8={_`WJFbjEs$e-sSaX5Oci6i zP;@2f4pV&TZ8yuAArf)-33?DbFBob{GTviqsK5uQ`w==y3NPmJe~XHu=uv?F2RizNu;Cq$Ki z{}a@#^yXIF93E-uwy`0Ax51sENnQgf{v~Fy8XLC;J1&HAY;c??c~*o7+f*DTiUI|e zVTF>-1&ha^j4C~X?r-Ug$J z$0E8oc!NqwD(j-boRh>B*~?2N2m=E!>^PVKR zp3e_aC6CbfiyG)n-0L?3!Q;c5dYvICBELM>M}_dbIlG;);(9rsatiPAyV(v;6%KEI z(gRJNy+}*!EsNAnB8T;~(uJsQ1jj9kQuh1q;{4H5afV3v@8yfPe))J$DIXQav3G?1 z``{MOxS4rl)0x)+4@uENxKp4J?ZJIT--egnk2?&S9W!Ka#&CvE<5%@cgmvxRq+!zU zPkc@rpx5(A(TzTIQ7K19R*)FwSL2iAMz5Ef1q^I#Dcgl9?da8MUgyqR#_u=BYw&X< z-eMlhE!yLW<;yKN^`E@vGC|uT$(4ubvW{gQwQ7=z-+rKvZ ztn93$6cjiWX*r=`V7_Li#+Z+2lXEk>M->Tv`?k%OsS&D1fe1;5G!d`mZh{`+y0&%= z=xvq}OWNT7WWwW*`FMH5QuTVio2GR1$edorfKP&_8EaRYZzE$aS5Q3?G-k7=!13)< z!3}llDw9Lhi*{Q7jrKG}^ps%qhAPcRBi^ibaQve;dKlem2gw>t8&OUARhSeNh?6Z6 zVo^)mYlCaCyhXALXZ-!e%=lixhlk*xAJs3$uo50Buuv4e4U3`_i-NWqw^|hOBoJL< zu#EoQTr^$Ck25TEbnVqlfeh-g`>nquXmHTAsuA)SKQM^ch%f>O5~BVl$Ukpv6Ih>Y zI|`cRPtSf>4u2@#g*-7ny>E1n@1(|pD5 zG2yxCg0J#S+2=z2#KnQMkB^TDBHkQ|GB%$h+fNN6MLX^(mGcC`_wu}PH8nNSo<7z% zK^ZYuS@IFsTji*O1|pGSx-6A3avtw-sg?C$?5N z?}};}*EiDE8ph&&moBm;$q+O9A|S*XiC0(6S-$}JPDS=JVc~Oz=33*(8>m>gi;&^%6Z3wi|wuJ`w-R%xj@2 z*j%(HQJn@fAk*$^m^i3qC7*UgOgorn%V<RsYXrtPP6&0&=a*XvM|s|wZ}qhLLJrb48v?4UaPo-tw$;B4m*46?9|5zK{`{_P z=q18=yieNln81U~3_5i~@b>Oi@}cR9;GCRK_chAE5VW@pUBxC%-jFPfz~ugse1B%H zsLB<6*>^l&R@?F>B1qF#LX7D;qDu(Qr1J`LoP!8GIwdNfD5ZC1J;ai!KX1kTjXK)~ zxm#_7Lw{BP<}!wnZ-ERC<5-pcyOl#<^w#^*7jQUqsYFkBxd0bqJIQC;Fn%>W!eq zf6x+qO*m9m%^&@Rs`*+8UvV~!?1|+K=OWk@gN$oo=V52TzAugLjJVDtD0)8E^rt5q zQCH>#j(V?lTggOUB-&>C+w)MPcSFNQ!PD80ilC>oR$9=3$FPg`fsu7d~_ z{%>Bd>@vY(6sXd-Ep!mPbX1jS)@7ezr>5Nx`gZqUwLE2C9sfCDwb#(oQjuJYKHd5t zNZUZbrv{Vv$9+}I_ySnHWFhJwLfSs(F>GU^nIg3`puyXVyDY*$5L^7afD*U+AyeC! zlKUk~9D)W%r3dxC$Wc1r#P&Rej$+u6;S;IJB!R%xqC5DcV-Eoi8h&$R=gvkf$OZfL zqjgxj|8Nr-RDQ-Oadu)$6fy-VlXPQI+LuszGwKVTocrXD#m#_kV3 ziJ%!VN&hHE7>XILb-IL&eDQPMFV-YaXBR=^9e*Dg>2;n?4VBGU-P5!>GfAX$Z?(=Y z6gYBop32|(+Lq}QtE#H{Mg}0zpuZ>>xa9l%!+BOP{@v~m(E+Su{85i6*_h5^$@ zvWyQB^FD&|^S;!sK+w`R=BXPJ4vxn7@$rcxos0ha8~W$u&fvi+0Ax}F zr&W8;iOzrkuiaM5f7Pki@MCx~O9`ft{{zihz!fuo{Pm6B#lxQ0-&)zq8(Aduuh1jv zMlwF51_?1wXGab2ILg|AAqL?SQ(QtIrzRi9s=RC7U&8w`dE6v)z?5<7dPm(!2_@I3 zxzI;?sq_~|aRd05NT=wX?R$St>;Btkx{ITS*7-Vu3v`NAPb9a99>$=tRt0NlJG%`# zCiw?Wd$fQ{d|0p6JzWbfX9lJ-HrY&!JI`yY@~M(e6um#7A5B3psf#eT{1`ul%^(L| z=^w~$hW&q4`_Bm=oXVul%7wpkKgieW-d$Tay9|+;7GaPEKZT`h=<6pgx*P# zn-WadQZl0kUZzvlOe#-wMYhE~#XXiojwU~2J#koBS^ZwdcvfAGDwAQ>7ADY%NWEdJ zJ09T8+H-SBi3L6E7Pkc@sFSxgv;EUk?xQrxL64!z=3jiF??zw0PT9{`&2PGFdkl(X zh={_1f%{CqVeq%;g)f#~Q)H|Lpal$7L`c+aUa&JoY=x+z(Uc)+PqIKNS(Ub-8Ob15 zw}-XwpAu#WQt>a+H`3xQvBO1d9D##OoDTa9(yUj^5J!ZbP1}n4GfJc*!?P~38U^$* zntUuh0-8oH6FR>oI5!vboF?%M{0|D~&xZ4tIHT0O&5oX~`T6*zBjr$_`a~=?I`iX~ zN-!8Y!0=_gz+X3~uWj!g{MBhiCi~K+UTQAHzLBm)Rnt?cH4;tlA6!*xP5>Kp?5sjQ zYl&$qn4=tdveLX=l4jUD!rB$*|8zU0v0WgN^!xEqc+UMo+Pf z+KjTUH<0!15C%mrMStNq6Mt!@YD3Uwv_rr(@m*~FU93xsIB@KoMUD?9aCrJF_&2%B zBeGDHT&BhG^NfW>J?Vpcb*~g5ezlqv23TAdD@v^q?piK<9JaEjti8qp-X4m%aI@Rb z$DNzrVcBH>F0`avxFf1$Lyx5K>>*5jJ7eBCbL~t+FyWHKUq5QEiY#arZ zc>x~Clfc&=q(96gKE1&xRLhL)Lpe8c*cIdhFneVjIc$2$x$Qj9)9p!Li}QAk&D^`^ zP4vL-s|nFB1F?8!hqGn$OvP)ALkK$?!N&%1ku`8RwFp2dTe~-%cm8nE2nuS-#mp{y z4b#%5MeOe5f5JFExp(-DaUt${(Y^X6-Ve#Vkvky5?)gMtq2DoVopY4RV$S4fYfCi0 zIG#@EA2}mnCn5f&DML$Z!8<~&y>}p;i^qwlD5A9RW-{uuXAUV%(r&WO{~p#yX#hy@ z@4jq{>2ABGjaN z#P+>ka^It_|J&?PCtG# zk3Iku`uq3q#%$RKIYq^W)FC*F?b>juf|qQXE9Y7r-<&q;JbPlO z2VH0j@>`!M%?!dLU1JTQC_>U?AxdD|69&R5;TFPL;qU2o$Q1}#jdi|J$UiX*4l$E? z9Nn$-x~*S90~;)=o2xkKu+hU!Z#9HE%c)CkDjdtWHN}xl;|cPd8#a@as!7rxBb!FE zdYVA6!7T3VFifXXOuw(h7+4P{e zOI!@R`Obyafx@Cg=kVs{rd5bmTS|hV6eW?Aa<`PXGWhgt0+-WNw#TuyXroy&vvhtE zHK#VxowQ3%XZ5e$NCaU~H^GO(K!lVwV&*^I^Pyw6R@>0l9hdla2aF0X%T;4|NJ%7Dmz;0e2aC*G9#>Nv=`P^W;z~bgM z`NtmcfU)YYL3moTCf>`8=P7D1dJu9pej5PkawX85zDis~&uD7mS@pmB0|tY;uMcLv zW@JR?=aUP)-fz!O{=R+ye-#d(s{E<961M!gfaW9N$fiQ8=#sNh*x)2LrK`K_WT7o0 z%b-)T!Am3E=8bl9pZ$k4Z|AR5cc=Q}hx=Fp{=cUGv*RAx4g?^-ifbmew+F1fOW-y+(xi z4IC8};)0Dg;$Kh>yz1kByC` zvzx&;IsAD)W%_;2#%{R*k)NNxBBbZKx;>&6dzxFGDACITL)w0%ECie*Rl93p3BO&;DnepF+ZHG$O-~t_F`% z#VD|xkFWRqZnu03`4RU%RKBKN!JY=XLY@Tk{T%MoKCGZt(6v2AI3~ILQoF*^>HMO66PP@Nu zE@JG$H1E#q!?!3PO4Hf@y;6O;RiuQf+6YLb)lD zRB-P}D?+Q``)4Y-uTZT?>93%TlZ>E>Fm30Gs>}rOXicO;n^zOU&E0%=IOrDbdLj$)$3EBo1>}dS z>jhy`vw@ut!{%OA);u&VQ;pp<+S?4bCDLt2^Bn6~6WRP&(7_;!`t9>VB7v}bcFKdfKV0-;{CFz|UPV58e1r+i>^?btW>%U#g z`(aFt>rKUQ?GW~`%ZHyNWw|bm%?t2t>sCvOnfAd7XDRM3vm7BlqT=<;HGhhlPB|Sg zkObO%8OpCyS%zBSiFbtaPYLl)KulJseaP}>%t@y99<)uz!ZE|UJjB0P2MZZKcZ|bA zL*KFF#uudx*X%u;Ut2X;Y2N7kA!K&vz%r|K8T@NI+&$S_6CMJphRsw}UH68M3QX7H z6%r!$T=S*_UcAGaiFhJj2__4kA$8yd8y-w%wP73z$6j6A zQxOu%JK$~s>HZkWOLqu~HL_hC3ymq+r-3e{*2sF*CD`YkvR`pejGxoqa0Sfh>YIW_ z2<`2Kx()PdBexa-H zP9-D?_Qe~M$WuoDDkjFMWgY@b5OyZEpry;NYqe)nD3pr=IkHmAMo2iL65uX#+7*xk zRT%RR1E7vfD9&K7`}0jH%jx0LJe$AoShvOJ#zyE0#kE`y08iSKeL&giK^ZXe&bd+w zcQpO*<1&65`b-)3mP{(X1DnrEw5N9tiUr|NV%;_46hvw-y-}#?aJ7?3xP-06-n=^y zCpwoehlxFjolmm0-Lz@06n6L#smb!))Jx`wjBj9cu(aO%CIUDrdb#p zbu34_PhHV#Z2?54}y;U=< z4Qd&!%Y~dQk@nRb)dJvZ=&ztq%&Om46^kj#J=@BHZUdqKHrLEEi5R2VrFLE zKR%ZF_>q9r_m~~5qc4j>%k46Z1pp<=+x295?bo!%yx#RAXWx6YO z`3=Fbeqwq%oHe+g$Utze@WWEp2nfPTA=_}9ldT}&sHUx$KOpYCbeY3tBhg8*Uk5}# z^-p-lX*yBlppL9AJG&lkuWTmAy<9XAQAQ|#h zCy%foiH@Pvk#QYb5qboyUad(9iyt>RUU%KZieCaj+aY)ow1`3K2B82BkpJ@rWM*U{ zQYIJwb86EI@J41vQwj|(q4J?gn%6Lu1xL{HivE@j&DQse*2s$*`}+7o+53kl$c=A7 z)q7rH4+7|#!pRSMc2I4zGys)$S`+=nk`kL#ZGUcIh7w1hO@=-5(U`*)0o_}>c0%Sr z*DPiXlw}`0M?S5!MINM@*BXWTv!}y%;wrX>8Snd5Qos;?hofar{dWsC6t|SCMx#wD z{uhOhw1{E=9zibFt%AGVkY_O`{SS%NJZVk71mpvMcXmu}?b;%H>&!=-p6{LKC;Oz8 zL^&hQ;*p6n8OK;tR=x$sh_U*x5f*)n{CVklVZ9db^`?r}Dbn~^lU!9!vese#PI<;U z@=4i)aA_Jhr#8#=4?FT^W>a^#!+#Q*O`Qj`{i(Jom`I9mVzOk-#r%jA&HVGYZ2CnB zY=s})-Q+$r#%OsgLo3_M?`1+M5LB|rg*}W)>uJoWBYZP5rkrcc{@wt*o|`MQt-k)kQ zCvcaWP}2rqk#SEy^Nfg;zVWRU>r+e*u3zMVs#}>#DvW~T@gNX33Otj<20kHA778GC z{PuoR3sBf(j(7dY_ReM%CBAcDr&YZd1D#VfiE|}H{7xQo6M*mWys%gy=P1a{(gr#L z2Q+|nhSgPPs#kH|ZiP}Qu!%kp08A6KnT0rQLxjU~a@SW6W)ljAYk^NRC~d;S)KKXe zHKd$a^P$vZ067;OI`C#ZtFa&Nf4{%~UZ{4TE95xwuDH0EoF-hax452Lk04LCRI1Yb z(3d&oJq0$B^%}YhK{H5&Bh|+as_+wvwIpIlQ0Zt=fj+2KRR$S{dDBXW0pquLP(BI1 zdhND8Y-gR-vrhmETFNt)?8q9BFC4g40CJ+bA;}{27g3EX;X7 z>oZBecf+S0s1b#=xq;6H3R!bkqFDi9>|FIYWOQ*K2`(v#Tg&60g-ElW^v-`~qtCy7 z=yF}r5a!cS4LJuHjmnz@d_Ooo={uOV;c|29`NTsdMj@+j1(`ObibyV)#(+H-T>R#q zQ+lyh(8`vX?y7*`Lt_dcd!PC%LRUkV3}i~mhN5Evx21Lv8txHrv8Pr4&9m(W5i+1Q zsJ#r_-0QU@=)mpur`x^=FXg9ss4ZOd(A@`t9}J$f_~)Y%DApF+Ho# zu0a3K*T}=$qU4%7MDZ)g&cv8kS&KJbW*hthEhH)dvngbz69jWmlXaK>^MqWtJr+<_ ziaBaX(q2TaQi0W8XCP#Jco*XQ4$f3|XUvkmDg-`m%=>d_0jK>EDvI%)p*L4PjWzYg z?3rz!KNg*Td~GCt7gmA-`%OwTlwoRu@3^||MDTBlsDtaB{X(fyAV3`S&+hLWb+CC5inDqP&2G5n*$~w3U4nSDErQkwtPYwMV z?(mt9SwV6%rqAyz;nmF!BCUbyOJprAa@^1--K13`#As|p;yw1&-0|%PxWzo9unP=| z`n7F*Y>NPb#<0Jri#NWJm~z_l5EjX%-K! zDDTI#4Be}xx6#{bN(3HM%g-qsMBijokN_gYs-R_%yx!!GN^$=^2dM{E*JY7WOBRd% zu8{gURtd+$T908G2Pk7eB8m7_b=L_cv@ESIO){q51h34=iOxQR85Qzfj?6h&45*qr z^3V(FOVPKfBx;zba5kYj*)kRRwAEtsCa@)J*(R6WudlaW3?d8W55DF?;-x>(k}nPs z<#hP-=^2K0?6I1Sj!{ZU9%im_=29;ENIAX`VD0`gT>TUgDh7#(biJioTyg&X-ghh1#c4g8B<^tMWgswe{Fm_M z!axMsc{NOZ{xlccxZ^=_8sFVOpQ|**A^_vGx7(l@eB9g;@GQTCYq)cIgSHG|9tW7! zyUNH~&JgSww!q%fsB67XAB7r4=i?%~!)k6I$RtKI*4OsO>~Jlnf>n*ofBZ42gTwOm zWL6_KR868a$ro$vNKGZ7$uSTqt`dxu-&4N2J-z?*Iku#6#eJ6&@UnG@TeS{7ieXPr z64cn_i?YE-oU|qYPL@7u@$9t~GQ$si3Na%cd{rp1I@I{cGt4kH?C{sHu&wqmp7z;o z`r-af*cpnpo?xW3#8hb>8}^Bw91{t0dKRm<3v#68*U%D(Qus< z(7>RRazlW~$k;ZmgpN-u#lD(Jgy8HCwUSlE$=x?0nv7iRhiq3cBu7e^oO8#@aFRcK`hHs-z$f)5&^3{7ZToJv-31t)-*> zI;{@VK20NQ#upU!yP|KZ4i(T}7?6RRc+m0v63KMF=&r{-w=MA9?EnfdpV2dR(T5Mz z@at&dmMD*eG9$R3bH!?IT%ss*-~Fv8T8$$IoKY@75hYh)Jv%Bn= zkEX0@pH^uA6+n*>g=gDHfFTF$1APs!%WWmDQM3yT{BV0N5-vqI^l%;xaEeTFW=#q@ z85!`vNSZ3ajH1DI?_^jfM1|cSXo#?&7Z-TQXe5Y^Qb+&uSM@vG`%_l@jYRLLQr8#& z0w`C<4ZgL+aZBuCq=B4M|KcL9ds?2Ag6h_xO(34yd9xP|8xK#CHpygnGy`p$^k41z zcfr1BK#))GiedNjIyhf6*+p`}+Y0ratd9Z z6;|I}zCpC6C2VI7f7z+{uL)Cbi~XV$J(=|(uTOeu z!O@9$cTY&O5w1}gJ*NlET88k`eDthQOpJVXf@nv{UiG$olgez|C?SOyJ!dK?6pkic zL338gK#K>u#a5>D*qMKB+!@RF@qx*LrV~-y|Ar}&W~JlpAv28};}PHun-pH4gXHK# zztHh|YVqi{s}!iL!^(zW;Mx7wvTS8YK;*ODv2+ zi`6f&8~Q{0NB!qbD^Z6WPSom`8iaKc!AP8pGCl*X_yc+8C{)Ca=T@Z{@WAXneKvD- z51?xzW?>4C=Gu;^D6>*8C$1s`wap!?>|1am0uv1#6#46-R)CouKAyyYPgOMz2*t4} zoWy%j7pdM`(VEC=>jhHs{+#a#byb1DO{StRre?s&!{qA7cf^>ZE=PLvK zQ@PqwZfxqQ!oK=~gzY|*I0Q(vAsZ<{YY{}=4b32v54GW0X_K!$@ zBI6*1r*CwV)RmSZ3`k2SjO@tevITQfH~hAea&~492?@m@CPqo5R&#sccz%8%2xa?d zh>naLnb$>z0uS>(CKdyi^m9-UNY@`w(#2f5fP!?G#6M|J|6nW*Ut2TO(AMl(Ss|KY zKMwwunVC5`wWiE~6-yIGN^pDhsi>$3X2;wd;2^}d_&m%cM%uQwvB;A~CKnbW%ZK0h z_YX=GI2|i{YZ~Llc6G&7`EL_jMTQ;`pQus?9p0Xtr_fN)_=VceG{;(5RWq|?p#H~H z`n%#(^B9SQMnBCL6+f6A@o*x^=)02Xp!v7Ug~!2v)egF!6wChceW3M{kZEkggN6wg z2?OWvm*pm9{qGfK4gBA2tYu^Ch+N7LLi|bI{KKD~Ovl5dh+8!xXwC72XiO_c8-DBc zO<9;J&1>ja+4lY;cPi&i0cYC#>_53$LbxBXrczT4ZK#e;@B5@WH$85J?cG!e?^A>^ zCH|V7yQ;0IB57uc7ju+#(dX>Y#Pby@APVKYulG0B*EsD5-W?m=sOt@*Mlk=N`1 zB&F95&hBv(uo$MuQ_S^_=KdEY|3e!VIUArl6lyGX($R*$TX$uhuh2Jt`Pgd559k{4 zk4_gmN$S5%be}X=4V3RN*4Qk$G&FJOjcRFWncl>tc-fM3x$4<6YB3uHrx`vYo4t&W zD_7`tLO1_$L2l4xX2HcZ9UIoXzk7Nc(s{FTsB#}4Nn?@Topt;9vYUmr@VJ%t8ph42 zZE~t z%hq{h0rwQI_d}u_0mgthtgz&XqohK?0Oo5qOh6sJ+qEW#V`?|d2eW|i)@`1)u_*Nd z)s3^adxW51V+j=Ro;iOTp||6Dz&+gW*;a2}?u>Z7{qelmwXNRX37ihkE0D?k4lK~g zs_qItN)^C$V=@~3u}~p?yV`nxIKo(uzD!>PrLJo z?k-y&5$OZBR`thFrL&*$(~0UoUw5^5}G=VEa!@I#|4Lx+6S zGv>)k(7wRNE%ie-TJnlCyEz~%nBu`jr_Y>@??SbDVwNyQ?m$5eGua5C&f?S{doa^s&RDc?<^0|K$3JMX>ofr6w( z*H;conCE<6LbbJ7Bp~=_M8ewEeN$a78IUo(n8iL84kUP6bH@h?4e-T{jI0XT;j*9! zPtq^BxSkL$2bNX!Xnfv@aj^hRVGAYvN6m@V=xw!?`RFplDK0AcrmpLUGI^{0{obWZ z!k-uG9Z^WvvOyA`Q>z@fJluj|YrJfbrcHC&hxHP?<<2+Um%OfqIsiUD&K*dn0SO6m zBJ5x|0#*M45uk~~QL^9PKj`b;u5QQ|BR(~`y$EA>&Qf1e2K@kQ- zTZf-eU*2|kK|!3Q8JvWq0#1;Qjt)T6Hng$18GIsIE9wn^tWKdc-y>0PkY`OeVs2SC26YKdPfawn&JLT0k{jTpF6Llt(1Y+4=o?8kksS2eWrD(zy`yOU?LLS`~UpS4xnF$tU^p|^gG zO1kaSh0vKG?JMBF) z&C~f`1g#WJ(yANl+EJfo=1EU60V%eZPmOF=vbP+0BmXm8$ZP4fM%J%z$^sAvI^vHL z7;#98YXC0rVEq!U`%#=TJDCj+pN3J#510&ENp?(ws!xO!9{w6JL>l2D}0y<-Sn4RH+2z;661~m zd1=iU{^PI0lwUD{;ExyR2DLYL*Vh}hvZPQTAZ8oC6-YJ+$e4A#gnrrxWrcvrabrz5 z2uN;E4m}5Fuia)fw9RH#9NTBd`-;{f8r2Kzotcm%X!kpuk)okKyV5LKr=)IHFp67tc@71AVWxwq4p6GE-;qI<`3ZySgB%&|^mbmzjUCeSmo-fM#`t!=OXzd-hcE-3!Ft-9m|jtq{L7 zM6e7#&U{nao@7i&NC1h7?slUoenc%PWZYge`%+!aD)LbA_B!)numTJ03HS`FxFb-vx(?_PtY#KPwvv0o#4Zcq{-DmMyxFr^!ma zSj)yCVlBi`YPofz-TKt|#r>G`j{gdiEz{RIf1d>qgNaAoF+sw>B>sC-g#sok4AR{- z%`wf*xy@vKrd$gpEQ3|OJK&pK7^9DvVbBbZZt+&94?alwr~)O?*VgEU?@g;@?I)eABl=9DX zNE8c(3t{ZUg>IoV``o4~27&%X4Yfbhs5X2gFu`U~nf241NWRsPQs*Wg-$D`V3VUPkTO{TR|uC_^?mb?y9o^qtu>yKAYN=H zoB8%lvQkZw8gnyM&oP`6m)S7RoH2EKk3=wM-v1^=`}{CS+)_dUTC4IK-M6KuF5yX= zMx7Qf3{0%2GbquAeY4}mPXGS!kg|gVxrtxh6M`OFI;U#ax3|4BGgzLh?xizL=)%uz zw_GUJM7FafjtPlVNMG+>q4;MX&ED=mh9S1CD*^hspKI&sD{C*8Q`G?(pWTU$& zC=3w@x|3_ZYb@1Jni~bjL}FNRRd9NExwS*zk>No9;>t+;tnV&R)6WM|T@UaU6B+TGr4wSCmEFU0 zKiIuF+5?Ze)_l=dIv)w~kdZe}wnK@4pu>Qbq3P_>N_@m?*WFSa5L%7*cBw5T`+wRv z_jo4vIF4_I(d0H3nq}xvsIf-8Xf?N@yk5`q{9eE3`F+2i_nSxg$auq;NAmKx+_kbE z!Juat^lIW)MntyRZ3U%BpZw)@Ny=ZoeBQtQ(IeBV)z#_4VIi^A`x-+C1HbIev}@m_ znU#?t9$h`jEbsj-!dFLGJlIF%K0T0}J$5m03BL5#YjUJLNs9e(tFEr<%JphW@L08M ziI5yQMtqT!{`j%c*I}B)P-27ud}w=kDfh*KOJcoA7=OwyY5*l5J|Mq@djmA~crSSJ zelpHbdAvduMuTCKp7rM*l0Ja-QKA4^y5$i>@fb?*_LUfw?;j5BYs@9H^%c)?FJBLG z+(y{Ibk*sMez?&Wx$fNAvto;TA673+4a3;CzfQFAc<{9r5gmxlcUuo9)-TTHuKZ;_ z-UCS^)`Vq{w5){WHZETGQ%nHH&yxq~G)T_XLYwbx@1D;b95TPd3b$zvNv*fsQo=^L z3$3~$#%nNaRBK3B=?A4o44FKM87>k9*Tr~l?lRIw>In!Jt*vbXc2ifKA=PmeT~A5@ z&~jwpR(+t^`<4iIch$IQ;mM&PF95l2-*jG&Ku`~aGaFW-?Vg`Xy)ekFxohMSl2sU) z9Ef>P)2u%6j8IBgcz2or;}ZBgsfe1ygVA|o0Nc&teP%Q}>-JX|V6jw8t@W*`@W-)M zarp>EMTfXJ^MS!Z?N!ncN%Satd}yH1b$4zH=;o#ZvY#n;!tlc?V|WpJD)ThI*lf|Q zQ|IcutPwlZ`qC?k_(E4U_6|0&(^htImvco%nIX48@?hKjE$46SVC7h4#^z6r7#}i+ zDFo}XmM;q4NqjVrBq#(_(Cv-zg$_xv6D}82#$20R7YUU`D_n)c&1fzjQ(mo^@`17x z#<68gvZynYPL|!;XMJVhXJU>Uw!alup>tgLf-w}%J{oNH0o+A^g!(g5ik%Hwq?t6TrYIqy z4mC!P2(hWhsf9z^{K>d}*cBMfvrz`1^R|yrg0SWzbmp`2LVbBC*6#N;>@*XlKAHxe z1U^1A(Oj)W`4M|9C+0wV)n{MBL(iw(D*$^XdSB5F;_zuDm*6$K1V)MetIe~mNaPgE<@eyuPsP~?oFbM{U8W=LnzxjyQQ;ssUd z_N3?nwPr;rI+)82nWh`m3UnH(C6;L8UVQiuFN>uyQY#(|$cviyD0*j~HlLc+C zI{_XN1x%PZ{M>X~$kBIf8#Sn_qCFpJW11jfa;}SRvZzp;pf`#}-iI&=hZ80`#2V(o zoO>dK0w>5r8lg~Esuh#1fv$*B!AUj-Bo-@SCWhu$_wqxYPAi^>qMax*%W1{;=ch_i zNm{weGc_tbOI=(T!;RJ2K765eX>_ErU-Mbvs4+jq>MdU=OFTA3-JJKsxnf#%QVKFY zUfpr}(-m67C=a`cRbQqY Date: Sat, 13 Jun 2026 02:18:44 +0000 Subject: [PATCH 24/82] Increment Version to 0.19.0a2 --- ovoscope/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovoscope/version.py b/ovoscope/version.py index d880880..9ebb2ab 100644 --- a/ovoscope/version.py +++ b/ovoscope/version.py @@ -2,7 +2,7 @@ VERSION_MAJOR = 0 VERSION_MINOR = 19 VERSION_BUILD = 0 -VERSION_ALPHA = 1 +VERSION_ALPHA = 2 # END_VERSION_BLOCK __version__ = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}" + ( From 56b4e1f9da4892df0508eb599edf94f41b873184 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 13 Jun 2026 02:19:13 +0000 Subject: [PATCH 25/82] Update Changelog --- CHANGELOG.md | 50 +++++++++++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac96aa8..51c996f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,60 +1,68 @@ # Changelog -## [0.19.0a1](https://github.com/TigreGotico/ovoscope/tree/0.19.0a1) (2026-06-12) +## [0.19.0a2](https://github.com/OpenVoiceOS/ovoscope/tree/0.19.0a2) (2026-06-13) -[Full Changelog](https://github.com/TigreGotico/ovoscope/compare/0.18.0a1...0.19.0a1) +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.19.0a1...0.19.0a2) **Merged pull requests:** -- feat: MiniVoiceLoop + simple/classic listener bus-sequence harnesses [\#67](https://github.com/TigreGotico/ovoscope/pull/67) ([JarbasAl](https://github.com/JarbasAl)) +- docs: standardize NGI0 Commons Fund attribution [\#69](https://github.com/OpenVoiceOS/ovoscope/pull/69) ([JarbasAl](https://github.com/JarbasAl)) -## [0.18.0a1](https://github.com/TigreGotico/ovoscope/tree/0.18.0a1) (2026-06-10) +## [0.19.0a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.19.0a1) (2026-06-12) -[Full Changelog](https://github.com/TigreGotico/ovoscope/compare/0.17.1a1...0.18.0a1) +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.18.0a1...0.19.0a1) **Merged pull requests:** -- feat\(phal\): plugin\_factories for MiniPHAL and PHALTest [\#65](https://github.com/TigreGotico/ovoscope/pull/65) ([JarbasAl](https://github.com/JarbasAl)) +- feat: MiniVoiceLoop + simple/classic listener bus-sequence harnesses [\#67](https://github.com/OpenVoiceOS/ovoscope/pull/67) ([JarbasAl](https://github.com/JarbasAl)) -## [0.17.1a1](https://github.com/TigreGotico/ovoscope/tree/0.17.1a1) (2026-05-20) +## [0.18.0a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.18.0a1) (2026-06-10) -[Full Changelog](https://github.com/TigreGotico/ovoscope/compare/0.17.0a1...0.17.1a1) +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.17.1a1...0.18.0a1) **Merged pull requests:** -- fix\(pipeline-harness\): default \_SinkSkill bus to FakeBus [\#62](https://github.com/TigreGotico/ovoscope/pull/62) ([JarbasAl](https://github.com/JarbasAl)) +- feat\(phal\): plugin\_factories for MiniPHAL and PHALTest [\#65](https://github.com/OpenVoiceOS/ovoscope/pull/65) ([JarbasAl](https://github.com/JarbasAl)) -## [0.17.0a1](https://github.com/TigreGotico/ovoscope/tree/0.17.0a1) (2026-05-14) +## [0.17.1a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.17.1a1) (2026-05-20) -[Full Changelog](https://github.com/TigreGotico/ovoscope/compare/0.16.0a1...0.17.0a1) +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.17.0a1...0.17.1a1) **Merged pull requests:** -- feat\(intent-cases\): markdown reporter, baseline diff, auto-discovery, deterministic m2v warmup [\#60](https://github.com/TigreGotico/ovoscope/pull/60) ([JarbasAl](https://github.com/JarbasAl)) +- fix\(pipeline-harness\): default \_SinkSkill bus to FakeBus [\#62](https://github.com/OpenVoiceOS/ovoscope/pull/62) ([JarbasAl](https://github.com/JarbasAl)) -## [0.16.0a1](https://github.com/TigreGotico/ovoscope/tree/0.16.0a1) (2026-05-14) +## [0.17.0a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.17.0a1) (2026-05-14) -[Full Changelog](https://github.com/TigreGotico/ovoscope/compare/0.15.0a1...0.16.0a1) +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.16.0a1...0.17.0a1) **Merged pull requests:** -- feat\(intent-cases\): file-based intent test layout + pytest accuracy gate [\#58](https://github.com/TigreGotico/ovoscope/pull/58) ([JarbasAl](https://github.com/JarbasAl)) +- feat\(intent-cases\): markdown reporter, baseline diff, auto-discovery, deterministic m2v warmup [\#60](https://github.com/OpenVoiceOS/ovoscope/pull/60) ([JarbasAl](https://github.com/JarbasAl)) -## [0.15.0a1](https://github.com/TigreGotico/ovoscope/tree/0.15.0a1) (2026-05-14) +## [0.16.0a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.16.0a1) (2026-05-14) -[Full Changelog](https://github.com/TigreGotico/ovoscope/compare/0.14.0a1...0.15.0a1) +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.15.0a1...0.16.0a1) **Merged pull requests:** -- feat\(e2e\): reusable harness, bus helpers, and intent-registration shims [\#55](https://github.com/TigreGotico/ovoscope/pull/55) ([JarbasAl](https://github.com/JarbasAl)) +- feat\(intent-cases\): file-based intent test layout + pytest accuracy gate [\#58](https://github.com/OpenVoiceOS/ovoscope/pull/58) ([JarbasAl](https://github.com/JarbasAl)) -## [0.14.0a1](https://github.com/TigreGotico/ovoscope/tree/0.14.0a1) (2026-05-14) +## [0.15.0a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.15.0a1) (2026-05-14) -[Full Changelog](https://github.com/TigreGotico/ovoscope/compare/0.13.1...0.14.0a1) +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.14.0a1...0.15.0a1) **Merged pull requests:** -- feat: add NEBULENTO\_PIPELINE and PALAVREADO\_PIPELINE stage groups [\#54](https://github.com/TigreGotico/ovoscope/pull/54) ([JarbasAl](https://github.com/JarbasAl)) +- feat\(e2e\): reusable harness, bus helpers, and intent-registration shims [\#55](https://github.com/OpenVoiceOS/ovoscope/pull/55) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.14.0a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.14.0a1) (2026-05-14) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.13.1...0.14.0a1) + +**Merged pull requests:** + +- feat: add NEBULENTO\_PIPELINE and PALAVREADO\_PIPELINE stage groups [\#54](https://github.com/OpenVoiceOS/ovoscope/pull/54) ([JarbasAl](https://github.com/JarbasAl)) From 02845370c5066f650a6c9e5e0b1a3c13ac919247 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 13 Jun 2026 16:52:38 +0100 Subject: [PATCH 26/82] chore: remove agent-audit scratch files from repo root (#71) --- AUDIT.md | 184 ----------------- FAQ.md | 445 ---------------------------------------- MAINTENANCE_REPORT.md | 464 ------------------------------------------ QUICK_FACTS.md | 55 ----- SUGGESTIONS.md | 153 -------------- downstream_report.txt | 1 - 6 files changed, 1302 deletions(-) delete mode 100644 AUDIT.md delete mode 100644 FAQ.md delete mode 100644 MAINTENANCE_REPORT.md delete mode 100644 QUICK_FACTS.md delete mode 100644 SUGGESTIONS.md delete mode 100644 downstream_report.txt diff --git a/AUDIT.md b/AUDIT.md deleted file mode 100644 index d4dd424..0000000 --- a/AUDIT.md +++ /dev/null @@ -1,184 +0,0 @@ -# ovoscope โ€” Audit Report -## Documentation Status -- [x] AGENTS.md Header Format -- [x] QUICK_FACTS.md -- [x] FAQ.md -- [x] MAINTENANCE_REPORT.md -- [x] AUDIT.md -- [x] SUGGESTIONS.md -- [x] docs/index.md -- [x] docs/usage-guide.md -- [x] docs/ci-integration.md ---- -## Technical Debt & Issues -### ~~[CRITICAL] No unit tests for ovoscope itself~~ โœ… FIXED -62 unit tests across `test/unittests/test_capture_session.py`, `test/unittests/test_end2end.py`, -`test/unittests/test_minicroft.py`, `test/unittests/test_pydantic_helpers.py`, and -`test/unittests/test_audio_harness.py`. -All pass (62 passed, 0 failed). ---- -### ~~[MAJOR] Missing LICENSE file~~ โœ… FIXED -LICENSE file (Apache-2.0) added to repo root. `pyproject.toml` correctly references it. ---- -### ~~[MAJOR] `End2EndTest.execute()` returns `None`~~ โœ… FIXED -`execute()` now returns `List[Message]`. Signature changed to -`def execute(self, timeout: int = 30) -> List[Message]`. `assert_spoke()` builds on this. ---- -### ~~[MAJOR] No type annotations on any public method~~ โœ… FIXED -All public methods in `ovoscope/__init__.py` and new modules now have PEP 484 annotations. ---- -### [MODERATE] `CaptureSession.capture()` returns `None` โ€” captured messages inaccessible inline -**Evidence**: `ovoscope/__init__.py` โ€” `capture()` returns nothing. Captured -messages are only accessible via `self.responses` after `finish()` is called. The pattern -`capture.capture(msg); messages = capture.finish()` is functional but forces a two-step flow. -**Impact**: Cannot chain captures or do inline assertions without accessing the attribute -directly. Slightly awkward for advanced multi-turn test composition. -**Recommended fix**: Return `List[Message]` (the current `self.responses` snapshot) from -`capture()`, or add a `messages` property. No breaking change required. ---- -### ~~[MODERATE] `get_minicroft()` busy-waits with no timeout~~ โœ… FIXED -`get_minicroft()` now accepts `max_wait: float = 60` and raises `TimeoutError` if the deadline -is exceeded. Signature: `def get_minicroft(skill_ids, *args, max_wait=60, **kwargs) -> MiniCroft`. ---- -### ~~[MINOR] `setup.py` should migrate to `pyproject.toml`~~ โœ… FIXED -`setup.py` removed. `pyproject.toml` uses `build-backend = "setuptools.build_meta"`, -`dynamic = ["version"]`, and `[tool.setuptools.dynamic] version = {attr = "ovoscope.version.__version__"}`. -`version.py` now exports `__version__`. Optional pydantic extras added: -`pip install ovoscope[pydantic]`. ---- -### [MINOR] CI action pins at `@master` -**Evidence**: GitHub Actions workflows use `pypa/gh-action-pypi-publish@master` and -`ad-m/github-push-action@master`. The `@master` ref is not pinned to a specific SHA, making -builds non-reproducible if the upstream action changes. -**Recommended fix**: Pin to `pypa/gh-action-pypi-publish@release/v1` and a specific SHA for -`ad-m/github-push-action`. ---- -## Next Steps (Priority Order) -1. Address CaptureSession.capture() return value โ€” usability -2. Pin CI action refs โ€” reproducibility - ---- - -## Audit [2026-03-12] โ€” Correctness Bugs, Coverage Gaps, Docs - -### ~~[CRITICAL] `pipeline.py` race condition in `match()`~~ โœ… FIXED - -**Evidence**: `ovoscope/pipeline.py:159โ€“181` โ€” a single `threading.Event` was -shared for both success (`intent.service.skills.activated`) and failure -(`intent_failure`, `mycroft.skill.handler.start`) handlers. If `intent_failure` -fired first, `captured[0]` would be a failure message returned as success. -Additionally, `done.wait()` return value was not checked; a timeout silently -returned `captured[0]` (or raised `IndexError` on empty list). - -**Fix**: Separate `_matched` and `_failed` events; only the success handler -populates `captured`. Timeout and failure both return `None`. -Source: `ovoscope/pipeline.py:149`. - -### ~~[MAJOR] `diff.py` โ€” subset comparison silently ignores extra keys~~ โœ… FIXED - -**Evidence**: `ovoscope/diff.py:121โ€“138` โ€” `_dict_diff()` only iterated keys -in `expected`, so unexpected keys in `actual` were never flagged. - -**Fix**: Added `strict: bool = False` parameter to `_dict_diff()` and -`diff_fixtures()`. When `strict=True`, extra keys in `actual` are included in -the diff detail. Default `False` preserves existing behaviour. -Source: `ovoscope/diff.py:121`. - -### ~~[MINOR] `bus_coverage.py` โ€” dead method `_skill_id_for_handler()`~~ โœ… FIXED - -**Evidence**: `ovoscope/bus_coverage.py:745โ€“763` โ€” `_skill_id_for_handler()` -was never called anywhere in the codebase (verified by grep). Its logic is -a subset of `_skill_id_for_closure()` which is the method actually used. - -**Fix**: Method deleted. -Source: formerly `ovoscope/bus_coverage.py:745`. - -### ~~[MAJOR] No unit tests for `media.py` (`MockOCPBackend`, `OCPCaptureSession`, `OCPPlayerHarness`)~~ โœ… FIXED - -**Evidence**: `ovoscope/media.py` had no corresponding test file. - -**Fix**: Created `test/unittests/test_media.py` โ€” 20 tests covering -`MockOCPBackend` state transitions, `OCPCaptureSession` message accumulation, -and assertion helpers. - -### ~~[MAJOR] No unit tests for `remote_recorder.py`~~ โœ… FIXED - -**Evidence**: `ovoscope/remote_recorder.py` had no corresponding test file. - -**Fix**: Created `test/unittests/test_remote_recorder.py` โ€” 15 tests covering -constructor defaults, `_parse_url`, connect/disconnect lifecycle, `record()` with -mocked bus client, timeout handling, and fixture serialization. - -### ~~[MINOR] Deprecated `ovos_utils.messagebus.Message` import in `test_phal.py`~~ โœ… FIXED - -**Evidence**: `test/unittests/test_phal.py:23` โ€” imported `Message` from -deprecated `ovos_utils.messagebus` instead of canonical `ovos_bus_client.message`. - -**Fix**: Import changed to `from ovos_bus_client.message import Message`. -Source: `test/unittests/test_phal.py:23`. - -### ~~[MINOR] `SUGGESTIONS.md` missing~~ โœ… FIXED - -**Evidence**: `SUGGESTIONS.md` was absent despite being required by `AGENTS.md`. - -**Fix**: `SUGGESTIONS.md` created with 10 structured proposals. - ---- -## Bus Coverage Module โ€” Full Audit [2026-03-12] - -### ~~[CRITICAL] async_responses excluded from emitter coverage~~ โœ… FIXED -`execute()` now passes `messages + list(capture.async_responses)` to `record_session()`. Source: `ovoscope/__init__.py:666`. - -### ~~[CRITICAL] Unattributed expected_messages silently disappear~~ โœ… FIXED -Both observed and expected messages with no `skill_id` in context now fall back to the `"__core__"` sentinel bucket. Source: `BusCoverageTracker.record_session` โ€” `ovoscope/bus_coverage.py:510`. - -### [CRITICAL] Registration-time handlers always show NOT TESTED โ€” misleading -**Evidence**: `ovoscope/bus_coverage.py:368` / `ovoscope/__init__.py:647` โ€” `snapshot_listeners()` is called after `MiniCroft` reaches READY. Handlers invoked *during* skill loading were called *before* the snapshot and will always show 0 invocations. -**Impact**: Users see `register_intent: NOT TESTED` and conclude their intent registration failed. -**Status**: Documented in `docs/bus-coverage.md` Limitations as a known structural constraint. A `LOAD_TIME` tag or pre-READY tracking remains a future enhancement. - -### ~~[CRITICAL] Non-skill messages silently excluded from emitter coverage~~ โœ… FIXED -Messages with no `skill_id` in context are now attributed to the `"__core__"` bucket in `record_session()`. Source: `ovoscope/bus_coverage.py:510`. - -### ~~[MAJOR] Unused `skill_map` variable in `record_session()`~~ โœ… FIXED -Dead `skill_map = self._skill_instance_map()` call removed from `record_session()`. - -### ~~[MAJOR] `_get_bus_events()` called three times in `snapshot_listeners()`~~ โœ… FIXED -`bus_events = self._get_bus_events()` is now called once and reused across all three passes. Source: `ovoscope/bus_coverage.py:432`. - -### ~~[MAJOR] Double-stop risk in `cmd_bus_coverage()`~~ โœ… FIXED -`test.managed = False` is now set explicitly before `test.execute()`. The `finally` block is the sole owner of `mc.stop()`. The redundant `mc.stop()` in the `except Exception` branch was also removed. Source: `ovoscope/cli.py:349`. - -### ~~[MAJOR] `pytest_terminal_summary` hook uses private pytest internals~~ โœ… FIXED -`bus_coverage_session` fixture now stores merged reports on `request.config._bus_coverage_reports` in its teardown. `pytest_terminal_summary` reads that list โ€” no private attrs. Source: `ovoscope/pytest_plugin.py:191`. - -### [MAJOR] `once()` handlers invisible after firing -**Evidence**: `bus_coverage.py:368` โ€” one-shot handlers fired during skill loading are de-registered before the snapshot. -**Status**: Documented in `docs/bus-coverage.md` Limitations. Pre-READY tracking is a future enhancement. - -### [MAJOR] `ignore_messages` list silently excludes messages from emitter coverage -**Evidence**: `ovoscope/__init__.py:504-505` โ€” messages in `ignore_messages` never reach `responses`. -**Status**: Documented in `docs/bus-coverage.md` Limitations. Passing ignored messages as a separate bucket is a future enhancement. - -### ~~[MAJOR] No JSON schema version field~~ โœ… FIXED -`to_json()` now includes `"schema_version": "1"` as the first key. Source: `ovoscope/bus_coverage.py:297`. - -### ~~[MINOR] Column widths hardcoded in `print_report()`~~ โœ… FIXED -`col_w = max(len(s.skill_id) for s in self.skills) + 2` now drives column width dynamically. Source: `ovoscope/bus_coverage.py:237`. - -### ~~[MINOR] Coverage summary printed before test assertions~~ โœ… FIXED -`print_bus_coverage` block moved to after all assertions, just before `managed` teardown. Source: `ovoscope/__init__.py:795`. - -### ~~[MINOR] `bus_coverage_report` type hint uses `Optional[Any]`~~ โœ… FIXED -Type hint changed to `Optional["BusCoverageReport"]`. Source: `ovoscope/__init__.py:589`. - -### ~~[NITPICK] `_SKIP` set has fragile `"type"` entry~~ โœ… FIXED -Pass 2 now uses `isinstance(owner, type)` check to skip class objects, and the `"type"` string entry removed from `_SKIP`. Source: `ovoscope/bus_coverage.py:450`. - -### ~~Docs gaps (bus-coverage.md)~~ โœ… FIXED -All five missing Limitations items added to `docs/bus-coverage.md`: -- Registration-time handlers always show NOT TESTED -- Pipeline matching is not bus-driven -- `async_responses` now included (fix note) -- `ignore_messages` types excluded -- Core services use `__core__` bucket (not 0/0 โ€” fix note) diff --git a/FAQ.md b/FAQ.md deleted file mode 100644 index 4c92f15..0000000 --- a/FAQ.md +++ /dev/null @@ -1,445 +0,0 @@ -# FAQ โ€” `ovoscope` -## How do I measure which bus message handlers my tests actually exercise? - -Use bus coverage: set `track_bus_coverage=True` on `End2EndTest`. After -`execute()`, `test.bus_coverage_report` contains a `BusCoverageReport` with -per-skill listener coverage (which `bus.on()` registrations were triggered) -and emitter coverage (which message types were observed / asserted). - -```python -test = End2EndTest( - skill_ids=["my-skill.author"], - source_message=message, - expected_messages=[...], - track_bus_coverage=True, - print_bus_coverage=True, # print inline summary -) -test.execute() -print(test.bus_coverage_report.to_json()) -``` - -See [docs/bus-coverage.md](docs/bus-coverage.md) for the full reference. -`BusCoverageTracker` โ€” `ovoscope/bus_coverage.py:242`. - -## How do I get an aggregate bus coverage report across an entire test suite? - -Use the `bus_coverage_session` pytest fixture. Each test calls -`bus_coverage_session.add(test.bus_coverage_report)` after `execute()`. A -merged table is printed automatically at session end. See -[docs/bus-coverage.md](docs/bus-coverage.md). - -## How do I run bus coverage from the command line without writing pytest tests? - -Use the `ovoscope bus-coverage` subcommand: - -```bash -ovoscope bus-coverage path/to/fixtures/ # table report -ovoscope bus-coverage path/to/fixtures/ --format json -ovoscope bus-coverage path/to/fixtures/ --verbose # per-msg detail -``` - -`cmd_bus_coverage` โ€” `ovoscope/cli.py`. - - -## How do I test AudioService or PlaybackService without real audio hardware? -Use `AudioServiceHarness` or `PlaybackServiceHarness` from `ovoscope.audio`. Both run on a -`FakeBus` with `MockAudioBackend`/`MockTTS` respectively โ€” no real audio device, TTS engine, -or network required. See [docs/audio-testing.md](docs/audio-testing.md) for the full API -reference. Requires `pip install ovoscope[audio]` (or `ovos-audio` installed separately). - -## Why does AudioService.stop() silently ignore my stop() call in tests? -`AudioService._stop()` โ€” `ovos-audio/ovos_audio/audio.py` โ€” has a 1-second stop guard: -it does nothing if called within 1 second of `play()`. Tests must `time.sleep(1.1)` after -`play()` before calling `stop()`. - -## Why doesn't FakeBus.wait_for_response() work in audio harness tests? -`FakeBus.wait_for_response()` does not work for synchronous in-process handlers because the -reply is emitted before the internal listener is registered. Use subscribe-emit-wait with a -`threading.Event` instead. `AudioServiceHarness.get_track_info()` and `list_backends()` -implement this pattern โ€” `ovoscope/audio.py`. - -## How do I test VAD (Voice Activity Detection) without a real microphone? - -Use `MockVADEngine` from `ovoscope.listener`. It classifies all-zero bytes as silence and -any non-zero byte as speech. Inject it into `MiniListener(config, vad_instance=MockVADEngine())` -or use the declarative `VADTest` dataclass. No microphone, audio driver, or OPM plugin required. - -```python -from ovoscope.listener import MockVADEngine, VADTest -VADTest(vad_instance=MockVADEngine(), audio_input=b"\\x01" * 512, expect_silence=False).execute() -``` - -## How do I test Wake Word detection without loading a real model? - -Use `MockHotWordEngine(trigger_after=N)` from `ovoscope.listener`. It fires after exactly N -`update()` calls and auto-resets. Inject via `MiniListener(config, ww_instances={"hey_mycroft": engine})` -or use the declarative `WakeWordTest` dataclass. - -```python -from ovoscope.listener import MockHotWordEngine, WakeWordTest -WakeWordTest( - ww_instances={"hey_mycroft": MockHotWordEngine(trigger_after=2)}, - audio_chunks=[b"\\x00" * 512] * 4, - expect_detected=True, - expected_detection_frame=1, -).execute() -``` - -## What is `ovoscope`? -`ovoscope` is End-to-end test framework for OpenVoiceOS skills. -## How do I install it? -```bash -pip install ovoscope -``` -Or for development: -```bash -uv pip install -e ovoscope/ -``` -## Where do I report bugs? -Open an issue on the GitHub repository. Ensure you are targeting the `dev` branch for fixes. -## How do I run tests? -```bash -uv run pytest ovoscope/test/ --cov=ovoscope -``` -## How do I contribute? -1. Fork the repository and create a feature branch from `dev`. -2. Write tests for your changes. -3. Open a PR targeting the `dev` branch. -4. Ensure CI passes before requesting review. -## What CI workflows does ovoscope run? -Seven workflows: `unit_tests.yml` (pytest + coverage on PRs), `build_tests.yml` (sdist/wheel matrix build), `license_tests.yml` (dependency license audit via gh-automations), `pipaudit.yml` (CVE scanning), `release_workflow.yml` (test-gated alpha release), `publish_stable.yml` (stable release), and `conventional-label.yaml` (PR label automation). -## Does the release workflow run tests before publishing? -Yes. The `release_workflow.yml` has a `build_tests` job that runs the full test suite. The `publish_alpha` job depends on it via `needs: build_tests`, so a failing test blocks the alpha release. -## How does ovoscope's coverage reporting work in CI? -The `unit_tests.yml` workflow runs `pytest --cov=ovoscope --cov-report xml` and uses `py-cov-action/python-coverage-comment-action@v3` to post a coverage summary as a PR comment. -## What test coverage does ovoscope have? -104 tests across 6 test files achieving 89% overall coverage. Key areas tested: End2EndTest execute/assertions/serialization/routing/active skills/boot sequence/final session/from_message recording, CaptureSession lifecycle, MiniCroft config isolation/lang/pipeline, pytest_plugin fixture logic, pydantic_helpers bridge. -## What Python versions are supported? -See `QUICK_FACTS.md` โ€” currently `>=3.10`. -## My tests pass locally but fail on CI โ€” why? -Usually one of three causes: -1. **Different pipeline plugins installed** โ€” The default session pipeline includes whatever - pipeline plugins happen to be installed. On CI, Gemma/Ollama/persona plugins may not be - installed (or vice versa), changing which plugin handles the utterance. - **Fix**: always pass an explicit `default_pipeline` to `get_minicroft()` (or use the default - `DEFAULT_TEST_PIPELINE` by leaving `isolate_config=True`). -2. **User locale affecting intent matching** โ€” `isolate_config=True` (the default) removes the - user's `~/.config/mycroft/mycroft.conf` from the config chain so the test environment locale - does not affect results. Always leave this enabled. -3. **Skill plugin not discoverable** โ€” The skill must be registered under the `opm.skill` entry - point group. Old-style `ovos.plugin.skill` entries are warned but not loaded by - `find_skill_plugins()`. Use `extra_skills={SKILL_ID: SkillClass}` to inject skills that - lack a proper entry point. -## A persona / AI plugin is intercepting my test utterances -This happens because `SessionManager.default_session` is initialized at import time from the -full system config (which may include persona pipeline stages). -`MiniCroft` solves this with `default_pipeline` (default: `DEFAULT_TEST_PIPELINE`): -```python -from ovoscope import get_minicroft, DEFAULT_TEST_PIPELINE, ADAPT_PIPELINE -# Default โ€” all standard stages, no AI/persona/OCP (recommended) -mc = get_minicroft([]) -# Adapt-only for fast unit-style end2end tests -mc = get_minicroft([SKILL_ID], default_pipeline=ADAPT_PIPELINE) -# Opt in to persona explicitly -from ovoscope import PERSONA_PIPELINE -mc = get_minicroft([SKILL_ID], default_pipeline=DEFAULT_TEST_PIPELINE + PERSONA_PIPELINE) -``` -`DEFAULT_TEST_PIPELINE` excludes persona, Ollama, OCP, and m2v stages. -The original pipeline is restored when `mc.stop()` is called. -## Why does `SessionManager.default_session.pipeline` matter? -When a message is emitted without an explicit `session` in its context, ovos-core creates a -session by copying `SessionManager.default_session`. That copy inherits its `pipeline` list, -which controls which pipeline plugins are consulted for intent matching. -`MiniCroft.run()` overrides `SessionManager.default_session.pipeline` to `default_pipeline` -(set after FakeBus init messages are processed, just before `READY`), and restores it on -`stop()`. -## How is `isolate_config` different from `default_pipeline`? -- `isolate_config=True` โ€” clears `Configuration.xdg_configs` so `~/.config/mycroft/mycroft.conf` - is not read. Prevents user locale, custom wake words, and user-level pipeline *config* from - affecting tests. -- `default_pipeline` โ€” overrides `SessionManager.default_session.pipeline` directly. Necessary - because the default session is initialized at module import time (before config isolation takes - effect) and may already have the user's pipeline. -Both are enabled by default. They are complementary. -## How do I know whether to use ADAPT_PIPELINE or PADATIOUS_PIPELINE for my test? -It depends on how the skill registers its intent: -| Decorator | Pipeline | -|-----------|----------| -| `@intent_handler(IntentBuilder(...))` | `ADAPT_PIPELINE` | -| `@intent_handler("my.intent")` (string ending in `.intent`) | `PADATIOUS_PIPELINE` | -| `@fallback_handler(priority=N)` | `FALLBACK_PIPELINE` | -| `@converse_handler` | `CONVERSE_PIPELINE` | -When in doubt, look at the intent files in `locale/en-us/` โ€” if there is a file named `*.intent`, -it is Padatious. If there is a file named `*.voc` or `*.rx`, it is Adapt. -## My skill emits extra messages (enclosure.eyes.*, add_context, configuration.patch) โ€” how do I handle them? -Some skills emit low-level hardware events or internal context messages that are not part of the -utterance handling protocol. Add them to `ignore_messages`: -```python -test = End2EndTest( - ... - ignore_messages=[ - "enclosure.eyes.level", # Mark 1 LED animation - "enclosure.eyes.look", - "add_context", # set_context() calls - "configuration.patch", # disable_confirm_listening() etc. - ], - ... -) -``` -## A skill emits a raw message (no source/dest) like recognizer_loop:sleep โ€” how do I test it? -Some skills call `self.bus.emit(Message("some.message"))` without inheriting source/dest. -These messages have `source=None` and will fail source-checking in the framework. -Use `async_messages` to assert they were received without checking order or source: -```python -test = End2EndTest( - ... - ignore_messages=["recognizer_loop:sleep"], # exclude from ordered sequence - async_messages=["recognizer_loop:sleep"], # assert it was received somewhere - ... -) -``` -## My test passes locally but the user's blacklisted skills cause failures on CI -Users may have skills like `skill-ovos-stop.openvoiceos` in `blacklisted_skills` in their -`~/.config/mycroft/mycroft.conf`. `Session.__init__` reads this from the live -`Configuration()` singleton dict cache (not invalidated by `reload()`). -`MiniCroft` solves this: when `isolate_config=True` (the default), it patches -`Configuration()["skills"]["blacklisted_skills"] = []` and -`Configuration()["intents"]["blacklisted_intents"] = []` in `run()`, and restores them in `stop()`. -This is complementary to the `xdg_configs = []` isolation applied in `__init__`. -## Can I use typed pydantic models instead of raw Message objects? -Yes. Install the optional pydantic extras: -```bash -pip install ovoscope[pydantic] -``` -Then use the bridge in `ovoscope.pydantic_helpers`: -```python -from ovoscope.pydantic_helpers import to_bus_message, from_bus_message -from ovos_pydantic_models import RecognizerLoopUtteranceMessage, RecognizerLoopUtteranceData, SpeakMessage -# Build a typed source message โ€” validated at construction -utterance = to_bus_message(RecognizerLoopUtteranceMessage( - data=RecognizerLoopUtteranceData(utterances=["hello"], lang="en-us") -)) -# Parse a received message into a typed model for richer assertions -messages = test.execute() -speak = from_bus_message(messages[0], SpeakMessage) -assert "hello" in speak.data.utterance.lower() -``` -A typo in a field name (`"utterance"` vs `"utterances"`) raises `ValidationError` at -construction time instead of silently producing a wrong test. -## How do I validate a JSON fixture file before loading it? -Use `validate_fixture()` from `ovoscope.pydantic_helpers` (requires `ovoscope[pydantic]`): -```python -from ovoscope.pydantic_helpers import validate_fixture -from ovoscope import End2EndTest -test = End2EndTest.deserialize(validate_fixture("test/fixtures/hello_world.json")) -test.execute() -``` -If any message in the fixture is malformed, a clear `ValidationError` is raised pointing -to the offending field โ€” instead of a cryptic `KeyError` inside `deserialize()`. -## How do I trigger non-utterance events during a test? -Use `MiniCroft.inject_message(msg)`: -```python -from ovos_bus_client.message import Message -mc.inject_message(Message("mycroft.gui.connected", {"connected": True})) -``` -This emits an arbitrary message on the FakeBus during a test without going through the -utterance pipeline โ€” useful for timer events, GUI events, or skill API calls. -## How do I assert a skill spoke a specific phrase without checking the full message sequence? -Use `End2EndTest.assert_spoke(text, lang)`: -```python -test = End2EndTest( - skill_ids=["my-skill.author"], - source_message=Message("recognizer_loop:utterance", {"utterances": ["hello"], "lang": "en-US"}, {}), - expected_messages=[], # not used by assert_spoke -) -test.assert_spoke("Hello, world!", lang="en-US") -``` -`assert_spoke()` calls `execute()` internally and scans captured messages for a `speak` -message with the matching utterance and lang. -## `get_minicroft()` hangs forever โ€” what do I do? -Pass `max_wait` to set a timeout: -```python -mc = get_minicroft(["my-skill.author"], max_wait=30) -``` -If `MiniCroft` does not reach `READY` within `max_wait` seconds, a `TimeoutError` is raised -with the skill IDs โ€” pointing you at the skill startup logs. The default is 60 seconds. -## How do I use the `minicroft` pytest fixture? -The fixture is registered automatically when ovoscope is installed (via the `pytest11` entry -point). Just declare `skill_ids` on your test class: -```python -class TestMySkill: - skill_ids = ["my-skill.author"] - def test_something(self, minicroft): - from ovoscope import End2EndTest - from ovos_bus_client.message import Message - test = End2EndTest( - minicroft=minicroft, - skill_ids=self.skill_ids, - source_message=Message( - "recognizer_loop:utterance", - {"utterances": ["hello"], "lang": "en-US"}, - {}, - ), - expected_messages=[...], - ) - test.execute() -``` -The `MiniCroft` is started once per class and stopped in teardown โ€” no `setUp`/`tearDown` -boilerplate needed. -## How do I test a pipeline plugin (not a skill) like PersonaService? -Pipeline plugins are loaded by `MiniCroft` automatically via `IntentService`. Access them via: -```python -mc = get_minicroft([], default_pipeline=PERSONA_PIPELINE) -persona_svc = mc.intents.pipeline_plugins["ovos-persona-pipeline-plugin"] -``` -Inject mocks directly into the plugin's state before each test: -```python -def setUp(self): - persona_svc.personas.clear() # remove real solvers (Gemma, Ollama, etc.) - persona_svc.active_persona = None # reset pipeline state - persona_svc.personas["TestBot"] = MockPersona("TestBot", "forty two") -``` -The `skill_ids=[]` parameter tells MiniCroft to load no skills โ€” only pipeline plugins. -See `ovos-persona/test/end2end/test_persona.py` for a full working example. ---- -## How do I test skills in non-English languages? -Pass `secondary_langs` to `get_minicroft()`: -```python -croft = get_minicroft( - [SKILL_ID], - secondary_langs=["pt-PT", "de-DE", "es-ES"], -) -``` -This patches `Configuration()["secondary_langs"]` before Adapt/Padatious initialize, so they create per-language engines and register vocab for all specified languages. Without this, only the system's default language has vocab registered. -## Why does `End2EndTest.from_message()` crash with `TypeError: argument of type 'NoneType' is not iterable`? -This was a bug where `async_messages` defaulted to `None` and was passed to `CaptureSession`, which tried `msg.msg_type in None`. Fixed by defaulting to `[]`. -## Why do JSON fixture replays fail on session context? -Session context includes timestamps (e.g., `active_skills` activation time) that differ between recording and replay. Set `test_msg_context=False` on fixture tests. For skills with random dialog rendering (like quote pools), also set `test_msg_data=False`. -## Does `from_message()` filter GUI messages during recording? -Yes โ€” `from_message()` now accepts `ignore_gui=True` (default), which adds `GUI_IGNORED` messages to the capture filter. This prevents GUI namespace messages from appearing in recorded fixtures. -## How do I override pipeline plugin config in a test (e.g. M2V model path)? -Pass `pipeline_config` to `get_minicroft()`. It is a `dict` keyed by the plugin's config key under `Configuration()["intents"]`: -```python -croft = get_minicroft( - [SKILL_ID], - default_pipeline=M2V_PIPELINE, - pipeline_config={ - "ovos_m2v_pipeline": { - "model": "Jarbas/ovos-model2vec-intents-distiluse-base-multilingual-cased-v2" - } - }, -) -``` -The override is patched into `Configuration()["intents"]` before `super().__init__()`, so the pipeline plugin reads the test value in its `__init__`. It is restored in `stop()`. This is useful for forcing a specific model regardless of what `mycroft.conf` says locally. -## Why do M2V tests skip when the multilingual model is not cached? -`ovos-m2v-pipeline` classifies utterances using a pre-trained model whose `classes_` are fixed intent labels. A language-specific model (e.g. Portuguese-only) won't contain English intent names and will always return no match. The multilingual model (`Jarbas/ovos-model2vec-intents-distiluse-base-multilingual-cased-v2`) covers all OVOS skill intent names. Tests that use M2V should skip when this model is not cached locally (to avoid downloading a large model in CI). Download it once with: -```bash -python -c "from model2vec.inference import StaticModelPipeline; StaticModelPipeline.from_pretrained('Jarbas/ovos-model2vec-intents-distiluse-base-multilingual-cased-v2')" -``` - ---- - -## Bus Coverage - -### Why do I see `Thread-1`, `Thread-2` entries in my bus coverage report? - -**This was fixed in ovoscope 0.14.0.** Previously, OVOS components that inherit -from `Thread` (such as `SkillManager`, `PlaybackService`, `OVOSDinkumVoiceService`, -and `MediaService`) had their handlers attributed to generic names like `Thread-1`, -`Thread-2`, etc. - -The fix uses Python's Method Resolution Order (MRO) to automatically resolve -Thread subclasses to their actual class names. The `_get_component_name()` method -walks the MRO chain and skips the `Thread` class, returning the first non-Thread -class name. - -**Special case:** `MiniCroft` (the test harness) is automatically renamed to -`SkillManager` in reports for clarity, since MiniCroft is just a test wrapper -around SkillManager. - -This approach is automatic and requires no manual maintenance of message type patterns. -Any future Thread-based components will be correctly attributed without code changes. - -See [docs/bus-coverage.md](docs/bus-coverage.md) for the full reference. - ---- - -## CLI - -### How do I record a fixture from the command line? -```bash -ovoscope record --skill-id ovos-skill-hello-world.openvoiceos \ - --utterance "hello" --output fixture.json -``` - -### How do I replay a fixture? -```bash -ovoscope run fixture.json --verbose -``` - -### How do I compare two fixture files? -```bash -ovoscope diff expected.json actual.json -``` -Exit code 0 = identical, 1 = differences found. - -### How do I scan my workspace for E2E coverage gaps? -```bash -ovoscope coverage "OpenVoiceOS Workspace/" --format table -``` - ---- - -## PHAL Testing - -### Can I test PHAL plugins with ovoscope? -Yes โ€” any PHAL plugin that communicates only via the MessageBus (no physical -hardware) is testable with `MiniPHAL` or `PHALTest` from `ovoscope.phal`. - -### Which PHAL plugins require real hardware? -`ovos-PHAL-plugin-alsa`, `ovos-PHAL-plugin-mk1`, `ovos-PHAL-plugin-dotstar`. -These should use hardware-in-the-loop integration tests instead. - ---- - -## OCP Testing - -### How do I test an OCP skill without a real HTTP server? -Use `OCPTest` with `mock_responses` โ€” keys are URL substrings matched -against actual requests, values are the JSON bodies returned. - -### What message flow does OCP testing drive? -`recognizer_loop:utterance` โ†’ `ovos.common_play.query` โ†’ `ovos.common_play.query.response` โ†’ `ovos.common_play.start` - ---- - -## GUI Assertions - -### How do I assert that a skill showed a GUI page? -```python -from ovoscope import GUICaptureSession -with GUICaptureSession(mc.bus) as gui: - # ... trigger interaction ... - gui.assert_page_shown("my_skill", "main.qml") -``` - -### How do I assert that a skill set a specific session data key in a namespace? -Use `assert_namespace_has_key()`: -```python -with GUICaptureSession(mc.bus) as gui: - # ... trigger interaction ... - gui.assert_namespace_has_key("my_skill", "temperature") -``` -This checks that a `mycroft.session.set` message was captured containing the given key in the specified namespace. See [docs/gui-testing.md](docs/gui-testing.md). - ---- - -## Coverage Scanner - -### What entry-point groups does the scanner detect? -`opm.skill`, `opm.pipeline`, `opm.phal`, `opm.plugin.tts`, `opm.plugin.stt`, -`opm.plugin.audio`, `opm.common_play`, `opm.solver`. - -### How is "covered" defined? -A repo is considered covered when `test/end2end/` (or `tests/end2end/`) -exists and contains at least one `.py` file (excluding `__init__.py`). diff --git a/MAINTENANCE_REPORT.md b/MAINTENANCE_REPORT.md deleted file mode 100644 index f17aadd..0000000 --- a/MAINTENANCE_REPORT.md +++ /dev/null @@ -1,464 +0,0 @@ -# Maintenance Report โ€” `ovoscope` - -## [2026-03-13] โ€” GUICaptureSession: assert_namespace_has_key + unit tests - -- **AI Model**: Claude Opus 4.6 -- **Actions Taken**: - - Added `assert_namespace_has_key()` method to `GUICaptureSession` for asserting that a skill set a specific session data key in a GUI namespace - - Updated `docs/gui-testing.md` with documentation for the new method - - Created `test/unittests/test_gui_capture.py` with 10 unit tests covering the new assertion method - - Updated `FAQ.md` with Q&A for `assert_namespace_has_key()` -- **Oversight**: Human-reviewed plan execution - ---- - -## [2026-03-12] โ€” Full Audit Improvements (Correctness, Coverage, Docs, Packaging) - -- **AI Model**: claude-sonnet-4-6 -- **Actions Taken**: - - **P1.1** Fixed `pipeline.py` race condition: split success/failure into - separate `threading.Event` objects; `match()` now returns `None` on timeout - or failure instead of returning a failure message as success. - Source: `ovoscope/pipeline.py:149โ€“200`. - - **P1.2** Added `strict: bool = False` to `diff.py` `_dict_diff()` and - `diff_fixtures()`. When `strict=True`, extra keys in `actual` not in - `expected` are flagged. Default `False` preserves existing behaviour. - Source: `ovoscope/diff.py:121`. - - **P1.3** Deleted dead `_skill_id_for_handler()` from `bus_coverage.py` - (lines 745โ€“763, never called anywhere in the codebase). - - **P2.1** Created `test/unittests/test_media.py` โ€” 20 unit tests for - `MockOCPBackend` state transitions and `OCPCaptureSession` accumulation. - - **P2.2** Created `test/unittests/test_remote_recorder.py` โ€” 15 unit tests - for `RemoteRecorder` using mocked `MessageBusClient`. - - **P2.3** Fixed deprecated `ovos_utils.messagebus.Message` import in - `test/unittests/test_phal.py` โ†’ `ovos_bus_client.message.Message`. - - **P3.1** Created `SUGGESTIONS.md` with 10 structured proposals. - - **P3.2** Updated `QUICK_FACTS.md` test count: 243 โ†’ 306. - - **P3.3** Expanded `docs/pipeline.md` to full API reference with `pipeline.py:LINE` - citations, `_SinkSkill` explanation, Adapt/Padatious examples, and - pipeline success/failure signal documentation. - - **P3.4** Fixed `docs/ocp.md` to correctly reference `OCPTest` (the class - in `ocp.py`) and add cross-reference to `OCPPlayerHarness` in `media.py`. - - **P3.5** Updated `AUDIT.md` with 7 new findings (5 fixed, 2 pre-existing). - - **P4.1** Updated `pyproject.toml`: added Documentation and Issue Tracker - URLs, `[tool.setuptools.package-data]`, `timeout = 60` in pytest options, - and comment explaining the `ovos-core>=2.0.4a2` alpha pin. - - **P5.1** Made `_count_fixtures()` in `coverage.py` use `Path.rglob("*.json")` - for recursive fixture counting instead of `os.listdir()`. - - **P5.2** Added `TYPE_CHECKING` guard and proper `List["BusCoverageReport"]` - type annotation to `BusCoverageCollector._reports` in `pytest_plugin.py`. -- **Oversight**: Human review pending. 348 unit tests pass locally (was 301; +35 new, +12 from new files). - ---- - -## [2026-03-12] โ€” Bus Coverage Report Feature - -- **AI Model**: Claude Sonnet 4.6 -- **Actions Taken**: - - Created `ovoscope/bus_coverage.py` โ€” `BusCoverageTracker`, `BusCoverageReport`, `SkillBusCoverage`, `HandlerEntry`, `EmitterEntry` dataclasses. Tracks listener invocations (via `bus.emit` monkey-patch) and emitter observed/asserted counts per skill_id. Handler attribution via `handler.__self__` โ†’ `minicroft.plugin_skills`. Handles pyee v9 `OrderedDict` storage format. - - Modified `ovoscope/__init__.py`: added `track_bus_coverage`, `print_bus_coverage`, `bus_coverage_report` fields to `End2EndTest`; hooked `BusCoverageTracker` into `execute()` around the capture block. - - Modified `ovoscope/pytest_plugin.py`: added `BusCoverageCollector`, `bus_coverage_session` session fixture, `pytest_terminal_summary` hook for merged end-of-session report. - - Modified `ovoscope/cli.py`: added `cmd_bus_coverage` subcommand and `bus-coverage` parser entry. - - Created `docs/bus-coverage.md` โ€” full API reference with source citations. - - Updated `FAQ.md` with three new Q&A entries. - - Created `test/unittests/test_bus_coverage.py` โ€” 32 unit tests, all passing. -- **Oversight**: 301 unit tests pass locally. `bus_coverage.py` at 97% coverage. - - -## [2026-03-11] โ€” Add ovoscope-setup entrypoint for AI assistant skill installation - -- **AI Model**: Claude Sonnet 4.6 -- **Actions Taken**: - - Created `ovoscope/setup_skill.py` โ€” `ovoscope-setup` CLI with install/uninstall for Claude, Gemini, OpenCode; auto-detect mode; `--list`, `--path`, `--uninstall` flags. - - Created `ovoscope/skill_data/` package with bundled skill definitions: - - `claude/SKILL.md` + `claude/scripts/ovoscope.sh` + `claude/assets/docs/` + `claude/assets/FAQ.md|QUICK_FACTS.md` - - `gemini/` โ€” identical structure (Gemini uses same SKILL.md format, project-level install) - - `opencode/ovoscope.md` โ€” YAML frontmatter agent definition for OpenCode - - Updated `pyproject.toml`: added `ovoscope-setup` script entrypoint, `[tool.setuptools.packages.find]`, and `[tool.setuptools.package-data]` to bundle `skill_data/`. - - Added 26 unit tests in `test/unittests/test_setup_skill.py` โ€” all passing. -- **Oversight**: 269 unit tests pass locally. - -## [2026-03-11] โ€” Docs Gap Review and Fixes - -- **AI Model**: Claude Sonnet 4.6 -- **Actions Taken**: - - `docs/ocp.md`: Documented `execute()` return type (`List[Message]`), clarified `patch_targets` format (dotted Python path where symbol is used), added aiohttp example. - - `docs/pipeline.md`: Documented `assert_matches(intent_type=...)` as substring check with example; added `ovoscope/pipeline.py:LINE` citations to all methods. - - `docs/cli.md`: Corrected `--ignore-context` โ†’ `--include-context`, explained when/why to use it; clarified `validate` pydantic fallback trigger. - - `docs/end2end-test.md`, `docs/minicroft.md`, `docs/capture-session.md`: Added `ovoscope/__init__.py:LINE` source citations to class and key method definitions. - - `docs/capture-session.md`: Documented `finish()` idempotency. - - `docs/listener.md`: Added full VAD/WakeWord API section (`MockVADEngine`, `MockHotWordEngine`, `is_silence`, `extract_speech`, `detect_wakeword`, `scan_for_wakeword`, `VADTest`, `WakeWordTest`) with examples and `ovoscope/listener.py:LINE` citations. Updated constructor parameter table. Fixed stale line references. - - `docs/index.md`: Added `gui-testing.md` link; updated Public API section with `GUICaptureSession`, VAD/WW helpers; fixed "Does NOT Do" section for VAD/WW. - - `QUICK_FACTS.md`: Added entry-point groups table; updated test count (243) and coverage note. -- **Oversight**: No new code changes โ€” docs only. - -## [2026-03-11] โ€” Add VAD and WakeWord Support to MiniListener - -- **AI Model**: Claude Haiku 4.5 -- **Actions Taken**: - - Extended `ovoscope/listener.py` with `MockVADEngine`, `MockHotWordEngine`, `VADTest`, `WakeWordTest`. - - Extended `MiniListener` with `vad_instance` / `ww_instances` constructor params. - - Added `is_silence()`, `extract_speech()`, `detect_wakeword()`, `scan_for_wakeword()` methods to `MiniListener` โ€” `listener.py:466โ€“600`. - - Extended `get_mini_listener()` factory with `vad_plugin`, `vad_instance`, `ww_plugin`, `ww_instances` params. - - Made `ovos_dinkum_listener` import lazy (graceful `ImportError`) so VAD/WW tests work without the full listener stack installed. - - Added 41 unit tests in `test/unittests/test_listener_vad_ww.py`. - - Updated `FAQ.md` with VAD and WakeWord testing Q&A. -- **Oversight**: 243 unit tests pass locally. - -## [2026-03-11] โ€” Enhance Audio Testing Robustness and CI - -- **AI Model**: Claude Sonnet 4.6 -- **Actions Taken**: - - Added `LIGHT_TEST_PIPELINE` as a lightweight fallback when Adapt/Padatious are missing. - - Updated `MiniCroft` to auto-fallback to `LIGHT_TEST_PIPELINE` if stages are missing. - - Refactored `PlaybackServiceHarness` for better robustness (proper patch cleanup, timeout handling). - - Added skip guard to audio harness tests to prevent failures when `ovos-audio` is not installed. - - Fixed documentation path references and added prerequisites. - - Added missing `LICENSE` (Apache-2.0) file. - - Updated CI workflows to include `audio` extra for unit tests. -- **Oversight**: All 147 unit tests pass locally. - -## [2026-03-10] โ€” Add Audio Testing Harnesses - -- **AI Model**: Claude Sonnet 4.6 -- **Actions Taken**: - - Created `ovoscope/audio.py` โ€” 5 new classes: - - `MockAudioBackend` (inherits `AudioBackend`) โ€” no-op backend tracking state - - `AudioServiceHarness` โ€” context manager wrapping `AudioService` with `MockAudioBackend` - - `MockTTS` (inherits `TTS`) โ€” writes 44-byte silent WAV, records spoken utterances - - `PlaybackServiceHarness` โ€” context manager wrapping `PlaybackService` with `MockTTS` - - `AudioCaptureSession` โ€” records bus messages matching configurable prefix list - - Updated `ovoscope/__init__.py` โ€” guarded import of audio harness classes - - Updated `pyproject.toml` โ€” added `[audio]` optional dependency - - Created `test/unittests/test_audio_harness.py` โ€” 38 unit tests (all passing) - - Created `ovos-audio/test/end2end/__init__.py` โ€” empty marker - - Created `ovos-audio/test/end2end/test_audio_service_e2e.py` โ€” 11 E2E tests (all passing) - - Created `ovos-audio/test/end2end/test_playback_service_e2e.py` โ€” 7 E2E tests (all passing) - - Created `docs/audio-testing.md` โ€” full API reference with source citations - - Updated `docs/index.md` โ€” link to audio-testing.md - - Updated `FAQ.md` โ€” 3 new Q&As for audio testing - - Updated `QUICK_FACTS.md` โ€” new audio harness classes, updated test count -- **Key design decisions**: - - `AudioServiceHarness` uses `autoload=False` then manually injects `MockAudioBackend` - - `PlaybackServiceHarness` patches `ovos_audio.playback.play_audio` to prevent real audio - - `TTS.queue` is class-level; harness drains it before each `PlaybackService` construction - - `stop()` MUST return `True` to trigger `mycroft.stop.handled` in `AudioService` - - `FakeBus.wait_for_response()` does not work in-process; subscribe-emit-wait pattern used -- **Oversight**: All 38 ovoscope unit tests + 18 ovos-audio E2E tests pass - -## [2026-03-10] โ€” Add `pipeline_config` parameter to `MiniCroft` -- **AI Model**: Claude Sonnet 4.6 -- **Actions Taken**: - - Added `pipeline_config: Optional[Dict[str, Dict]] = None` parameter to `MiniCroft.__init__` โ€” `ovoscope/__init__.py` - - Patches `Configuration()["intents"][plugin_key]` before `super().__init__()` so pipeline plugins read overridden config in their own `__init__` - - Restores all overrides in `MiniCroft.stop()` โ€” `ovoscope/__init__.py` - - Updated `docs/minicroft.md`: added `pipeline_config` to constructor table and added "Pipeline Plugin Config Overrides" section with usage example - - Updated `FAQ.md`: added Q&A for `pipeline_config` and M2V multilingual model skip behaviour - - Added 5 unit tests in `test/unittests/test_minicroft.py::TestMiniCroftPipelineConfig`: patch active, restore after stop, existing key preserved, None is no-op, multiple keys -- **Oversight**: 18/18 minicroft unit tests pass; confucius e2e suite: 20 passed, 2 skipped. -- **Motivation**: Needed to force the M2V multilingual model in `TestConfuciusM2VEN` regardless of what `mycroft.conf` says locally. Language-specific models (e.g. Portuguese) don't contain English intent labels and always return no match. -- **Oversight**: All ovoscope unit tests pass; confucius e2e suite: 20 passed, 2 skipped (M2V โ€” multilingual model not cached locally). - -## [2026-03-10] โ€” Test coverage improvement (78% โ†’ 89%) -### Changes -- Created `test/unittests/test_end2end_extended.py` โ€” 46 new tests covering: - - **Routing internals**: flip_points, entry_points, keep_original_src assertions - - **Active skills**: inject_active, activation_points, deactivation_points, disallow_extra_active_skills - - **Boot sequence**: correct/incorrect boot message assertions - - **Final session**: lang mismatch raises, matching session passes - - **Async messages**: captured separately, missing raises, count mismatch raises - - **Context assertions**: wrong context raises - - **GUI filtering**: ignore_gui=True/False behavior - - **Serialization**: JSON string input, flip_points/flags preservation, anonymize_message - - **from_message recording**: captures sequence, wraps single message - - **Pipeline constants**: composition validation - - **MiniCroft lang config**: override and restore - - **Verbose output**: exercises all print branches - - **Message count verbose**: first differing message output -- Created `test/unittests/test_pytest_plugin.py` โ€” 6 new tests for minicroft fixture logic via `__wrapped__` -- Updated `FAQ.md` โ€” added coverage FAQ entry -### AI Transparency Report -- **AI Model**: Claude Opus 4.6 -- **Actions Taken**: Created 2 new test files with 52 total new tests -- **Oversight**: All tests verified passing. Coverage: 78% โ†’ 89% overall, `__init__.py` 54% โ†’ 68%, `pytest_plugin.py` 0% โ†’ 64% ---- -## [2026-03-09] โ€” CI workflows and test-gated releases -### Changes -- Created `.github/workflows/unit_tests.yml` โ€” runs 58 unit tests with `pytest --cov=ovoscope` on PRs/pushes to `dev`, posts coverage comment via `py-cov-action/python-coverage-comment-action@v3` -- Created `.github/workflows/build_tests.yml` โ€” matrix build (Python 3.10, 3.11) with `python -m build`, tests sdist/wheel creation and package install -- Created `.github/workflows/license_tests.yml` โ€” calls `OpenVoiceOS/gh-automations/.github/workflows/license-check.yml@dev` reusable workflow -- Created `.github/workflows/pipaudit.yml` โ€” CVE scanning via `pypa/gh-action-pip-audit@v1.0.0` on Python 3.10/3.11 matrix -- Updated `.github/workflows/release_workflow.yml` โ€” added `build_tests` job that runs full test suite; `publish_alpha` now depends on `build_tests` via `needs:`, gating alpha releases on test success -- Updated `docs/ci-integration.md` โ€” documented ovoscope's own CI workflow table -- Updated `FAQ.md` โ€” added 3 new CI-related Q&A entries -### AI Transparency Report -- **AI Model**: Claude Opus 4.6 -- **Actions Taken**: Created 4 new workflow files, updated 1 existing workflow, updated docs and FAQ -- **Oversight**: All workflows follow established OVOS conventions (actions/checkout@v4, actions/setup-python@v5, python-version 3.11, python -m build). 58 existing tests verified passing. ---- -## [2026-03-09] โ€” pytest_plugin: safe teardown guard -### Changes -- `ovoscope/pytest_plugin.py` โ€” `minicroft` fixture: initialise `mc = None` before calling - `get_minicroft()`, then wrap `yield mc` in `try/finally` with `if mc is not None: mc.stop()`. - Previously, if `get_minicroft()` raised (e.g. `TimeoutError`), teardown would hit a - `NameError: name 'mc' is not defined`, masking the original exception in pytest output. -### AI Transparency Report -- **AI Model**: Claude Sonnet 4.6 -- **Actions Taken**: Applied targeted edit to `pytest_plugin.py` lines 57โ€“62. -- **Oversight**: No logic change โ€” only teardown safety. Existing tests unaffected. ---- -## [2026-03-09] โ€” pydantic_helpers: typing, docstrings, tests, bug fix, pyproject.toml migration -### Changes -**`ovoscope/pydantic_helpers.py`** (new module, renamed from the initial `pydantic.py`): -- Full module docstring with install instructions and usage example. -- `TYPE_CHECKING`-guarded import of `OpenVoiceOSMessage` โ€” type checkers see full annotations; - no `ImportError` at runtime when `ovos-pydantic-models` is absent. -- All three public functions have complete Google-style docstrings with `Args`, `Returns`, - `Raises`, and `Example` blocks; all parameter and return types are annotated. -- **Bug fixed** in `validate_fixture()`: was constructing `raise ValidationError(str, ...)` which - pydantic v2 does not support (it is not user-constructible). Changed to `raise ValueError(...) from exc`. -- **Bug fixed** in `validate_fixture()`: normalisation fallback changed from `""` to `None` so - messages missing both `"type"` and `"message_type"` keys are correctly rejected by pydantic - (an empty string passes `message_type: str = Field(...)` validation silently). -**`test/unittests/test_pydantic_helpers.py`** (new, 20 tests): -| Class | Tests | What is covered | -|-------|-------|----------------| -| `TestToBusMessage` | 6 | `msg_type`, data fields, return type, utterance msg, empty context, roundtrip | -| `TestFromBusMessage` | 5 | valid speak, valid utterance, return type, invalid raises `ValidationError`, base model leniency | -| `TestValidateFixture` | 9 | valid fixture, source/expected preserved, missing file, malformed source, malformed expected, error chains `ValidationError`, `message_type` key accepted, empty lists | -All 20 tests pass. Full suite now 58 tests (38 pre-existing + 20 new), all passing. -**`pyproject.toml`** โ€” completed migration: -- `build-backend` changed from `setuptools.backends.legacy:build` to `setuptools.build_meta`. -- `dynamic = ["version"]` added; `[tool.setuptools.dynamic] version = {attr = "ovoscope.version.__version__"}`. -- `[project.optional-dependencies] pydantic = ["ovos-pydantic-models>=0.1.0"]` added. -- `setup.py` removed. -**`ovoscope/version.py`**: -- Added `__version__` computed from `VERSION_MAJOR`, `VERSION_MINOR`, `VERSION_BUILD`, `VERSION_ALPHA` - so `pyproject.toml` dynamic versioning works without `setup.py`. -**`AUDIT.md`**, **`SUGGESTIONS.md`**, **`FAQ.md`**: -- All module references updated from `ovoscope.pydantic` โ†’ `ovoscope.pydantic_helpers`. -- AUDIT unit-test count updated to 58; setup.py fix marked fully complete. -- SUGGESTIONS.md item 6 file path corrected. -- FAQ.md pydantic section import paths corrected. -### AI Transparency Report -- **AI Model**: Claude Sonnet 4.6 -- **Actions Taken**: Read existing module and all test files; identified two bugs in `validate_fixture()` - via live `python -c` verification; wrote 20 tests and iterated until all pass; migrated pyproject.toml. -- **Oversight**: `validate_fixture()` validates top-level message structure only (`message_type`, - `data`, `context` shape) via `OpenVoiceOSMessage` โ€” it does not validate data-field-level schemas - (e.g. `utterances` type). Use `from_bus_message(msg, SpecificModel)` for field-level validation. ---- -## [2026-03-09] โ€” End2end tests written for ovos-persona (11 tests) -### Tests Created -New `ovos-persona/test/end2end/test_persona.py` โ€” 4 test classes, 11 tests: -| Class | Tests | Intents/triggers | -|-------|-------|-----------------| -| `TestPersonaList` | 2 | `list_personas.intent` (no personas / 2 personas) | -| `TestPersonaCheck` | 2 | `active_persona.intent` (no active / active) | -| `TestPersonaSummon` | 2 | `summon.intent` (known / unknown persona) | -| `TestPersonaRelease` | 1 | `Release.voc` via `voc_match()` | -| `TestPersonaQuery` | 4 | `ask.intent` explicit / active fallback / error / no-match | -### Key Patterns Discovered -- Pipeline plugins accessed via `mc.intents.pipeline_plugins["ovos-persona-pipeline-plugin"]` -- Inject mock personas into `persona_svc.personas` dict โ€” bypasses real solver loading -- `setUp()` clears real personas (Gemma etc.) and resets `active_persona = None` -- `skill_ids=[]` โ€” no skills needed; pipeline plugin loads automatically -- `ovos.utterance.handled` data is `{"name": "PersonaService.handle_persona_*"}` โ€” not empty -- `speak` with dialog template checked only by `context={"skill_id": SKILL_ID}` (text varies) -- Direct speaks from query answers checked with `data={"utterance": "forty two"}` -### AI Transparency Report -- **AI Model**: Claude Sonnet 4.6 -- **Actions Taken**: Traced all message sequences via live FakeBus capture; wrote tests iteratively; all 11 pass. -- **Oversight**: Dialog text assertions omitted โ€” locale-dependent. `test_list_two_personas` relies on dict insertion order (Python 3.7+). ---- -## [2026-03-09] โ€” End2end tests written for 6 skills (18 tests) -### Skills Covered -New `test/end2end/` directories and test files created for: -| Skill | Test file | Tests | Intents tested | -|-------|-----------|-------|----------------| -| `ovos-skill-hello-world` | `test_hello_world.py` | 4 | HelloWorldIntent (Adapt), Greetings.intent (Padatious), no-match cases | -| `ovos-skill-fallback-unknown` | `test_fallback_unknown.py` | 2 | fallback-low match, Adapt no-match | -| `ovos-skill-naptime` | `test_naptime.py` | 2 | naptime.intent (Padatious), no-match | -| `ovos-skill-volume` | `test_volume.py` | 4 | volume.max.intent, volume.mute.intent, volume.unmute.intent (Padatious), no-match | -| `ovos-skill-count` | `test_count.py` | 3 | count_to_N.intent (Padatious), no-match | -| `ovos-skill-parrot` | `test_parrot.py` | 3 | speak.intent, repeat.tts.intent, no-match | -### Key Patterns Discovered -- Intents registered with string `"name.intent"` โ†’ Padatious; `IntentBuilder(...)` โ†’ Adapt -- Skills emitting raw `Message(...)` without `forward(...)`/`reply()` have `source=None` โ€” use `async_messages` + `ignore_messages` -- Enclosure/LED messages and `add_context`/`configuration.patch` must be in `ignore_messages` -- `message.forward(...)` inherits the post-flip source/dest โ€” do NOT add these to `keep_original_src` -### AI Transparency Report -- **AI Model**: Claude Sonnet 4.6 -- **Actions Taken**: Read each skill's `__init__.py` and locale files; wrote tests iteratively with test runs to fix pipeline selection, ignore_messages, meta dict content. All 18 tests pass. -- **Oversight**: naptime test skips dialog content check (varies with `listener.wake_word` config). ---- -## [2026-03-09] โ€” Config isolation extended: blacklisted_skills + blacklisted_intents -### Problem Addressed -After pipeline isolation, `test_stop.py` (6 tests) and `test_cancel_plugin.py` (1 test) still failed. -Root cause: `Session.__init__` reads `Configuration()["skills"]["blacklisted_skills"]` and -`Configuration()["intents"]["blacklisted_intents"]` from the live singleton dict cache, same problem -as the pipeline. The user had `skill-ovos-stop.openvoiceos` blacklisted in `~/.config/mycroft/mycroft.conf`. -Additionally, `ovos-skill-count` and `ovos-utterance-plugin-cancel` were not installed in the workspace venv. -### Changes -- `ovoscope/__init__.py` โ€” `MiniCroft.__init__`: added `_original_blacklisted_skills` and - `_original_blacklisted_intents` state variables. -- `ovoscope/__init__.py` โ€” `MiniCroft.run()`: when `isolate_config=True`, patches - `Configuration()["skills"]["blacklisted_skills"] = []` and - `Configuration()["intents"]["blacklisted_intents"] = []` in the live singleton dict cache. -- `ovoscope/__init__.py` โ€” `MiniCroft.stop()`: restores both lists from saved originals. -- `ovos-core/test/end2end/test_stop.py` โ€” Added `"ovos-hivemind-pipeline-plugin.stop.response"` to - `ignore_messages` in both `TestStopNoSkills` and `TestCountSkills` (hivemind responds to `mycroft.stop`). -- Installed `Skills/ovos-skill-count` and `Transformer plugins/ovos-utterance-plugin-cancel` with `uv pip install --no-deps -e`. -### Result -27/27 ovos-core end2end tests pass (was 20/27). -### AI Transparency Report -- **AI Model**: Claude Sonnet 4.6 -- **Actions Taken**: Diagnosed `blacklisted_skills` stale cache issue; extended `run()`/`stop()` patch pattern; - identified hivemind stop response as uncaught message; installed missing plugins. -- **Oversight**: `blacklisted_skills` patch only runs when `isolate_config=True` (same condition as xdg isolation). ---- -## [2026-03-09] โ€” Pipeline isolation and reproducible test pipelines -### Problem Addressed -`MiniCroft.isolate_config=True` cleared `Configuration.xdg_configs` to remove the user's -`~/.config/mycroft/mycroft.conf`, but `SessionManager.default_session` is a singleton -initialised at module import time โ€” it already held the user's pipeline (which could include -`ovos-persona-pipeline-plugin-high`, Ollama, OCP, etc.). Any utterance emitted without an -explicit session in its context would inherit this pipeline and be intercepted by AI plugins -non-deterministically, making tests environment-dependent. -### Changes -- `ovoscope/__init__.py` โ€” Added pipeline stage constants: - `STOP_PIPELINE`, `CONVERSE_PIPELINE`, `ADAPT_PIPELINE`, `PADATIOUS_PIPELINE`, - `FALLBACK_PIPELINE`, `COMMON_QUERY_PIPELINE`, `PERSONA_PIPELINE`. -- `ovoscope/__init__.py` โ€” Added `DEFAULT_TEST_PIPELINE`: all standard built-in pipeline stages - (stop/converse/adapt/padatious/fallback/common-query), **no** AI/LLM/persona/OCP stages. -- `ovoscope/__init__.py` โ€” `MiniCroft.__init__`: added `default_pipeline: Optional[List[str]]` - parameter (default `DEFAULT_TEST_PIPELINE`). Stored as `_default_pipeline`; original - pipeline stored for restoration. -- `ovoscope/__init__.py` โ€” `MiniCroft.run()`: after `load_plugin_skills()` and before - `set_ready()`, sets `SessionManager.default_session.pipeline = self._default_pipeline` when - not `None`. Setting it here (post-FakeBus-sync) is the correct point โ€” after all init bus - messages have been processed. -- `ovoscope/__init__.py` โ€” `MiniCroft.stop()`: restores `SessionManager.default_session.pipeline` - to its pre-test value, enabling test isolation within a single process. -- `test/unittests/test_minicroft.py` โ€” Added `TestMiniCroftPipelineIsolation` (5 tests): - pipeline overrides default session, restored after stop, `isolate_config=True` uses - `DEFAULT_TEST_PIPELINE`, persona/ollama/m2v absent from `DEFAULT_TEST_PIPELINE`, - `default_pipeline=None` leaves session unchanged. -### Use Cases Unblocked -- `get_minicroft([])` โ†’ `complete_intent_failure` tests now pass without Gemma/persona intercepting. -- `get_minicroft([SKILL_ID], default_pipeline=ADAPT_PIPELINE)` โ€” Adapt-only testing. -- `get_minicroft([SKILL_ID], default_pipeline=ADAPT_PIPELINE + FALLBACK_PIPELINE)` โ€” intent+fallback. -- `get_minicroft([SKILL_ID], default_pipeline=PERSONA_PIPELINE)` โ€” explicitly test persona behaviour. -- `get_minicroft([SKILL_ID], default_pipeline=None)` โ€” use system default (includes all installed plugins). -### AI Transparency Report -- **AI Model**: Claude Sonnet 4.6 -- **Actions Taken**: Diagnosed root cause (singleton default_session pre-initialised from user config); - added constants + `default_pipeline` param; updated `run()` and `stop()`; added 5 unit tests. -- **Oversight**: `DEFAULT_TEST_PIPELINE` does not include OCP or m2v stages โ€” repos that test - media skills should pass an explicit pipeline including those stages. ---- -## [2026-03-08] โ€” Code improvements from SUGGESTIONS.md -### Changes -- `ovoscope/__init__.py` โ€” `get_minicroft()`: added `max_wait: float = 60` parameter; raises - `TimeoutError` if MiniCroft does not reach READY within the deadline. Return type annotated - as `-> MiniCroft`. -- `ovoscope/__init__.py` โ€” `MiniCroft.inject_message(msg: Message) -> None`: new helper method - for emitting arbitrary messages during a test without going through the utterance pipeline. -- `ovoscope/__init__.py` โ€” `End2EndTest.execute()`: now returns `List[Message]` (was `None`). - Return type annotated. Enables test composition and `assert_spoke()`. -- `ovoscope/__init__.py` โ€” `End2EndTest.assert_spoke(text, lang, timeout)`: new sugar method; - calls `execute()` and asserts a matching `speak` message was emitted. -- `ovoscope/__init__.py` โ€” `End2EndTest.save()`: return type annotated as `-> None`. -- `ovoscope/pytest_plugin.py` (NEW): class-scoped `minicroft` pytest fixture; reads `skill_ids` - from the test class attribute; handles startup and teardown automatically. -- `setup.py`: registered `pytest11` entry point so the fixture is auto-discovered. -- `pyproject.toml` (NEW): `[build-system]`, `[project]`, `[project.entry-points."pytest11"]`, - and `[tool.pytest.ini_options]` tables. `setup.py` retained for dynamic version reading. -- `AUDIT.md`: marked 3 issues as FIXED; updated Next Steps. -### AI Transparency Report -- **AI Model**: Claude Sonnet 4.6 -- **Actions Taken**: Read `__init__.py` fully; made targeted edits; created `pytest_plugin.py` - and `pyproject.toml`. No logic was changed โ€” only additions and the `return messages` fix. -- **Oversight**: `assert_spoke()` depends on `execute()` returning messages โ€” verify against - a live install. `pyproject.toml` dynamic version section has a TODO comment for when - `version.py` exports `__version__`. ---- -## [2026-03-08] โ€” Documentation enrichment and audit deepening -### Changes -- Created `docs/usage-guide.md` โ€” full tutorial from install to 8 test patterns; references - hello-world canonical examples and real class/method signatures from `ovoscope/__init__.py`. -- Created `docs/ci-integration.md` โ€” directory layout, pytest config, GitHub Actions job - template, fixture management, and CI gotchas. -- Updated `docs/index.md` โ€” added `usage-guide.md` and `ci-integration.md` to navigation table; - added "Who Uses ovoscope" section; added gh-automations cross-reference. -- Replaced `AUDIT.md` โ€” shallow CI-pin findings replaced with 7 evidence-based issues - (CRITICAL/MAJOR/MODERATE/MINOR) all traced to specific lines in `__init__.py`. -- Replaced `SUGGESTIONS.md` โ€” 4 generic stubs replaced with 7 concrete, repo-specific proposals - with code snippets pointing to specific lines. -### Rationale -The previous docs scaffold was boilerplate with no practical value. This pass enriches docs to the -level where every OVOS repo can adopt ovoscope end-to-end testing without reading source code. -### Verification -- `ls ovoscope/docs/` shows 7 files (5 pre-existing + `usage-guide.md` + `ci-integration.md`). -- All code examples in `usage-guide.md` use real imports and class names verified from source. -- All `AUDIT.md` findings reference specific file:line evidence. -### AI Transparency Report -- **AI Model**: Claude Sonnet 4.6 -- **Actions Taken**: Read `ovoscope/__init__.py` (485 lines), `test/test_helloworld.py`, - `ovos-core/test/end2end/test_adapt.py`, and all existing docs; then generated enriched content. -- **Oversight**: Code examples are illustrative but not executed. Verify against live skill install before treating as runnable. - ---- - -## 2026-03-11 โ€” Phase 1โ€“3 Feature Additions - -**AI Model**: claude-sonnet-4-6 -**Oversight**: Human review pending - -### Actions Taken - -Added CLI, PHAL harness, fixture differ, OCP harness, pipeline harness, -ecosystem coverage scanner, GUI capture session, and remote recorder. - -**New modules:** -- `ovoscope/cli.py` โ€” `ovoscope` CLI with `record`, `run`, `diff`, `validate`, `coverage` -- `ovoscope/diff.py` โ€” `MessageDiff`, `FixtureDiffResult`, `diff_fixtures` -- `ovoscope/phal.py` โ€” `MiniPHAL`, `PHALTest` -- `ovoscope/ocp.py` โ€” `OCPTest`, `assert_ocp_query_response` -- `ovoscope/pipeline.py` โ€” `PipelineHarness` -- `ovoscope/coverage.py` โ€” `RepoCoverage`, `EcosystemCoverageReport`, `scan_workspace` -- `ovoscope/remote_recorder.py` โ€” `RemoteRecorder` - -**Extended modules:** -- `ovoscope/__init__.py` โ€” added `GUICaptureSession` - -**New docs:** -- `docs/cli.md`, `docs/phal.md`, `docs/ocp.md`, `docs/pipeline.md` -- `docs/usage-guide.md` โ€” Patterns 9โ€“12 appended - -**New tests:** -- `test/unittests/test_diff.py` โ€” 7 test methods -- `test/unittests/test_phal.py` โ€” 8 test methods -- `test/unittests/test_coverage.py` โ€” 11 test methods -- `test/unittests/test_cli.py` โ€” 14 test methods - -**pyproject.toml changes:** -- Added `[project.scripts] ovoscope = "ovoscope.cli:main"` - -All 202 tests pass. No regressions introduced. - ---- - -## [2026-03-08] โ€” Initial compliance scaffold -### Changes -- Created `QUICK_FACTS.md` with machine-readable package metadata. -- Created `FAQ.md` with common Q&A. -- Created `MAINTENANCE_REPORT.md` (this file) as the change log. -- Created `SUGGESTIONS.md` with initial improvement proposals. -- Created `docs/index.md` as the documentation entry point (if missing). -### Rationale -Establishing the required file set mandated by `AGENTS.md` for all active workspace repositories. -### AI Transparency Report -- **AI Model**: Claude Sonnet 4.6 -- **Actions Taken**: Generated boilerplate compliance scaffold (QUICK_FACTS, FAQ, MAINTENANCE_REPORT, SUGGESTIONS, docs/index). -- **Oversight**: Files were stubs โ€” enriched in the 2026-03-08 documentation pass above. diff --git a/QUICK_FACTS.md b/QUICK_FACTS.md deleted file mode 100644 index 7a9a2c6..0000000 --- a/QUICK_FACTS.md +++ /dev/null @@ -1,55 +0,0 @@ -# Quick Facts โ€” `ovoscope` -End-to-end test framework for OpenVoiceOS skills -## Core Information -| Feature | Details | -|---------|---------| -| Package Name | `ovoscope` | -| Version | `0.7.2` | -| License | Apache-2.0 | -| Repository | [https://github.com/TigreGotico/ovoscope](https://github.com/TigreGotico/ovoscope) | -| Python Support | >=3.10 | -| Status | Active development | -## Entry Points -| Group | Value | Description | -|-------|-------|-------------| -| `console_scripts` | `ovoscope = ovoscope.cli:main` | CLI entry point | -| `pytest11` | `ovoscope = ovoscope.pytest_plugin` | pytest plugin (auto-loaded by pytest) | - -## Testing & CI -| Feature | Details | -|---------|---------| -| Unit Tests | 348 tests across `test/unittests/` (all passing) | -| Coverage | 53% overall (transformer/remote code excluded โ€” requires optional deps) | -| Test Framework | pytest with custom fixtures | -| Coverage Reporter | py-cov-action/python-coverage-comment-action@v3 | -## CI Workflows -| Workflow | Trigger | Status | -|----------|---------|--------| -| `unit_tests.yml` | Push to dev | Uses coverage.yml@dev | -| `build_tests.yml` | Push to master, PR to dev | Uses build-tests.yml@dev | -| `license_check.yml` | Push to master/dev, PR | Uses license-check.yml@dev | -| `pip_audit.yml` | Push to master/dev, PR | Uses pip-audit.yml@dev | -| `release_workflow.yml` | PR merge to dev | Gates on build_tests, calls publish-alpha.yml@dev | -| `publish_stable.yml` | Push to master | Calls publish-stable.yml@dev | -| `release_preview.yml` | PR to dev | Uses release-preview.yml@dev | -| `repo_health.yml` | PR to dev | Uses repo-health.yml@dev | -## Key Features -- **End-to-end testing framework** for OpenVoiceOS skills -- **MiniCroft fixture** โ€” pytest integration with class-scoped skill testing -- **Message capture** โ€” CaptureSession for recording skill responses -- **Assertions** โ€” End2EndTest with assertions (assert_spoke, etc.) -- **Audio harnesses** โ€” AudioServiceHarness, PlaybackServiceHarness, MockAudioBackend, MockTTS, AudioCaptureSession (optional `[audio]` extra) -- **Pydantic integration** โ€” Optional typed bridge with ovos-pydantic-models -- **Version from pyproject.toml** โ€” Full migration from setup.py - -## Audio Harness Classes (ovoscope.audio) -| Class | Description | -|---|---| -| `MockAudioBackend` | No-op AudioBackend tracking state (is_playing, is_paused, played_tracks, ducking counters) | -| `AudioServiceHarness` | Context manager: AudioService + MockAudioBackend on FakeBus | -| `MockTTS` | No-op TTS writing silent WAV, recording spoken_utterances | -| `PlaybackServiceHarness` | Context manager: PlaybackService + MockTTS on FakeBus | -| `AudioCaptureSession` | Records bus messages matching prefix list for sequence assertions | -## Test-Gated Releases -โœ… Alpha releases gate on `build_tests` passing (100+ unit tests) -โœ… Stable releases gate on master push (must pass alpha CI first) diff --git a/SUGGESTIONS.md b/SUGGESTIONS.md deleted file mode 100644 index 19ff107..0000000 --- a/SUGGESTIONS.md +++ /dev/null @@ -1,153 +0,0 @@ -# Suggestions โ€” `ovoscope` - -Agent-generated proposals for refactors and enhancements. -Each item includes a rationale, affected file, and implementation sketch. - ---- - -## 1. Share MiniCroft Across Fixtures in `cmd_bus_coverage()` [PERFORMANCE] - -**File**: `ovoscope/cli.py` โ€” `cmd_bus_coverage()` - -**Problem**: The current implementation creates a new `MiniCroft` for every -fixture file it runs. When a workspace has many fixtures for the same skill, -this means repeated skill loading, plugin initialisation, and READY-wait -overhead for each fixture โ€” typically 5โ€“20 seconds per fixture. - -**Suggestion**: Group fixture files by their `skill_ids` list, create a single -`MiniCroft` per unique skill set, then replay all fixtures against that shared -instance. Expected speedup: 10โ€“50ร— for typical skill test suites. - -**Sketch**: -```python -from itertools import groupby -fixtures_by_skills = groupby(sorted(fixtures, key=lambda f: f.skill_ids_key), ...) -for skill_key, group in fixtures_by_skills: - mc = get_minicroft(skill_key) - for fixture in group: - fixture.execute(minicroft=mc) - mc.stop() -``` - ---- - -## 2. Add `PipelineHarness.assert_no_match()` Convenience Method [DONE] - -**File**: `ovoscope/pipeline.py` - -**Status**: Already implemented โ€” `assert_no_match(utterance, timeout=2.0)` is -present at `pipeline.py:213`. No further action needed. - ---- - -## 3. Add `LOAD_TIME` Tag for Registration-Time Handlers [BUS COVERAGE] - -**File**: `ovoscope/bus_coverage.py` - -**Problem**: Handlers invoked during skill loading (before the snapshot) always -show `0 invocations` and `NOT TESTED` in bus coverage reports. This is -misleading because intent registration handlers *are* exercised โ€” just before -the snapshot window. - -**Suggestion**: Capture a pre-READY handler snapshot in `MiniCroft.__init__` -(before `super().__init__()`), then tag any handler present in both the -pre-READY and post-READY snapshots with a `LOAD_TIME` label in the report. -These handlers should be excluded from the `NOT TESTED` count. - ---- - -## 4. `diff.py` โ€” Strict Mode for Extra Keys [DONE] - -**File**: `ovoscope/diff.py` - -**Status**: Implemented in this audit cycle. `_dict_diff()` now accepts -`strict: bool = False`; when `True`, keys present in `actual` but not in -`expected` are flagged as unexpected extras. `diff_fixtures()` exposes the -same `strict` parameter. Default `False` preserves existing behaviour. - ---- - -## 5. Make Coverage Fixture Search Recursive [DONE] - -**File**: `ovoscope/coverage.py` โ€” `_count_fixtures()` - -**Status**: Implemented in this audit cycle. The search now uses -`Path.rglob("*.json")` instead of `os.listdir()`, so fixtures in -sub-directories are counted correctly. - ---- - -## 6. Add Noise-Floor Tolerance to `MockVADEngine.is_silence()` [LISTENER] - -**File**: `ovoscope/listener.py` - -**Problem**: `MockVADEngine.is_silence()` currently returns a fixed value -configured at construction time. Real VAD engines apply a noise-floor -threshold; tests that simulate borderline audio may need to replicate this -behaviour. - -**Suggestion**: Add `noise_floor: float = 0.0` parameter to -`MockVADEngine.__init__()`. When `noise_floor > 0`, `is_silence()` returns -`True` only if the sample RMS is below `noise_floor`. Default `0.0` -preserves existing behaviour (fixed return value). - ---- - -## 7. Make OCP HTTP Patch Targets Configurable [OCP] - -**File**: `ovoscope/ocp.py` - -**Problem**: The default patch targets (`requests.Session.get` and -`requests.get`) are hardcoded in `_apply_patches`. Skills that use -`httpx`, `aiohttp`, or a custom HTTP wrapper cannot be mocked without -specifying `patch_targets`. - -**Suggestion**: Expose `default_patch_targets: List[str]` as a class-level -constant on `OCPTest` so subclasses can override it without rewriting each -test instance: - -```python -class OCPTest: - default_patch_targets: List[str] = ["requests.Session.get", "requests.get"] -``` - ---- - -## 8. Support Env Var for GitHub URL in `setup_skill.py` [SETUP] - -**File**: `ovoscope/setup_skill.py` - -**Problem**: The GitHub URL for skill assets is hardcoded. CI environments or -forks may need to point to a different repository. - -**Suggestion**: Read `OVOSCOPE_SKILL_URL` environment variable as an override: - -```python -import os -SKILL_URL = os.environ.get("OVOSCOPE_SKILL_URL", DEFAULT_SKILL_URL) -``` - ---- - -## 9. Add `RemoteRecorder` Usage to Docs [DOCUMENTATION] - -**File**: `docs/index.md`, `docs/usage-guide.md` - -**Problem**: `RemoteRecorder` โ€” `ovoscope/remote_recorder.py:46` โ€” is not -documented in any public-facing doc file. Users who want to capture fixtures -from a live OVOS instance have no guide. - -**Suggestion**: Add a "Pattern 13: Recording from a Live OVOS Instance" section -to `docs/usage-guide.md` showing the `connect()`/`record()`/`disconnect()` -workflow and the `--live` CLI flag. - ---- - -## 10. Expand `docs/pipeline.md` to Full API Reference [DONE] - -**File**: `docs/pipeline.md` - -**Status**: Expanded in this audit cycle. The file now includes the full -`PipelineHarness` API table with `pipeline.py:LINE` citations, examples for -Adapt and Padatious pipelines, an explanation of `_SinkSkill`, and notes on -pipeline stage ordering and success/failure signals. diff --git a/downstream_report.txt b/downstream_report.txt deleted file mode 100644 index 85cec10..0000000 --- a/downstream_report.txt +++ /dev/null @@ -1 +0,0 @@ -ovoscope==0.7.2 From 3cb08af0de3f6453b0bf1d7732ce116ed65c5eef Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 13 Jun 2026 15:52:50 +0000 Subject: [PATCH 27/82] Increment Version to 0.19.0a3 --- ovoscope/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovoscope/version.py b/ovoscope/version.py index 9ebb2ab..78930c7 100644 --- a/ovoscope/version.py +++ b/ovoscope/version.py @@ -2,7 +2,7 @@ VERSION_MAJOR = 0 VERSION_MINOR = 19 VERSION_BUILD = 0 -VERSION_ALPHA = 2 +VERSION_ALPHA = 3 # END_VERSION_BLOCK __version__ = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}" + ( From 619e93606176155eef6c96a5dc4351941f5f4f32 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 13 Jun 2026 15:53:15 +0000 Subject: [PATCH 28/82] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 51c996f..94aa7c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.19.0a3](https://github.com/OpenVoiceOS/ovoscope/tree/0.19.0a3) (2026-06-13) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.19.0a2...0.19.0a3) + +**Merged pull requests:** + +- chore: remove agent-audit scratch files [\#71](https://github.com/OpenVoiceOS/ovoscope/pull/71) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.19.0a2](https://github.com/OpenVoiceOS/ovoscope/tree/0.19.0a2) (2026-06-13) [Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.19.0a1...0.19.0a2) From 9e8074d535e7cc4a09967b448c18b6c07702e315 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Sun, 14 Jun 2026 19:00:55 +0100 Subject: [PATCH 29/82] fix: drop removed 'path' arg from pytest_pycollect_makemodule hook (pytest>=8 compat) (#73) * fix: pytest_pycollect_makemodule hook signature for pytest>=8 (drop removed 'path' arg) * fix: declare pytest>=8 as a core dependency ovoscope registers a pytest11 plugin, so pytest is a runtime dependency, not just a test extra. Pin >=8: the pytest_pycollect_makemodule hook dropped the 'path' arg in pytest 8 (the bug this PR fixes). --- ovoscope/pytest_plugin.py | 2 +- pyproject.toml | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/ovoscope/pytest_plugin.py b/ovoscope/pytest_plugin.py index 389444d..bdae9ad 100644 --- a/ovoscope/pytest_plugin.py +++ b/ovoscope/pytest_plugin.py @@ -422,7 +422,7 @@ def _autodiscover_intent_cases(config): @pytest.hookimpl(hookwrapper=True) -def pytest_pycollect_makemodule(module_path, path, parent): +def pytest_pycollect_makemodule(module_path, parent): """Auto-register intent-case tests on shim modules that declare ``ovoscope_intent_cases = {...}``. diff --git a/pyproject.toml b/pyproject.toml index e061f24..6ec33d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,10 @@ dependencies = [ # Alpha pin: 2.0.4 stable not yet released; 2.0.4a2 is the minimum that # includes the FakeBus-compatible SkillManager changes ovoscope depends on. "ovos-core>=2.0.4a2", + # ovoscope ships a pytest plugin (pytest11 entry point) -> pytest is a runtime + # dependency. >=8 is required: the pytest_pycollect_makemodule hook dropped the + # 'path' argument in pytest 8. + "pytest>=8", ] classifiers = [ "Programming Language :: Python :: 3", @@ -31,7 +35,6 @@ audio = ["ovos-audio>=1.2.0"] dev = [ "ovos-audio>=1.2.0", "ovos-pydantic-models>=0.1.0", - "pytest", "pytest-cov", ] From c6e1ecd3e158979dca56cfb9c562bae45975989d Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sun, 14 Jun 2026 18:01:09 +0000 Subject: [PATCH 30/82] Increment Version to 0.19.1a1 --- ovoscope/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ovoscope/version.py b/ovoscope/version.py index 78930c7..29ca8b6 100644 --- a/ovoscope/version.py +++ b/ovoscope/version.py @@ -1,8 +1,8 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 VERSION_MINOR = 19 -VERSION_BUILD = 0 -VERSION_ALPHA = 3 +VERSION_BUILD = 1 +VERSION_ALPHA = 1 # END_VERSION_BLOCK __version__ = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}" + ( From 194c83f5d582badead3aa37eb43ec825777c25a6 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sun, 14 Jun 2026 18:01:35 +0000 Subject: [PATCH 31/82] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94aa7c0..e2b2a77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.19.1a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.19.1a1) (2026-06-14) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.19.0a3...0.19.1a1) + +**Merged pull requests:** + +- fix: drop removed 'path' arg from pytest\_pycollect\_makemodule hook \(pytest\>=8 compat\) [\#73](https://github.com/OpenVoiceOS/ovoscope/pull/73) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.19.0a3](https://github.com/OpenVoiceOS/ovoscope/tree/0.19.0a3) (2026-06-13) [Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.19.0a2...0.19.0a3) From 536c932081bfb7674535342bdfaf389cfede6214 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Mon, 15 Jun 2026 21:50:21 +0100 Subject: [PATCH 32/82] feat: TTS end-to-end intelligibility harness (#75) Add ovoscope/tts_intelligibility.py: synthesise speech with a TTS plugin under test, transcribe the rendered audio back with a reference STT (faster-whisper tiny), and score the round-trip with WER/CER. - score_tts_intelligibility() + TTSIntelligibilityHarness context manager returning an IntelligibilityReport (per-utterance UtteranceScore, mean WER/CER, to_dict/to_markdown_row). - mode="playback" drives the full ovos-audio stack and captures the rendered WAV via a play_audio side_effect; mode="direct" calls tts.get_tts directly. - Extend PlaybackServiceHarness with a tts= arg (default MockTTS, backward compatible) and a captured_wavs list. - Add [tts] optional extra; graceful optional import in __init__. - Unit tests (MockTTS + MockSTT, no model download) cover WER/CER math, report aggregation, playback wav capture, and graceful import. Co-authored-by: Claude Opus 4.8 --- ovoscope/__init__.py | 22 ++ ovoscope/audio.py | 31 +- ovoscope/tts_intelligibility.py | 410 +++++++++++++++++++++ pyproject.toml | 12 + test/unittests/test_tts_intelligibility.py | 192 ++++++++++ 5 files changed, 662 insertions(+), 5 deletions(-) create mode 100644 ovoscope/tts_intelligibility.py create mode 100644 test/unittests/test_tts_intelligibility.py diff --git a/ovoscope/__init__.py b/ovoscope/__init__.py index 8816468..56eccda 100644 --- a/ovoscope/__init__.py +++ b/ovoscope/__init__.py @@ -1095,6 +1095,28 @@ def assert_spoke(self, text: str, lang: str = "en-US", timeout: int = 30) -> Non else: raise +try: + from ovoscope.tts_intelligibility import ( # noqa: F401 + TTSIntelligibilityHarness, + IntelligibilityReport, + UtteranceScore, + score_tts_intelligibility, + ) +except ImportError as e: + # Optional [tts] extra. Silence only when the missing module is one of the + # optional TTS-scoring deps; a logic error in a present lib must re-raise. + _TTS_OPTIONAL_MODULES = ( + "jiwer", + "ovos_audio", "ovos_audio.audio", + "ovos_utterance_normalizer", + "ovos_stt_plugin_fasterwhisper", + "faster_whisper", + ) + if isinstance(e, ModuleNotFoundError) and e.name in _TTS_OPTIONAL_MODULES: + pass + else: + raise + try: from ovoscope.listener import ( # noqa: F401 MiniListener, diff --git a/ovoscope/audio.py b/ovoscope/audio.py index d7e7e81..fd82dba 100644 --- a/ovoscope/audio.py +++ b/ovoscope/audio.py @@ -518,21 +518,32 @@ class PlaybackServiceHarness: Args: validate_source: Enable session-source validation in the service. disable_ocp: Disable legacy OCP in the encapsulated AudioService. + tts: TTS instance to drive the PlaybackService with. Defaults to a + fresh ``MockTTS()`` (backward compatible). Pass a real TTS plugin + to synthesise actual audio โ€” the rendered WAV path of each + utterance is captured in :attr:`captured_wavs`. """ def __init__(self, validate_source: bool = False, - disable_ocp: bool = True) -> None: + disable_ocp: bool = True, + tts: Optional[TTS] = None) -> None: """Initialise harness parameters. Args: validate_source: Enable session-source validation. disable_ocp: Disable OCP audio plugin. + tts: TTS instance to inject. Defaults to ``MockTTS()`` when None. """ self.validate_source: bool = validate_source self.disable_ocp: bool = disable_ocp self.bus: Optional[FakeBus] = None self.svc = None # PlaybackService instance - self.mock_tts: Optional[MockTTS] = None + # ``mock_tts`` keeps its historic name for backward compatibility but + # holds whatever TTS was injected (real plugin or MockTTS). + self.tts: Optional[TTS] = tts + self.mock_tts: Optional[TTS] = None + # Paths captured from the ``play_audio`` side_effect, in playback order. + self.captured_wavs: List[str] = [] self._play_audio_patcher = None self._audio_enabled_patcher = None self._audio_output_start = threading.Event() @@ -558,15 +569,25 @@ def __enter__(self) -> "PlaybackServiceHarness": TTS.queue = Queue() self.bus = FakeBus() - self.mock_tts = MockTTS() + # Inject the provided TTS (real plugin) or fall back to MockTTS. + self.mock_tts = self.tts if self.tts is not None else MockTTS() - # Patch play_audio so no real audio device is accessed + # Patch play_audio so no real audio device is accessed. The side_effect + # records the first positional arg โ€” the rendered WAV path + # (ovos_audio/playback.py: ``self.p = play_audio(data)``) โ€” so callers + # can round-trip the synthesised audio through a reference STT. mock_proc = MagicMock() mock_proc.communicate.return_value = (b"", b"") mock_proc.wait.return_value = 0 + self.captured_wavs = [] + + def _capture_play_audio(data, *args, **kwargs): + self.captured_wavs.append(data) + return mock_proc + self._play_audio_patcher = patch( - "ovos_audio.playback.play_audio", return_value=mock_proc + "ovos_audio.playback.play_audio", side_effect=_capture_play_audio ) self._play_audio_patcher.start() diff --git a/ovoscope/tts_intelligibility.py b/ovoscope/tts_intelligibility.py new file mode 100644 index 0000000..6489b67 --- /dev/null +++ b/ovoscope/tts_intelligibility.py @@ -0,0 +1,410 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""End-to-end TTS intelligibility scoring for ovoscope. + +Synthesises speech with a TTS plugin under test, transcribes the rendered +audio back with a reference STT, and scores the round-trip with word- and +character-error-rate (WER/CER). This catches regressions that file-existence +unit tests miss โ€” garbled audio, wrong sample rate, broken transforms, silent +output โ€” and gives every TTS plugin a comparable intelligibility number. + +Two synthesis modes: + +* ``"playback"`` (default) drives the full ovos-audio stack + (``speak`` -> PlaybackService -> tts.execute -> get_tts -> tts_transform -> + play_audio) via :class:`ovoscope.audio.PlaybackServiceHarness`, with the real + plugin injected. The rendered WAV is captured from the patched ``play_audio``. +* ``"direct"`` calls ``tts.get_tts(utterance, wav_path, ...)`` directly with no + bus โ€” a fallback for engines that hang under the playback thread or when + ``ovos_audio`` is unavailable. + +Public API: + score_tts_intelligibility -- convenience function returning an IntelligibilityReport + TTSIntelligibilityHarness -- context manager form of the above + IntelligibilityReport -- aggregate report with mean WER/CER + serialisation + UtteranceScore -- per-utterance result +""" + +import dataclasses +import os +import re +import shutil +import string +import tempfile +import threading +from typing import Any, List, Optional + +import jiwer + +from ovos_plugin_manager.utils.audio import AudioFile + +# Module-level singleton for the reference STT โ€” model load is expensive. +_REFERENCE_STT: Optional[Any] = None +_REFERENCE_STT_LOCK = threading.Lock() + +# Module-level singleton for the utterance normaliser (cheap, but shared). +_NORMALIZER: Optional[Any] = None + + +def get_reference_stt() -> Any: + """Return a lazily-instantiated faster-whisper ``tiny`` reference STT. + + The model is loaded once per process and reused. ``beam_size=1`` and + ``compute_type="int8"`` keep it deterministic and light enough for CI. + + Returns: + A ready-to-use ``FasterWhisperSTT`` instance. + """ + global _REFERENCE_STT + if _REFERENCE_STT is None: + with _REFERENCE_STT_LOCK: + if _REFERENCE_STT is None: + from ovos_stt_plugin_fasterwhisper import FasterWhisperSTT + _REFERENCE_STT = FasterWhisperSTT({ + "model": "tiny", + "compute_type": "int8", + "beam_size": 1, + }) + return _REFERENCE_STT + + +def _normalize(text: str, lang: str) -> str: + """Normalise a transcript for fair WER/CER scoring. + + Uses ``ovos_utterance_normalizer`` (lowercase, number expansion, + contraction expansion, punctuation strip) so cosmetic differences between + reference and hypothesis don't inflate the error rate. The normaliser + yields several variants per input; the last one is the fully-normalised + form, which is what we score against. + + Args: + text: The raw text to normalise. + lang: BCP-47 language tag (e.g. ``"en-US"``). + + Returns: + A single normalised string (whitespace-collapsed, lowercased). + """ + global _NORMALIZER + if _NORMALIZER is None: + from ovos_utterance_normalizer import UtteranceNormalizerPlugin + _NORMALIZER = UtteranceNormalizerPlugin() + if not text: + return "" + variants, _ = _NORMALIZER.transform([text], {"lang": lang}) + # transform() emits [contraction-expanded, original, number-normalized] + # per utterance, deduplicated and order-preserving. The first variant + # (contractions expanded, punctuation stripped, words kept as words) is the + # stable choice โ€” the number-collapsed variant ("two" -> "2") would create + # spurious mismatches against a word-emitting STT. Lowercasing/whitespace + # collapse are applied here so both reference and hypothesis align. + normalized = variants[0] if variants else text + # The normaliser only strips leading/trailing punctuation of the whole + # string; interior punctuation (e.g. a tokenised comma) survives and would + # inflate WER. Strip all punctuation characters here, then collapse space. + normalized = re.sub(rf"[{re.escape(string.punctuation)}]", " ", normalized) + return " ".join(normalized.lower().split()) + + +def _score_pair(reference: str, hypothesis: str) -> "tuple[float, float]": + """Compute (WER, CER) for a reference/hypothesis pair. + + jiwer raises on an empty reference, so empty references are handled + explicitly: a non-empty hypothesis against an empty reference scores 1.0 + (fully wrong), two empties score 0.0 (trivially correct). + + Args: + reference: Normalised ground-truth string. + hypothesis: Normalised transcript from the reference STT. + + Returns: + Tuple of ``(wer, cer)`` as floats. + """ + if not reference: + return (0.0, 0.0) if not hypothesis else (1.0, 1.0) + wer = float(jiwer.wer(reference, hypothesis)) + cer = float(jiwer.cer(reference, hypothesis)) + return wer, cer + + +@dataclasses.dataclass +class UtteranceScore: + """Per-utterance intelligibility result. + + Attributes: + utterance: The text that was synthesised (the ground truth). + transcript: What the reference STT heard back. + wer: Word error rate of transcript vs utterance (0.0 = perfect). + cer: Character error rate of transcript vs utterance. + wav_path: Path to the captured rendered WAV (may be None on failure). + lang: BCP-47 language tag used for synthesis and scoring. + voice: Voice identifier used, if any. + """ + + utterance: str + transcript: str + wer: float + cer: float + wav_path: Optional[str] = None + lang: str = "en-US" + voice: Optional[str] = None + + def to_dict(self) -> dict: + """Return a JSON-serialisable dict of this score.""" + return { + "utterance": self.utterance, + "transcript": self.transcript, + "wer": round(self.wer, 4), + "cer": round(self.cer, 4), + "wav_path": self.wav_path, + "lang": self.lang, + "voice": self.voice, + } + + +@dataclasses.dataclass +class IntelligibilityReport: + """Aggregate intelligibility report over a set of utterances. + + Attributes: + scores: Per-utterance :class:`UtteranceScore` results. + lang: BCP-47 language tag the run used. + voice: Voice identifier the run used, if any. + mode: Synthesis mode used (``"playback"`` or ``"direct"``). + """ + + scores: List[UtteranceScore] = dataclasses.field(default_factory=list) + lang: str = "en-US" + voice: Optional[str] = None + mode: str = "playback" + + @property + def mean_wer(self) -> float: + """Mean word error rate across all scored utterances (0.0 if empty).""" + if not self.scores: + return 0.0 + return sum(s.wer for s in self.scores) / len(self.scores) + + @property + def mean_cer(self) -> float: + """Mean character error rate across all scored utterances (0.0 if empty).""" + if not self.scores: + return 0.0 + return sum(s.cer for s in self.scores) / len(self.scores) + + def to_dict(self) -> dict: + """Return a JSON-serialisable dict of the full report.""" + return { + "lang": self.lang, + "voice": self.voice, + "mode": self.mode, + "mean_wer": round(self.mean_wer, 4), + "mean_cer": round(self.mean_cer, 4), + "n_utterances": len(self.scores), + "scores": [s.to_dict() for s in self.scores], + } + + def to_markdown_row(self) -> str: + """Return a single markdown table row: ``| voice | lang | mean_wer | mean_cer | n |``.""" + voice = self.voice or "default" + return ( + f"| {voice} | {self.lang} | " + f"{self.mean_wer:.3f} | {self.mean_cer:.3f} | {len(self.scores)} |" + ) + + +class TTSIntelligibilityHarness: + """Context manager that scores TTS intelligibility end-to-end. + + Usage:: + + with TTSIntelligibilityHarness(tts, lang="en-US") as h: + report = h.score(["hello world", "what time is it"]) + print(report.mean_wer) + + In ``mode="playback"`` the harness owns a :class:`PlaybackServiceHarness` + for its lifetime; in ``mode="direct"`` no bus is started. A temp directory + holds copies of the rendered WAVs (the TTS cache may delete originals); it + is cleaned up on exit. + + Args: + tts: The TTS plugin under test. + lang: BCP-47 language tag for synthesis and scoring. + voice: Optional voice identifier passed to ``get_tts``. + reference_stt: STT used to transcribe. Defaults to the lazy + faster-whisper ``tiny`` singleton. + mode: ``"playback"`` (full ovos-audio stack) or ``"direct"`` + (``tts.get_tts`` only). + speak_timeout: Per-utterance timeout for playback mode. + """ + + def __init__(self, tts: Any, *, lang: str = "en-US", + voice: Optional[str] = None, + reference_stt: Optional[Any] = None, + mode: str = "playback", + speak_timeout: float = 30.0) -> None: + if mode not in ("playback", "direct"): + raise ValueError(f"mode must be 'playback' or 'direct', got {mode!r}") + self.tts = tts + self.lang = lang + self.voice = voice + self._reference_stt = reference_stt + self.mode = mode + self.speak_timeout = speak_timeout + self._tmpdir: Optional[str] = None + self._playback = None # PlaybackServiceHarness in playback mode + + @property + def reference_stt(self) -> Any: + """The reference STT, lazily resolved to the faster-whisper singleton.""" + if self._reference_stt is None: + self._reference_stt = get_reference_stt() + return self._reference_stt + + def __enter__(self) -> "TTSIntelligibilityHarness": + self._tmpdir = tempfile.mkdtemp(prefix="ovoscope-tts-") + if self.mode == "playback": + from ovoscope.audio import PlaybackServiceHarness + self._playback = PlaybackServiceHarness(tts=self.tts) + self._playback.__enter__() + return self + + def __exit__(self, *args) -> None: + if self._playback is not None: + try: + self._playback.__exit__(*args) + except Exception: + pass + self._playback = None + if self._tmpdir and os.path.isdir(self._tmpdir): + shutil.rmtree(self._tmpdir, ignore_errors=True) + self._tmpdir = None + + # ------------------------------------------------------------------ + # Synthesis + # ------------------------------------------------------------------ + + def _render_playback(self, utterance: str) -> Optional[str]: + """Synthesise via the full ovos-audio stack; return a copied WAV path.""" + before = len(self._playback.captured_wavs) + self._playback.speak(utterance, timeout=self.speak_timeout) + captured = self._playback.captured_wavs[before:] + if not captured: + return None + return self._copy_out(captured[-1]) + + def _render_direct(self, utterance: str) -> Optional[str]: + """Synthesise via ``tts.get_tts`` directly; return the WAV path.""" + wav_path = os.path.join( + self._tmpdir, f"direct_{abs(hash(utterance)) & 0xffffffff}.wav" + ) + self.tts.get_tts(utterance, wav_path, lang=self.lang, voice=self.voice) + return wav_path if os.path.isfile(wav_path) else None + + def _copy_out(self, wav_path: str) -> Optional[str]: + """Copy a rendered WAV into the harness temp dir before the cache prunes it.""" + if not wav_path or not os.path.isfile(wav_path): + return wav_path if wav_path and os.path.isfile(wav_path) else None + dst = os.path.join( + self._tmpdir, f"play_{len(os.listdir(self._tmpdir))}_{os.path.basename(wav_path)}" + ) + try: + shutil.copyfile(wav_path, dst) + return dst + except OSError: + return wav_path + + # ------------------------------------------------------------------ + # Scoring + # ------------------------------------------------------------------ + + def _transcribe(self, wav_path: str) -> str: + """Round-trip a WAV through the reference STT and return the transcript.""" + with AudioFile(wav_path) as source: + audio = source.read() + return self.reference_stt.execute(audio, language=self.lang) or "" + + def score_one(self, utterance: str) -> UtteranceScore: + """Synthesise, transcribe, and score a single utterance. + + Args: + utterance: The text to synthesise and score. + + Returns: + An :class:`UtteranceScore`. On synthesis/transcription failure the + transcript is empty and WER/CER reflect a total miss. + """ + if self.mode == "playback": + wav_path = self._render_playback(utterance) + else: + wav_path = self._render_direct(utterance) + + transcript = "" + if wav_path and os.path.isfile(wav_path): + try: + transcript = self._transcribe(wav_path) + except Exception: + transcript = "" + + ref = _normalize(utterance, self.lang) + hyp = _normalize(transcript, self.lang) + wer, cer = _score_pair(ref, hyp) + return UtteranceScore( + utterance=utterance, + transcript=transcript, + wer=wer, + cer=cer, + wav_path=wav_path, + lang=self.lang, + voice=self.voice, + ) + + def score(self, utterances: List[str]) -> IntelligibilityReport: + """Score a list of utterances and return the aggregate report. + + Args: + utterances: Phrases to synthesise and score. + + Returns: + An :class:`IntelligibilityReport`. + """ + report = IntelligibilityReport(lang=self.lang, voice=self.voice, mode=self.mode) + for utt in utterances: + report.scores.append(self.score_one(utt)) + return report + + +def score_tts_intelligibility(tts: Any, utterances: List[str], *, + lang: str = "en-US", + voice: Optional[str] = None, + reference_stt: Optional[Any] = None, + mode: str = "playback", + speak_timeout: float = 30.0) -> IntelligibilityReport: + """Synthesise, transcribe, and score a set of utterances in one call. + + Args: + tts: The TTS plugin under test. + utterances: Phrases to synthesise and score. + lang: BCP-47 language tag for synthesis and scoring. + voice: Optional voice identifier passed to ``get_tts``. + reference_stt: STT used to transcribe. Defaults to faster-whisper tiny. + mode: ``"playback"`` (full ovos-audio stack) or ``"direct"``. + speak_timeout: Per-utterance timeout for playback mode. + + Returns: + An :class:`IntelligibilityReport` with per-utterance and mean scores. + """ + with TTSIntelligibilityHarness( + tts, lang=lang, voice=voice, reference_stt=reference_stt, + mode=mode, speak_timeout=speak_timeout, + ) as harness: + return harness.score(utterances) diff --git a/pyproject.toml b/pyproject.toml index 6ec33d0..a729214 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,9 +32,21 @@ classifiers = [ [project.optional-dependencies] pydantic = ["ovos-pydantic-models>=0.1.0"] audio = ["ovos-audio>=1.2.0"] +# End-to-end TTS intelligibility scoring (WER/CER round-trip via reference STT). +# faster-whisper itself is pulled by the plugin โ€” don't list it here to avoid +# version skew. +tts = [ + "ovos-audio>=1.2.0", + "jiwer", + "ovos-utterance-normalizer", + "ovos-stt-plugin-fasterwhisper", +] dev = [ "ovos-audio>=1.2.0", "ovos-pydantic-models>=0.1.0", + "jiwer", + "ovos-utterance-normalizer", + "ovos-stt-plugin-fasterwhisper", "pytest-cov", ] diff --git a/test/unittests/test_tts_intelligibility.py b/test/unittests/test_tts_intelligibility.py new file mode 100644 index 0000000..283f40e --- /dev/null +++ b/test/unittests/test_tts_intelligibility.py @@ -0,0 +1,192 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for ovoscope.tts_intelligibility. + +These tests use a MockTTS (silent WAV) and a MockSTT that echoes a fixed +transcript โ€” no model download, no real audio. They cover WER/CER math, report +aggregation, serialisation, and that playback interception captures a wav path. +""" + +import importlib.util +import subprocess +import sys +import unittest + +TTS_AVAILABLE = ( + importlib.util.find_spec("jiwer") is not None + and importlib.util.find_spec("ovos_audio") is not None + and importlib.util.find_spec("ovos_utterance_normalizer") is not None +) + + +@unittest.skipUnless(TTS_AVAILABLE, "tts extra (jiwer/ovos-audio/normalizer) not installed") +class TestScoring(unittest.TestCase): + """WER/CER math, normalisation and report aggregation โ€” no synthesis.""" + + def setUp(self): + from ovoscope import tts_intelligibility as ti + self.ti = ti + + def test_perfect_match_is_zero(self): + wer, cer = self.ti._score_pair("hello world", "hello world") + self.assertEqual(wer, 0.0) + self.assertEqual(cer, 0.0) + + def test_one_wrong_word_half_wer(self): + wer, _ = self.ti._score_pair("hello world", "hello there") + self.assertAlmostEqual(wer, 0.5, places=3) + + def test_empty_reference_handling(self): + self.assertEqual(self.ti._score_pair("", ""), (0.0, 0.0)) + self.assertEqual(self.ti._score_pair("", "noise"), (1.0, 1.0)) + + def test_normalize_strips_case_and_punctuation(self): + # "Hello, World!" should normalise to "hello world" so it scores 0 + # against the lowercased ground truth. + wer, _ = self.ti._score_pair( + self.ti._normalize("Hello, World!", "en-US"), + self.ti._normalize("hello world", "en-US"), + ) + self.assertEqual(wer, 0.0) + + def test_report_aggregation(self): + UtteranceScore = self.ti.UtteranceScore + IntelligibilityReport = self.ti.IntelligibilityReport + report = IntelligibilityReport(lang="en-US", voice="alan") + report.scores.append(UtteranceScore("a", "a", 0.0, 0.0, lang="en-US")) + report.scores.append(UtteranceScore("b", "x", 1.0, 1.0, lang="en-US")) + self.assertAlmostEqual(report.mean_wer, 0.5) + self.assertAlmostEqual(report.mean_cer, 0.5) + + def test_empty_report_means_zero(self): + report = self.ti.IntelligibilityReport() + self.assertEqual(report.mean_wer, 0.0) + self.assertEqual(report.mean_cer, 0.0) + + def test_to_dict_and_markdown_row(self): + UtteranceScore = self.ti.UtteranceScore + report = self.ti.IntelligibilityReport(lang="en-US", voice="alan") + report.scores.append(UtteranceScore("a", "a", 0.0, 0.0, lang="en-US")) + d = report.to_dict() + self.assertEqual(d["lang"], "en-US") + self.assertEqual(d["voice"], "alan") + self.assertEqual(d["n_utterances"], 1) + self.assertEqual(d["mean_wer"], 0.0) + self.assertIn("scores", d) + row = report.to_markdown_row() + self.assertIn("alan", row) + self.assertIn("en-US", row) + self.assertTrue(row.startswith("|") and row.endswith("|")) + + +class MockSTT: + """Reference STT stub that echoes a fixed transcript regardless of audio.""" + + def __init__(self, transcript="hello world"): + self.transcript = transcript + self.calls = 0 + + def execute(self, audio, language=None): + self.calls += 1 + return self.transcript + + +@unittest.skipUnless(TTS_AVAILABLE, "tts extra (jiwer/ovos-audio/normalizer) not installed") +class TestHarnessDirectMode(unittest.TestCase): + """Direct mode: tts.get_tts only, MockSTT echo โ€” no bus, no model.""" + + def test_direct_mode_perfect_score(self): + from ovoscope.audio import MockTTS + from ovoscope.tts_intelligibility import score_tts_intelligibility + + stt = MockSTT("hello world") + report = score_tts_intelligibility( + MockTTS(), ["hello world"], + reference_stt=stt, mode="direct", + ) + self.assertEqual(len(report.scores), 1) + self.assertEqual(report.mean_wer, 0.0) + self.assertEqual(stt.calls, 1) + self.assertIsNotNone(report.scores[0].wav_path) + + def test_direct_mode_mismatch_scores_high(self): + from ovoscope.audio import MockTTS + from ovoscope.tts_intelligibility import score_tts_intelligibility + + report = score_tts_intelligibility( + MockTTS(), ["completely different text here"], + reference_stt=MockSTT("hello world"), mode="direct", + ) + self.assertGreater(report.mean_wer, 0.0) + + +@unittest.skipUnless(TTS_AVAILABLE, "tts extra (jiwer/ovos-audio/normalizer) not installed") +class TestHarnessPlaybackMode(unittest.TestCase): + """Playback mode: full ovos-audio stack drives MockTTS; wav is captured.""" + + def test_playback_captures_wav_and_scores(self): + from ovoscope.audio import MockTTS + from ovoscope.tts_intelligibility import TTSIntelligibilityHarness + + tts = MockTTS() + stt = MockSTT("hello world") + with TTSIntelligibilityHarness( + tts, reference_stt=stt, mode="playback", speak_timeout=15.0, + ) as h: + report = h.score(["hello world"]) + + self.assertEqual(len(report.scores), 1) + score = report.scores[0] + # Playback interception must have captured a rendered wav path. + self.assertIsNotNone(score.wav_path, "no wav captured from playback") + self.assertIn("hello world", tts.spoken_utterances) + self.assertEqual(report.mean_wer, 0.0) + + +class TestGracefulImport(unittest.TestCase): + """Core ``import ovoscope`` must succeed even without the [tts] extra.""" + + def test_import_without_tts_extra(self): + # Run in a subprocess with the optional tts deps blocked at import time, + # simulating an environment that never installed ovoscope[tts]. + code = ( + "import sys, importlib.abc, importlib.machinery\n" + "BLOCKED = {'jiwer', 'ovos_utterance_normalizer', " + "'ovos_stt_plugin_fasterwhisper', 'faster_whisper'}\n" + "class _Block(importlib.abc.MetaPathFinder):\n" + " def find_spec(self, name, path, target=None):\n" + " if name.split('.')[0] in BLOCKED:\n" + " raise ModuleNotFoundError(name=name.split('.')[0])\n" + " return None\n" + "sys.meta_path.insert(0, _Block())\n" + "for m in list(sys.modules):\n" + " if m.split('.')[0] in BLOCKED:\n" + " del sys.modules[m]\n" + "import ovoscope\n" + "assert not hasattr(ovoscope, 'TTSIntelligibilityHarness'), " + "'harness should be absent without the tts extra'\n" + "print('OK')\n" + ) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, text=True, + ) + self.assertEqual( + result.returncode, 0, + f"core import failed without tts extra:\nstdout={result.stdout}\nstderr={result.stderr}", + ) + self.assertIn("OK", result.stdout) + + +if __name__ == "__main__": + unittest.main() From e95fa7374c2d2d02695a6117cbb5ea129f388a9d Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Mon, 15 Jun 2026 21:10:14 +0000 Subject: [PATCH 33/82] Increment Version to 0.19.1a2 --- ovoscope/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovoscope/version.py b/ovoscope/version.py index 29ca8b6..1b30799 100644 --- a/ovoscope/version.py +++ b/ovoscope/version.py @@ -2,7 +2,7 @@ VERSION_MAJOR = 0 VERSION_MINOR = 19 VERSION_BUILD = 1 -VERSION_ALPHA = 1 +VERSION_ALPHA = 2 # END_VERSION_BLOCK __version__ = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}" + ( From e4fedf5d0598a412da29ee9a945391296968b10f Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Mon, 15 Jun 2026 21:16:04 +0000 Subject: [PATCH 34/82] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2b2a77..d7e0d75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.19.1a2](https://github.com/OpenVoiceOS/ovoscope/tree/0.19.1a2) (2026-06-15) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.19.1a1...0.19.1a2) + +**Merged pull requests:** + +- feat: TTS end-to-end intelligibility harness [\#75](https://github.com/OpenVoiceOS/ovoscope/pull/75) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.19.1a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.19.1a1) (2026-06-14) [Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.19.0a3...0.19.1a1) From bad04fd7fc31c1a4ff1abc116d912a5b241600bb Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Wed, 17 Jun 2026 00:52:55 +0100 Subject: [PATCH 35/82] fix(tts-intelligibility): score synthesis failures as total miss instead of aborting (#77) A get_tts() crash (missing model/binary, bad audio, engine error) propagated out of score()/score_tts_intelligibility, so the per-plugin test never reached its ::TTS-INTELLIGIBILITY:: marker print and the PR comment showed 'No intelligibility scores reported'. Wrap the render in score_one: on failure the utterance scores as a total miss (WER 1.0), the report is still produced and a marker is emitted, and the gate correctly fails the unintelligible engine with the score visible. --- ovoscope/tts_intelligibility.py | 17 +++++++++--- test/unittests/test_tts_intelligibility.py | 30 ++++++++++++++++++++++ 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/ovoscope/tts_intelligibility.py b/ovoscope/tts_intelligibility.py index 6489b67..ae25db6 100644 --- a/ovoscope/tts_intelligibility.py +++ b/ovoscope/tts_intelligibility.py @@ -47,6 +47,7 @@ import jiwer from ovos_plugin_manager.utils.audio import AudioFile +from ovos_utils.log import LOG # Module-level singleton for the reference STT โ€” model load is expensive. _REFERENCE_STT: Optional[Any] = None @@ -343,10 +344,18 @@ def score_one(self, utterance: str) -> UtteranceScore: An :class:`UtteranceScore`. On synthesis/transcription failure the transcript is empty and WER/CER reflect a total miss. """ - if self.mode == "playback": - wav_path = self._render_playback(utterance) - else: - wav_path = self._render_direct(utterance) + try: + if self.mode == "playback": + wav_path = self._render_playback(utterance) + else: + wav_path = self._render_direct(utterance) + except Exception as exc: + # A synthesis failure (engine crash, missing model/binary, bad audio) + # is itself an intelligibility failure: score it as a total miss so the + # report is still produced and a marker is still emitted, instead of + # letting the exception abort the run with "no scores reported". + LOG.error(f"TTS synthesis failed for {utterance!r}: {exc}") + wav_path = None transcript = "" if wav_path and os.path.isfile(wav_path): diff --git a/test/unittests/test_tts_intelligibility.py b/test/unittests/test_tts_intelligibility.py index 283f40e..7f8f247 100644 --- a/test/unittests/test_tts_intelligibility.py +++ b/test/unittests/test_tts_intelligibility.py @@ -188,5 +188,35 @@ def test_import_without_tts_extra(self): self.assertIn("OK", result.stdout) +@unittest.skipUnless(TTS_AVAILABLE, "tts extra (jiwer/ovos-audio/normalizer) not installed") +class TestSynthesisFailureResilience(unittest.TestCase): + """A get_tts crash must score as a total miss, not abort the whole run.""" + + def test_synthesis_failure_scores_total_miss(self): + from ovoscope import tts_intelligibility as ti + + class BoomTTS: + def get_tts(self, *args, **kwargs): + raise RuntimeError("synthesis exploded") + + class EchoSTT: + def execute(self, audio, language=None): + return "anything" + + harness = ti.TTSIntelligibilityHarness( + BoomTTS(), lang="en-US", reference_stt=EchoSTT(), mode="direct", + ) + with harness: + report = harness.score(["hello world", "good morning"]) + + # The run completes and every utterance is scored a total miss (WER 1.0) + # rather than the exception propagating and emitting no marker at all. + self.assertEqual(len(report.scores), 2) + self.assertEqual(report.mean_wer, 1.0) + # The report still serialises, so the test's marker can be emitted. + import json + json.loads(json.dumps(report.to_dict())) + + if __name__ == "__main__": unittest.main() From fcd23b34b62997f32e92a6739949bc86bba65c4e Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Tue, 16 Jun 2026 23:53:13 +0000 Subject: [PATCH 36/82] Increment Version to 0.19.2a1 --- ovoscope/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ovoscope/version.py b/ovoscope/version.py index 1b30799..ca56a70 100644 --- a/ovoscope/version.py +++ b/ovoscope/version.py @@ -1,8 +1,8 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 VERSION_MINOR = 19 -VERSION_BUILD = 1 -VERSION_ALPHA = 2 +VERSION_BUILD = 2 +VERSION_ALPHA = 1 # END_VERSION_BLOCK __version__ = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}" + ( From 09d06f8efb269d9a701d7c888747686f8726dc1d Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Tue, 16 Jun 2026 23:53:47 +0000 Subject: [PATCH 37/82] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7e0d75..6d84648 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.19.2a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.19.2a1) (2026-06-16) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.19.1a2...0.19.2a1) + +**Merged pull requests:** + +- fix\(tts-intelligibility\): score synthesis failures as total miss, not abort [\#77](https://github.com/OpenVoiceOS/ovoscope/pull/77) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.19.1a2](https://github.com/OpenVoiceOS/ovoscope/tree/0.19.1a2) (2026-06-15) [Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.19.1a1...0.19.1a2) From eea9b2e51019d85094fbe331b08443d084d96419 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Wed, 17 Jun 2026 01:57:21 +0100 Subject: [PATCH 38/82] fix(tts-intelligibility): transcode non-WAV engine output before scoring (#79) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Engines that natively emit a non-PCM container (gTTS, edge-tts declare audio_ext=mp3) wrote mp3 bytes into a hardcoded .wav path, which the WAV reader could not decode โ€” the round-trip produced an empty transcript and a spurious WER of 1.0. Render to the engine's real audio_ext and transcode any non-WAV output to 16 kHz mono PCM (via PyAV, already a faster-whisper dep) before handing it to the reference STT, so mp3 engines get a real score. Co-authored-by: Claude Opus 4.8 --- ovoscope/tts_intelligibility.py | 58 +++++++++++++++++++++++++++++---- 1 file changed, 51 insertions(+), 7 deletions(-) diff --git a/ovoscope/tts_intelligibility.py b/ovoscope/tts_intelligibility.py index ae25db6..b134d54 100644 --- a/ovoscope/tts_intelligibility.py +++ b/ovoscope/tts_intelligibility.py @@ -304,12 +304,20 @@ def _render_playback(self, utterance: str) -> Optional[str]: return self._copy_out(captured[-1]) def _render_direct(self, utterance: str) -> Optional[str]: - """Synthesise via ``tts.get_tts`` directly; return the WAV path.""" - wav_path = os.path.join( - self._tmpdir, f"direct_{abs(hash(utterance)) & 0xffffffff}.wav" + """Synthesise via ``tts.get_tts`` directly; return the rendered audio path. + + The output extension follows the engine's ``audio_ext`` (``wav`` by + default) so engines that natively emit a non-PCM container (e.g. gTTS / + edge-tts write ``mp3``) produce a valid file instead of mp3 bytes in a + ``.wav`` that the WAV reader can't decode. ``_transcribe`` transcodes + non-WAV output to PCM before scoring. + """ + ext = (getattr(self.tts, "audio_ext", "wav") or "wav").lstrip(".") + out_path = os.path.join( + self._tmpdir, f"direct_{abs(hash(utterance)) & 0xffffffff}.{ext}" ) - self.tts.get_tts(utterance, wav_path, lang=self.lang, voice=self.voice) - return wav_path if os.path.isfile(wav_path) else None + self.tts.get_tts(utterance, out_path, lang=self.lang, voice=self.voice) + return out_path if os.path.isfile(out_path) else None def _copy_out(self, wav_path: str) -> Optional[str]: """Copy a rendered WAV into the harness temp dir before the cache prunes it.""" @@ -328,12 +336,48 @@ def _copy_out(self, wav_path: str) -> Optional[str]: # Scoring # ------------------------------------------------------------------ - def _transcribe(self, wav_path: str) -> str: - """Round-trip a WAV through the reference STT and return the transcript.""" + def _transcribe(self, audio_path: str) -> str: + """Round-trip rendered audio through the reference STT. + + ``AudioFile`` decodes PCM WAV only, so any non-WAV container the engine + produced (mp3 from gTTS / edge-tts, etc.) is transcoded to a temporary + 16 kHz mono PCM WAV first. The transcode uses PyAV, which is already a + faster-whisper dependency, so no extra requirement is introduced. + """ + wav_path = self._ensure_wav(audio_path) with AudioFile(wav_path) as source: audio = source.read() return self.reference_stt.execute(audio, language=self.lang) or "" + def _ensure_wav(self, audio_path: str) -> str: + """Return a PCM-WAV path for ``audio_path``, transcoding if needed.""" + if audio_path.lower().endswith(".wav"): + return audio_path + wav_path = os.path.splitext(audio_path)[0] + ".transcoded.wav" + import av # bundled with faster-whisper + from av.audio.resampler import AudioResampler + + in_container = av.open(audio_path) + out_container = av.open(wav_path, mode="w", format="wav") + out_stream = out_container.add_stream("pcm_s16le", rate=16000) + out_stream.layout = "mono" + resampler = AudioResampler(format="s16", layout="mono", rate=16000) + try: + for frame in in_container.decode(audio=0): + frame.pts = None + for rframe in resampler.resample(frame): + for packet in out_stream.encode(rframe): + out_container.mux(packet) + for rframe in resampler.resample(None): + for packet in out_stream.encode(rframe): + out_container.mux(packet) + for packet in out_stream.encode(None): + out_container.mux(packet) + finally: + out_container.close() + in_container.close() + return wav_path + def score_one(self, utterance: str) -> UtteranceScore: """Synthesise, transcribe, and score a single utterance. From a7cff3a8da7a5767fc2209356bf73293112bdce6 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Wed, 17 Jun 2026 00:57:34 +0000 Subject: [PATCH 39/82] Increment Version to 0.19.3a1 --- ovoscope/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovoscope/version.py b/ovoscope/version.py index ca56a70..9f5680b 100644 --- a/ovoscope/version.py +++ b/ovoscope/version.py @@ -1,7 +1,7 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 VERSION_MINOR = 19 -VERSION_BUILD = 2 +VERSION_BUILD = 3 VERSION_ALPHA = 1 # END_VERSION_BLOCK From 155a5c3aa2ec9a8582c3818a6015a30f3fba5c4d Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Wed, 17 Jun 2026 00:57:57 +0000 Subject: [PATCH 40/82] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d84648..19ce955 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.19.3a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.19.3a1) (2026-06-17) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.19.2a1...0.19.3a1) + +**Merged pull requests:** + +- fix\(tts-intelligibility\): transcode non-WAV engine output before scoring [\#79](https://github.com/OpenVoiceOS/ovoscope/pull/79) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.19.2a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.19.2a1) (2026-06-16) [Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.19.1a2...0.19.2a1) From 1bc5fe2bd092ecbc673876aa2512a217ab4a8a6b Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Wed, 17 Jun 2026 02:07:16 +0100 Subject: [PATCH 41/82] fix(tts-intelligibility): normalise rendered audio to 16kHz mono before STT (#81) Engines that render a WAV at a non-16kHz rate (mimic's ap voice is 44.1kHz) were read back at the wrong rate by the AudioFile -> STT path and heard sped-up, producing word-salad transcripts and a spurious high WER. Normalise every rendered file to 16kHz mono 16-bit PCM (not just non-WAV containers) before transcription; already-conforming WAVs pass through untouched. Co-authored-by: Claude Opus 4.8 --- ovoscope/tts_intelligibility.py | 35 +++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/ovoscope/tts_intelligibility.py b/ovoscope/tts_intelligibility.py index b134d54..826b6f9 100644 --- a/ovoscope/tts_intelligibility.py +++ b/ovoscope/tts_intelligibility.py @@ -339,10 +339,13 @@ def _copy_out(self, wav_path: str) -> Optional[str]: def _transcribe(self, audio_path: str) -> str: """Round-trip rendered audio through the reference STT. - ``AudioFile`` decodes PCM WAV only, so any non-WAV container the engine - produced (mp3 from gTTS / edge-tts, etc.) is transcoded to a temporary - 16 kHz mono PCM WAV first. The transcode uses PyAV, which is already a - faster-whisper dependency, so no extra requirement is introduced. + The rendered file is normalised to a 16 kHz mono PCM WAV first. This + covers two cases the ``AudioFile`` -> STT path otherwise mishandles: + non-WAV containers (mp3 from gTTS / edge-tts, which ``AudioFile`` cannot + decode) and WAVs at a non-16 kHz rate (e.g. mimic's 44.1 kHz ``ap`` + voice, which the STT reads at the wrong rate and hears sped-up). The + normalisation uses PyAV, already a faster-whisper dependency, so no + extra requirement is introduced. """ wav_path = self._ensure_wav(audio_path) with AudioFile(wav_path) as source: @@ -350,8 +353,13 @@ def _transcribe(self, audio_path: str) -> str: return self.reference_stt.execute(audio, language=self.lang) or "" def _ensure_wav(self, audio_path: str) -> str: - """Return a PCM-WAV path for ``audio_path``, transcoding if needed.""" - if audio_path.lower().endswith(".wav"): + """Return a 16 kHz mono PCM-WAV path for ``audio_path``. + + Already-conforming WAVs (16 kHz, mono, 16-bit PCM) are returned as-is; + anything else (other container, rate, channel count, or sample format) + is transcoded with PyAV. + """ + if self._is_pcm16k_mono(audio_path): return audio_path wav_path = os.path.splitext(audio_path)[0] + ".transcoded.wav" import av # bundled with faster-whisper @@ -378,6 +386,21 @@ def _ensure_wav(self, audio_path: str) -> str: in_container.close() return wav_path + @staticmethod + def _is_pcm16k_mono(audio_path: str) -> bool: + """True if ``audio_path`` is already a 16 kHz mono 16-bit PCM WAV.""" + if not audio_path.lower().endswith(".wav"): + return False + import wave + + try: + with wave.open(audio_path, "rb") as w: + return (w.getframerate() == 16000 + and w.getnchannels() == 1 + and w.getsampwidth() == 2) + except (wave.Error, EOFError, OSError): + return False + def score_one(self, utterance: str) -> UtteranceScore: """Synthesise, transcribe, and score a single utterance. From 622b13f33729e463372a9bdd96c9b9bb220c29a7 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Wed, 17 Jun 2026 01:07:28 +0000 Subject: [PATCH 42/82] Increment Version to 0.19.4a1 --- ovoscope/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovoscope/version.py b/ovoscope/version.py index 9f5680b..dcc3cc4 100644 --- a/ovoscope/version.py +++ b/ovoscope/version.py @@ -1,7 +1,7 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 VERSION_MINOR = 19 -VERSION_BUILD = 3 +VERSION_BUILD = 4 VERSION_ALPHA = 1 # END_VERSION_BLOCK From dcbcf4ce302d80ba7e63eea4368cd50832bfed74 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Wed, 17 Jun 2026 01:07:49 +0000 Subject: [PATCH 43/82] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19ce955..ad2be28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.19.4a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.19.4a1) (2026-06-17) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.19.3a1...0.19.4a1) + +**Merged pull requests:** + +- fix\(tts-intelligibility\): normalise rendered audio to 16kHz mono before STT [\#81](https://github.com/OpenVoiceOS/ovoscope/pull/81) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.19.3a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.19.3a1) (2026-06-17) [Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.19.2a1...0.19.3a1) From 807cf9ed902e8a509a97081828e8ae36369cec1c Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Wed, 24 Jun 2026 18:32:45 +0100 Subject: [PATCH 44/82] feat: assert_template_shown for SYSTEM_* GUI templates (#83) * feat: add GUICaptureSession.assert_template_shown for SYSTEM_* templates Ergonomic GUI assertion for the template-based GUI: asserts a built-in SYSTEM_* template was shown (prefix optional) plus its accompanying gui.value.set session data, in one call. Backend-agnostic. Adds unit tests and a docs section. * fix: normalize single source_message robustly across Message classes End2EndTest normalized source_message to a list via isinstance(Message), but the message class can come from ovos_bus_client / ovos_spec_tools / ovos_utils.fakebus depending on installed versions; a cross-class isinstance is False, leaving a single non-iterable Message that broke later iteration (TypeError: 'Message' object is not iterable on newer stacks). Normalize on 'not a list' instead. Fixes the test_remote_recorder build failure. --- docs/gui-testing.md | 24 ++++++++++++++++ ovoscope/__init__.py | 35 ++++++++++++++++++++++-- test/unittests/test_gui_capture.py | 44 ++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 2 deletions(-) diff --git a/docs/gui-testing.md b/docs/gui-testing.md index 7a04675..cf1ae47 100644 --- a/docs/gui-testing.md +++ b/docs/gui-testing.md @@ -225,6 +225,30 @@ Setting `ignore_gui=True` (the default on `End2EndTest`) keeps the ordered message sequence clean while `GUICaptureSession` captures the GUI events independently. +## Template-Based GUI (`SYSTEM_*` templates) + +Skills that use the typed template API (`self.gui.show_weather(...)`, +`show_text(...)`, `show_list(...)`, โ€ฆ) emit a `gui.page.show` for a built-in +`SYSTEM_*` template plus `gui.value.set` for its data keys. `assert_template_shown` +checks both in one call: + +```python +with GUICaptureSession(mc.bus) as gui: + mc.bus.emit(Message("recognizer_loop:utterance", + {"utterances": ["what's the weather"], "lang": "en-US"})) + import time; time.sleep(2) + # the SYSTEM_ prefix is optional ("weather" == "SYSTEM_weather") + gui.assert_template_shown( + "ovos-skill-weather.openvoiceos", + "weather", + values={"current_temp": 22, "condition": "Sunny"}, + ) +``` + +This is the recommended assertion for the template-based GUI: it does not care +which display backend (Qt, pyhtmx, โ€ฆ) renders the template โ€” only that the skill +requested the right `SYSTEM_*` template with the right session data. + ## What `GUICaptureSession` Does NOT Cover - Full GUI rendering โ€” only bus messages are captured; no QML engine is run. diff --git a/ovoscope/__init__.py b/ovoscope/__init__.py index 56eccda..e1c94af 100644 --- a/ovoscope/__init__.py +++ b/ovoscope/__init__.py @@ -733,8 +733,12 @@ def __post_init__(self): if GLOBAL_BUS_COVERAGE: self.track_bus_coverage = True - # standardize to be a list - if isinstance(self.source_message, Message): + # standardize to be a list. Use "not a list" rather than an + # isinstance(Message) check: depending on installed versions the + # message class may come from ovos_bus_client / ovos_spec_tools / + # ovos_utils.fakebus, and a cross-class isinstance can be False โ€” which + # would leave a single (non-iterable) Message and break later iteration. + if not isinstance(self.source_message, list): self.source_message = [self.source_message] if self.ignore_gui: # ensure we don't mutate a shared default list @@ -1249,6 +1253,33 @@ def assert_page_shown(self, namespace: str, page: str, timeout: float = 2.0) -> f"but no matching gui.page.show message was captured.\nGot: {captured}" ) + def assert_template_shown(self, namespace: str, template: str, + values: Optional[Dict[str, Any]] = None, + timeout: float = 2.0) -> None: + """Assert that a built-in ``SYSTEM_*`` template was shown. + + Ergonomic helper for the template-based GUI: a skill calling a typed + method such as ``self.gui.show_weather(...)`` emits a + ``gui.page.show`` for the ``SYSTEM_weather`` template plus + ``gui.value.set`` for its data keys. This asserts both in one call. + + Args: + namespace: GUI namespace (typically the skill ID). + template: Template name, with or without the ``SYSTEM_`` prefix + (``"weather"`` and ``"SYSTEM_weather"`` are equivalent). + values: Optional mapping of session-data keys to expected values; + each is checked via :meth:`assert_namespace_value`. + timeout: Maximum seconds to wait for the page-show message. + + Raises: + AssertionError: If the template was not shown, or a listed value + was not set. + """ + name = template if template.startswith("SYSTEM_") else f"SYSTEM_{template}" + self.assert_page_shown(namespace, name, timeout=timeout) + for key, value in (values or {}).items(): + self.assert_namespace_value(namespace, key, value) + def assert_namespace_value(self, namespace: str, key: str, value: Any) -> None: """Assert that a namespace key was set to a specific value. diff --git a/test/unittests/test_gui_capture.py b/test/unittests/test_gui_capture.py index 11cc11d..70fc994 100644 --- a/test/unittests/test_gui_capture.py +++ b/test/unittests/test_gui_capture.py @@ -106,6 +106,50 @@ def test_assert_namespace_has_key_from_field(self) -> None: )] session.assert_namespace_has_key("weather", "current_temp") + # -- assert_template_shown (SYSTEM_* template model) -- + + def test_assert_template_shown_with_prefix(self) -> None: + """Full SYSTEM_ name should match the shown template.""" + session = self._make_session() + session.messages = [self._page_show_msg("weatherskill", "SYSTEM_weather")] + session.assert_template_shown("weatherskill", "SYSTEM_weather") + + def test_assert_template_shown_without_prefix(self) -> None: + """Short name is normalized to the SYSTEM_ prefix.""" + session = self._make_session() + session.messages = [self._page_show_msg("weatherskill", "SYSTEM_weather")] + session.assert_template_shown("weatherskill", "weather") + + def test_assert_template_shown_with_values(self) -> None: + """Template + accompanying session-data values both asserted.""" + session = self._make_session() + session.messages = [ + self._page_show_msg("weatherskill", "SYSTEM_weather"), + self._value_set_msg("weatherskill", {"current_temp": 22, + "condition": "Sunny"}), + ] + session.assert_template_shown("weatherskill", "weather", + values={"current_temp": 22, + "condition": "Sunny"}) + + def test_assert_template_shown_missing_template(self) -> None: + """Template never shown should raise.""" + session = self._make_session() + session.messages = [self._page_show_msg("weatherskill", "SYSTEM_text")] + with self.assertRaises(AssertionError): + session.assert_template_shown("weatherskill", "weather", timeout=0.1) + + def test_assert_template_shown_wrong_value(self) -> None: + """Template shown but a listed value differs should raise.""" + session = self._make_session() + session.messages = [ + self._page_show_msg("weatherskill", "SYSTEM_weather"), + self._value_set_msg("weatherskill", {"current_temp": 22}), + ] + with self.assertRaises(AssertionError): + session.assert_template_shown("weatherskill", "weather", + values={"current_temp": 99}) + # -- assert_namespace_cleared -- def test_assert_namespace_cleared_match(self) -> None: From b65ba6a196ab47b243ece438f5ba348e31dafe5f Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Wed, 24 Jun 2026 17:32:56 +0000 Subject: [PATCH 45/82] Increment Version to 0.20.0a1 --- ovoscope/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ovoscope/version.py b/ovoscope/version.py index dcc3cc4..bc191ae 100644 --- a/ovoscope/version.py +++ b/ovoscope/version.py @@ -1,7 +1,7 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 -VERSION_MINOR = 19 -VERSION_BUILD = 4 +VERSION_MINOR = 20 +VERSION_BUILD = 0 VERSION_ALPHA = 1 # END_VERSION_BLOCK From f3283daf283591a0d42e76d670bc3c588e6e9f83 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Wed, 24 Jun 2026 17:33:19 +0000 Subject: [PATCH 46/82] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad2be28..15d94a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.20.0a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.20.0a1) (2026-06-24) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.19.4a1...0.20.0a1) + +**Merged pull requests:** + +- feat: assert\_template\_shown for SYSTEM\_\* GUI templates [\#83](https://github.com/OpenVoiceOS/ovoscope/pull/83) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.19.4a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.19.4a1) (2026-06-17) [Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.19.3a1...0.19.4a1) From b32c86e070d055250737b9b6f5d3b2ee935bfa17 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Thu, 25 Jun 2026 18:32:34 +0100 Subject: [PATCH 47/82] feat: export ovos-media OCP harness from the package + add [media] extra (#89) * feat: export ovos-media OCP harness from the package + add [media] extra ovoscope.media (OCPPlayerHarness, OCPCaptureSession, MockOCPBackend) was importable only via the submodule path, while the ovos-audio AudioServiceHarness is re-exported from the top-level package. Mirror that: re-export the media harness from ovoscope/__init__.py (guarded like the audio block) and add a [media] extra declaring ovos-media, so OCP/ovos-media backends get a first-class 'from ovoscope import OCPPlayerHarness' entry point alongside the legacy audio one. ovos-media is imported lazily inside the harness, so the export needs no extra at import time; the [media] extra is required only to run the harness. Co-Authored-By: Claude Opus 4.8 * feat: MediaProviderHarness + injectable backend for OCPPlayerHarness Add ovos-media test-harness support for the two e2e shapes that previously had to be hand-rolled by consumers: - MediaProviderHarness (ovoscope/media_provider.py): a dependency-free, duck-typed harness for opm.media.provider catalog/search plugins. from_entrypoint/from_class constructors, drivers mirroring the pipeline path (serves -> search_safe), and assert helpers. Imports neither mediavocab nor opm's MediaProvider, so it needs no extra and is re-exported unconditionally. - OCPPlayerHarness backend_factory: optional bus->AudioBackend callable; when given, the harness wires a real AudioService (no autoload) with the injected backend so the player's play->load_track->LOADED_MEDIA->backend.play() path actually drives a real OCP backend (e.g. a Music Assistant audio backend). Default stays MockOCPBackend. Tests + docs included; CI build-tests/coverage extras gain 'media' so the new harness tests run. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- .github/workflows/build-tests.yml | 2 +- .github/workflows/coverage.yml | 1 + docs/index.md | 3 + docs/media-provider-testing.md | 97 ++++++++++++ docs/media-testing.md | 45 ++++++ ovoscope/__init__.py | 19 +++ ovoscope/media.py | 92 ++++++++--- ovoscope/media_provider.py | 194 +++++++++++++++++++++++ pyproject.toml | 3 + test/unittests/test_media.py | 96 +++++++++++- test/unittests/test_media_provider.py | 217 ++++++++++++++++++++++++++ 11 files changed, 746 insertions(+), 23 deletions(-) create mode 100644 docs/media-provider-testing.md create mode 100644 ovoscope/media_provider.py create mode 100644 test/unittests/test_media_provider.py diff --git a/.github/workflows/build-tests.yml b/.github/workflows/build-tests.yml index e1fc761..439aded 100644 --- a/.github/workflows/build-tests.yml +++ b/.github/workflows/build-tests.yml @@ -11,5 +11,5 @@ jobs: secrets: inherit with: python_versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]' - install_extras: 'audio,pydantic' + install_extras: 'audio,pydantic,media' test_path: 'test/unittests/' diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 22cc219..49263df 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -13,4 +13,5 @@ jobs: python_version: '3.11' coverage_source: 'ovoscope' test_path: 'test/unittests/' + test_extras: 'audio,pydantic,media' min_coverage: 0 diff --git a/docs/index.md b/docs/index.md index b9833ed..2d23772 100644 --- a/docs/index.md +++ b/docs/index.md @@ -10,6 +10,9 @@ | [end2end-test.md](end2end-test.md) | `End2EndTest` โ€” full test runner reference | | [pydantic-integration.md](pydantic-integration.md) | Using `ovos-pydantic-models` with OvoScope | | [audio-testing.md](audio-testing.md) | `AudioServiceHarness`, `PlaybackServiceHarness` โ€” testing audio services | +| [media-testing.md](media-testing.md) | `OCPPlayerHarness`, `OCPCaptureSession`, `MockOCPBackend` โ€” testing the `ovos-media` OCP player (and driving a real OCP backend) | +| [media-provider-testing.md](media-provider-testing.md) | `MediaProviderHarness` โ€” testing `opm.media.provider` catalog/search plugins | +| [ocp.md](ocp.md) | `OCPTest` โ€” testing legacy OCP search skills (`@ocp_search`) | | [listener.md](listener.md) | `MiniListener`, `get_mini_listener`, `ListenerTest`, `MockVADEngine`, `MockHotWordEngine`, `VADTest`, `WakeWordTest` โ€” testing audio transformer plugins, STT pipeline, VAD, and wake-word | | [voice-loop.md](voice-loop.md) | `MiniVoiceLoop` / `MiniSimpleListener` / `MiniClassicListener` โ€” file-driven bus-sequence testing for the ovos-dinkum, ovos-simple, and mycroft-classic listener services (wake-word โ†’ record-begin โ†’ utterance), with verifier-chain gating | | [gui-testing.md](gui-testing.md) | `GUICaptureSession` โ€” asserting GUI page navigation and namespace values | diff --git a/docs/media-provider-testing.md b/docs/media-provider-testing.md new file mode 100644 index 0000000..4e84a98 --- /dev/null +++ b/docs/media-provider-testing.md @@ -0,0 +1,97 @@ +# MediaProvider (Search) Testing with ovoscope + +`MediaProviderHarness` (`ovoscope.media_provider`) tests `opm.media.provider` +plugins โ€” the in-process **catalog/search** providers introduced by the +ovos-media sprint to replace OCP search skills. It is the search-side counterpart +to [`OCPPlayerHarness`](media-testing.md), which drives the *player*. + +> **No extra required.** Unlike the player harness, `MediaProviderHarness` is +> **duck-typed**: it imports neither `mediavocab` nor `ovos-plugin-manager`'s +> `MediaProvider` / `QueryContext`. Your *test* supplies the `Signals` / +> `QueryContext` objects, so ovoscope itself stays dependency-free. The provider +> package and `mediavocab` only need to be installed for the test that uses it. + +## Why a dedicated harness + +There is no released loader for the `opm.media.provider` entry-point group, and +the OCP pipeline does not yet load providers in-process. So the harness models the +pipeline's *intended* path and removes the boilerplate every provider e2e would +otherwise repeat: + +``` +discover the entry-point -> serves(signals, context) gate -> search_safe() +``` + +## Basic usage + +```python +from ovoscope import MediaProviderHarness +from mediavocab import MediaType, Signals +from ovos_plugin_manager.templates.media_provider import QueryContext + +h = MediaProviderHarness.from_entrypoint( + "music_assistant", # opm.media.provider entry-point name + config={"url": "http://mass.local:8095"}, + mock_api=my_mock_client, # injected onto provider._api +) + +# packaging registered the plugin +h.assert_entrypoint_registered() + +# three-axis + context routing gate +h.assert_routes(Signals(medium=MediaType.MUSIC), + QueryContext(supported_playback_types={"audio"})) +h.assert_not_routes(Signals(medium=MediaType.MUSIC), + QueryContext(supported_playback_types={"video"})) # audio-only provider +h.assert_not_routes(Signals(medium=MediaType.MOVIE)) + +# the never-raising search the pipeline calls โ€” ranked, playable results +releases = h.assert_returns_playables(Signals(title="worms")) +assert all(r.uri.startswith("library://") for r in releases) +``` + +## Constructing the harness + +| Constructor | Use | +|---|---| +| `MediaProviderHarness.from_entrypoint(name, config=None, group="opm.media.provider", mock_api=None, api_attr="_api")` | Discover the provider through its installed entry-point (the real e2e). Raises `AssertionError` if the entry-point is missing or ambiguous. | +| `MediaProviderHarness.from_class(provider_cls, config=None, mock_api=None, api_attr="_api")` | Wrap a class you already hold (no packaging needed) โ€” handy for unit tests. | + +`mock_api` is set onto `provider.` (default `_api`) to bypass the lazy, +network-backed client a provider builds on first use. + +## Drivers + +Thin pass-throughs mirroring the `MediaProvider` contract: + +| Method | Delegates to | +|---|---| +| `is_available()` | `provider.is_available()` | +| `serves(signals, context=None)` | `provider.serves(...)` | +| `search(signals, lang="en-us")` | `provider.search(...)` (may raise) | +| `search_safe(signals, context=None, lang="en-us")` | `provider.search_safe(...)` (never raises) | +| `featured_media(lang="en-us")` | `provider.featured_media(...)` | + +## Assertions + +| Method | Checks | +|---|---| +| `assert_entrypoint_registered(name=None, group=None)` | the provider is discoverable under its entry-point group | +| `assert_routes(signals, context=None)` | `serves(...)` is `True` | +| `assert_not_routes(signals, context=None)` | `serves(...)` is `False` | +| `assert_returns_playables(signals, context=None, lang="en-us")` | `search_safe` returns results, each with a truthy `uri`, a `match_confidence` in `[0,1]`, and a `work`; returns the results | + +## Exposed attributes + +| Attribute | Description | +|---|---| +| `provider` | the wrapped provider instance | +| `api` | the injected mock client (for custom `assert_called_with` checks) | +| `entrypoint_name` / `entrypoint_group` | set when built via `from_entrypoint` | + +## Cross-references + +- `MediaProvider` / `QueryContext` โ€” `ovos_plugin_manager.templates.media_provider` +- `Signals`, `Release`, `MediaType` โ€” `mediavocab` +- Player-side harness โ€” [media-testing.md](media-testing.md) +- OCP *search skill* testing (legacy stack) โ€” [ocp.md](ocp.md) diff --git a/docs/media-testing.md b/docs/media-testing.md index 3516676..5ab02b3 100644 --- a/docs/media-testing.md +++ b/docs/media-testing.md @@ -190,6 +190,45 @@ with OCPPlayerHarness() as h: # Player auto-advances or stops depending on queue + autoplay config ``` +### Driving a Real OCP Backend + +By default `OCPPlayerHarness` injects a `MockOCPBackend` and mocks out +`AudioService`, so it exercises the **player state machine** but never the real +backend routing. To test a **real** OCP audio backend end-to-end โ€” e.g. assert +that playing a uri makes a Music Assistant backend call its server โ€” pass a +`backend_factory`: a `bus -> AudioBackend` callable. The harness then wires a +*real* `AudioService` (no autoload) with your backend as its sole service, so the +player's `play -> load_track -> LOADED_MEDIA -> backend.play()` path actually +reaches it. + +```python +from ovoscope.media import OCPPlayerHarness +from ovos_utils.ocp import MediaEntry, PlaybackType + +def make_backend(bus): + backend = MAssOCPAudioService(config={"url": "http://mass.local:8095"}, bus=bus) + backend.api = mock_client # mock the network client the backend reaches + backend.player_state = {"available": True} + return backend + +with OCPPlayerHarness(backend_factory=make_backend) as h: + h.play(MediaEntry(uri="library://track/42", playback=PlaybackType.AUDIO)) + h.backend.api.play_media.assert_called_once_with(queue_id, "library://track/42") +``` + +Notes: + +- The factory **owns mocking** any network client the real backend would reach. +- Deferred uris (`library://`, `{sei}//โ€ฆ`) are resolved by the OCP pipeline's + stream extractors *before* the player in production; the harness loads no + extractor plugins, so it bypasses the player's stream validation when a backend + factory is used. +- `name`/`namespace` are supplied by the harness if the backend lacks them + (normally set by `BaseMediaService.load_services()`, which the harness bypasses). +- The mock-only helpers (`assert_backend_paused`, `backend.played_uris`) assume a + `MockOCPBackend` and may not apply to a real backend โ€” assert on the backend's + own state/spies instead. + ## OCPCaptureSession `OCPCaptureSession` โ€” `ovoscope/media.py` @@ -245,6 +284,12 @@ with OCPPlayerHarness() as h: `OCPPlayerHarness` โ€” `ovoscope/media.py` +**Constructor:** `OCPPlayerHarness(backend_namespace="audio", backend_factory=None)`. +`backend_factory` is an optional `bus -> AudioBackend` callable; when given, the +harness drives that real backend through a real `AudioService` (see +[Driving a Real OCP Backend](#driving-a-real-ocp-backend)) instead of the default +`MockOCPBackend`. + **Control methods** (each emits the corresponding bus message + `time.sleep(0.05)`): | Method | Bus message emitted | diff --git a/ovoscope/__init__.py b/ovoscope/__init__.py index e1c94af..26c4fe3 100644 --- a/ovoscope/__init__.py +++ b/ovoscope/__init__.py @@ -1099,6 +1099,25 @@ def assert_spoke(self, text: str, lang: str = "en-US", timeout: int = 30) -> Non else: raise +try: + from ovoscope.media import ( # noqa: F401 + MockOCPBackend, + OCPPlayerHarness, + OCPCaptureSession, + ) +except ImportError as e: + # Optional [media] extra (ovos-media). Silence only when the missing module + # is ovos-media itself; a logic error in a present lib must re-raise. + if isinstance(e, ModuleNotFoundError) and e.name in ("ovos_media", "ovos_media.player"): + pass + else: + raise + +# MediaProvider (catalog/search) harness โ€” duck-typed, stdlib-only at import time, +# so it needs no optional dependency guard (the provider package + mediavocab are +# only needed by the *test* that uses it, not by ovoscope itself). +from ovoscope.media_provider import MediaProviderHarness # noqa: F401,E402 + try: from ovoscope.tts_intelligibility import ( # noqa: F401 TTSIntelligibilityHarness, diff --git a/ovoscope/media.py b/ovoscope/media.py index 0c07ee1..4016331 100644 --- a/ovoscope/media.py +++ b/ovoscope/media.py @@ -26,7 +26,7 @@ import dataclasses import time -from typing import List, Optional +from typing import Callable, List, Optional from unittest.mock import MagicMock, patch from ovos_bus_client.message import Message @@ -268,16 +268,28 @@ class OCPPlayerHarness: backend_namespace: Namespace for ``MockOCPBackend``; default ``"audio"``. """ - def __init__(self, backend_namespace: str = "audio") -> None: + def __init__(self, backend_namespace: str = "audio", + backend_factory: Optional[Callable[[FakeBus], AudioBackend]] = None) -> None: """Initialise harness parameters. Args: backend_namespace: Namespace prefix passed to ``MockOCPBackend``. + backend_factory: Optional ``bus -> AudioBackend`` callable used to build + the injected backend instead of the default :class:`MockOCPBackend`. + The harness owns the ``FakeBus``, so a *factory* (not a pre-built + instance) is taken: it is called with the harness bus inside + ``__enter__``. Use it to drive a **real** OCP backend (e.g. a + Music Assistant audio backend) through the real ``OCPMediaPlayer``; + the factory is responsible for mocking any network client the real + backend would otherwise reach. Note the mock-only assertion helpers + (:meth:`assert_backend_paused`, ``backend.played_uris``) assume a + :class:`MockOCPBackend` and may not apply to a real backend. """ self.backend_namespace: str = backend_namespace + self.backend_factory = backend_factory self.bus: Optional[FakeBus] = None self.player = None # OCPMediaPlayer instance - self.backend: Optional[MockOCPBackend] = None + self.backend: Optional[AudioBackend] = None self.gui: Optional[MagicMock] = None self._patches: list = [] @@ -290,9 +302,19 @@ def __enter__(self) -> "OCPPlayerHarness": from ovos_media.player import OCPMediaPlayer self.bus = FakeBus() - self.backend = MockOCPBackend( - config={}, bus=self.bus, namespace=self.backend_namespace - ) + if self.backend_factory is not None: + self.backend = self.backend_factory(self.bus) + # A config-loaded backend gets name/namespace from + # BaseMediaService.load_services(), which the harness bypasses โ€” supply + # sane defaults so the service's bookkeeping (shutdown, routing) works. + if not getattr(self.backend, "name", None): + self.backend.name = "test-backend" + if not getattr(self.backend, "namespace", None): + self.backend.namespace = self.backend_namespace + else: + self.backend = MockOCPBackend( + config={}, bus=self.bus, namespace=self.backend_namespace + ) # Build patch targets gui_mock = MagicMock() @@ -341,22 +363,50 @@ def __init__(self, *args, **kwargs): # Instantiate the real player (all heavy deps are now mocked) self.player = OCPMediaPlayer(self.bus, config={}) - # Inject MockOCPBackend as the sole audio backend - audio_svc = self.player.audio_service - audio_svc.services = [self.backend] - audio_svc.default = self.backend - self.backend.set_track_start_callback(audio_svc.track_start) - - # Register the audio service bus handlers manually - # (normally done inside BaseMediaService.load_services) ns = self.backend_namespace - self.bus.on(f"ovos.{ns}.service.play", audio_svc.handle_play) - self.bus.on(f"ovos.{ns}.service.pause", audio_svc.pause) - self.bus.on(f"ovos.{ns}.service.resume", audio_svc.resume) - self.bus.on(f"ovos.{ns}.service.stop", audio_svc.stop) - self.bus.on("ovos.common_play.media.state", - audio_svc.handle_media_state_change) - audio_svc._loaded.set() + if self.backend_factory is not None: + # Real-backend mode: the mocked AudioService (a MagicMock) never routes + # play()->load_track()->backend.play(), so swap in a *real* AudioService + # with autoload off and the injected backend as its sole service. Now the + # player's playback path actually drives the real backend (e.g. asserting + # a Music Assistant client's play_media() call). + from ovos_media.media_backends.audio import AudioService as _RealAudioService + audio_svc = _RealAudioService(self.bus, config={"audio_players": {}}, + autoload=False, validate_source=False) + self.player.audio_service = audio_svc + # Deferred uris (e.g. library://, {sei}//) are resolved by the OCP + # pipeline's stream extractors *before* the player sees them; this + # harness drives the backend directly, so bypass the player's + # stream-extraction validation (no extractor plugins are loaded). + self.player.validate_stream = lambda: True + audio_svc.services = [self.backend] + audio_svc.default = self.backend + self.backend.set_track_start_callback(audio_svc.track_start) + # load_services() (skipped with autoload=False) would register these. + self.bus.on(f"ovos.{ns}.service.play", audio_svc.handle_play) + self.bus.on(f"ovos.{ns}.service.pause", audio_svc.pause) + self.bus.on(f"ovos.{ns}.service.resume", audio_svc.resume) + self.bus.on(f"ovos.{ns}.service.stop", audio_svc.stop) + # NB: BaseMediaService.__init__ already wired ovos.common_play.media.state + # -> handle_media_state_change; re-registering it would fire backend.play() + # twice, so it is deliberately omitted here. + audio_svc._loaded.set() + else: + # Mock-backend mode: drive the player state-machine against the MagicMock + # AudioService; the backend is exposed for manual simulate_*/state asserts. + audio_svc = self.player.audio_service + audio_svc.services = [self.backend] + audio_svc.default = self.backend + self.backend.set_track_start_callback(audio_svc.track_start) + # Register the audio service bus handlers manually + # (normally done inside BaseMediaService.load_services) + self.bus.on(f"ovos.{ns}.service.play", audio_svc.handle_play) + self.bus.on(f"ovos.{ns}.service.pause", audio_svc.pause) + self.bus.on(f"ovos.{ns}.service.resume", audio_svc.resume) + self.bus.on(f"ovos.{ns}.service.stop", audio_svc.stop) + self.bus.on("ovos.common_play.media.state", + audio_svc.handle_media_state_change) + audio_svc._loaded.set() return self diff --git a/ovoscope/media_provider.py b/ovoscope/media_provider.py new file mode 100644 index 0000000..c70f476 --- /dev/null +++ b/ovoscope/media_provider.py @@ -0,0 +1,194 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""MediaProvider (catalog/search) test harness for ovoscope. + +``ovoscope.media`` drives the OCP *player* state-machine; this module is its +catalog/search counterpart โ€” a harness for ``opm.media.provider`` plugins +(``MediaProvider`` subclasses introduced by the ovos-media sprint). + +There is no released loader for the ``opm.media.provider`` entry-point group, and +the OCP pipeline does not yet load providers in-process, so this harness models +the pipeline's *intended* path: + + discover the provider -> gate it with ``serves(signals, context)`` + -> call the never-raising ``search_safe`` + +It is deliberately **duck-typed**: it imports neither mediavocab (``Signals`` / +``Release``) nor ovos-plugin-manager's ``MediaProvider`` / ``QueryContext``. The +test supplies whatever ``signals`` / ``context`` objects the provider expects, so +ovoscope stays dependency-free and usable even without the (currently branch-only) +``opm.media.provider`` plugin type installed. + +Usage:: + + from ovoscope import MediaProviderHarness + + h = MediaProviderHarness.from_entrypoint( + "music_assistant", + config={"url": "http://mass.local:8095"}, + mock_api=my_mock_client, # injected onto provider._api + ) + h.assert_entrypoint_registered() + h.assert_routes(Signals(medium=MediaType.MUSIC), + QueryContext(supported_playback_types={"audio"})) + h.assert_not_routes(Signals(medium=MediaType.MOVIE)) + releases = h.assert_returns_playables(Signals(title="worms")) +""" +from __future__ import annotations + +from importlib.metadata import entry_points +from typing import Any, List, Optional + +DEFAULT_GROUP = "opm.media.provider" + + +class MediaProviderHarness: + """Wrap a ``MediaProvider`` instance and drive it the way the OCP pipeline does. + + Build one with :meth:`from_entrypoint` (real plugin discovery) or + :meth:`from_class` (a class you already hold โ€” used by ovoscope's own tests). + The wrapped instance is exposed as :attr:`provider` and the injected mock + client as :attr:`api` for custom assertions. + """ + + def __init__(self, provider: Any, api: Any = None, + entrypoint_name: Optional[str] = None, + entrypoint_group: str = DEFAULT_GROUP) -> None: + self.provider = provider + self.api = api + self.entrypoint_name = entrypoint_name + self.entrypoint_group = entrypoint_group + + # ------------------------------------------------------------------ + # Constructors + # ------------------------------------------------------------------ + + @classmethod + def from_class(cls, provider_cls: Any, config: Optional[dict] = None, + mock_api: Any = None, api_attr: str = "_api") -> "MediaProviderHarness": + """Instantiate ``provider_cls(config)`` and (optionally) inject ``mock_api``. + + Args: + provider_cls: a ``MediaProvider`` subclass (duck-typed โ€” anything with + ``is_available`` / ``serves`` / ``search`` / ``search_safe`` / + ``featured_media``). + config: config dict passed to the provider constructor. + mock_api: object set onto ``provider.`` to bypass the real + (network) client built lazily by the provider. + api_attr: attribute name the provider reads its client from + (default ``"_api"``). + """ + provider = provider_cls(config or {}) + if mock_api is not None: + setattr(provider, api_attr, mock_api) + return cls(provider, api=mock_api) + + @classmethod + def from_entrypoint(cls, name: str, config: Optional[dict] = None, + group: str = DEFAULT_GROUP, mock_api: Any = None, + api_attr: str = "_api") -> "MediaProviderHarness": + """Discover the provider through its installed entry-point and wrap it. + + Resolves ``name`` in the ``group`` entry-point group (default + ``opm.media.provider``), loads the class, and delegates to + :meth:`from_class`. Raises ``AssertionError`` if the entry-point is not + installed (or is ambiguous) โ€” that *is* the e2e signal that the plugin's + packaging registered it correctly. + """ + matches = [ep for ep in entry_points(group=group) if ep.name == name] + if not matches: + raise AssertionError( + f"no {group!r} entry-point named {name!r} is installed โ€” " + f"is the provider package installed (pip install -e .)?" + ) + if len(matches) > 1: + raise AssertionError( + f"multiple {group!r} entry-points named {name!r}: {matches!r}" + ) + provider_cls = matches[0].load() + harness = cls.from_class(provider_cls, config=config, + mock_api=mock_api, api_attr=api_attr) + harness.entrypoint_name = name + harness.entrypoint_group = group + return harness + + # ------------------------------------------------------------------ + # Drivers โ€” mirror the pipeline's discover -> gate -> search path + # ------------------------------------------------------------------ + + def is_available(self) -> bool: + """Provider self-check (server reachable / keys present).""" + return self.provider.is_available() + + def serves(self, signals: Any, context: Any = None) -> bool: + """Context-aware routing gate (three-axis ``matches`` + device/policy).""" + return self.provider.serves(signals, context) + + def search(self, signals: Any, lang: str = "en-us") -> List[Any]: + """Raw search โ€” may raise, mirroring a direct provider call.""" + return self.provider.search(signals, lang=lang) + + def search_safe(self, signals: Any, context: Any = None, + lang: str = "en-us") -> List[Any]: + """The never-raising entry the pipeline's thread-pool dispatch calls.""" + return self.provider.search_safe(signals, context=context, lang=lang) + + def featured_media(self, lang: str = "en-us") -> List[Any]: + """Curated/home content (recently-played, recommendations, โ€ฆ).""" + return self.provider.featured_media(lang=lang) + + # ------------------------------------------------------------------ + # Assertions + # ------------------------------------------------------------------ + + def assert_entrypoint_registered(self, name: Optional[str] = None, + group: Optional[str] = None) -> None: + """Assert the provider is discoverable under its entry-point group.""" + name = name or self.entrypoint_name + group = group or self.entrypoint_group + assert name, ("no entry-point name to check โ€” pass name=, or build the " + "harness with from_entrypoint()") + names = [ep.name for ep in entry_points(group=group)] + assert name in names, ( + f"{name!r} is not registered under {group!r}; found {names!r}" + ) + + def assert_routes(self, signals: Any, context: Any = None) -> None: + """Assert the provider serves ``signals`` under ``context``.""" + assert self.serves(signals, context), ( + f"provider does not serve {signals!r} (context={context!r})" + ) + + def assert_not_routes(self, signals: Any, context: Any = None) -> None: + """Assert the provider is gated out for ``signals`` under ``context``.""" + assert not self.serves(signals, context), ( + f"provider unexpectedly serves {signals!r} (context={context!r})" + ) + + def assert_returns_playables(self, signals: Any, context: Any = None, + lang: str = "en-us") -> List[Any]: + """Assert ``search_safe`` returns ranked, playable results and return them. + + Each result must carry a truthy ``uri``, a ``match_confidence`` in + ``[0.0, 1.0]``, and a ``work``. + """ + results = self.search_safe(signals, context=context, lang=lang) + assert results, f"provider returned no results for {signals!r}" + for r in results: + assert getattr(r, "uri", None), f"result has no uri: {r!r}" + mc = getattr(r, "match_confidence", None) + assert mc is not None and 0.0 <= mc <= 1.0, ( + f"result match_confidence out of [0,1]: {mc!r} ({r!r})" + ) + assert getattr(r, "work", None) is not None, f"result has no work: {r!r}" + return results diff --git a/pyproject.toml b/pyproject.toml index a729214..cfac501 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,8 @@ classifiers = [ [project.optional-dependencies] pydantic = ["ovos-pydantic-models>=0.1.0"] audio = ["ovos-audio>=1.2.0"] +# OCP / ovos-media player harnesses (ovoscope.media: OCPPlayerHarness, ...). +media = ["ovos-media>=0.0.2a3"] # End-to-end TTS intelligibility scoring (WER/CER round-trip via reference STT). # faster-whisper itself is pulled by the plugin โ€” don't list it here to avoid # version skew. @@ -43,6 +45,7 @@ tts = [ ] dev = [ "ovos-audio>=1.2.0", + "ovos-media>=0.0.2a3", "ovos-pydantic-models>=0.1.0", "jiwer", "ovos-utterance-normalizer", diff --git a/test/unittests/test_media.py b/test/unittests/test_media.py index 5c0e405..79fe260 100644 --- a/test/unittests/test_media.py +++ b/test/unittests/test_media.py @@ -20,7 +20,17 @@ from ovos_utils.fakebus import FakeBus from ovos_utils.ocp import MediaState -from ovoscope.media import MockOCPBackend, OCPCaptureSession +from ovoscope.media import MockOCPBackend, OCPCaptureSession, OCPPlayerHarness + + +def test_media_harness_reexported_from_package() -> None: + """The ovos-media harness is reachable from the top-level package, like the + ovos-audio harness. (media.py imports ovos-media lazily, so the export does + not require the [media] extra to be installed.)""" + import ovoscope + assert ovoscope.OCPPlayerHarness is OCPPlayerHarness + assert ovoscope.OCPCaptureSession is OCPCaptureSession + assert ovoscope.MockOCPBackend is MockOCPBackend # --------------------------------------------------------------------------- @@ -202,3 +212,87 @@ def test_custom_prefixes(self) -> None: self.bus.emit(Message("ovos.common_play.play")) # should not be captured session.stop() assert session.message_types == ["custom.prefix.event"] + + +try: + import ovos_media # noqa: F401 + _HAS_OVOS_MEDIA = True +except ImportError: + _HAS_OVOS_MEDIA = False + + +if _HAS_OVOS_MEDIA: + from ovos_plugin_manager.templates.media import AudioPlayerBackend + + class _RecordingBackend(AudioPlayerBackend): + """A real OCP ``MediaBackend`` stand-in built by a factory. + + Subclasses the genuine ``AudioPlayerBackend`` (not ``MockOCPBackend``) so + its ``load_track`` emits the real ``ovos.common_play.media.state`` + ``LOADED_MEDIA`` event the live ``AudioService`` routes on. Records the uri + its ``play()`` is driven with โ€” analogous to a Music Assistant backend + calling ``client.play_media(uri)`` โ€” so a test can assert the player's play + path actually reached the injected backend. + """ + + def __init__(self, bus): + super().__init__(config={}, bus=bus) + self.play_calls = [] + self.is_playing = False + + def supported_uris(self): + return ["library", "http", "https"] + + def play(self, repeat: bool = False): + self.is_playing = True + self.play_calls.append(self._now_playing) + + def stop(self): + self.is_playing = False + return True + + def pause(self): + pass + + def resume(self): + pass + + def lower_volume(self): + pass + + def restore_volume(self): + pass + + def get_track_length(self): + return 0 + + def get_track_position(self): + return 0 + + def set_track_position(self, milliseconds): + pass + + +@pytest.mark.skipif(not _HAS_OVOS_MEDIA, + reason="requires the [media] extra (ovos-media)") +class TestOCPPlayerHarnessBackendInjection: + """OCPPlayerHarness(backend_factory=...) drives a real injected backend.""" + + def test_default_factory_is_mock_backend(self) -> None: + with OCPPlayerHarness() as h: + assert isinstance(h.backend, MockOCPBackend) + assert type(h.backend) is MockOCPBackend + + def test_injected_backend_is_used(self) -> None: + with OCPPlayerHarness(backend_factory=_RecordingBackend) as h: + assert isinstance(h.backend, _RecordingBackend) + # name is supplied by the harness when the backend lacks one + assert getattr(h.backend, "name", None) + + def test_player_drives_injected_backend_play(self) -> None: + from ovos_utils.ocp import MediaEntry, PlaybackType + with OCPPlayerHarness(backend_factory=_RecordingBackend) as h: + h.play(MediaEntry(uri="library://track/42", + playback=PlaybackType.AUDIO)) + assert h.backend.is_playing is True + assert h.backend.play_calls == ["library://track/42"] diff --git a/test/unittests/test_media_provider.py b/test/unittests/test_media_provider.py new file mode 100644 index 0000000..478955e --- /dev/null +++ b/test/unittests/test_media_provider.py @@ -0,0 +1,217 @@ +"""Tests for ovoscope.media_provider.MediaProviderHarness. + +The harness is duck-typed, so these tests use a dependency-free ``_DummyProvider`` +(no mediavocab / ovos-plugin-manager import) and ``SimpleNamespace`` stand-ins for +``Signals`` / ``QueryContext`` / ``Release``. +""" +import types + +import pytest + +from ovoscope import MediaProviderHarness +from ovoscope.media_provider import DEFAULT_GROUP + + +# -------------------------------------------------------------------------- +# duck-typed fixtures (no real mediavocab / opm types) +# -------------------------------------------------------------------------- + +def _signals(title="x", medium="music", artist=None): + return types.SimpleNamespace(title=title, medium=medium, artist=artist) + + +def _context(supported_playback_types=None, blocked_genres=None): + return types.SimpleNamespace( + supported_playback_types=supported_playback_types or set(), + blocked_genres=blocked_genres or set(), + ) + + +def _release(uri, title="T", mc=0.5): + return types.SimpleNamespace( + uri=uri, match_confidence=mc, + work=types.SimpleNamespace(title=title), + ) + + +class _DummyProvider: + """Minimal MediaProvider look-alike: serves music, returns library:// uris.""" + + name = "dummy" + + def __init__(self, config=None): + self.config = config or {} + self._api = None + self._served = {"music", "radio"} + + def is_available(self): + return self._api is not None + + def serves(self, signals, context=None): + if getattr(signals, "medium", None) not in self._served: + return False + if context is not None: + spt = getattr(context, "supported_playback_types", set()) + if spt and "audio" not in spt: + return False + return True + + def search(self, signals, lang="en-us"): + if self._api is None: + return [] + return [_release("library://track/1", "Hit", 0.9), + _release("library://track/2", "Miss", 0.3)] + + def search_safe(self, signals, context=None, lang="en-us"): + try: + return self.search(signals, lang=lang) + except Exception: + return [] + + def featured_media(self, lang="en-us"): + return [_release("library://track/3", "Featured", 0.0)] + + +class _ExplodingProvider(_DummyProvider): + def search(self, signals, lang="en-us"): + raise RuntimeError("backend exploded") + + +def _harness(mock_api="client", provider_cls=_DummyProvider): + return MediaProviderHarness.from_class( + provider_cls, config={"url": "http://x"}, mock_api=mock_api) + + +# -------------------------------------------------------------------------- +# from_class + injection +# -------------------------------------------------------------------------- + +def test_from_class_instantiates_and_injects_api(): + sentinel = object() + h = _harness(mock_api=sentinel) + assert isinstance(h.provider, _DummyProvider) + assert h.provider._api is sentinel + assert h.api is sentinel + assert h.provider.config == {"url": "http://x"} + + +def test_from_class_custom_api_attr(): + sentinel = object() + h = MediaProviderHarness.from_class(_DummyProvider, mock_api=sentinel, + api_attr="config") + assert h.provider.config is sentinel + + +def test_is_available_reflects_injected_client(): + assert _harness(mock_api="client").is_available() is True + assert MediaProviderHarness.from_class(_DummyProvider).is_available() is False + + +# -------------------------------------------------------------------------- +# routing / serves drivers + asserts +# -------------------------------------------------------------------------- + +def test_serves_and_assert_routes(): + h = _harness() + assert h.serves(_signals(medium="music")) is True + h.assert_routes(_signals(medium="music")) + h.assert_routes(_signals(medium="music"), + _context(supported_playback_types={"audio"})) + + +def test_assert_not_routes_unserved_medium(): + h = _harness() + assert h.serves(_signals(medium="movie")) is False + h.assert_not_routes(_signals(medium="movie")) + + +def test_assert_not_routes_video_only_device(): + h = _harness() + h.assert_not_routes(_signals(medium="music"), + _context(supported_playback_types={"video"})) + + +def test_assert_routes_raises_when_not_served(): + h = _harness() + with pytest.raises(AssertionError): + h.assert_routes(_signals(medium="movie")) + + +# -------------------------------------------------------------------------- +# search / search_safe drivers + playable asserts +# -------------------------------------------------------------------------- + +def test_search_safe_returns_playables(): + h = _harness() + results = h.assert_returns_playables(_signals()) + assert [r.work.title for r in results] == ["Hit", "Miss"] + assert all(r.uri.startswith("library://") for r in results) + + +def test_search_safe_swallows_backend_error(): + h = _harness(provider_cls=_ExplodingProvider) + assert h.search_safe(_signals()) == [] + + +def test_assert_returns_playables_fails_on_empty(): + h = MediaProviderHarness.from_class(_DummyProvider) # no api -> search returns [] + with pytest.raises(AssertionError): + h.assert_returns_playables(_signals()) + + +def test_assert_returns_playables_fails_on_bad_confidence(): + class _BadProvider(_DummyProvider): + def search(self, signals, lang="en-us"): + return [_release("library://track/9", "Bad", mc=5.0)] # out of [0,1] + + h = _harness(provider_cls=_BadProvider) + with pytest.raises(AssertionError): + h.assert_returns_playables(_signals()) + + +def test_featured_media(): + h = _harness() + feats = h.featured_media() + assert [r.work.title for r in feats] == ["Featured"] + + +# -------------------------------------------------------------------------- +# from_entrypoint (monkeypatched discovery) +# -------------------------------------------------------------------------- + +def _fake_entry_points(monkeypatch, eps): + monkeypatch.setattr("ovoscope.media_provider.entry_points", + lambda group=None: eps) + + +def test_from_entrypoint_discovers_and_loads(monkeypatch): + ep = types.SimpleNamespace(name="dummy", load=lambda: _DummyProvider) + _fake_entry_points(monkeypatch, [ep]) + + h = MediaProviderHarness.from_entrypoint("dummy", config={"url": "u"}, + mock_api="client") + assert isinstance(h.provider, _DummyProvider) + assert h.provider._api == "client" + assert h.entrypoint_name == "dummy" + assert h.entrypoint_group == DEFAULT_GROUP + h.assert_entrypoint_registered() # "dummy" present in the patched group + + +def test_from_entrypoint_missing_raises(monkeypatch): + _fake_entry_points(monkeypatch, []) + with pytest.raises(AssertionError): + MediaProviderHarness.from_entrypoint("nope") + + +def test_from_entrypoint_ambiguous_raises(monkeypatch): + ep1 = types.SimpleNamespace(name="dummy", load=lambda: _DummyProvider) + ep2 = types.SimpleNamespace(name="dummy", load=lambda: _DummyProvider) + _fake_entry_points(monkeypatch, [ep1, ep2]) + with pytest.raises(AssertionError): + MediaProviderHarness.from_entrypoint("dummy") + + +def test_assert_entrypoint_registered_without_name_raises(): + h = MediaProviderHarness.from_class(_DummyProvider) # no entrypoint name + with pytest.raises(AssertionError): + h.assert_entrypoint_registered() From 9b07e4bed909c81169de23a579e7a1fdd914cd95 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Thu, 25 Jun 2026 17:32:48 +0000 Subject: [PATCH 48/82] Increment Version to 0.21.0a1 --- ovoscope/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovoscope/version.py b/ovoscope/version.py index bc191ae..207b1e9 100644 --- a/ovoscope/version.py +++ b/ovoscope/version.py @@ -1,6 +1,6 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 -VERSION_MINOR = 20 +VERSION_MINOR = 21 VERSION_BUILD = 0 VERSION_ALPHA = 1 # END_VERSION_BLOCK From 615fe57de69255ba878370c443f39824cb0ec9ee Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Thu, 25 Jun 2026 17:33:19 +0000 Subject: [PATCH 49/82] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 15d94a0..f7eee30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.21.0a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.21.0a1) (2026-06-25) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.20.0a1...0.21.0a1) + +**Merged pull requests:** + +- feat: export ovos-media OCP harness from the package + add \[media\] extra [\#89](https://github.com/OpenVoiceOS/ovoscope/pull/89) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.20.0a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.20.0a1) (2026-06-24) [Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.19.4a1...0.20.0a1) From 90166b0a197afbb9de1ffcd03e7aeee177a6832f Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Thu, 25 Jun 2026 20:50:09 +0100 Subject: [PATCH 50/82] fix: repair ovoscope record in-process path (default_pipeline kwarg + from_message skill_ids) (#85) --- ovoscope/__init__.py | 2 +- ovoscope/cli.py | 35 +++++++++++++++++++---------------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/ovoscope/__init__.py b/ovoscope/__init__.py index 26c4fe3..d6cbbdf 100644 --- a/ovoscope/__init__.py +++ b/ovoscope/__init__.py @@ -1029,7 +1029,7 @@ def from_message(cls, message: Union[Message, List[Message]], async_messages=async_messages) for idx, source_message in enumerate(message): - if "session" not in source_message.context: + if "session" not in source_message.context and len(capture.responses): # propagate session updates as a client would do source_message.context["session"] = capture.responses[-1].context["session"] capture.capture(source_message, timeout) diff --git a/ovoscope/cli.py b/ovoscope/cli.py index 722d91d..5548b86 100644 --- a/ovoscope/cli.py +++ b/ovoscope/cli.py @@ -78,7 +78,7 @@ def _record_inprocess(args: argparse.Namespace) -> int: Exit code (0 = success, 1 = failure). """ try: - from ovoscope import End2EndTest, get_minicroft + from ovoscope import End2EndTest from ovos_utils.messagebus import Message except ImportError as exc: _die(f"ovoscope import failed: {exc}") @@ -88,26 +88,29 @@ def _record_inprocess(args: argparse.Namespace) -> int: pipeline: Optional[List[str]] = args.pipeline.split(",") if args.pipeline else None timeout: float = args.timeout + src_msg = Message( + "recognizer_loop:utterance", + data={"utterances": [args.utterance], "lang": lang}, + ) + + # from_message owns the MiniCroft lifecycle: it loads the skills, emits the + # source utterance, captures the response sequence, and stops MiniCroft. + # Loading a MiniCroft here as well would double-load the skill plugins. print(f"[record] Loading skills: {skill_ids}") + print(f"[record] Sending utterance: {args.utterance!r}") try: - mc = get_minicroft(skill_ids, lang=lang, pipeline=pipeline, max_wait=60) + from_message_kwargs = {"lang": lang, "timeout": timeout} + if pipeline is not None: + # MiniCroft's pipeline override kwarg is `default_pipeline`; it is + # forwarded through from_message -> get_minicroft -> MiniCroft. + from_message_kwargs["default_pipeline"] = pipeline + test = End2EndTest.from_message(src_msg, skill_ids, **from_message_kwargs) except TimeoutError: _die("MiniCroft did not reach READY state in time.") - try: - src_msg = Message( - "recognizer_loop:utterance", - data={"utterances": [args.utterance], "lang": lang}, - ) - - print(f"[record] Sending utterance: {args.utterance!r}") - test = End2EndTest.from_message(src_msg, mc, timeout=timeout) - - test.save(args.output) - print(f"[record] Fixture saved to {args.output}") - return 0 - finally: - mc.stop() + test.save(args.output) + print(f"[record] Fixture saved to {args.output}") + return 0 def _record_live(args: argparse.Namespace) -> int: From 0a37608e76eda39a8a8c19aaa27e8020a0496a52 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Thu, 25 Jun 2026 19:50:20 +0000 Subject: [PATCH 51/82] Increment Version to 0.21.1a1 --- ovoscope/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovoscope/version.py b/ovoscope/version.py index 207b1e9..ed9dc5e 100644 --- a/ovoscope/version.py +++ b/ovoscope/version.py @@ -1,7 +1,7 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 VERSION_MINOR = 21 -VERSION_BUILD = 0 +VERSION_BUILD = 1 VERSION_ALPHA = 1 # END_VERSION_BLOCK From 7ea78d951e62e741f8ac8cc8b14bd7368989050f Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Thu, 25 Jun 2026 19:50:44 +0000 Subject: [PATCH 52/82] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7eee30..0fe4645 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.21.1a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.21.1a1) (2026-06-25) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.21.0a1...0.21.1a1) + +**Merged pull requests:** + +- fix: repair ovoscope record in-process path \(default\_pipeline kwarg + from\_message skill\_ids\) [\#85](https://github.com/OpenVoiceOS/ovoscope/pull/85) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.21.0a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.21.0a1) (2026-06-25) [Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.20.0a1...0.21.0a1) From a7f0490391ec11e5962661031640125a60637c96 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Thu, 25 Jun 2026 20:51:30 +0100 Subject: [PATCH 53/82] feat: stream audio frames through MiniListener for multi-frame decoders (#86) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: stream audio frames through MiniListener for multi-frame decoders Add ``MiniListener.feed_audio_stream`` which feeds a sequence of audio frames in order and aggregates every message emitted across the whole stream, instead of clearing the capture buffer per call. This is required to test transformers whose decoder only fires after accumulating many frames (e.g. ggwave data-over-sound). - ``ListenerTest`` gains ``feed_method="feed_audio_stream"`` + ``chunk_size`` - document the real-ggwave streaming pattern in docs/listener.md - add unit tests using a stub accumulating transformer (no native deps) Co-Authored-By: Claude Opus 4.8 (1M context) * ci: install ovos-dinkum-listener so streaming tests run test_listener_stream exercises the MiniListener.feed_audio_stream plugin_instances path, which needs the dinkum AudioTransformersService. Add a [listener] extra (pinned >=0.7.2a1 โ€” the first release that allows ovos-bus-client 2.x, older pins cap it <2.0.0 and conflict with ovos-core) and install it in the build-tests and coverage workflows so the tests run instead of erroring on the missing dependency. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 (1M context) --- .github/workflows/build-tests.yml | 2 +- .github/workflows/coverage.yml | 2 +- docs/listener.md | 27 +++++ ovoscope/listener.py | 73 +++++++++++- pyproject.toml | 5 + test/unittests/test_listener_stream.py | 154 +++++++++++++++++++++++++ 6 files changed, 257 insertions(+), 6 deletions(-) create mode 100644 test/unittests/test_listener_stream.py diff --git a/.github/workflows/build-tests.yml b/.github/workflows/build-tests.yml index 439aded..6af0db7 100644 --- a/.github/workflows/build-tests.yml +++ b/.github/workflows/build-tests.yml @@ -11,5 +11,5 @@ jobs: secrets: inherit with: python_versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]' - install_extras: 'audio,pydantic,media' + install_extras: 'audio,pydantic,media,listener' test_path: 'test/unittests/' diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 49263df..eb94e5f 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -13,5 +13,5 @@ jobs: python_version: '3.11' coverage_source: 'ovoscope' test_path: 'test/unittests/' - test_extras: 'audio,pydantic,media' + test_extras: 'audio,pydantic,media,listener' min_coverage: 0 diff --git a/docs/listener.md b/docs/listener.md index 3fe008e..c19749b 100644 --- a/docs/listener.md +++ b/docs/listener.md @@ -68,6 +68,32 @@ assert any(m.msg_type == "recognizer_loop:utterance" for m in msgs) listener.shutdown() ``` +**Streaming real ggwave audio** โ€” the ggwave decoder only fires after it has +accumulated enough frames, so feed the whole waveform with `feed_audio_stream`, +which keeps every message emitted across the stream (unlike `feed_audio`, which +clears its buffer on each call): + +```python +import ggwave, numpy as np +from ovos_audio_transformer_plugin_ggwave import GGWavePlugin +from ovoscope.listener import get_mini_listener + +# real ggwave waveform (48kHz float32 โ†’ 16kHz int16, as the mic would deliver it) +wf = np.frombuffer(ggwave.encode("UTT:turn on the lights", protocolId=1, volume=20), + dtype=np.float32) +mic = np.interp(np.linspace(0, len(wf) - 1, int(len(wf) * 16000 / 48000)), + np.arange(len(wf)), wf) +audio = (np.clip(mic, -1, 1) * 32767).astype(np.int16).tobytes() + +plugin = GGWavePlugin(config={"start_enabled": True, "sample_rate": 16000}) +listener = get_mini_listener( + plugin_instances={"ovos-audio-transformer-plugin-ggwave": plugin} +) +msgs = listener.feed_audio_stream(audio, chunk_size=2048) +assert any(m.msg_type == "recognizer_loop:utterance" for m in msgs) +listener.shutdown() +``` + **Full pipeline testing** (STT with real WAV): ```python @@ -105,6 +131,7 @@ listener.shutdown() |--------|-----------|-------------| | `feed_audio(chunk)` โ€” `ovoscope/listener.py:351` | `(bytes) โ†’ List[Message]` | Calls `AudioTransformersService.feed_audio()`. Requires `ovos-dinkum-listener`. | | `feed_speech(chunk)` โ€” `ovoscope/listener.py:371` | `(bytes) โ†’ List[Message]` | Calls `AudioTransformersService.feed_speech()`. Requires `ovos-dinkum-listener`. | +| `feed_audio_stream(chunks, feed, chunk_size)` | `(bytes\|list[bytes], str, int) โ†’ List[Message]` | Streams frames in order **without** clearing between them; aggregates all emitted messages. Use for decoders that fire after many frames (ggwave). | | `transform(chunk)` โ€” `ovoscope/listener.py:390` | `(bytes) โ†’ tuple[bytes, dict, List[Message]]` | Full transform pipeline; returns `(audio, ctx, messages)`. Requires `ovos-dinkum-listener`. | | `listen(audio, ...)` โ€” `ovoscope/listener.py:410` | `(audio, language, stt_instance, ...) โ†’ List[Message]` | Full pipeline: audio โ†’ transformers โ†’ STT โ†’ utterance message. Requires `ovos-dinkum-listener`. | diff --git a/ovoscope/listener.py b/ovoscope/listener.py index 4fe93fb..bdd3772 100644 --- a/ovoscope/listener.py +++ b/ovoscope/listener.py @@ -402,6 +402,59 @@ def transform(self, chunk: bytes) -> Tuple[bytes, dict, List[Message]]: audio, ctx = self.transformers.transform(chunk) return audio, ctx, list(self._messages) + def feed_audio_stream( + self, + chunks: Union[bytes, List[bytes]], + feed: str = "feed_audio", + chunk_size: int = 2048, + ) -> List[Message]: + """Stream a sequence of audio frames and aggregate emitted messages. + + Unlike :meth:`feed_audio` / :meth:`feed_speech`, which clear the + capture buffer on every call, this feeds each frame in order and keeps + every message emitted across the **whole** stream. This is required + for transformers whose decoder only fires after accumulating many + frames of audio (e.g. ggwave data-over-sound). + + Args: + chunks: Either a flat ``bytes`` object (split into *chunk_size* + frames internally) or a pre-segmented ``List[bytes]`` of frames. + feed: Which transformer feed to drive per frame โ€” + ``"feed_audio"`` (non-speech) or ``"feed_speech"``. + chunk_size: Bytes per frame when *chunks* is a flat ``bytes`` + object (ignored when *chunks* is already a list). + + Returns: + All ``Message`` objects emitted on the bus across every frame. + + Raises: + RuntimeError: If ``ovos-dinkum-listener`` is not installed. + ValueError: If *feed* is not a recognised feed method. + """ + if self.transformers is None: + raise RuntimeError( + "ovos-dinkum-listener is required for feed_audio_stream. " + "Install it with: pip install ovos-dinkum-listener" + ) + if feed not in ("feed_audio", "feed_speech"): + raise ValueError( + f"feed must be 'feed_audio' or 'feed_speech', got {feed!r}" + ) + + if isinstance(chunks, bytes): + frames = [ + chunks[i:i + chunk_size] + for i in range(0, len(chunks), chunk_size) + ] + else: + frames = list(chunks) + + feeder = getattr(self.transformers, feed) + self._messages.clear() + for frame in frames: + feeder(frame) + return list(self._messages) + def listen( self, audio: Union[bytes, str, Path], @@ -758,8 +811,17 @@ class ListenerTest: """Raw audio bytes to inject into the pipeline.""" feed_method: str = "feed_audio" - """Which feed method to call: ``"feed_audio"``, ``"feed_speech"``, or - ``"transform"``.""" + """Which feed method to call: ``"feed_audio"``, ``"feed_speech"``, + ``"feed_audio_stream"``, ``"transform"``, or ``"listen"``. + + Use ``"feed_audio_stream"`` for transformers that decode only after + accumulating many frames (e.g. ggwave): *audio_input* is split into + *chunk_size* frames, fed in order, and all emitted messages are + aggregated across the whole stream.""" + + chunk_size: int = 2048 + """Frame size (bytes) used to split *audio_input* when *feed_method* is + ``"feed_audio_stream"``.""" expected_types: List[str] = field(default_factory=list) """Message types that MUST appear in the captured output.""" @@ -787,11 +849,14 @@ def execute(self) -> List[Message]: stt_instance=self.stt_instance, ) try: - method = getattr(listener, self.feed_method) if self.feed_method == "listen": messages = listener.listen(self.audio_input) + elif self.feed_method == "feed_audio_stream": + messages = listener.feed_audio_stream( + self.audio_input, chunk_size=self.chunk_size + ) else: - result = method(self.audio_input) + result = getattr(listener, self.feed_method)(self.audio_input) if self.feed_method == "transform": messages: List[Message] = result[2] else: diff --git a/pyproject.toml b/pyproject.toml index cfac501..5d5c426 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,10 @@ pydantic = ["ovos-pydantic-models>=0.1.0"] audio = ["ovos-audio>=1.2.0"] # OCP / ovos-media player harnesses (ovoscope.media: OCPPlayerHarness, ...). media = ["ovos-media>=0.0.2a3"] +# MiniListener plugin_instances path (feed_audio_stream) needs the dinkum +# AudioTransformersService. >=0.7.2a1 is the first release that allows +# ovos-bus-client 2.x (older pins cap it <2.0.0 and conflict with ovos-core). +listener = ["ovos-dinkum-listener>=0.7.2a1"] # End-to-end TTS intelligibility scoring (WER/CER round-trip via reference STT). # faster-whisper itself is pulled by the plugin โ€” don't list it here to avoid # version skew. @@ -46,6 +50,7 @@ tts = [ dev = [ "ovos-audio>=1.2.0", "ovos-media>=0.0.2a3", + "ovos-dinkum-listener>=0.7.2a1", "ovos-pydantic-models>=0.1.0", "jiwer", "ovos-utterance-normalizer", diff --git a/test/unittests/test_listener_stream.py b/test/unittests/test_listener_stream.py new file mode 100644 index 0000000..ef4d491 --- /dev/null +++ b/test/unittests/test_listener_stream.py @@ -0,0 +1,154 @@ +# Copyright 2024 Jarbas AI +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for MiniListener.feed_audio_stream and ListenerTest streaming. + +Uses a self-contained stub transformer (no native ggwave dependency) that only +emits its message after accumulating a threshold number of frames โ€” exactly the +behaviour ``feed_audio_stream`` exists to support. +""" +import unittest + +from ovos_bus_client.message import Message +from ovoscope.listener import ListenerTest, MiniListener, get_mini_listener + + +_BASE_CONFIG = {"listener": {"audio_transformers": {}}} + + +class _AccumulatingTransformer: + """Stub audio transformer that fires only after N fed frames. + + Mirrors a real streaming decoder: a single ``feed_audio`` call never + produces output, so the aggregating ``feed_audio_stream`` is required to + observe the emitted message. + """ + + def __init__(self, trigger_after=3, msg_type="recognizer_loop:utterance"): + self.name = "stub" + self.priority = 10 + self.trigger_after = trigger_after + self.msg_type = msg_type + self.count = 0 + self.bus = None + + def bind(self, bus): + self.bus = bus + + def feed_audio_chunk(self, chunk): + self.count += 1 + if self.count == self.trigger_after: + self.bus.emit(Message(self.msg_type, {"utterances": ["streamed"]})) + + def feed_speech_chunk(self, chunk): + self.feed_audio_chunk(chunk) + + def shutdown(self): + pass + + +def _listener(trigger_after=3): + plugin = _AccumulatingTransformer(trigger_after=trigger_after) + return get_mini_listener( + config=_BASE_CONFIG, + plugin_instances={"stub": plugin}, + ) + + +class TestFeedAudioStream(unittest.TestCase): + """MiniListener.feed_audio_stream aggregation behaviour.""" + + def test_single_feed_audio_misses_late_decode(self): + """A lone feed_audio call before the threshold yields nothing.""" + listener = _listener(trigger_after=3) + try: + self.assertEqual(listener.feed_audio(b"\x00" * 100), []) + finally: + listener.shutdown() + + def test_stream_aggregates_across_frames(self): + """feed_audio_stream keeps the message emitted on a later frame.""" + listener = _listener(trigger_after=3) + try: + msgs = listener.feed_audio_stream([b"\x00" * 100] * 5) + types = [m.msg_type for m in msgs] + self.assertIn("recognizer_loop:utterance", types) + finally: + listener.shutdown() + + def test_stream_splits_flat_bytes_by_chunk_size(self): + """A flat bytes object is split into chunk_size frames.""" + listener = _listener(trigger_after=4) + try: + # 4 frames of 50 bytes -> fourth frame triggers + msgs = listener.feed_audio_stream(b"\x00" * 200, chunk_size=50) + self.assertTrue( + any(m.msg_type == "recognizer_loop:utterance" for m in msgs) + ) + finally: + listener.shutdown() + + def test_stream_feed_speech_path(self): + """feed='feed_speech' drives the speech feed instead of audio.""" + listener = _listener(trigger_after=2) + try: + msgs = listener.feed_audio_stream( + [b"\x00" * 100] * 3, feed="feed_speech" + ) + self.assertTrue( + any(m.msg_type == "recognizer_loop:utterance" for m in msgs) + ) + finally: + listener.shutdown() + + def test_invalid_feed_raises(self): + """An unknown feed method is rejected.""" + listener = _listener() + try: + with self.assertRaises(ValueError): + listener.feed_audio_stream([b"\x00" * 10], feed="nope") + finally: + listener.shutdown() + + +class TestListenerTestStreaming(unittest.TestCase): + """ListenerTest declarative streaming support.""" + + def test_listener_test_feed_audio_stream(self): + """ListenerTest with feed_method='feed_audio_stream' aggregates.""" + plugin = _AccumulatingTransformer(trigger_after=3) + ListenerTest( + config=_BASE_CONFIG, + plugin_instances={"stub": plugin}, + audio_input=b"\x00" * 300, + feed_method="feed_audio_stream", + chunk_size=100, + expected_types=["recognizer_loop:utterance"], + ).execute() + + def test_listener_test_forbidden_absent(self): + """forbidden_types passes when the type never appears.""" + plugin = _AccumulatingTransformer(trigger_after=2) + ListenerTest( + config=_BASE_CONFIG, + plugin_instances={"stub": plugin}, + audio_input=b"\x00" * 200, + feed_method="feed_audio_stream", + chunk_size=100, + expected_types=["recognizer_loop:utterance"], + forbidden_types=["speak"], + ).execute() + + +if __name__ == "__main__": + unittest.main() From 855573e3b4ae2c2e0631173024716c0e642434cd Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Thu, 25 Jun 2026 19:51:40 +0000 Subject: [PATCH 54/82] Increment Version to 0.22.0a1 --- ovoscope/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ovoscope/version.py b/ovoscope/version.py index ed9dc5e..ac25f67 100644 --- a/ovoscope/version.py +++ b/ovoscope/version.py @@ -1,7 +1,7 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 -VERSION_MINOR = 21 -VERSION_BUILD = 1 +VERSION_MINOR = 22 +VERSION_BUILD = 0 VERSION_ALPHA = 1 # END_VERSION_BLOCK From 4b06ce3fb63c2106b865a037e65f47cb23d5075b Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Thu, 25 Jun 2026 19:52:11 +0000 Subject: [PATCH 55/82] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0fe4645..84fa51b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.22.0a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.22.0a1) (2026-06-25) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.21.1a1...0.22.0a1) + +**Merged pull requests:** + +- feat: stream audio frames through MiniListener for multi-frame decoders [\#86](https://github.com/OpenVoiceOS/ovoscope/pull/86) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.21.1a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.21.1a1) (2026-06-25) [Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.21.0a1...0.21.1a1) From cf5122bf6d4bce73c61e13dd3e5b64ac0d773d78 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Thu, 25 Jun 2026 20:52:24 +0100 Subject: [PATCH 56/82] fix: migrate pytest_pycollect_makemodule to wrapper=True for pytest 9 (#88) The pytest11 plugin used the legacy hookwrapper=True / outcome.get_result() protocol. That style is deprecated and slated for removal; pytest 9 standardizes on the wrapper=True return-style. Adopt it so the auto-loaded plugin keeps importing cleanly under pytest 8 and 9 without consumers needing -p no:ovoscope. Behavior is unchanged: the downstream collector now arrives directly from yield and is returned unchanged after intent-case auto-discovery runs. Closes #87 --- ovoscope/pytest_plugin.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/ovoscope/pytest_plugin.py b/ovoscope/pytest_plugin.py index bdae9ad..7f51d1a 100644 --- a/ovoscope/pytest_plugin.py +++ b/ovoscope/pytest_plugin.py @@ -421,33 +421,39 @@ def _autodiscover_intent_cases(config): return None -@pytest.hookimpl(hookwrapper=True) +@pytest.hookimpl(wrapper=True) def pytest_pycollect_makemodule(module_path, parent): """Auto-register intent-case tests on shim modules that declare ``ovoscope_intent_cases = {...}``. - The hookwrapper imports the module first, lets pytest build the + The hook wrapper imports the module first, lets pytest build the collector, then injects the generated TestCase classes into the module's namespace so the standard Python-class collector finds them. + + Uses the modern ``wrapper=True`` hook-wrapper protocol (pluggy >=1.2 / + pytest >=7.2): the downstream result arrives from ``yield`` and is + returned unchanged. The legacy ``hookwrapper=True`` / + ``outcome.get_result()`` protocol is removed in pytest 10, so we adopt + the new one to stay loadable under pytest 8 and 9. """ from ovoscope.intent_cases import autodiscover_from_conftest - outcome = yield - collector = outcome.get_result() + collector = yield if collector is None: - return + return collector try: mod = collector.obj # imports the module if not already loaded except Exception: - return + return collector if not hasattr(mod, "ovoscope_intent_cases"): - return + return collector if getattr(mod, "_ovoscope_intent_cases_registered", False): - return + return collector try: autodiscover_from_conftest(Path(mod.__file__).parent, mod.__dict__) except Exception as exc: # noqa: BLE001 print(f"ovoscope auto-discovery skipped for {mod.__file__}: {exc}") + return collector def pytest_collection_modifyitems(config, items): From d47847995081be282ef52355865af3f473b00d38 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Thu, 25 Jun 2026 19:52:35 +0000 Subject: [PATCH 57/82] Increment Version to 0.22.1a1 --- ovoscope/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovoscope/version.py b/ovoscope/version.py index ac25f67..515f5df 100644 --- a/ovoscope/version.py +++ b/ovoscope/version.py @@ -1,7 +1,7 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 VERSION_MINOR = 22 -VERSION_BUILD = 0 +VERSION_BUILD = 1 VERSION_ALPHA = 1 # END_VERSION_BLOCK From 1fa77cfcf3c15c9f901340980f2f1d6764a15fbb Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Thu, 25 Jun 2026 19:52:57 +0000 Subject: [PATCH 58/82] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 84fa51b..ffb1ede 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.22.1a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.22.1a1) (2026-06-25) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.22.0a1...0.22.1a1) + +**Merged pull requests:** + +- fix: pytest 9 compatibility for the pytest11 plugin [\#88](https://github.com/OpenVoiceOS/ovoscope/pull/88) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.22.0a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.22.0a1) (2026-06-25) [Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.21.1a1...0.22.0a1) From a83317ccdc5b0a1defa68b2aa093a43e7649fbbe Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Thu, 25 Jun 2026 22:24:32 +0100 Subject: [PATCH 59/82] feat!: audio harness on OVOS spec bus namespace (#92) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat!: audio harness on OVOS spec bus namespace Migrate the AudioServiceHarness / PlaybackServiceHarness / AudioCaptureSession to the ovos.* spec topics via SpecMessage, matching ovos-audio's spec-bus migration (PIPELINE-1 ยง9.6): - emit speak as SpecMessage.SPEAK (ovos.utterance.speak) - subscribe/capture SpecMessage.AUDIO_OUTPUT_STARTED / AUDIO_OUTPUT_ENDED - subscribe SpecMessage.MIC_LISTEN The harness runs on a plain FakeBus (no modernize bridging), so it must emit and observe the spec topics directly. Bump the audio/tts extras to the spec-migrated ovos-audio and add an explicit ovos-spec-tools dependency. Co-Authored-By: Claude Opus 4.8 * fix: bump ovos-audio dev floor to 1.3.0a1 for bus-client 2.x Co-Authored-By: Claude Opus 4.8 * fix: lower ovos-audio audio/tts extra floors to 1.3.0a1 (highest on PyPI) Co-Authored-By: Claude Opus 4.8 * feat: test BOTH legacy and ovos.* bus namespaces via bridging FakeBus The audio harness migrated to the ovos.* spec topics (PR #92) while ovos-audio still emits the legacy topics; on the old non-bridging FakeBus a legacy producer never reached the spec-subscribed harness handler, so several test_audio_harness assertions (ducking, speak lifecycle, capture sequence) failed. ovos-utils #381 makes FakeBus mirror MessageBusClient's legacy<->ovos.* migration, which reconnects them โ€” these were never genuine harness bugs. - pin ovos-utils>=0.12.0a1 (first FakeBus with namespace migration; #381) - thread modernize=/emit_legacy= through MiniCroft, AudioServiceHarness and PlaybackServiceHarness so harness users can exercise either namespace, both, or a single isolated namespace - AudioCaptureSession captures BOTH the legacy and ovos.* audio topics: it observes the raw "message" wire stream (which carries the producer's ORIGINAL topic only โ€” the bridge re-dispatches the counterpart as a typed event, not a second "message"), so it must list both namespaces to record either producer - add test/unittests/test_namespace_bridging.py: legacy->spec, spec->legacy, dual-subscribe dedup, two-genuine-events, and no-bridging isolation - add TestAudioHarnessNamespaceBridging to test_audio_harness.py: ducking via the bridge, ducking via the spec topic natively, single-namespace isolation, and the speak lifecycle through the bridge Stacked on ovos-utils#381 and ovos-spec-tools#26 (NamespaceTranslator); CI stays red until both publish. Co-Authored-By: Claude Opus 4.8 * feat: dual-namespace bus tests across all core-service harnesses Extends the audio-harness namespace work (#92) to every core service so each proves its migrated bus topics travel on BOTH the legacy and the ovos.* spec namespace via FakeBus bridging. Each harness gains modernize=/emit_legacy= kwargs (threaded to FakeBus or to MiniCroft) so callers can exercise either namespace, both, or a single isolated one. Per service (harness + TestNamespaceBridging): - listener / voice_loop / simple_listener: recognizer_loop:utterance -> ovos.utterance.handle, record_begin/end -> ovos.listener.record.started/ended - classic_listener: + mycroft.awoken -> ovos.listener.awoken - e2e MiniCroft / pipeline / ocp: recognizer_loop:utterance -> ovos.utterance.handle - media (OCPPlayer): cork/duck via record + audio.output topics, spec->legacy reaches the legacy-subscribed handlers via emit_legacy - phal: no migrated topics โ€” verifies the harness bus itself bridges Each class covers: legacy->spec bridged, spec->legacy bridged, spec native, and single-namespace isolation with both flags off. Handlers subscribe on both topics (the bridge re-dispatches the counterpart as a typed event, not a second wire 'message'). * fix: raise audio extra ovos-spec-tools floor to 0.10.0a1 The FakeBus namespace bridging (ovos-utils >=0.12.0a1) unconditionally imports NamespaceTranslator, which first ships in ovos-spec-tools 0.10.0a1. The previous >=0.9.0a1 floor could resolve to a version without it. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- ovoscope/__init__.py | 15 +- ovoscope/audio.py | 70 ++++++--- ovoscope/classic_listener.py | 15 +- ovoscope/e2e.py | 9 ++ ovoscope/listener.py | 21 ++- ovoscope/media.py | 29 +++- ovoscope/ocp.py | 15 +- ovoscope/phal.py | 21 ++- ovoscope/pipeline.py | 15 ++ ovoscope/simple_listener.py | 22 ++- ovoscope/voice_loop.py | 35 ++++- pyproject.toml | 15 +- test/unittests/test_audio_harness.py | 54 +++++++ test/unittests/test_classic_listener.py | 103 ++++++++++++++ test/unittests/test_listener_vad_ww.py | 78 ++++++++++ test/unittests/test_media.py | 64 +++++++++ test/unittests/test_minicroft.py | 74 ++++++++++ test/unittests/test_namespace_bridging.py | 166 ++++++++++++++++++++++ test/unittests/test_ocp_namespace.py | 125 ++++++++++++++++ test/unittests/test_phal.py | 66 +++++++++ test/unittests/test_pipeline_harness.py | 82 ++++++++++- test/unittests/test_simple_listener.py | 90 ++++++++++++ test/unittests/test_voice_loop.py | 75 ++++++++++ 23 files changed, 1226 insertions(+), 33 deletions(-) create mode 100644 test/unittests/test_namespace_bridging.py create mode 100644 test/unittests/test_ocp_namespace.py diff --git a/ovoscope/__init__.py b/ovoscope/__init__.py index d6cbbdf..0f1eddc 100644 --- a/ovoscope/__init__.py +++ b/ovoscope/__init__.py @@ -301,7 +301,19 @@ def __init__(self, skill_ids, lang: Optional[str] = None, secondary_langs: Optional[List[str]] = None, pipeline_config: Optional[Dict[str, Dict]] = None, + modernize: bool = True, + emit_legacy: bool = True, *args, **kwargs): + # Namespace-migration flags forwarded to the harness FakeBus so callers + # can choose which bus namespace(s) to exercise: + # modernize=True emitting a legacy topic ALSO emits the ovos.* spec + # topic (legacy producer -> spec listener) + # emit_legacy=True emitting an ovos.* spec topic ALSO emits the legacy + # topic (spec producer -> legacy listener) + # Both default on (mirrors MessageBusClient). Set BOTH False to isolate a + # single namespace and assert no cross-namespace bridging occurs. + self._modernize = modernize + self._emit_legacy = emit_legacy self._isolated_config = isolate_config self._original_xdg_configs: Optional[List[LocalConf]] = None @@ -376,7 +388,8 @@ def __init__(self, skill_ids, LOG.debug(f"ovoscope: pipeline_config patched '{plugin_key}'") self.boot_messages: List[Message] = [] - bus = FakeBus() + bus = FakeBus(modernize=self._modernize, + emit_legacy=self._emit_legacy) bus.on("message", self.handle_boot_message) self.skill_ids = skill_ids self.extra_skills = extra_skills or {} diff --git a/ovoscope/audio.py b/ovoscope/audio.py index fd82dba..a1d816e 100644 --- a/ovoscope/audio.py +++ b/ovoscope/audio.py @@ -33,6 +33,7 @@ from unittest.mock import MagicMock, patch from ovos_bus_client.message import Message +from ovos_spec_tools import SpecMessage from ovos_plugin_manager.templates.audio import AudioBackend from ovos_plugin_manager.templates.tts import TTS from ovos_utils.fakebus import FakeBus @@ -227,17 +228,28 @@ class AudioServiceHarness: def __init__(self, backend_name: str = "mock", validate_source: bool = False, - disable_ocp: bool = True) -> None: + disable_ocp: bool = True, + modernize: bool = True, + emit_legacy: bool = True) -> None: """Initialise harness parameters. Args: backend_name: Name for the MockAudioBackend instance. validate_source: Enable source-session validation in AudioService. disable_ocp: Disable OCP plugin during tests. + modernize: FakeBus also emits the ovos.* spec topic when a legacy + topic is emitted (legacy producer -> spec listener). ovos-audio + emits legacy audio_output_start/end; the harness subscribes on + the spec topics, so this bridge is what connects them. + emit_legacy: FakeBus also emits the legacy topic when an ovos.* spec + topic is emitted (spec producer -> legacy listener). Set both + False to exercise a single namespace with no bridging. """ self.backend_name: str = backend_name self.validate_source: bool = validate_source self.disable_ocp: bool = disable_ocp + self.modernize: bool = modernize + self.emit_legacy: bool = emit_legacy self.bus: Optional[FakeBus] = None self.service = None # AudioService instance self.backend: Optional[MockAudioBackend] = None @@ -250,7 +262,8 @@ def __enter__(self) -> "AudioServiceHarness": """ from ovos_audio.audio import AudioService - self.bus = FakeBus() + self.bus = FakeBus(modernize=self.modernize, + emit_legacy=self.emit_legacy) self.backend = MockAudioBackend(config={}, bus=self.bus, name=self.backend_name) @@ -282,9 +295,9 @@ def __enter__(self) -> "AudioServiceHarness": self.service._get_track_length) self.bus.on("mycroft.audio.service.seek_forward", self.service._seek_forward) self.bus.on("mycroft.audio.service.seek_backward", self.service._seek_backward) - self.bus.on("recognizer_loop:audio_output_start", + self.bus.on(SpecMessage.AUDIO_OUTPUT_STARTED, self.service._lower_volume_on_speak) - self.bus.on("recognizer_loop:audio_output_end", + self.bus.on(SpecMessage.AUDIO_OUTPUT_ENDED, self.service._restore_volume_on_speak) self.bus.on("recognizer_loop:record_begin", self.service._lower_volume_on_record) @@ -508,8 +521,8 @@ class PlaybackServiceHarness: """Context manager wrapping PlaybackService with a MockTTS on a FakeBus. PlaybackService is a ``Thread``; this harness starts it and wires it to the - provided FakeBus so tests can emit ``speak`` messages and observe the - resulting ``recognizer_loop:audio_output_start/end`` events. + provided FakeBus so tests can emit ``ovos.utterance.speak`` messages and + observe the resulting ``ovos.audio.output.started/ended`` events. The harness patches ``ovos_utils.sound.play_audio`` so no actual audio device is accessed. It also drains ``TTS.queue`` before construction to @@ -526,16 +539,27 @@ class PlaybackServiceHarness: def __init__(self, validate_source: bool = False, disable_ocp: bool = True, - tts: Optional[TTS] = None) -> None: + tts: Optional[TTS] = None, + modernize: bool = True, + emit_legacy: bool = True) -> None: """Initialise harness parameters. Args: validate_source: Enable session-source validation. disable_ocp: Disable OCP audio plugin. tts: TTS instance to inject. Defaults to ``MockTTS()`` when None. + modernize: FakeBus also emits the ovos.* spec topic when a legacy + topic is emitted (legacy producer -> spec listener). PlaybackService + emits legacy audio_output_start/end and mic.listen; the harness + subscribes on the spec topics, so this bridge connects them. + emit_legacy: FakeBus also emits the legacy topic when an ovos.* spec + topic is emitted (spec producer -> legacy listener). Set both + False to exercise a single namespace with no bridging. """ self.validate_source: bool = validate_source self.disable_ocp: bool = disable_ocp + self.modernize: bool = modernize + self.emit_legacy: bool = emit_legacy self.bus: Optional[FakeBus] = None self.svc = None # PlaybackService instance # ``mock_tts`` keeps its historic name for backward compatibility but @@ -568,7 +592,8 @@ def __enter__(self) -> "PlaybackServiceHarness": break TTS.queue = Queue() - self.bus = FakeBus() + self.bus = FakeBus(modernize=self.modernize, + emit_legacy=self.emit_legacy) # Inject the provided TTS (real plugin) or fall back to MockTTS. self.mock_tts = self.tts if self.tts is not None else MockTTS() @@ -603,11 +628,11 @@ def _capture_play_audio(data, *args, **kwargs): self.mock_tts.init(self.bus, self.svc.playback_thread) # Subscribe lifecycle events for synchronisation - self.bus.on("recognizer_loop:audio_output_start", + self.bus.on(SpecMessage.AUDIO_OUTPUT_STARTED, lambda m: self._audio_output_start.set()) - self.bus.on("recognizer_loop:audio_output_end", + self.bus.on(SpecMessage.AUDIO_OUTPUT_ENDED, lambda m: self._audio_output_end.set()) - self.bus.on("mycroft.mic.listen", + self.bus.on(SpecMessage.MIC_LISTEN, lambda m: self._mic_listen.set()) except Exception: @@ -660,7 +685,7 @@ def speak(self, utterance: str, expect_response: bool = False, self._audio_output_end.clear() self._mic_listen.clear() - self.bus.emit(Message("speak", { + self.bus.emit(Message(SpecMessage.SPEAK, { "utterance": utterance, "lang": "en-US", "expect_response": expect_response, @@ -692,31 +717,31 @@ def assert_spoke(self, text: str) -> None: ) def assert_audio_output_started(self, timeout: float = 3.0) -> None: - """Assert that recognizer_loop:audio_output_start was emitted. + """Assert that ovos.audio.output.started was emitted. Args: timeout: Seconds to wait for the event. """ assert self._audio_output_start.wait(timeout), \ - "recognizer_loop:audio_output_start was not emitted" + "ovos.audio.output.started was not emitted" def assert_audio_output_ended(self, timeout: float = 3.0) -> None: - """Assert that recognizer_loop:audio_output_end was emitted. + """Assert that ovos.audio.output.ended was emitted. Args: timeout: Seconds to wait for the event. """ assert self._audio_output_end.wait(timeout), \ - "recognizer_loop:audio_output_end was not emitted" + "ovos.audio.output.ended was not emitted" def assert_mic_listen(self, timeout: float = 3.0) -> None: - """Assert that mycroft.mic.listen was emitted after speech. + """Assert that ovos.mic.listen was emitted after speech. Args: timeout: Seconds to wait for the event. """ assert self._mic_listen.wait(timeout), \ - "mycroft.mic.listen was not emitted" + "ovos.mic.listen was not emitted" # --------------------------------------------------------------------------- @@ -740,9 +765,18 @@ class AudioCaptureSession: """ bus: FakeBus + # capture BOTH the legacy and the ovos.* spec topics of the migrating audio + # messages. The capture session observes the raw "message" wire stream, which + # carries the producer's ORIGINAL topic only (FakeBus' namespace bridging + # re-dispatches the counterpart as a typed event, not a second "message" + # event). Listing both namespaces lets the session record the sequence + # whether the producer emits legacy or spec, so harness users can assert on + # either namespace. track_prefixes: List[str] = dataclasses.field(default_factory=lambda: [ "mycroft.audio.", + "ovos.audio.output", "recognizer_loop:audio_output", + "ovos.mic.listen", "mycroft.mic.listen", ]) messages: List[Message] = dataclasses.field(default_factory=list) diff --git a/ovoscope/classic_listener.py b/ovoscope/classic_listener.py index ecf9a8a..6eed52d 100644 --- a/ovoscope/classic_listener.py +++ b/ovoscope/classic_listener.py @@ -297,7 +297,14 @@ class MiniClassicListener(ListenerHarness): :class:`MockHotWordEngine`). stt_instance: STT engine for the built loop (defaults to :class:`MockStreamingSTT`). - bus: Optional :class:`FakeBus` to capture on. + bus: Optional :class:`FakeBus` to capture on. When ``None`` a fresh + :class:`FakeBus` is built using *modernize* / *emit_legacy*. + modernize: When a fresh bus is built, have :class:`FakeBus` also emit the + ovos.* spec topic whenever a legacy topic is emitted (legacy -> + spec bridging). Ignored when *bus* is supplied. Defaults to True. + emit_legacy: When a fresh bus is built, have :class:`FakeBus` also emit + the legacy topic whenever an ovos.* spec topic is emitted (spec -> + legacy bridging). Ignored when *bus* is supplied. Defaults to True. Raises: RuntimeError: If *recognizer_loop* is ``None`` and @@ -311,7 +318,13 @@ def __init__( wakeword: Optional[Any] = None, stt_instance: Optional[Any] = None, bus: Optional[FakeBus] = None, + modernize: bool = True, + emit_legacy: bool = True, ) -> None: + if bus is None: + bus = FakeBus(modernize=modernize, emit_legacy=emit_legacy) + self.modernize: bool = modernize + self.emit_legacy: bool = emit_legacy super().__init__(bus) self._built = recognizer_loop is None self._wakeword = wakeword diff --git a/ovoscope/e2e.py b/ovoscope/e2e.py index 04e1672..6de66b1 100644 --- a/ovoscope/e2e.py +++ b/ovoscope/e2e.py @@ -228,6 +228,13 @@ class E2EPipelineHarness(unittest.TestCase): SKILL_ID: ClassVar[str] = "test_skill_ovoscope" DEFAULT_LANG: ClassVar[str] = "en-US" STARTUP_MAX_WAIT: ClassVar[float] = 60.0 + # Namespace-migration flags forwarded to the harness MiniCroft / FakeBus. + # MODERNIZE on: a legacy emit (recognizer_loop:utterance) also dispatches its + # ovos.* spec counterpart (ovos.utterance.handle); EMIT_LEGACY on: a spec emit + # also dispatches the legacy topic. Both default on. Subclasses set BOTH False + # to drive a single isolated namespace. + MODERNIZE: ClassVar[bool] = True + EMIT_LEGACY: ClassVar[bool] = True mc: ClassVar[Any] pipeline: ClassVar[Any] @@ -252,6 +259,8 @@ def setUpClass(cls) -> None: lang=cls.DEFAULT_LANG, default_pipeline=[cls.PIPELINE_ID], max_wait=cls.STARTUP_MAX_WAIT, + modernize=cls.MODERNIZE, + emit_legacy=cls.EMIT_LEGACY, ) cls.pipeline = cls.mc.intents.pipeline_plugins[cls.PIPELINE_ID] diff --git a/ovoscope/listener.py b/ovoscope/listener.py index bdd3772..a5ec659 100644 --- a/ovoscope/listener.py +++ b/ovoscope/listener.py @@ -283,6 +283,13 @@ class MiniListener: ww_instances: Optional mapping of hotword name โ†’ :class:`HotWordEngine` (or mock) instance. Multiple wake-word engines can be registered simultaneously. + modernize: FakeBus also emits the ovos.* spec topic when a legacy + topic is emitted (legacy producer -> spec listener). The listener + pipeline emits legacy ``recognizer_loop:*`` topics; this bridge is + what lets a spec-topic subscriber observe them. + emit_legacy: FakeBus also emits the legacy topic when an ovos.* spec + topic is emitted (spec producer -> legacy listener). Set both False + to exercise a single namespace with no bridging. Example:: @@ -308,8 +315,11 @@ def __init__( stt_instance: Optional[Any] = None, vad_instance: Optional[Any] = None, ww_instances: Optional[Dict[str, Any]] = None, + modernize: bool = True, + emit_legacy: bool = True, ) -> None: - self.bus: FakeBus = FakeBus() + self.bus: FakeBus = FakeBus(modernize=modernize, + emit_legacy=emit_legacy) self._messages: List[Message] = [] self._stt_instance: Optional[Any] = stt_instance self._vad: Optional[Any] = vad_instance @@ -688,6 +698,8 @@ def get_mini_listener( vad_instance: Optional[Any] = None, ww_plugin: Optional[str] = None, ww_instances: Optional[Dict[str, Any]] = None, + modernize: bool = True, + emit_legacy: bool = True, ) -> MiniListener: """Factory: create a ready-to-use :class:`MiniListener`. @@ -717,6 +729,11 @@ def get_mini_listener( ww_instances: Mapping of hotword name โ†’ engine instance. Supports multiple simultaneous wake-word engines. Takes precedence over *ww_plugin*. + modernize: FakeBus also emits the ovos.* spec topic when a legacy topic + is emitted (legacy producer -> spec listener). + emit_legacy: FakeBus also emits the legacy topic when an ovos.* spec + topic is emitted (spec producer -> legacy listener). Set both False + to exercise a single namespace with no bridging. Returns: A fully initialised :class:`MiniListener` ready to receive audio. @@ -770,6 +787,8 @@ def get_mini_listener( stt_instance=stt_instance, vad_instance=resolved_vad, ww_instances=resolved_ww, + modernize=modernize, + emit_legacy=emit_legacy, ) diff --git a/ovoscope/media.py b/ovoscope/media.py index 4016331..36b3683 100644 --- a/ovoscope/media.py +++ b/ovoscope/media.py @@ -269,7 +269,9 @@ class OCPPlayerHarness: """ def __init__(self, backend_namespace: str = "audio", - backend_factory: Optional[Callable[[FakeBus], AudioBackend]] = None) -> None: + backend_factory: Optional[Callable[[FakeBus], AudioBackend]] = None, + modernize: bool = True, + emit_legacy: bool = True) -> None: """Initialise harness parameters. Args: @@ -284,9 +286,22 @@ def __init__(self, backend_namespace: str = "audio", backend would otherwise reach. Note the mock-only assertion helpers (:meth:`assert_backend_paused`, ``backend.played_uris``) assume a :class:`MockOCPBackend` and may not apply to a real backend. + modernize: FakeBus also emits the ovos.* spec topic when a legacy + topic is emitted (legacy producer -> spec listener). OCPMediaPlayer + subscribes to the LEGACY duck/cork topics + (recognizer_loop:audio_output_start/end, record_begin/end); this + bridge lets a spec-namespace producer's ovos.audio.output.* / + ovos.listener.record.* reach those legacy handlers. + emit_legacy: FakeBus also emits the legacy topic when an ovos.* spec + topic is emitted (spec producer -> legacy listener). Because the + player subscribes on the legacy topics, this is the bridge that + connects a spec producer to the player. Set both False to exercise + a single namespace with no bridging. """ self.backend_namespace: str = backend_namespace self.backend_factory = backend_factory + self.modernize: bool = modernize + self.emit_legacy: bool = emit_legacy self.bus: Optional[FakeBus] = None self.player = None # OCPMediaPlayer instance self.backend: Optional[AudioBackend] = None @@ -301,7 +316,8 @@ def __enter__(self) -> "OCPPlayerHarness": """ from ovos_media.player import OCPMediaPlayer - self.bus = FakeBus() + self.bus = FakeBus(modernize=self.modernize, + emit_legacy=self.emit_legacy) if self.backend_factory is not None: self.backend = self.backend_factory(self.bus) # A config-loaded backend gets name/namespace from @@ -624,9 +640,18 @@ class OCPCaptureSession: """ bus: FakeBus + # capture BOTH the legacy and the ovos.* spec topics of the duck/cork + # messages the player consumes. The session observes the raw "message" wire + # stream, which carries the producer's ORIGINAL topic only (FakeBus' namespace + # bridging re-dispatches the counterpart as a typed event, not a second + # "message" event). Listing both namespaces lets the session record the + # sequence whether the producer emits legacy or spec. track_prefixes: List[str] = dataclasses.field(default_factory=lambda: [ "ovos.common_play.", "ovos.audio.", + "recognizer_loop:audio_output", + "ovos.listener.record", + "recognizer_loop:record", ]) messages: List[Message] = dataclasses.field(default_factory=list) diff --git a/ovoscope/ocp.py b/ovoscope/ocp.py index 9d5d8b5..826bfd5 100644 --- a/ovoscope/ocp.py +++ b/ovoscope/ocp.py @@ -67,6 +67,15 @@ class OCPTest: patch_targets: Additional ``requests``-like module paths to patch (e.g. ``["my_skill.http_client.requests"]``). The default target is ``"requests.Session.get"``. + modernize: Forwarded to the harness ``MiniCroft`` / ``FakeBus``. When + on (default), emitting a LEGACY topic also dispatches its ovos.* + spec counterpart (legacy producer -> spec listener). The OCP flow + is driven by ``recognizer_loop:utterance``; bridging lets it be + observed on / driven from ``ovos.utterance.handle`` too. + emit_legacy: Forwarded to the harness. When on (default), emitting an + ovos.* spec topic also dispatches the legacy one (spec producer -> + legacy listener). Set BOTH False to exercise a single namespace + with no cross-namespace bridging. Example:: @@ -86,6 +95,8 @@ class OCPTest: lang: str = "en-US" timeout: float = 20.0 patch_targets: List[str] = field(default_factory=list) + modernize: bool = True + emit_legacy: bool = True def execute(self) -> List[Message]: """Run the OCP test with optional HTTP mocking. @@ -100,7 +111,9 @@ def execute(self) -> List[Message]: captured: List[Message] = [] - mc = get_minicroft(self.skill_ids, lang=self.lang, max_wait=60) + mc = get_minicroft(self.skill_ids, lang=self.lang, max_wait=60, + modernize=self.modernize, + emit_legacy=self.emit_legacy) mc.bus.on("message", lambda m: captured.append( Message.deserialize(m) if isinstance(m, str) else m )) diff --git a/ovoscope/phal.py b/ovoscope/phal.py index 7ef0901..156ac90 100644 --- a/ovoscope/phal.py +++ b/ovoscope/phal.py @@ -58,6 +58,14 @@ class MiniPHAL: is always wired to the harness ``FakeBus``. Takes precedence over *plugin_instances* for the same plugin_id. config: Per-plugin configuration overrides keyed by plugin_id. + modernize: FakeBus also emits the ovos.* spec topic when a legacy topic + is emitted (legacy producer -> spec listener). PHAL plugins do not use + any of the migrated audio/listener topics, so this only matters for a + plugin that happens to consume/produce a migrated topic; it is threaded + for consistency with the audio/media harnesses. + emit_legacy: FakeBus also emits the legacy topic when an ovos.* spec topic + is emitted (spec producer -> legacy listener). Set both False to + exercise a single namespace with no bridging. Example:: @@ -86,12 +94,16 @@ def __init__( plugin_instances: Optional[Dict[str, Any]] = None, plugin_factories: Optional[Dict[str, Callable[[FakeBus], Any]]] = None, config: Optional[Dict[str, Dict[str, Any]]] = None, + modernize: bool = True, + emit_legacy: bool = True, ) -> None: self.plugin_ids: List[str] = plugin_ids or [] self.plugin_instances: Dict[str, Any] = plugin_instances or {} self.plugin_factories: Dict[str, Callable[[FakeBus], Any]] = plugin_factories or {} self.config: Dict[str, Dict[str, Any]] = config or {} - self._bus: FakeBus = FakeBus() + self.modernize: bool = modernize + self.emit_legacy: bool = emit_legacy + self._bus: FakeBus = FakeBus(modernize=modernize, emit_legacy=emit_legacy) self._captured: List[Message] = [] self._loaded: Dict[str, Any] = {} @@ -249,6 +261,9 @@ class PHALTest: Use this when the plugin must be constructed with the harness bus. config: Per-plugin config overrides. timeout: Maximum seconds to wait for expected messages (default 5.0). + modernize: Forwarded to :class:`MiniPHAL` โ€” FakeBus bridges legacy->spec. + emit_legacy: Forwarded to :class:`MiniPHAL` โ€” FakeBus bridges spec->legacy. + Set both False to exercise a single namespace with no bridging. Example:: @@ -271,6 +286,8 @@ class PHALTest: plugin_factories: Dict[str, Callable[[FakeBus], Any]] = field(default_factory=dict) config: Dict[str, Dict[str, Any]] = field(default_factory=dict) timeout: float = 5.0 + modernize: bool = True + emit_legacy: bool = True def execute(self) -> List[Message]: """Run the test: load plugins, emit trigger, assert expectations. @@ -286,6 +303,8 @@ def execute(self) -> List[Message]: plugin_instances=self.plugin_instances, plugin_factories=self.plugin_factories, config=self.config, + modernize=self.modernize, + emit_legacy=self.emit_legacy, ) as phal: phal.emit(self.trigger_message, wait=0.1) diff --git a/ovoscope/pipeline.py b/ovoscope/pipeline.py index 4ad0ba1..de21cb2 100644 --- a/ovoscope/pipeline.py +++ b/ovoscope/pipeline.py @@ -95,6 +95,15 @@ class PipelineHarness: pipeline: List of OPM pipeline stage IDs to load. pipeline_config: Per-stage config overrides keyed by stage ID. lang: Language tag (default ``"en-US"``). + modernize: Forwarded to the harness ``MiniCroft`` / ``FakeBus``. When + on (default), emitting a LEGACY topic also dispatches its ovos.* + spec counterpart (legacy producer -> spec listener). Utterances are + injected via ``recognizer_loop:utterance``; bridging lets them also + drive / be observed on ``ovos.utterance.handle``. + emit_legacy: Forwarded to the harness. When on (default), emitting an + ovos.* spec topic also dispatches the legacy one (spec producer -> + legacy listener). Set BOTH False to exercise a single namespace + with no cross-namespace bridging. Example:: @@ -111,10 +120,14 @@ def __init__( pipeline: Optional[List[str]] = None, pipeline_config: Optional[Dict[str, Dict[str, Any]]] = None, lang: str = "en-US", + modernize: bool = True, + emit_legacy: bool = True, ) -> None: self.pipeline: List[str] = pipeline or [] self.pipeline_config: Dict[str, Dict[str, Any]] = pipeline_config or {} self.lang: str = lang + self.modernize: bool = modernize + self.emit_legacy: bool = emit_legacy self._mc: Any = None # ------------------------------------------------------------------ @@ -135,6 +148,8 @@ def __enter__(self) -> "PipelineHarness": default_pipeline=self.pipeline or None, extra_skills={"__ovoscope_sink__": sink_skill}, max_wait=60, + modernize=self.modernize, + emit_legacy=self.emit_legacy, ) # Update sink skill's bus reference now that MiniCroft is created diff --git a/ovoscope/simple_listener.py b/ovoscope/simple_listener.py index 2317cb5..3ea2d34 100644 --- a/ovoscope/simple_listener.py +++ b/ovoscope/simple_listener.py @@ -100,6 +100,14 @@ class MiniSimpleListener(ListenerHarness): max_silence_seconds: Trailing silence that ends a command. max_speech_seconds: Hard cap on command length. bus: Optional :class:`FakeBus` to capture on. + modernize: When a fresh bus is created, also emit the ovos.* spec topic + whenever a legacy topic is emitted (legacy producer -> spec + listener). The ``_SimpleBusCallbacks`` emit legacy + ``recognizer_loop:*`` topics; this bridge lets a spec-topic + subscriber observe them. Ignored when *bus* is supplied. + emit_legacy: When a fresh bus is created, also emit the legacy topic + whenever an ovos.* spec topic is emitted. Set both False to exercise + a single namespace with no bridging. Ignored when *bus* is supplied. Raises: RuntimeError: If ovos-simple-listener is not installed. @@ -119,8 +127,10 @@ def __init__( max_silence_seconds: float = 0.1, max_speech_seconds: float = 8.0, bus: Optional[FakeBus] = None, + modernize: bool = True, + emit_legacy: bool = True, ) -> None: - super().__init__(bus) + super().__init__(bus, modernize=modernize, emit_legacy=emit_legacy) try: from ovos_simple_listener import SimpleListener @@ -202,6 +212,8 @@ def get_mini_simple_listener( vad_instance: Optional[Any] = None, stt_instance: Optional[Any] = None, bus: Optional[FakeBus] = None, + modernize: bool = True, + emit_legacy: bool = True, ) -> MiniSimpleListener: """Factory: create a ready-to-feed :class:`MiniSimpleListener`. @@ -211,6 +223,12 @@ def get_mini_simple_listener( stt_instance: Streaming STT engine (defaults to :class:`MockStreamingSTT`). bus: Optional :class:`FakeBus` to capture on. + modernize: When a fresh bus is created, also emit the ovos.* spec topic + whenever a legacy topic is emitted (legacy producer -> spec + listener). Ignored when *bus* is supplied. + emit_legacy: When a fresh bus is created, also emit the legacy topic + whenever an ovos.* spec topic is emitted. Set both False to exercise + a single namespace with no bridging. Ignored when *bus* is supplied. Returns: A fully initialised :class:`MiniSimpleListener`. @@ -220,4 +238,6 @@ def get_mini_simple_listener( vad_instance=vad_instance, stt_instance=stt_instance, bus=bus, + modernize=modernize, + emit_legacy=emit_legacy, ) diff --git a/ovoscope/voice_loop.py b/ovoscope/voice_loop.py index ec208bb..a46406b 100644 --- a/ovoscope/voice_loop.py +++ b/ovoscope/voice_loop.py @@ -501,10 +501,21 @@ class ListenerHarness: Args: bus: Optional :class:`FakeBus` to capture on. Defaults to a fresh bus. + modernize: When a fresh bus is created, also emit the ovos.* spec topic + whenever a legacy topic is emitted (legacy producer -> spec + listener). The listener callbacks emit legacy ``recognizer_loop:*`` + topics; this bridge is what lets a spec-topic subscriber observe + them. Ignored when *bus* is supplied. + emit_legacy: When a fresh bus is created, also emit the legacy topic + whenever an ovos.* spec topic is emitted (spec producer -> legacy + listener). Set both False to exercise a single namespace with no + bridging. Ignored when *bus* is supplied. """ - def __init__(self, bus: Optional[FakeBus] = None) -> None: - self.bus: FakeBus = bus if bus is not None else FakeBus() + def __init__(self, bus: Optional[FakeBus] = None, + modernize: bool = True, emit_legacy: bool = True) -> None: + self.bus: FakeBus = bus if bus is not None else FakeBus( + modernize=modernize, emit_legacy=emit_legacy) self._messages: List[Message] = [] self._last_messages: List[Message] = [] self.bus.on("message", self._capture) @@ -696,6 +707,12 @@ class MiniVoiceLoop(ListenerHarness): :class:`MockStreamingSTT` returning no transcript). Used by :meth:`feed_file`. bus: Optional :class:`FakeBus` to capture on. Defaults to a fresh bus. + modernize: When a fresh bus is created, also emit the ovos.* spec topic + whenever a legacy topic is emitted (legacy producer -> spec + listener). Ignored when *bus* is supplied. + emit_legacy: When a fresh bus is created, also emit the legacy topic + whenever an ovos.* spec topic is emitted. Set both False to exercise + a single namespace with no bridging. Ignored when *bus* is supplied. Raises: RuntimeError: If *voice_loop* is ``None`` and ovos-dinkum-listener is not @@ -720,8 +737,10 @@ def __init__( vad_instance: Optional[Any] = None, stt_instance: Optional[Any] = None, bus: Optional[FakeBus] = None, + modernize: bool = True, + emit_legacy: bool = True, ) -> None: - super().__init__(bus) + super().__init__(bus, modernize=modernize, emit_legacy=emit_legacy) self.hotwords: Optional[MiniHotwordContainer] = None if voice_loop is not None: @@ -924,6 +943,8 @@ def get_mini_voice_loop( vad_instance: Optional[Any] = None, stt_instance: Optional[Any] = None, bus: Optional[FakeBus] = None, + modernize: bool = True, + emit_legacy: bool = True, ) -> MiniVoiceLoop: """Factory: create a ready-to-feed :class:`MiniVoiceLoop`. @@ -935,6 +956,12 @@ def get_mini_voice_loop( stt_instance: Optional streaming STT engine (defaults to :class:`MockStreamingSTT`). bus: Optional :class:`FakeBus` to capture on. + modernize: When a fresh bus is created, also emit the ovos.* spec topic + whenever a legacy topic is emitted (legacy producer -> spec + listener). Ignored when *bus* is supplied. + emit_legacy: When a fresh bus is created, also emit the legacy topic + whenever an ovos.* spec topic is emitted. Set both False to exercise + a single namespace with no bridging. Ignored when *bus* is supplied. Returns: A fully initialised :class:`MiniVoiceLoop`. @@ -957,6 +984,8 @@ def get_mini_voice_loop( vad_instance=vad_instance, stt_instance=stt_instance, bus=bus, + modernize=modernize, + emit_legacy=emit_legacy, ) diff --git a/pyproject.toml b/pyproject.toml index 5d5c426..6a12195 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,12 @@ dependencies = [ # Alpha pin: 2.0.4 stable not yet released; 2.0.4a2 is the minimum that # includes the FakeBus-compatible SkillManager changes ovoscope depends on. "ovos-core>=2.0.4a2", + # The audio harness emits/observes the ovos.* spec topics while ovos-audio + # still produces the legacy topics. >=0.12.0a1 is the first FakeBus that + # mirrors MessageBusClient's legacy<->ovos.* namespace migration (ovos-utils + # #381), so a legacy producer reaches a spec listener (and vice-versa) on the + # test double. Prerelease floor pin: resolves without --pre. + "ovos-utils>=0.12.0a1", # ovoscope ships a pytest plugin (pytest11 entry point) -> pytest is a runtime # dependency. >=8 is required: the pytest_pycollect_makemodule hook dropped the # 'path' argument in pytest 8. @@ -31,7 +37,10 @@ classifiers = [ [project.optional-dependencies] pydantic = ["ovos-pydantic-models>=0.1.0"] -audio = ["ovos-audio>=1.2.0"] +# The audio harness emits/asserts on the OVOS spec bus namespace via SpecMessage, +# and the FakeBus bridging (ovos-utils) imports NamespaceTranslator โ€” new in +# ovos-spec-tools 0.10.0a1, so that is the real floor. +audio = ["ovos-audio>=1.3.0a1", "ovos-spec-tools>=0.10.0a1"] # OCP / ovos-media player harnesses (ovoscope.media: OCPPlayerHarness, ...). media = ["ovos-media>=0.0.2a3"] # MiniListener plugin_instances path (feed_audio_stream) needs the dinkum @@ -42,13 +51,13 @@ listener = ["ovos-dinkum-listener>=0.7.2a1"] # faster-whisper itself is pulled by the plugin โ€” don't list it here to avoid # version skew. tts = [ - "ovos-audio>=1.2.0", + "ovos-audio>=1.3.0a1", "jiwer", "ovos-utterance-normalizer", "ovos-stt-plugin-fasterwhisper", ] dev = [ - "ovos-audio>=1.2.0", + "ovos-audio>=1.3.0a1", "ovos-media>=0.0.2a3", "ovos-dinkum-listener>=0.7.2a1", "ovos-pydantic-models>=0.1.0", diff --git a/test/unittests/test_audio_harness.py b/test/unittests/test_audio_harness.py index 354459a..2405fde 100644 --- a/test/unittests/test_audio_harness.py +++ b/test/unittests/test_audio_harness.py @@ -439,5 +439,59 @@ def test_assert_sequence_fails_for_missing_type(self) -> None: cap.assert_sequence("recognizer_loop:audio_output_end") +# --------------------------------------------------------------------------- +# TestAudioHarnessNamespaceBridging +# --------------------------------------------------------------------------- + +@unittest.skipUnless(AUDIO_AVAILABLE, "ovos-audio (audio extra) not installed") +class TestAudioHarnessNamespaceBridging(unittest.TestCase): + """The audio harness subscribes on the ovos.* SPEC topics while ovos-audio + emits the LEGACY topics. These tests pin that the FakeBus namespace bridging + is what connects them, and that turning it off isolates a single namespace. + """ + + def test_ducking_works_via_bridging_default(self) -> None: + """Default harness (bridging on): ovos-audio's legacy + recognizer_loop:audio_output_start reaches the spec-subscribed + _lower_volume_on_speak via modernize bridging.""" + with AudioServiceHarness() as h: # modernize/emit_legacy default on + h.play(["http://example.com/song.mp3"]) + h.bus.emit(Message("recognizer_loop:audio_output_start")) + start = time.monotonic() + while h.backend.lower_volume_calls == 0 and time.monotonic() - start < 2.0: + time.sleep(0.01) + h.assert_volume_lowered() + + def test_ducking_via_spec_topic_directly(self) -> None: + """A SPEC producer (ovos.audio.output.started) also reaches the + spec-subscribed ducking handler โ€” the harness exercises the new namespace + natively too.""" + from ovos_spec_tools import SpecMessage + with AudioServiceHarness() as h: + h.play(["http://example.com/song.mp3"]) + h.bus.emit(Message(str(SpecMessage.AUDIO_OUTPUT_STARTED))) + start = time.monotonic() + while h.backend.lower_volume_calls == 0 and time.monotonic() - start < 2.0: + time.sleep(0.01) + h.assert_volume_lowered() + + def test_no_bridging_isolates_legacy_from_spec(self) -> None: + """With bridging OFF, a legacy emit does NOT reach the spec-subscribed + ducking handler โ€” proving the harness can exercise a single namespace.""" + with AudioServiceHarness(modernize=False, emit_legacy=False) as h: + h.play(["http://example.com/song.mp3"]) + h.bus.emit(Message("recognizer_loop:audio_output_start")) + time.sleep(0.3) # give any (incorrect) bridge a chance to fire + self.assertEqual(h.backend.lower_volume_calls, 0) + + def test_speak_lifecycle_via_bridging(self) -> None: + """PlaybackService emits legacy audio_output_start/end; the harness + observes them on the spec topics via bridging (default on).""" + with PlaybackServiceHarness() as h: + h.speak("namespace test") + h.assert_audio_output_started() + h.assert_audio_output_ended() + + if __name__ == "__main__": unittest.main() diff --git a/test/unittests/test_classic_listener.py b/test/unittests/test_classic_listener.py index bda322e..3b7607e 100644 --- a/test/unittests/test_classic_listener.py +++ b/test/unittests/test_classic_listener.py @@ -21,6 +21,7 @@ import unittest import wave +from ovos_bus_client.message import Message from ovos_utils.fakebus import FakeBus from ovoscope.classic_listener import ( @@ -36,6 +37,12 @@ except ImportError: HAS_PYEE = False +try: + from ovos_spec_tools import SpecMessage + HAS_SPEC = True +except ImportError: + HAS_SPEC = False + HAS_CLASSIC = classic_listener_available() @@ -106,5 +113,101 @@ def test_full_sequence_with_utterance(self): cl.assert_utterance_emitted("hello world", msgs) +@unittest.skipUnless(HAS_PYEE, "pyee not installed") +@unittest.skipUnless(HAS_SPEC, "ovos-spec-tools not installed") +class TestClassicListenerNamespaceBridging(unittest.TestCase): + """The classic listener bridge emits the LEGACY recognizer_loop:* / + mycroft.awoken topics, while OVOS migrates them to the ovos.* SPEC namespace. + These tests pin that the harness FakeBus namespace bridging connects the two, + and that turning it off isolates a single namespace. + + Migrated pairs exercised here (legacy -> spec): + recognizer_loop:utterance -> ovos.utterance.handle + recognizer_loop:record_begin -> ovos.listener.record.started + recognizer_loop:record_end -> ovos.listener.record.ended + mycroft.awoken -> ovos.listener.awoken + """ + + def _harness(self, **kwargs): + """Build a MiniClassicListener around a plain EventEmitter loop. + + No classic listener install is needed โ€” the bridge drives any + EventEmitter and re-emits its events onto the harness FakeBus. + """ + return MiniClassicListener(recognizer_loop=EventEmitter(), **kwargs) + + def test_utterance_legacy_reaches_spec_default(self): + """Default harness (bridging on): the bridge's legacy + recognizer_loop:utterance reaches an ovos.utterance.handle subscriber.""" + h = self._harness() # modernize/emit_legacy default on + seen = [] + h.bus.on(str(SpecMessage.UTTERANCE), lambda m: seen.append(m.msg_type)) + h.loop.emit("recognizer_loop:utterance", + {"utterances": ["hello world"], "lang": "en-us"}) + self.assertIn(str(SpecMessage.UTTERANCE), seen) + + def test_utterance_spec_reaches_legacy_default(self): + """Default harness (bridging on): a SPEC producer of + ovos.utterance.handle reaches a legacy recognizer_loop:utterance + subscriber โ€” the new namespace is exercised natively too.""" + h = self._harness() + seen = [] + h.bus.on("recognizer_loop:utterance", lambda m: seen.append(m.msg_type)) + h.bus.emit(Message(str(SpecMessage.UTTERANCE), + {"utterances": ["hello world"]})) + self.assertIn("recognizer_loop:utterance", seen) + + def test_record_begin_end_bridge_to_spec(self): + """The bridge's record_begin/record_end reach their spec counterparts.""" + h = self._harness() + started, ended = [], [] + h.bus.on(str(SpecMessage.LISTENER_RECORD_STARTED), + lambda m: started.append(m.msg_type)) + h.bus.on(str(SpecMessage.LISTENER_RECORD_ENDED), + lambda m: ended.append(m.msg_type)) + h.loop.emit("recognizer_loop:record_begin") + h.loop.emit("recognizer_loop:record_end") + self.assertIn(str(SpecMessage.LISTENER_RECORD_STARTED), started) + self.assertIn(str(SpecMessage.LISTENER_RECORD_ENDED), ended) + + def test_awoken_bridges_to_spec(self): + """The bridge maps recognizer_loop:awoken -> mycroft.awoken, which + migrates to ovos.listener.awoken via modernize bridging.""" + h = self._harness() + seen = [] + h.bus.on(str(SpecMessage.LISTENER_AWOKEN), lambda m: seen.append(m.msg_type)) + h.loop.emit("recognizer_loop:awoken") + self.assertIn(str(SpecMessage.LISTENER_AWOKEN), seen) + + def test_awoken_spec_reaches_legacy_default(self): + """A SPEC producer of ovos.listener.awoken reaches a legacy + mycroft.awoken subscriber via emit_legacy bridging.""" + h = self._harness() + seen = [] + h.bus.on("mycroft.awoken", lambda m: seen.append(m.msg_type)) + h.bus.emit(Message(str(SpecMessage.LISTENER_AWOKEN))) + self.assertIn("mycroft.awoken", seen) + + def test_no_bridging_isolates_legacy_from_spec(self): + """With bridging OFF, the bridge's legacy emits do NOT reach the + spec-only subscribers โ€” proving a single namespace can be exercised.""" + h = self._harness(modernize=False, emit_legacy=False) + utt, started, ended, awoken = [], [], [], [] + h.bus.on(str(SpecMessage.UTTERANCE), lambda m: utt.append(m)) + h.bus.on(str(SpecMessage.LISTENER_RECORD_STARTED), lambda m: started.append(m)) + h.bus.on(str(SpecMessage.LISTENER_RECORD_ENDED), lambda m: ended.append(m)) + h.bus.on(str(SpecMessage.LISTENER_AWOKEN), lambda m: awoken.append(m)) + + h.loop.emit("recognizer_loop:utterance", {"utterances": ["hi"]}) + h.loop.emit("recognizer_loop:record_begin") + h.loop.emit("recognizer_loop:record_end") + h.loop.emit("recognizer_loop:awoken") + + self.assertEqual(utt, []) + self.assertEqual(started, []) + self.assertEqual(ended, []) + self.assertEqual(awoken, []) + + if __name__ == "__main__": unittest.main() diff --git a/test/unittests/test_listener_vad_ww.py b/test/unittests/test_listener_vad_ww.py index d6cc398..20851c1 100644 --- a/test/unittests/test_listener_vad_ww.py +++ b/test/unittests/test_listener_vad_ww.py @@ -20,7 +20,13 @@ TestVADTest (5 tests) โ€” VADTest declarative helper TestWakeWordTest (5 tests) โ€” WakeWordTest declarative helper """ +import importlib.util +import time import unittest +from unittest.mock import MagicMock + +from ovos_bus_client.message import Message +from ovos_spec_tools import SpecMessage from ovoscope.listener import ( MockHotWordEngine, @@ -31,6 +37,10 @@ get_mini_listener, ) +# MiniListener.listen() routes audio through AudioTransformersService before +# emitting recognizer_loop:utterance, which requires ovos-dinkum-listener. +DINKUM_AVAILABLE = importlib.util.find_spec("ovos_dinkum_listener") is not None + _BASE_CONFIG = {"listener": {"audio_transformers": {}}} @@ -441,5 +451,73 @@ def test_no_engines_raises(self): WakeWordTest(audio_chunks=[b"\x00" * 512] * 3).execute() +# --------------------------------------------------------------------------- +# TestMiniListenerNamespaceBridging +# --------------------------------------------------------------------------- + +@unittest.skipUnless( + DINKUM_AVAILABLE, "ovos-dinkum-listener not installed" +) +class TestMiniListenerNamespaceBridging(unittest.TestCase): + """MiniListener.listen() emits the LEGACY ``recognizer_loop:utterance`` + (migrated to ``ovos.utterance.handle``). These tests pin that the FakeBus + namespace bridging connects the two namespaces, and that turning it off + isolates a single namespace. + """ + + @staticmethod + def _stt(transcript): + stt = MagicMock() + stt.execute.return_value = transcript + return stt + + def test_utterance_legacy_reaches_spec_via_bridging(self): + """Default harness (bridging on): the legacy utterance emitted by + listen() also reaches a subscriber on the spec topic.""" + listener = get_mini_listener(stt_instance=self._stt("hello world")) + legacy_hits, spec_hits = [], [] + listener.bus.on("recognizer_loop:utterance", lambda m: legacy_hits.append(m)) + listener.bus.on(str(SpecMessage.UTTERANCE), lambda m: spec_hits.append(m)) + try: + listener.listen(b"\x00" * 1024, language="en-us") + time.sleep(0.05) + self.assertTrue(legacy_hits, "legacy recognizer_loop:utterance not seen") + self.assertTrue(spec_hits, "spec ovos.utterance.handle not seen via bridge") + finally: + listener.shutdown() + + def test_utterance_spec_native(self): + """A SPEC producer (ovos.utterance.handle) reaches a spec subscriber + natively โ€” the harness bus exercises the new namespace too.""" + listener = get_mini_listener() + spec_hits = [] + listener.bus.on(str(SpecMessage.UTTERANCE), lambda m: spec_hits.append(m)) + try: + listener.bus.emit(Message(str(SpecMessage.UTTERANCE), + {"utterances": ["hi"], "lang": "en-us"})) + time.sleep(0.05) + self.assertTrue(spec_hits, "spec ovos.utterance.handle not delivered") + finally: + listener.shutdown() + + def test_no_bridging_isolates_legacy_from_spec(self): + """With bridging OFF, the legacy utterance emitted by listen() does NOT + reach a spec-only subscriber.""" + listener = get_mini_listener( + stt_instance=self._stt("hello world"), + modernize=False, emit_legacy=False, + ) + legacy_hits, spec_hits = [], [] + listener.bus.on("recognizer_loop:utterance", lambda m: legacy_hits.append(m)) + listener.bus.on(str(SpecMessage.UTTERANCE), lambda m: spec_hits.append(m)) + try: + listener.listen(b"\x00" * 1024, language="en-us") + time.sleep(0.1) + self.assertTrue(legacy_hits, "legacy utterance should still fire") + self.assertEqual(spec_hits, [], "spec topic must not fire with bridging off") + finally: + listener.shutdown() + + if __name__ == "__main__": unittest.main() diff --git a/test/unittests/test_media.py b/test/unittests/test_media.py index 79fe260..fb225b2 100644 --- a/test/unittests/test_media.py +++ b/test/unittests/test_media.py @@ -13,6 +13,8 @@ # limitations under the License. """Unit tests for ovoscope.media (MockOCPBackend, OCPCaptureSession, OCPPlayerHarness).""" +import time + import pytest from unittest.mock import MagicMock @@ -296,3 +298,65 @@ def test_player_drives_injected_backend_play(self) -> None: playback=PlaybackType.AUDIO)) assert h.backend.is_playing is True assert h.backend.play_calls == ["library://track/42"] + + +# --------------------------------------------------------------------------- +# Namespace bridging +# --------------------------------------------------------------------------- + +@pytest.mark.skipif(not _HAS_OVOS_MEDIA, + reason="requires the [media] extra (ovos-media)") +class TestOCPHarnessNamespaceBridging: + """OCPMediaPlayer subscribes to the LEGACY duck/cork topics + (``recognizer_loop:audio_output_start/end``, ``recognizer_loop:record_begin/end``) + which OVOS is migrating to the ``ovos.*`` spec namespace + (``ovos.audio.output.*`` / ``ovos.listener.record.*``). These tests pin that + the harness FakeBus namespace bridging connects a SPEC-namespace producer to + those legacy handlers, and that turning the bridge off isolates a single + namespace. + + The observable behaviour is the *cork* path: while PLAYING, a record-start + event pauses the player (``handle_cork_request``). + """ + + @staticmethod + def _play_then(h): + """Drive the player into PLAYING with an AUDIO MediaEntry.""" + from ovos_utils.ocp import MediaEntry, PlaybackType, PlayerState + h.play(MediaEntry(uri="http://example.com/song.mp3", + playback=PlaybackType.AUDIO)) + h.assert_player_state(PlayerState.PLAYING) + + def test_cork_via_legacy_topic_natively(self) -> None: + """The legacy ``recognizer_loop:record_begin`` corks (pauses) the player โ€” + the namespace the player subscribes on works natively.""" + from ovos_utils.ocp import PlayerState + with OCPPlayerHarness() as h: # bridging default on + self._play_then(h) + h.bus.emit(Message("recognizer_loop:record_begin")) + time.sleep(0.05) + h.assert_player_state(PlayerState.PAUSED) + + def test_cork_via_spec_topic_through_bridging(self) -> None: + """A SPEC producer emitting ``ovos.listener.record.started`` reaches the + legacy-subscribed ``handle_cork_request`` via emit_legacy bridging and + corks (pauses) the player.""" + from ovos_spec_tools import SpecMessage + from ovos_utils.ocp import PlayerState + with OCPPlayerHarness() as h: # bridging default on + self._play_then(h) + h.bus.emit(Message(str(SpecMessage.LISTENER_RECORD_STARTED))) + time.sleep(0.05) + h.assert_player_state(PlayerState.PAUSED) + + def test_no_bridging_isolates_spec_from_legacy(self) -> None: + """With bridging OFF, a SPEC ``ovos.listener.record.started`` emit does + NOT reach the legacy-subscribed cork handler โ€” the player stays PLAYING, + proving the harness can exercise a single namespace.""" + from ovos_spec_tools import SpecMessage + from ovos_utils.ocp import PlayerState + with OCPPlayerHarness(modernize=False, emit_legacy=False) as h: + self._play_then(h) + h.bus.emit(Message(str(SpecMessage.LISTENER_RECORD_STARTED))) + time.sleep(0.2) # give any (incorrect) bridge a chance to fire + h.assert_player_state(PlayerState.PLAYING) diff --git a/test/unittests/test_minicroft.py b/test/unittests/test_minicroft.py index c3f448f..f48e4ff 100644 --- a/test/unittests/test_minicroft.py +++ b/test/unittests/test_minicroft.py @@ -3,6 +3,7 @@ import unittest from ovos_bus_client.message import Message +from ovos_spec_tools import SpecMessage from ovos_utils.log import LOG from ovos_workshop.skills.ovos import OVOSSkill @@ -10,6 +11,9 @@ from ovoscope import MiniCroft, get_minicroft, DEFAULT_TEST_PIPELINE, LIGHT_TEST_PIPELINE, ADAPT_PIPELINE +LEGACY_UTTERANCE = "recognizer_loop:utterance" +SPEC_UTTERANCE = str(SpecMessage.UTTERANCE) # ovos.utterance.handle + # --------------------------------------------------------------------------- # Minimal inline skill used across tests @@ -324,5 +328,75 @@ def test_pipeline_config_multiple_keys(self): self.assertIsNone(cfg_after.get("plugin_b")) +class TestMiniCroftNamespaceBridging(unittest.TestCase): + """MiniCroft is the e2e harness bus. Utterances are injected on the LEGACY + topic (``recognizer_loop:utterance``); these tests pin that MiniCroft's + FakeBus bridges that to/from the ovos.* SPEC topic + (``ovos.utterance.handle``) so an e2e test can drive EITHER namespace, and + that disabling the bridge isolates a single namespace. + """ + + def setUp(self): + LOG.set_level("ERROR") + + def tearDown(self): + LOG.set_level("CRITICAL") + + def _emit_and_collect(self, mc, emit_topic, watch_topic, *, timeout=3.0): + seen = [] + got = threading.Event() + + def _on(msg): + if isinstance(msg, str): + msg = Message.deserialize(msg) + seen.append(msg) + got.set() + + mc.bus.on(watch_topic, _on) + try: + mc.bus.emit(Message( + emit_topic, + data={"utterances": ["hello world"], "lang": "en-US"}, + )) + got.wait(timeout) + finally: + mc.bus.remove(watch_topic, _on) + return seen + + def test_legacy_utterance_observed_on_spec_topic(self): + """Default (bridging on): a legacy recognizer_loop:utterance injected + into the e2e harness is observed on ovos.utterance.handle (modernize).""" + mc = get_minicroft([]) # modernize/emit_legacy default on + try: + seen = self._emit_and_collect(mc, LEGACY_UTTERANCE, SPEC_UTTERANCE) + self.assertTrue(seen, "legacy utterance was not bridged to the spec topic") + self.assertEqual(seen[0].data["utterances"], ["hello world"]) + finally: + mc.stop() + + def test_spec_utterance_reaches_legacy_listener(self): + """An utterance injected on the SPEC topic reaches a LEGACY listener + (emit_legacy) โ€” the intent pipeline keys off the legacy topic.""" + mc = get_minicroft([]) + try: + seen = self._emit_and_collect(mc, SPEC_UTTERANCE, LEGACY_UTTERANCE) + self.assertTrue(seen, "spec utterance was not bridged to the legacy topic") + self.assertEqual(seen[0].data["utterances"], ["hello world"]) + finally: + mc.stop() + + def test_no_bridging_isolates_legacy_from_spec(self): + """With bridging OFF, a legacy emit does NOT reach a spec-only + subscriber โ€” the e2e harness exercises a single isolated namespace.""" + mc = get_minicroft([], modernize=False, emit_legacy=False) + try: + seen = self._emit_and_collect(mc, LEGACY_UTTERANCE, SPEC_UTTERANCE, + timeout=0.5) + self.assertEqual(seen, [], + "legacy emit must not reach the spec topic when bridging is off") + finally: + mc.stop() + + if __name__ == "__main__": unittest.main() diff --git a/test/unittests/test_namespace_bridging.py b/test/unittests/test_namespace_bridging.py new file mode 100644 index 0000000..7dd1327 --- /dev/null +++ b/test/unittests/test_namespace_bridging.py @@ -0,0 +1,166 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""End-to-end tests for the FakeBus legacy<->ovos.* namespace migration. + +The ovoscope harness runs components on a :class:`ovos_utils.fakebus.FakeBus` +that mirrors ``MessageBusClient``'s namespace migration (ovos-utils #381): with +``modernize``/``emit_legacy`` on (the defaults), emitting a topic on one +namespace ALSO dispatches its counterpart on the other, and a handler subscribed +to both topics fires once. These tests pin that behaviour through the harness bus +so an ovoscope test can deliberately exercise EITHER namespace, BOTH, or a single +isolated namespace. + +The pairs come from ``ovos_spec_tools.MIGRATION_MAP`` (legacy -> SpecMessage), +e.g. ``speak`` <-> ``ovos.utterance.speak`` and +``recognizer_loop:utterance`` <-> ``ovos.utterance.handle``. +""" + +import threading +import time +import unittest + +from ovos_bus_client.message import Message +from ovos_utils.fakebus import FakeBus +from ovos_spec_tools import SpecMessage +from ovos_spec_tools.messages import MIGRATION_MAP + + +# Representative migrating pairs (legacy topic, spec topic). +LEGACY_SPEAK = "speak" +SPEC_SPEAK = str(SpecMessage.SPEAK) # ovos.utterance.speak +LEGACY_UTTERANCE = "recognizer_loop:utterance" +SPEC_UTTERANCE = str(SpecMessage.UTTERANCE) # ovos.utterance.handle + + +def _collect(bus, topic): + """Subscribe a counting handler on ``topic``; return its list of payloads.""" + received = [] + bus.on(topic, lambda m: received.append(m)) + return received + + +class TestFakeBusNamespaceBridging(unittest.TestCase): + """Both-namespace e2e coverage through the ovoscope harness FakeBus.""" + + def test_migration_map_is_populated(self) -> None: + """Sanity: the spec pairs this suite asserts on are actually migrated.""" + self.assertEqual(MIGRATION_MAP[LEGACY_SPEAK], SpecMessage.SPEAK) + self.assertEqual(MIGRATION_MAP[LEGACY_UTTERANCE], SpecMessage.UTTERANCE) + + # -- modernize: legacy producer reaches a spec listener ----------------- + + def test_legacy_emit_reaches_spec_listener(self) -> None: + """A component emitting a LEGACY topic is received on the ovos.* SPEC + topic (modernize bridging).""" + bus = FakeBus() # both flags default on + spec_seen = _collect(bus, SPEC_SPEAK) + + bus.emit(Message(LEGACY_SPEAK, {"utterance": "hello"})) + + self.assertEqual(len(spec_seen), 1) + self.assertEqual(spec_seen[0].msg_type, SPEC_SPEAK) + self.assertEqual(spec_seen[0].data["utterance"], "hello") + + def test_legacy_utterance_reaches_spec_listener(self) -> None: + """recognizer_loop:utterance is delivered on ovos.utterance.handle.""" + bus = FakeBus() + spec_seen = _collect(bus, SPEC_UTTERANCE) + + bus.emit(Message(LEGACY_UTTERANCE, {"utterances": ["turn on the light"]})) + + self.assertEqual(len(spec_seen), 1) + self.assertEqual(spec_seen[0].data["utterances"], ["turn on the light"]) + + # -- emit_legacy: spec producer reaches a legacy listener --------------- + + def test_spec_emit_reaches_legacy_listener(self) -> None: + """A component emitting the SPEC topic is received by a LEGACY listener + (emit_legacy bridging).""" + bus = FakeBus() + legacy_seen = _collect(bus, LEGACY_SPEAK) + + bus.emit(Message(SPEC_SPEAK, {"utterance": "goodbye"})) + + self.assertEqual(len(legacy_seen), 1) + self.assertEqual(legacy_seen[0].msg_type, LEGACY_SPEAK) + self.assertEqual(legacy_seen[0].data["utterance"], "goodbye") + + # -- dedup: a handler on BOTH topics fires once ------------------------- + + def test_handler_on_both_topics_fires_once(self) -> None: + """A handler subscribed to BOTH the legacy and the spec topic fires once + per real event (the mirror is dropped).""" + bus = FakeBus() + calls = [] + + def handler(message=None): + calls.append(message.msg_type) + + bus.on(LEGACY_SPEAK, handler) + bus.on(SPEC_SPEAK, handler) + + bus.emit(Message(LEGACY_SPEAK, {"utterance": "once"})) + + self.assertEqual(len(calls), 1, + f"dual-subscribed handler fired {len(calls)}x: {calls}") + + def test_two_genuine_events_each_fire(self) -> None: + """Dedup must not swallow two genuine events. A SINGLE handler on both + topics fires exactly once per real event, never zero.""" + bus = FakeBus() + calls = [] + + def handler(message=None): + calls.append(message.data.get("utterance")) + + bus.on(LEGACY_SPEAK, handler) + bus.on(SPEC_SPEAK, handler) + + bus.emit(Message(LEGACY_SPEAK, {"utterance": "first"})) + bus.emit(Message(LEGACY_SPEAK, {"utterance": "second"})) + + # shared mirror-guard dedupes each event's counterpart -> two calls + self.assertEqual(calls, ["first", "second"]) + + # -- single-namespace isolation (no bridging) --------------------------- + + def test_no_bridging_isolates_namespaces(self) -> None: + """FakeBus(modernize=False, emit_legacy=False) keeps each namespace + isolated, proving the harness can exercise ONE namespace explicitly.""" + bus = FakeBus(modernize=False, emit_legacy=False) + spec_seen = _collect(bus, SPEC_SPEAK) + legacy_seen = _collect(bus, LEGACY_SPEAK) + + bus.emit(Message(LEGACY_SPEAK, {"utterance": "legacy only"})) + bus.emit(Message(SPEC_SPEAK, {"utterance": "spec only"})) + + # legacy listener saw only the legacy emit; spec listener only the spec + self.assertEqual([m.data["utterance"] for m in legacy_seen], ["legacy only"]) + self.assertEqual([m.data["utterance"] for m in spec_seen], ["spec only"]) + + def test_modernize_only_does_not_emit_legacy(self) -> None: + """modernize=True, emit_legacy=False: legacy->spec bridges but spec->legacy + does not.""" + bus = FakeBus(modernize=True, emit_legacy=False) + spec_seen = _collect(bus, SPEC_SPEAK) + legacy_seen = _collect(bus, LEGACY_SPEAK) + + bus.emit(Message(LEGACY_SPEAK, {"utterance": "x"})) # bridges -> spec + bus.emit(Message(SPEC_SPEAK, {"utterance": "y"})) # must NOT -> legacy + + self.assertEqual([m.data["utterance"] for m in spec_seen], ["x", "y"]) + self.assertEqual([m.data["utterance"] for m in legacy_seen], ["x"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/unittests/test_ocp_namespace.py b/test/unittests/test_ocp_namespace.py new file mode 100644 index 0000000..85b5822 --- /dev/null +++ b/test/unittests/test_ocp_namespace.py @@ -0,0 +1,125 @@ +# Copyright 2024 Jarbas AI +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Namespace-bridging tests for the OCP harness (ovoscope.ocp.OCPTest). + +OCPTest drives the OCP query flow by emitting the LEGACY +``recognizer_loop:utterance`` topic into a MiniCroft. These tests pin that the +harness FakeBus bridges that topic to/from the ovos.* SPEC topic +(``ovos.utterance.handle``) so an OCP test can deliberately exercise EITHER +namespace, and that disabling the bridge isolates a single namespace. + +The OCP query/response path itself is covered elsewhere; here we assert only the +namespace plumbing OCPTest threads into the harness, driving the SAME real +utterance topic OCPTest emits. +""" + +import importlib.util +import threading +import unittest + +from ovos_bus_client.message import Message +from ovos_spec_tools import SpecMessage + +from ovoscope.ocp import OCPTest + +# OCPTest spins up a full MiniCroft; gate on ovos-core being importable. +CORE_AVAILABLE = importlib.util.find_spec("ovos_core") is not None + +LEGACY_UTTERANCE = "recognizer_loop:utterance" +SPEC_UTTERANCE = str(SpecMessage.UTTERANCE) # ovos.utterance.handle + + +class TestOCPTestNamespaceFields(unittest.TestCase): + """The bridging flags are exposed on OCPTest and default on (no extra deps).""" + + def test_defaults_on(self) -> None: + t = OCPTest(skill_ids=[], utterance="play jazz") + self.assertTrue(t.modernize) + self.assertTrue(t.emit_legacy) + + def test_flags_settable(self) -> None: + t = OCPTest(skill_ids=[], utterance="play jazz", + modernize=False, emit_legacy=False) + self.assertFalse(t.modernize) + self.assertFalse(t.emit_legacy) + + +@unittest.skipUnless(CORE_AVAILABLE, "ovos-core not installed") +class TestOCPTestNamespaceBridging(unittest.TestCase): + """Drive the real OCP utterance topic on a harness MiniCroft and assert the + legacy<->spec bridge OCPTest relies on.""" + + def _make_mc(self, **kwargs): + from ovoscope import get_minicroft + return get_minicroft([], lang="en-US", max_wait=60, **kwargs) + + def _emit_and_collect(self, mc, emit_topic, watch_topic, *, timeout=3.0): + seen = [] + got = threading.Event() + + def _on(msg): + if isinstance(msg, str): + msg = Message.deserialize(msg) + seen.append(msg) + got.set() + + mc.bus.on(watch_topic, _on) + try: + mc.bus.emit(Message( + emit_topic, + data={"utterances": ["play some jazz"], "lang": "en-US"}, + )) + got.wait(timeout) + finally: + mc.bus.remove(watch_topic, _on) + return seen + + def test_legacy_utterance_observed_on_spec_topic(self) -> None: + """Default (bridging on): OCPTest's legacy recognizer_loop:utterance is + observed on the spec ovos.utterance.handle topic (modernize).""" + mc = self._make_mc() # modernize/emit_legacy default on + try: + seen = self._emit_and_collect(mc, LEGACY_UTTERANCE, SPEC_UTTERANCE) + self.assertTrue(seen, "legacy utterance was not bridged to the spec topic") + self.assertEqual(seen[0].data["utterances"], ["play some jazz"]) + finally: + mc.stop() + + def test_spec_utterance_reaches_legacy_listener(self) -> None: + """An utterance emitted on the SPEC topic reaches a LEGACY listener + (emit_legacy) โ€” the OCP query path keys off the legacy topic.""" + mc = self._make_mc() + try: + seen = self._emit_and_collect(mc, SPEC_UTTERANCE, LEGACY_UTTERANCE) + self.assertTrue(seen, "spec utterance was not bridged to the legacy topic") + self.assertEqual(seen[0].data["utterances"], ["play some jazz"]) + finally: + mc.stop() + + def test_no_bridging_isolates_legacy_from_spec(self) -> None: + """With bridging OFF, a legacy emit does NOT reach a spec-only + subscriber โ€” OCPTest(modernize=False, emit_legacy=False) exercises a + single namespace.""" + mc = self._make_mc(modernize=False, emit_legacy=False) + try: + seen = self._emit_and_collect(mc, LEGACY_UTTERANCE, SPEC_UTTERANCE, + timeout=0.5) + self.assertEqual(seen, [], + "legacy emit must not reach the spec topic when bridging is off") + finally: + mc.stop() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/unittests/test_phal.py b/test/unittests/test_phal.py index e43168e..c0e9c89 100644 --- a/test/unittests/test_phal.py +++ b/test/unittests/test_phal.py @@ -235,3 +235,69 @@ def test_phal_test_plugin_factories_field(self): ) captured = t.execute() assert any(m.msg_type == "ovos.resp" for m in captured) + + +# --------------------------------------------------------------------------- +# Namespace bridging +# --------------------------------------------------------------------------- + +try: + from ovos_spec_tools import SpecMessage + _HAS_SPEC_TOOLS = True +except ImportError: + _HAS_SPEC_TOOLS = False + + +@pytest.mark.skipif(not _HAS_SPEC_TOOLS, + reason="requires ovos-spec-tools (SpecMessage)") +class TestMiniPHALNamespaceBridging: + """PHAL plugins communicate over arbitrary plugin-specific topics and touch + NONE of the legacy<->ovos.* migrated topics, so the harness has no migrated + topic of its own to drive. These tests instead verify that the harness + ``FakeBus`` performs the same namespace bridging as the audio/media harnesses + (so a PHAL plugin that *did* consume/produce a migrated topic would + interoperate across both namespaces), and that the ``modernize``/``emit_legacy`` + flags are threaded through ``MiniPHAL`` to that bus. + """ + + def test_bus_bridges_legacy_to_spec_by_default(self) -> None: + """Default harness (bridging on): a LEGACY emit on the harness bus is + delivered to a SPEC-topic subscriber (modernize bridging).""" + spec_topic = str(SpecMessage.SPEAK) # ovos.utterance.speak + with MiniPHAL() as phal: + seen = [] + phal._bus.on(spec_topic, lambda m: seen.append(m)) + phal._bus.emit(Message("speak", {"utterance": "hi"})) + time.sleep(0.05) + assert [m.data["utterance"] for m in seen] == ["hi"] + + def test_bus_bridges_spec_to_legacy_by_default(self) -> None: + """Default harness (bridging on): a SPEC emit on the harness bus is + delivered to a LEGACY-topic subscriber (emit_legacy bridging).""" + spec_topic = str(SpecMessage.SPEAK) + with MiniPHAL() as phal: + seen = [] + phal._bus.on("speak", lambda m: seen.append(m)) + phal._bus.emit(Message(spec_topic, {"utterance": "bye"})) + time.sleep(0.05) + assert [m.data["utterance"] for m in seen] == ["bye"] + + def test_no_bridging_isolates_namespaces(self) -> None: + """With modernize=False, emit_legacy=False the harness bus keeps each + namespace isolated โ€” a LEGACY emit does NOT reach a SPEC subscriber.""" + spec_topic = str(SpecMessage.SPEAK) + with MiniPHAL(modernize=False, emit_legacy=False) as phal: + seen = [] + phal._bus.on(spec_topic, lambda m: seen.append(m)) + phal._bus.emit(Message("speak", {"utterance": "legacy only"})) + time.sleep(0.1) + assert seen == [] + + def test_phal_test_threads_bridging_flags(self) -> None: + """PHALTest forwards modernize/emit_legacy to MiniPHAL (default True).""" + t = PHALTest( + plugin_ids=[], + trigger_message=Message("harmless.trigger"), + ) + assert t.modernize is True + assert t.emit_legacy is True diff --git a/test/unittests/test_pipeline_harness.py b/test/unittests/test_pipeline_harness.py index 24ee782..25cfa73 100644 --- a/test/unittests/test_pipeline_harness.py +++ b/test/unittests/test_pipeline_harness.py @@ -1,10 +1,23 @@ """Regression tests for ovoscope.pipeline._SinkSkill.""" +import importlib.util +import threading +import time +import unittest + import pytest +from ovos_bus_client.message import Message +from ovos_spec_tools import SpecMessage from ovos_utils.fakebus import FakeBus -from ovoscope.pipeline import _SinkSkill +from ovoscope.pipeline import PipelineHarness, _SinkSkill + +# PipelineHarness spins up a full MiniCroft pinned to the adapt pipeline. +ADAPT_AVAILABLE = importlib.util.find_spec("ovos_adapt") is not None + +LEGACY_UTTERANCE = "recognizer_loop:utterance" +SPEC_UTTERANCE = str(SpecMessage.UTTERANCE) # ovos.utterance.handle class _RecordingBus: @@ -56,3 +69,70 @@ def test_setting_bus_to_none_raises(self): sink = _SinkSkill() with pytest.raises(ValueError): sink.bus = None + + +# --------------------------------------------------------------------------- +# TestPipelineHarnessNamespaceBridging +# --------------------------------------------------------------------------- + +@unittest.skipUnless(ADAPT_AVAILABLE, "ovos-adapt pipeline plugin not installed") +class TestPipelineHarnessNamespaceBridging(unittest.TestCase): + """PipelineHarness injects utterances on the LEGACY topic + (``recognizer_loop:utterance``). These tests pin that the harness FakeBus + bridges that to/from the ovos.* SPEC topic (``ovos.utterance.handle``) so a + pipeline test can drive EITHER namespace, and that disabling the bridge + isolates a single namespace. + """ + + PIPELINE = ["ovos-adapt-pipeline-plugin-high"] + + def _emit_and_collect(self, harness, emit_topic, watch_topic, *, timeout=3.0): + """Subscribe on ``watch_topic``, emit one utterance on ``emit_topic``, + return the payloads observed on ``watch_topic``.""" + seen = [] + got = threading.Event() + + def _on(msg): + if isinstance(msg, str): + msg = Message.deserialize(msg) + seen.append(msg) + got.set() + + harness._mc.bus.on(watch_topic, _on) + try: + harness._mc.bus.emit(Message( + emit_topic, + data={"utterances": ["turn on the lights"], "lang": "en-US"}, + )) + got.wait(timeout) + finally: + harness._mc.bus.remove(watch_topic, _on) + return seen + + def test_legacy_utterance_observed_on_spec_topic(self): + """Default harness (bridging on): an utterance injected on the legacy + ``recognizer_loop:utterance`` topic is observed on the spec + ``ovos.utterance.handle`` topic (modernize bridging).""" + with PipelineHarness(pipeline=self.PIPELINE) as h: + seen = self._emit_and_collect(h, LEGACY_UTTERANCE, SPEC_UTTERANCE) + self.assertTrue(seen, "legacy utterance was not bridged to the spec topic") + self.assertEqual(seen[0].data["utterances"], ["turn on the lights"]) + + def test_spec_utterance_drives_pipeline_and_legacy_listener(self): + """An utterance injected on the SPEC topic still drives the pipeline + (a match is produced) and reaches a LEGACY listener (emit_legacy).""" + with PipelineHarness(pipeline=self.PIPELINE) as h: + legacy_seen = self._emit_and_collect(h, SPEC_UTTERANCE, LEGACY_UTTERANCE) + self.assertTrue(legacy_seen, + "spec utterance was not bridged to the legacy topic") + self.assertEqual(legacy_seen[0].data["utterances"], ["turn on the lights"]) + + def test_no_bridging_isolates_legacy_from_spec(self): + """With bridging OFF, a legacy utterance emit does NOT reach a + spec-only subscriber โ€” a single namespace is exercised in isolation.""" + with PipelineHarness(pipeline=self.PIPELINE, + modernize=False, emit_legacy=False) as h: + seen = self._emit_and_collect(h, LEGACY_UTTERANCE, SPEC_UTTERANCE, + timeout=0.5) + self.assertEqual(seen, [], + "legacy emit must not reach the spec topic when bridging is off") diff --git a/test/unittests/test_simple_listener.py b/test/unittests/test_simple_listener.py index 481c5f2..c099051 100644 --- a/test/unittests/test_simple_listener.py +++ b/test/unittests/test_simple_listener.py @@ -13,12 +13,30 @@ # limitations under the License. """Unit tests for the MiniSimpleListener harness (ovos-simple-listener).""" import io +import time import unittest import wave +from ovos_bus_client.message import Message +from ovos_spec_tools import SpecMessage + from ovoscope.simple_listener import MiniSimpleListener from ovoscope.voice_loop import MockHotWordEngine, MockStreamingSTT +def _mic_available() -> bool: + """SimpleListener eagerly builds a real microphone in __init__ (mic=None), + so the harness can only be constructed when a microphone plugin is present. + """ + try: + from ovos_plugin_manager.microphone import OVOSMicrophoneFactory + OVOSMicrophoneFactory.create() + return True + except Exception: + return False + + +HAS_MIC = _mic_available() + try: import ovos_simple_listener # noqa: F401 HAS_SIMPLE = True @@ -74,5 +92,77 @@ def test_record_begin_helper(self): sl.assert_record_begin_emitted() +# --------------------------------------------------------------------------- +# TestMiniSimpleListenerNamespaceBridging +# --------------------------------------------------------------------------- + +@unittest.skipUnless(HAS_SIMPLE, "ovos-simple-listener not installed") +@unittest.skipUnless(HAS_MIC, "no ovos microphone plugin available") +class TestMiniSimpleListenerNamespaceBridging(unittest.TestCase): + """The _SimpleBusCallbacks emit the LEGACY ``recognizer_loop:*`` topics; + ``record_begin``/``record_end``/``utterance`` are migrated to the ovos.* + spec namespace. These tests pin that the FakeBus namespace bridging + connects the two namespaces, and that turning it off isolates one. + """ + + def test_full_sequence_legacy_reaches_spec_via_bridging(self): + """Default harness (bridging on): the legacy record-begin/record-end/ + utterance emitted while running the listener are also delivered to + subscribers on the spec topics.""" + with MiniSimpleListener( + wakeword=MockHotWordEngine("hey_mycroft", trigger_after=2), + stt_instance=MockStreamingSTT(transcript="turn on the lights"), + ) as sl: + spec = {"begin": [], "end": [], "utt": []} + sl.bus.on(str(SpecMessage.LISTENER_RECORD_STARTED), + lambda m: spec["begin"].append(m)) + sl.bus.on(str(SpecMessage.LISTENER_RECORD_ENDED), + lambda m: spec["end"].append(m)) + sl.bus.on(str(SpecMessage.UTTERANCE), + lambda m: spec["utt"].append(m)) + + msgs = sl.feed_file(_wav(), silence_tail_chunks=20) + time.sleep(0.05) + legacy_types = [m.msg_type for m in msgs] + self.assertIn("recognizer_loop:record_begin", legacy_types) + self.assertTrue(spec["begin"], + "ovos.listener.record.started not seen via bridge") + self.assertTrue(spec["end"], + "ovos.listener.record.ended not seen via bridge") + self.assertTrue(spec["utt"], + "ovos.utterance.handle not seen via bridge") + + def test_record_begin_spec_native(self): + """A SPEC producer reaches a spec subscriber natively.""" + with MiniSimpleListener( + wakeword=MockHotWordEngine("hey_mycroft", trigger_after=2), + ) as sl: + spec_hits = [] + sl.bus.on(str(SpecMessage.LISTENER_RECORD_STARTED), + lambda m: spec_hits.append(m)) + sl.bus.emit(Message(str(SpecMessage.LISTENER_RECORD_STARTED))) + time.sleep(0.05) + self.assertTrue(spec_hits, + "ovos.listener.record.started not delivered natively") + + def test_no_bridging_isolates_legacy_from_spec(self): + """With bridging OFF, a legacy record-begin does NOT reach a spec-only + subscriber.""" + with MiniSimpleListener( + wakeword=MockHotWordEngine("hey_mycroft", trigger_after=2), + modernize=False, emit_legacy=False, + ) as sl: + legacy_hits, spec_hits = [], [] + sl.bus.on("recognizer_loop:record_begin", + lambda m: legacy_hits.append(m)) + sl.bus.on(str(SpecMessage.LISTENER_RECORD_STARTED), + lambda m: spec_hits.append(m)) + sl.bus.emit(Message("recognizer_loop:record_begin")) + time.sleep(0.1) + self.assertTrue(legacy_hits, "legacy record_begin should still fire") + self.assertEqual(spec_hits, [], + "spec topic must not fire with bridging off") + + if __name__ == "__main__": unittest.main() diff --git a/test/unittests/test_voice_loop.py b/test/unittests/test_voice_loop.py index 473c8c1..888360c 100644 --- a/test/unittests/test_voice_loop.py +++ b/test/unittests/test_voice_loop.py @@ -20,10 +20,14 @@ """ import inspect import io +import time import unittest import wave from unittest.mock import Mock +from ovos_bus_client.message import Message +from ovos_spec_tools import SpecMessage + from ovoscope.voice_loop import ( MiniHotwordContainer, MiniVoiceLoop, @@ -298,5 +302,76 @@ def test_audio_file_with_expected_utterance(self): ).execute() +# --------------------------------------------------------------------------- +# TestMiniVoiceLoopNamespaceBridging +# --------------------------------------------------------------------------- + +@unittest.skipUnless(HAS_DINKUM, "ovos-dinkum-listener not installed") +class TestMiniVoiceLoopNamespaceBridging(unittest.TestCase): + """The MiniVoiceLoop callbacks emit the LEGACY ``recognizer_loop:*`` topics; + ``record_begin``/``record_end``/``utterance`` are migrated to the ovos.* + spec namespace. These tests pin that the FakeBus namespace bridging + connects the two namespaces, and that turning it off isolates one. + """ + + def test_full_loop_legacy_reaches_spec_via_bridging(self): + """Default loop (bridging on): the legacy record-begin/record-end/ + utterance emitted while driving the full loop are also delivered to + subscribers on the spec topics.""" + with MiniVoiceLoop( + ww_instances={"hey_mycroft": MockHotWordEngine(trigger_after=2)}, + stt_instance=MockStreamingSTT(transcript="hello world"), + ) as vl: + spec = {"begin": [], "end": [], "utt": []} + vl.bus.on(str(SpecMessage.LISTENER_RECORD_STARTED), + lambda m: spec["begin"].append(m)) + vl.bus.on(str(SpecMessage.LISTENER_RECORD_ENDED), + lambda m: spec["end"].append(m)) + vl.bus.on(str(SpecMessage.UTTERANCE), + lambda m: spec["utt"].append(m)) + + msgs = vl.feed_file(_wav()) + time.sleep(0.05) + legacy_types = [m.msg_type for m in msgs] + self.assertIn("recognizer_loop:record_begin", legacy_types) + self.assertTrue(spec["begin"], + "ovos.listener.record.started not seen via bridge") + self.assertTrue(spec["end"], + "ovos.listener.record.ended not seen via bridge") + self.assertTrue(spec["utt"], + "ovos.utterance.handle not seen via bridge") + + def test_record_begin_spec_native(self): + """A SPEC producer reaches a spec subscriber natively.""" + with MiniVoiceLoop( + ww_instances={"hey_mycroft": MockHotWordEngine(trigger_after=2)}, + ) as vl: + spec_hits = [] + vl.bus.on(str(SpecMessage.LISTENER_RECORD_STARTED), + lambda m: spec_hits.append(m)) + vl.bus.emit(Message(str(SpecMessage.LISTENER_RECORD_STARTED))) + time.sleep(0.05) + self.assertTrue(spec_hits, + "ovos.listener.record.started not delivered natively") + + def test_no_bridging_isolates_legacy_from_spec(self): + """With bridging OFF, a legacy record-begin does NOT reach a spec-only + subscriber.""" + with MiniVoiceLoop( + ww_instances={"hey_mycroft": MockHotWordEngine(trigger_after=2)}, + modernize=False, emit_legacy=False, + ) as vl: + legacy_hits, spec_hits = [], [] + vl.bus.on("recognizer_loop:record_begin", + lambda m: legacy_hits.append(m)) + vl.bus.on(str(SpecMessage.LISTENER_RECORD_STARTED), + lambda m: spec_hits.append(m)) + vl.bus.emit(Message("recognizer_loop:record_begin")) + time.sleep(0.1) + self.assertTrue(legacy_hits, "legacy record_begin should still fire") + self.assertEqual(spec_hits, [], + "spec topic must not fire with bridging off") + + if __name__ == "__main__": unittest.main() From 7f5b29b9a43aae76516dc44704f685edcb50b007 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Thu, 25 Jun 2026 21:24:46 +0000 Subject: [PATCH 60/82] Increment Version to 1.0.0a1 --- ovoscope/version.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ovoscope/version.py b/ovoscope/version.py index 515f5df..ed2381b 100644 --- a/ovoscope/version.py +++ b/ovoscope/version.py @@ -1,7 +1,7 @@ # START_VERSION_BLOCK -VERSION_MAJOR = 0 -VERSION_MINOR = 22 -VERSION_BUILD = 1 +VERSION_MAJOR = 1 +VERSION_MINOR = 0 +VERSION_BUILD = 0 VERSION_ALPHA = 1 # END_VERSION_BLOCK From 0cbdf53e4c69f87530980348bb431d3d2b4d3933 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Thu, 25 Jun 2026 21:25:15 +0000 Subject: [PATCH 61/82] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ffb1ede..3dc7c98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [1.0.0a1](https://github.com/OpenVoiceOS/ovoscope/tree/1.0.0a1) (2026-06-25) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.22.1a1...1.0.0a1) + +**Breaking changes:** + +- feat!: audio harness on OVOS spec bus namespace [\#92](https://github.com/OpenVoiceOS/ovoscope/pull/92) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.22.1a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.22.1a1) (2026-06-25) [Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.22.0a1...0.22.1a1) From ae65a28edacdd3ee12e2d7a82b7da0b030296e52 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 27 Jun 2026 13:42:09 +0100 Subject: [PATCH 62/82] fix: guard None blacklisted_skills/intents in final-session check (#98) Under the 9.x session shape Session.blacklisted_skills and blacklisted_intents can be None rather than an empty list, which made the final-session equality check raise TypeError: 'NoneType' object is not iterable. Coalesce to [] before set() so the assertion compares cleanly. Co-authored-by: Claude Opus 4.8 --- ovoscope/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ovoscope/__init__.py b/ovoscope/__init__.py index 0f1eddc..88cdf21 100644 --- a/ovoscope/__init__.py +++ b/ovoscope/__init__.py @@ -941,8 +941,8 @@ def execute(self, timeout: int = 30) -> List[Message]: assert sess.time_format == expected_sess.time_format, f"โŒ final session time_format doesn't match" assert sess.site_id == expected_sess.site_id, f"โŒ final session site_id doesn't match" assert sess.session_id == expected_sess.session_id, f"โŒ final session session_id doesn't match" - assert set(sess.blacklisted_skills) == set(expected_sess.blacklisted_skills), f"โŒ final session blacklisted_skills doesn't match" - assert set(sess.blacklisted_intents) == set(expected_sess.blacklisted_intents), f"โŒ final session blacklisted_intents doesn't match" + assert set(sess.blacklisted_skills or []) == set(expected_sess.blacklisted_skills or []), f"โŒ final session blacklisted_skills doesn't match" + assert set(sess.blacklisted_intents or []) == set(expected_sess.blacklisted_intents or []), f"โŒ final session blacklisted_intents doesn't match" if self.verbose: print(f"โœ… final session matches: {expected_sess.serialize()}") From 1d4e3ca519a8809cfabaf124f39bf82e9cb06dea Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 27 Jun 2026 12:42:19 +0000 Subject: [PATCH 63/82] Increment Version to 1.0.1a1 --- ovoscope/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovoscope/version.py b/ovoscope/version.py index ed2381b..f5f70e5 100644 --- a/ovoscope/version.py +++ b/ovoscope/version.py @@ -1,7 +1,7 @@ # START_VERSION_BLOCK VERSION_MAJOR = 1 VERSION_MINOR = 0 -VERSION_BUILD = 0 +VERSION_BUILD = 1 VERSION_ALPHA = 1 # END_VERSION_BLOCK From 68f58379629e085bf0fc91de75f787472dd5afdb Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 27 Jun 2026 12:42:40 +0000 Subject: [PATCH 64/82] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3dc7c98..95932c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [1.0.1a1](https://github.com/OpenVoiceOS/ovoscope/tree/1.0.1a1) (2026-06-27) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/1.0.0a1...1.0.1a1) + +**Merged pull requests:** + +- fix: guard None blacklisted\_skills/intents in final-session check [\#98](https://github.com/OpenVoiceOS/ovoscope/pull/98) ([JarbasAl](https://github.com/JarbasAl)) + ## [1.0.0a1](https://github.com/OpenVoiceOS/ovoscope/tree/1.0.0a1) (2026-06-25) [Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.22.1a1...1.0.0a1) From 5afdc723bd040294d14ff39e01fe3944f50c44c1 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Sun, 28 Jun 2026 00:40:44 +0100 Subject: [PATCH 65/82] fix: MockTTS destructor must not stop the shared playback thread (#100) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TTS.playback is a class-level attribute shared by every TTS instance in the process. The inherited TTS.__del__ chains into TTS.stop() -> TTS.playback.stop(), so when an earlier PlaybackServiceHarness's MockTTS is garbage-collected its destructor terminated whatever PlaybackThread was *currently* registered there โ€” which by then belongs to a later, still-running harness. The victim thread had _terminated set and exited its loop, so its queued speak never played and ovos.audio.output.ended was never emitted, hanging the next speak() until timeout. GC timing made this a flaky TimeoutError that surfaced only after several harness create/destroy cycles (e.g. mid-file in a consumer's test/end2end suite). Override MockTTS.__del__ as a no-op: the harness already manages playback-thread lifecycle explicitly via PlaybackService.shutdown() on context exit, so a mock instance must never tear down the shared thread on collection. Add regression tests: a deterministic guard that fires a stale mock's destructor while a later harness owns TTS.playback and asserts the live thread is neither terminated nor unable to keep speaking, plus a many-sequential-harnesses smoke test. Co-authored-by: Claude Opus 4.8 --- ovoscope/audio.py | 20 ++++++++ test/unittests/test_audio_harness.py | 74 ++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/ovoscope/audio.py b/ovoscope/audio.py index a1d816e..bd504c1 100644 --- a/ovoscope/audio.py +++ b/ovoscope/audio.py @@ -512,6 +512,26 @@ def reset(self) -> None: """Clear the list of recorded spoken utterances.""" self.spoken_utterances.clear() + def __del__(self) -> None: + """No-op destructor. + + ``TTS.__del__`` chains into ``TTS.shutdown() -> TTS.stop() -> + TTS.playback.stop()``. ``TTS.playback`` is a **class-level** attribute + shared by every TTS instance in the process, so when an earlier + harness's MockTTS is garbage-collected its inherited destructor stops + whatever PlaybackThread is *currently* registered there โ€” which, by + then, belongs to a later, still-running harness. The victim thread sets + ``_terminated`` and exits mid-run, so its queued speak never plays and + ``ovos.audio.output.ended`` is never emitted, hanging the next + ``speak()`` wait. + + GC timing is nondeterministic, so the failure surfaces as a flaky + ``TimeoutError`` only after several harness instances have been created + and collected. The harness already manages thread lifecycle explicitly + via ``PlaybackService.shutdown()`` on context exit, so a MockTTS + instance must never tear down the shared playback thread on collection. + """ + # --------------------------------------------------------------------------- # PlaybackServiceHarness diff --git a/test/unittests/test_audio_harness.py b/test/unittests/test_audio_harness.py index 2405fde..009c05c 100644 --- a/test/unittests/test_audio_harness.py +++ b/test/unittests/test_audio_harness.py @@ -34,6 +34,7 @@ from ovos_utils.fakebus import FakeBus if AUDIO_AVAILABLE: + from ovos_plugin_manager.templates.tts import TTS from ovoscope.audio import ( AudioCaptureSession, AudioServiceHarness, @@ -493,5 +494,78 @@ def test_speak_lifecycle_via_bridging(self) -> None: h.assert_audio_output_ended() +@unittest.skipUnless(AUDIO_AVAILABLE, "ovos-audio (audio extra) not installed") +class TestPlaybackServiceHarnessIsolation(unittest.TestCase): + """Repeated, independent harness instances must not interfere. + + Regression for the shared ``TTS.playback`` class-attribute hazard: a + garbage-collected MockTTS from an earlier harness used to stop the + PlaybackThread of a *later*, still-running harness (via the inherited + ``TTS.__del__`` -> ``TTS.stop`` -> ``TTS.playback.stop()`` chain). The + victim thread terminated mid-run, its queued speak never played, and the + next ``speak()`` hung until timeout. Because GC timing is nondeterministic + this manifested as a flaky ``TimeoutError`` only after several + create/destroy cycles. + """ + + def test_many_sequential_harnesses_each_complete_speaks(self) -> None: + """Boot and tear down many harnesses, forcing GC between them, and + require every speak in every harness to complete deterministically.""" + import gc + + for i in range(12): + with PlaybackServiceHarness() as h: + for tag in ("a", "b", "c"): + # unique sentences so the persistent TTS cache never + # short-circuits synthesis โ€” each must drive real playback + h.speak(f"iter {i} part {tag}", timeout=8.0) + self.assertIn(f"iter {i} part {tag}", + h.mock_tts.spoken_utterances) + # provoke collection of the just-exited MockTTS *now*, while a + # fresh harness will shortly own TTS.playback. Pre-fix, this is + # exactly what killed the next harness's playback thread. + gc.collect() + + def test_stale_mock_destructor_does_not_kill_live_thread(self) -> None: + """A finished harness's MockTTS destructor must not terminate the + playback thread that a *later* harness now owns. + + Deterministic reproduction of the GC race: keep a reference to harness + A's MockTTS so it outlives A, open harness B (which registers its own + thread on the shared ``TTS.playback`` class attribute), then run A's + destructor. Pre-fix, ``MockTTS.__del__`` chained into + ``TTS.playback.stop()`` and terminated B's live thread; B's next speak + would then hang. With the no-op destructor, B is unaffected. + """ + # Harness A โ€” produce a MockTTS that survives the context exit. + with PlaybackServiceHarness() as ha: + ha.speak("harness A warmup", timeout=8.0) + stale_mock = ha.mock_tts + + # Harness B now owns the shared TTS.playback thread. + with PlaybackServiceHarness() as hb: + self.assertIs(TTS.playback, hb.svc.playback_thread) + self.assertTrue(hb.svc.playback_thread.is_alive()) + + # Fire harness A's destructor explicitly (what GC would do). + stale_mock.__del__() + + # The precise invariant: A's destructor must not have flagged B's + # thread for termination. ``_terminated`` is checked at the top of + # the playback loop, so a single in-flight speak can still slip + # through even when set โ€” but the thread would then exit on its next + # iteration, hanging a subsequent speak. Assert the flag directly. + self.assertFalse( + hb.svc.playback_thread._terminated, + "stale MockTTS destructor terminated the live playback thread", + ) + + # And B must keep working across multiple speaks (the loop must not + # have exited). + for n in range(3): + hb.speak(f"harness B speak {n}", timeout=8.0) + self.assertTrue(hb.svc.playback_thread.is_alive()) + + if __name__ == "__main__": unittest.main() From b19bef63c9702defdaf81aa1f4e7b8db50e22f26 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 27 Jun 2026 23:40:54 +0000 Subject: [PATCH 66/82] Increment Version to 1.0.2a1 --- ovoscope/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovoscope/version.py b/ovoscope/version.py index f5f70e5..605019b 100644 --- a/ovoscope/version.py +++ b/ovoscope/version.py @@ -1,7 +1,7 @@ # START_VERSION_BLOCK VERSION_MAJOR = 1 VERSION_MINOR = 0 -VERSION_BUILD = 1 +VERSION_BUILD = 2 VERSION_ALPHA = 1 # END_VERSION_BLOCK From 3075b6ab47f121fd415973d49cc413cacd89b890 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 27 Jun 2026 23:41:19 +0000 Subject: [PATCH 67/82] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95932c7..18c629a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [1.0.2a1](https://github.com/OpenVoiceOS/ovoscope/tree/1.0.2a1) (2026-06-27) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/1.0.1a1...1.0.2a1) + +**Merged pull requests:** + +- fix: MockTTS destructor must not stop the shared playback thread [\#100](https://github.com/OpenVoiceOS/ovoscope/pull/100) ([JarbasAl](https://github.com/JarbasAl)) + ## [1.0.1a1](https://github.com/OpenVoiceOS/ovoscope/tree/1.0.1a1) (2026-06-27) [Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/1.0.0a1...1.0.1a1) From 0256086e8aa8b4f6e884208be40638d34ffe750a Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Mon, 29 Jun 2026 02:57:28 +0100 Subject: [PATCH 68/82] =?UTF-8?q?feat:=20MockTTS=20=E2=80=94=20emit=20audi?= =?UTF-8?q?o=5Foutput=5Fend=20on=20delay=20for=20speak=5Fdialog(wait=3DTru?= =?UTF-8?q?e)=20(#102)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: MockTTS destructor must not stop the shared playback thread TTS.playback is a class-level attribute shared by every TTS instance in the process. The inherited TTS.__del__ chains into TTS.stop() -> TTS.playback.stop(), so when an earlier PlaybackServiceHarness's MockTTS is garbage-collected its destructor terminated whatever PlaybackThread was *currently* registered there โ€” which by then belongs to a later, still-running harness. The victim thread had _terminated set and exited its loop, so its queued speak never played and ovos.audio.output.ended was never emitted, hanging the next speak() until timeout. GC timing made this a flaky TimeoutError that surfaced only after several harness create/destroy cycles (e.g. mid-file in a consumer's test/end2end suite). Override MockTTS.__del__ as a no-op: the harness already manages playback-thread lifecycle explicitly via PlaybackService.shutdown() on context exit, so a mock instance must never tear down the shared thread on collection. Add regression tests: a deterministic guard that fires a stale mock's destructor while a later harness owns TTS.playback and asserts the live thread is neither terminated nor unable to keep speaking, plus a many-sequential-harnesses smoke test. Co-Authored-By: Claude Opus 4.8 * feat: MockTTS โ€” emit audio_output_end on delay for speak_dialog(wait=True) Skills calling speak_dialog(..., wait=True) block on recognizer_loop:audio_output_end via SessionManager.wait_while_speaking. Without a real TTS the handler thread blocks for 15+s, tripping the ยง8.3 10s handler backstop and spurious handler.error. MockTTS schedules audio_output_end on a 0.1s Timer from the speak handler. Uses bus.ee.emit (not bus.emit) to bypass FakeBus namespace-migration and on_message side effects so the synthetic event is invisible to test captures. * chore: drop agent scratch (AGENTS.md, TODO.md) from the PR Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- ovoscope/__init__.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/ovoscope/__init__.py b/ovoscope/__init__.py index 88cdf21..bdc5b78 100644 --- a/ovoscope/__init__.py +++ b/ovoscope/__init__.py @@ -15,6 +15,7 @@ from ovos_utils.fakebus import FakeBus from ovos_utils.log import LOG from ovos_utils.process_utils import ProcessState +from ovos_spec_tools import SpecMessage from ovos_workshop.skills.ovos import OVOSSkill SerializedMessage = Dict[str, Union[str, Dict[str, Any]]] @@ -391,6 +392,23 @@ def __init__(self, skill_ids, bus = FakeBus(modernize=self._modernize, emit_legacy=self._emit_legacy) bus.on("message", self.handle_boot_message) + + # TTS mock: speak_dialog(โ€ฆ, wait=True) blocks on + # recognizer_loop:audio_output_end. Since there is no real TTS we + # schedule a short-delay emit to unblock the handler. + # This uses bus.ee.emit (not bus.emit) to bypass FakeBus's + # namespace-migration and on_message side effects so the synthetic + # event does not appear in test captures or reset session state. + def _mock_tts(message): + sess = SessionManager.get(message) + threading.Timer(0.1, lambda: bus.ee.emit( + "recognizer_loop:audio_output_end", + Message("recognizer_loop:audio_output_end", + context={"session": sess.serialize()}) + )).start() + + bus.on(SpecMessage.SPEAK, _mock_tts) + self.skill_ids = skill_ids self.extra_skills = extra_skills or {} From a8c02d42e9d6bb59dc1a956d5d26f641e626511a Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Mon, 29 Jun 2026 01:57:44 +0000 Subject: [PATCH 69/82] Increment Version to 1.1.0a1 --- ovoscope/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ovoscope/version.py b/ovoscope/version.py index 605019b..c0ff163 100644 --- a/ovoscope/version.py +++ b/ovoscope/version.py @@ -1,7 +1,7 @@ # START_VERSION_BLOCK VERSION_MAJOR = 1 -VERSION_MINOR = 0 -VERSION_BUILD = 2 +VERSION_MINOR = 1 +VERSION_BUILD = 0 VERSION_ALPHA = 1 # END_VERSION_BLOCK From 303b98d6d0ac8304711d60b57c5dc6c8517a5a4d Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Mon, 29 Jun 2026 02:01:44 +0000 Subject: [PATCH 70/82] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 18c629a..709a354 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [1.1.0a1](https://github.com/OpenVoiceOS/ovoscope/tree/1.1.0a1) (2026-06-29) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/1.0.2a1...1.1.0a1) + +**Merged pull requests:** + +- feat: MockTTS โ€” emit audio\_output\_end on delay for speak\_dialog\(wait=True\) [\#102](https://github.com/OpenVoiceOS/ovoscope/pull/102) ([JarbasAl](https://github.com/JarbasAl)) + ## [1.0.2a1](https://github.com/OpenVoiceOS/ovoscope/tree/1.0.2a1) (2026-06-27) [Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/1.0.1a1...1.0.2a1) From 88fb7206d59cdcb10de024b9e94dd4feb9aef6ba Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Mon, 29 Jun 2026 13:38:59 +0100 Subject: [PATCH 71/82] docs: clarify why MockTTS uses bus.ee.emit (synthetic audio event, no session reset) (#104) Expands the comment to explain that audio_output_end is a synthetic hardware event, so it must bypass FakeBus.on_message's session rebuild/SessionManager.update (which mirrors the real MessageBusClient and would clobber transient is_speaking/active_skills). Co-authored-by: Claude Opus 4.8 --- ovoscope/__init__.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/ovoscope/__init__.py b/ovoscope/__init__.py index bdc5b78..bfd8a29 100644 --- a/ovoscope/__init__.py +++ b/ovoscope/__init__.py @@ -393,12 +393,19 @@ def __init__(self, skill_ids, emit_legacy=self._emit_legacy) bus.on("message", self.handle_boot_message) - # TTS mock: speak_dialog(โ€ฆ, wait=True) blocks on - # recognizer_loop:audio_output_end. Since there is no real TTS we - # schedule a short-delay emit to unblock the handler. - # This uses bus.ee.emit (not bus.emit) to bypass FakeBus's - # namespace-migration and on_message side effects so the synthetic - # event does not appear in test captures or reset session state. + # TTS mock: speak_dialog(โ€ฆ, wait=True) blocks in wait_while_speaking on + # recognizer_loop:audio_output_end. With no real TTS that event never + # arrives, so the handler stalls until the dispatcher's ยง8.3 timeout. We + # schedule a short-delay emit to unblock it. + # + # Deliberately bus.ee.emit, NOT bus.emit: audio_output_end is a synthetic + # *hardware/audio* event, not a message a component emitted with an + # authoritative session. bus.emit would run FakeBus.on_message, which โ€” + # like the real MessageBusClient โ€” rebuilds the Session from the message and + # calls SessionManager.update(), wholesale-replacing the cached session. A + # contentless synthetic event would thereby clobber transient state + # (is_speaking, active_skills). bus.ee.emit fires only the topic handlers, + # which is the correct semantics for injecting a fake audio event. def _mock_tts(message): sess = SessionManager.get(message) threading.Timer(0.1, lambda: bus.ee.emit( From 7ce13e52a909a334fb8bcca3b2eef5966eedb1e8 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Mon, 29 Jun 2026 12:39:12 +0000 Subject: [PATCH 72/82] Increment Version to 1.1.0a2 --- ovoscope/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovoscope/version.py b/ovoscope/version.py index c0ff163..587064f 100644 --- a/ovoscope/version.py +++ b/ovoscope/version.py @@ -2,7 +2,7 @@ VERSION_MAJOR = 1 VERSION_MINOR = 1 VERSION_BUILD = 0 -VERSION_ALPHA = 1 +VERSION_ALPHA = 2 # END_VERSION_BLOCK __version__ = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}" + ( From ca22916fc841451644bde2e42e5cf147d6a4176c Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Mon, 29 Jun 2026 12:39:33 +0000 Subject: [PATCH 73/82] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 709a354..6d694ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [1.1.0a2](https://github.com/OpenVoiceOS/ovoscope/tree/1.1.0a2) (2026-06-29) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/1.1.0a1...1.1.0a2) + +**Merged pull requests:** + +- docs: clarify MockTTS bus.ee.emit rationale [\#104](https://github.com/OpenVoiceOS/ovoscope/pull/104) ([JarbasAl](https://github.com/JarbasAl)) + ## [1.1.0a1](https://github.com/OpenVoiceOS/ovoscope/tree/1.1.0a1) (2026-06-29) [Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/1.0.2a1...1.1.0a1) From 31a5d778df00a891483e1f6b76657a215fdf59b0 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Mon, 29 Jun 2026 15:21:58 +0100 Subject: [PATCH 74/82] feat: MockTTS publishes audio_output_end via the full bus (faithful) (#106) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit recognizer_loop:audio_output_end is a real bus message in production (emitted by the audio service), so the harness now publishes it the same way โ€” plain bus.emit through on_message โ€” instead of bus.ee.emit, which bypassed namespace migration and the capture path. This is now safe and correct because ovos-bus-client's SessionManager keeps one live Session per id (mutates in place, no wholesale replace), so routing through on_message no longer clobbers transient session state; handle_audio_output_end flips is_speaking=False on the shared singleton. Capture position is faithful to a real deployment: with speak(wait=False) the handler emits its end-marker first, so audio_output_end lands after the EOF and is not captured; with speak(wait=True) the handler blocks in wait_while_speaking until it arrives, so it deterministically precedes the end-marker and is captured โ€” as any real bus observer would record it. Co-authored-by: Claude Opus 4.8 --- ovoscope/__init__.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/ovoscope/__init__.py b/ovoscope/__init__.py index bfd8a29..d931dc4 100644 --- a/ovoscope/__init__.py +++ b/ovoscope/__init__.py @@ -398,18 +398,20 @@ def __init__(self, skill_ids, # arrives, so the handler stalls until the dispatcher's ยง8.3 timeout. We # schedule a short-delay emit to unblock it. # - # Deliberately bus.ee.emit, NOT bus.emit: audio_output_end is a synthetic - # *hardware/audio* event, not a message a component emitted with an - # authoritative session. bus.emit would run FakeBus.on_message, which โ€” - # like the real MessageBusClient โ€” rebuilds the Session from the message and - # calls SessionManager.update(), wholesale-replacing the cached session. A - # contentless synthetic event would thereby clobber transient state - # (is_speaking, active_skills). bus.ee.emit fires only the topic handlers, - # which is the correct semantics for injecting a fake audio event. + # Full bus.emit, like the real audio service: recognizer_loop:audio_output_end + # is a genuine bus message in production, so the harness publishes it the + # same way โ€” through on_message (namespace migration + capture), where any + # bus observer would see it. SessionManager.handle_audio_output_end then + # flips is_speaking=False on the one live session object the ovos-bus-client + # SessionManager singleton keeps per id, so every holder of that session sees + # it. Capture position is faithful: with speak(wait=False) the handler emits + # its end-marker first, so this lands after the EOF and is not captured; with + # speak(wait=True) the handler blocks in wait_while_speaking until this + # arrives, so it deterministically precedes the end-marker and is captured โ€” + # exactly as a real deployment would record it. def _mock_tts(message): sess = SessionManager.get(message) - threading.Timer(0.1, lambda: bus.ee.emit( - "recognizer_loop:audio_output_end", + threading.Timer(0.1, lambda: bus.emit( Message("recognizer_loop:audio_output_end", context={"session": sess.serialize()}) )).start() From 9d70f9387e60ca8f29cba9d8922fc2f3d701c4b5 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Mon, 29 Jun 2026 14:22:14 +0000 Subject: [PATCH 75/82] Increment Version to 1.2.0a1 --- ovoscope/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ovoscope/version.py b/ovoscope/version.py index 587064f..b62bcfb 100644 --- a/ovoscope/version.py +++ b/ovoscope/version.py @@ -1,8 +1,8 @@ # START_VERSION_BLOCK VERSION_MAJOR = 1 -VERSION_MINOR = 1 +VERSION_MINOR = 2 VERSION_BUILD = 0 -VERSION_ALPHA = 2 +VERSION_ALPHA = 1 # END_VERSION_BLOCK __version__ = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}" + ( From 5e79e57a05da6a74c47cb6b72b1eb594b2d5099c Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Mon, 29 Jun 2026 14:22:37 +0000 Subject: [PATCH 76/82] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d694ec..fce9426 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [1.2.0a1](https://github.com/OpenVoiceOS/ovoscope/tree/1.2.0a1) (2026-06-29) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/1.1.0a2...1.2.0a1) + +**Merged pull requests:** + +- feat: MockTTS publishes audio\_output\_end via the full bus \(faithful\) [\#106](https://github.com/OpenVoiceOS/ovoscope/pull/106) ([JarbasAl](https://github.com/JarbasAl)) + ## [1.1.0a2](https://github.com/OpenVoiceOS/ovoscope/tree/1.1.0a2) (2026-06-29) [Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/1.1.0a1...1.1.0a2) From 42975d0e3be624ae8801fc3d37a2c692a4606e16 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Mon, 29 Jun 2026 18:42:51 +0100 Subject: [PATCH 77/82] feat: emit recognizer_loop:audio_output_start in _mock_tts alongside audio_output_end (#108) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: MockTTS publishes audio_output_end via the full bus (faithful) recognizer_loop:audio_output_end is a real bus message in production (emitted by the audio service), so the harness now publishes it the same way โ€” plain bus.emit through on_message โ€” instead of bus.ee.emit, which bypassed namespace migration and the capture path. This is now safe and correct because ovos-bus-client's SessionManager keeps one live Session per id (mutates in place, no wholesale replace), so routing through on_message no longer clobbers transient session state; handle_audio_output_end flips is_speaking=False on the shared singleton. Capture position is faithful to a real deployment: with speak(wait=False) the handler emits its end-marker first, so audio_output_end lands after the EOF and is not captured; with speak(wait=True) the handler blocks in wait_while_speaking until it arrives, so it deterministically precedes the end-marker and is captured โ€” as any real bus observer would record it. Co-Authored-By: Claude Opus 4.8 * feat: emit audio_output_start in _mock_tts alongside audio_output_end The TTS mock previously only emitted recognizer_loop:audio_output_end (unduck) after a delay, missing the recognizer_loop:audio_output_start (duck) that a real TTS playback emits when speech begins. Now _mock_tts emits audio_output_start synchronously on speak (TTS begins) and audio_output_end after 100ms (TTS finishes), properly simulating the full duck/unduck lifecycle. * Refactor TTS playback message handling * fix: update ovoscope unit tests to ignore audio_output_start/end signals The _mock_tts TTS mock now emits both recognizer_loop:audio_output_start (synchronous duck) and recognizer_loop:audio_output_end (delayed unduck) for every speak. Add both topics to HANDLER_LIFECYCLE ignore lists so unit tests that count messages and test routing don't see them. Also propagate source/destination from the original speak message into the mock's emitted context so routing assertions (source/dst matching) still pass. --------- Co-authored-by: Claude Opus 4.8 --- ovoscope/__init__.py | 24 ++++++++---------------- test/unittests/test_end2end.py | 4 +++- test/unittests/test_end2end_extended.py | 4 +++- 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/ovoscope/__init__.py b/ovoscope/__init__.py index d931dc4..fb2d48c 100644 --- a/ovoscope/__init__.py +++ b/ovoscope/__init__.py @@ -396,24 +396,16 @@ def __init__(self, skill_ids, # TTS mock: speak_dialog(โ€ฆ, wait=True) blocks in wait_while_speaking on # recognizer_loop:audio_output_end. With no real TTS that event never # arrives, so the handler stalls until the dispatcher's ยง8.3 timeout. We - # schedule a short-delay emit to unblock it. - # - # Full bus.emit, like the real audio service: recognizer_loop:audio_output_end - # is a genuine bus message in production, so the harness publishes it the - # same way โ€” through on_message (namespace migration + capture), where any - # bus observer would see it. SessionManager.handle_audio_output_end then - # flips is_speaking=False on the one live session object the ovos-bus-client - # SessionManager singleton keeps per id, so every holder of that session sees - # it. Capture position is faithful: with speak(wait=False) the handler emits - # its end-marker first, so this lands after the EOF and is not captured; with - # speak(wait=True) the handler blocks in wait_while_speaking until this - # arrives, so it deterministically precedes the end-marker and is captured โ€” - # exactly as a real deployment would record it. + # emit audio_output_start synchronously (duck) and schedule a short-delay + # audio_output_end (unduck) to simulate the full TTS playback lifecycle. def _mock_tts(message): - sess = SessionManager.get(message) + # TTS playback begins โ€” duck immediately. + # message.forward copies source/destination/session from the speak, + # matching what the real audio service would do. + bus.emit(message.forward("recognizer_loop:audio_output_start")) + # TTS playback ends after a short delay โ€” unduck threading.Timer(0.1, lambda: bus.emit( - Message("recognizer_loop:audio_output_end", - context={"session": sess.serialize()}) + message.forward("recognizer_loop:audio_output_end") )).start() bus.on(SpecMessage.SPEAK, _mock_tts) diff --git a/test/unittests/test_end2end.py b/test/unittests/test_end2end.py index b3696f5..0fd8e38 100644 --- a/test/unittests/test_end2end.py +++ b/test/unittests/test_end2end.py @@ -19,7 +19,9 @@ # Handler lifecycle messages emitted by add_event() wrappers. # Tests that don't care about lifecycle use this to filter noise. HANDLER_LIFECYCLE = ["mycroft.skill.handler.start", - "mycroft.skill.handler.complete"] + "mycroft.skill.handler.complete", + "recognizer_loop:audio_output_start", + "recognizer_loop:audio_output_end"] # Minimal pipeline: only Adapt-high so we get predictable no-match / match ADAPT_ONLY = ["ovos-adapt-pipeline-plugin-high"] diff --git a/test/unittests/test_end2end_extended.py b/test/unittests/test_end2end_extended.py index de388e2..8462d81 100644 --- a/test/unittests/test_end2end_extended.py +++ b/test/unittests/test_end2end_extended.py @@ -21,7 +21,9 @@ SKILL_ID = "ovoscope-extended-test.test" HANDLER_LIFECYCLE = ["mycroft.skill.handler.start", - "mycroft.skill.handler.complete"] + "mycroft.skill.handler.complete", + "recognizer_loop:audio_output_start", + "recognizer_loop:audio_output_end"] ADAPT_ONLY = ["ovos-adapt-pipeline-plugin-high"] From ecab39ca1bf1e16cbaaed2f0f3eba13e50cbb780 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Mon, 29 Jun 2026 17:43:05 +0000 Subject: [PATCH 78/82] Increment Version to 1.3.0a1 --- ovoscope/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovoscope/version.py b/ovoscope/version.py index b62bcfb..efeaeb9 100644 --- a/ovoscope/version.py +++ b/ovoscope/version.py @@ -1,6 +1,6 @@ # START_VERSION_BLOCK VERSION_MAJOR = 1 -VERSION_MINOR = 2 +VERSION_MINOR = 3 VERSION_BUILD = 0 VERSION_ALPHA = 1 # END_VERSION_BLOCK From 561fa8c9a101bab813bd6c6fd2445919afbc1ba8 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Mon, 29 Jun 2026 17:43:32 +0000 Subject: [PATCH 79/82] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fce9426..8f90deb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [1.3.0a1](https://github.com/OpenVoiceOS/ovoscope/tree/1.3.0a1) (2026-06-29) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/1.2.0a1...1.3.0a1) + +**Merged pull requests:** + +- feat: emit recognizer\_loop:audio\_output\_start in \_mock\_tts alongside audio\_output\_end [\#108](https://github.com/OpenVoiceOS/ovoscope/pull/108) ([JarbasAl](https://github.com/JarbasAl)) + ## [1.2.0a1](https://github.com/OpenVoiceOS/ovoscope/tree/1.2.0a1) (2026-06-29) [Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/1.1.0a2...1.2.0a1) From ca1987044b0aaf2dba2d47f9cccb14b2ddc3be9d Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Mon, 29 Jun 2026 23:47:10 +0100 Subject: [PATCH 80/82] feat: add skill_id lifecycle filter and eof_count to End2EndTest (#110) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two composable features for asserting scenarios that produce CONCURRENT dispatch lifecycles whose messages interleave non-deterministically (e.g. stopping a skill that is mid-dispatch โ€” the stop dispatch and the interrupted skill's own ยง8 trio + ยง9.5 terminal race). - skill_id: filter captured messages to a single context skill_id before asserting, isolating one lifecycle. Routing checks are skipped while filtering (the filtered stream is not a source/destination flip-chain). - eof_count: end capture only after an eof topic has been seen N times โ€” so capture spans all N lifecycles that each terminate on the same topic (e.g. two ovos.utterance.handled) before the filter runs. Run the same scenario once per skill_id to assert each lifecycle deterministically. Co-authored-by: Claude Opus 4.8 --- ovoscope/__init__.py | 34 +++++++++- test/unittests/test_capture_session.py | 44 +++++++++++++ test/unittests/test_end2end_extended.py | 88 +++++++++++++++++++++++++ 3 files changed, 164 insertions(+), 2 deletions(-) diff --git a/ovoscope/__init__.py b/ovoscope/__init__.py index fb2d48c..9dd6688 100644 --- a/ovoscope/__init__.py +++ b/ovoscope/__init__.py @@ -656,9 +656,16 @@ class CaptureSession: async_responses: List[Message] = dataclasses.field(default_factory=list) eof_msgs: List[str] = dataclasses.field(default_factory=lambda: DEFAULT_EOF) + # end capture only after an eof message has been seen this many times. Use >1 + # when the scenario produces N concurrent lifecycles that each terminate on the + # same eof topic (e.g. two ovos.utterance.handled โ€” one per utterance โ€” when a + # stop interrupts a running skill), so capture spans all of them. + eof_count: int = 1 ignore_messages: List[str] = dataclasses.field(default_factory=lambda: DEFAULT_IGNORED) async_messages: List[str] = dataclasses.field(default_factory=list) # these come from an external thread and might come in any order done: threading.Event = dataclasses.field(default_factory=lambda: threading.Event()) + _eof_lock: threading.Lock = dataclasses.field(default_factory=lambda: threading.Lock()) + _eof_seen: int = 0 def handle_message(self, msg: str): if self.done.is_set(): @@ -670,7 +677,10 @@ def handle_message(self, msg: str): self.responses.append(msg) def handle_end_of_test(self, msg: Message): - self.done.set() + with self._eof_lock: + self._eof_seen += 1 + if self._eof_seen >= self.eof_count: + self.done.set() def __post_init__(self): self.minicroft.bus.on("message", self.handle_message) @@ -680,6 +690,8 @@ def __post_init__(self): def capture(self, source_message: Message, timeout=20): test_message = deepcopy(source_message) # ensure object not mutated by ovos-core self.done.clear() + with self._eof_lock: + self._eof_seen = 0 self.minicroft.bus.emit(test_message) self.done.wait(timeout) @@ -709,7 +721,16 @@ class End2EndTest: # message type runtime modifiers ############################## eof_msgs: List[str] = dataclasses.field(default_factory=lambda: DEFAULT_EOF) # if received, end message capture + eof_count: int = 1 # end capture only after an eof message has been seen this many times (one per concurrent lifecycle terminating on the same topic) ignore_messages: List[str] = dataclasses.field(default_factory=lambda: DEFAULT_IGNORED) # pretend any message in this list was not emitted for testing purposes + # Assert only the messages belonging to a single dispatch lifecycle, identified + # by message.context["skill_id"]. When set, captured messages whose skill_id does + # not match are dropped before assertion. This isolates one lifecycle's ยง8 trio + + # ยง9 terminals from a CONCURRENT lifecycle whose messages interleave + # non-deterministically (e.g. stopping a skill that is mid-dispatch: the stop + # dispatch and the interrupted skill's own completion race). Run the same + # scenario once per skill_id to assert each lifecycle deterministically. + skill_id: Optional[str] = None ignore_gui: bool = True # ignore the gui namespace bus messages, usually unwanted unless explicitly testing gui integration async_messages: List[str] = dataclasses.field(default_factory=list) # these come from an external thread and might come in any order, validate they are received outside the main test @@ -823,6 +844,7 @@ def execute(self, timeout: int = 30) -> List[Message]: # the capture session will store all messages until capture.finish() # even if multiple messages are emitted capture = CaptureSession(self.minicroft, eof_msgs=self.eof_msgs, + eof_count=self.eof_count, ignore_messages=self.ignore_messages, async_messages=self.async_messages) for idx, source_message in enumerate(self.source_message): @@ -834,6 +856,14 @@ def execute(self, timeout: int = 30) -> List[Message]: # final message list messages = capture.finish() + # isolate a single dispatch lifecycle by skill_id โ€” drop messages from a + # concurrent (interleaving) lifecycle so the assertion is deterministic. + if self.skill_id is not None: + messages = [m for m in messages + if (m.context or {}).get("skill_id") == self.skill_id] + if self.verbose: + print(f"๐Ÿ’ก filtered to skill_id='{self.skill_id}': {len(messages)} messages") + if _bus_tracker is not None: _bus_tracker.stop_tracking() all_responses = messages + list(getattr(capture, "async_responses", [])) @@ -907,7 +937,7 @@ def execute(self, timeout: int = 30) -> List[Message]: assert received.context[k] == v, f"โŒ message context mismatch for key '{k}' - expected '{v}' | got '{received.context[k]}'" if self.verbose: print(f"โœ… got expected message context '{k}: '{v}'") - if self.test_routing: + if self.test_routing and self.skill_id is None: r_src = received.context.get("source") r_dst = received.context.get("destination") if expected.msg_type in self.keep_original_src: diff --git a/test/unittests/test_capture_session.py b/test/unittests/test_capture_session.py index bad5abd..f2ed5c5 100644 --- a/test/unittests/test_capture_session.py +++ b/test/unittests/test_capture_session.py @@ -161,6 +161,50 @@ def test_multiple_eof_types(self): self.assertIn("test.x", types) self.assertNotIn("test.late", types) + def test_eof_count_waits_for_n_occurrences(self): + """With eof_count>1, capture continues until an eof topic is seen that + many times โ€” for scenarios with N concurrent lifecycles each terminating + on the same eof topic.""" + cs = CaptureSession(self.mc, + eof_msgs=["test.eof"], + eof_count=2, + ignore_messages=[]) + self._emit_after(0.05, Message("test.eof")) # 1st eof โ€” must NOT stop + self._emit_after(0.10, Message("test.between")) # captured (after 1st eof) + self._emit_after(0.15, Message("test.eof")) # 2nd eof โ€” stops capture + self._emit_after(0.30, Message("test.after")) # must NOT appear + + cs.capture(Message("test.trigger"), timeout=3) + msgs = cs.finish() + types = [m.msg_type for m in msgs] + + self.assertIn("test.between", types, + "a message between the 1st and 2nd eof must be captured") + self.assertEqual(types.count("test.eof"), 2, + "both eof occurrences are captured") + self.assertNotIn("test.after", types, + "message after the Nth eof must not be captured") + + def test_eof_count_resets_between_captures(self): + """The eof counter resets per capture() call so eof_count applies fresh.""" + cs = CaptureSession(self.mc, + eof_msgs=["test.eof"], + eof_count=2, + ignore_messages=[]) + self._emit_after(0.05, Message("test.eof")) + self._emit_after(0.10, Message("test.eof")) + cs.capture(Message("test.trigger1"), timeout=3) + cs.finish() + # a second capture must again require 2 eofs, not be already-done + cs2 = CaptureSession(self.mc, eof_msgs=["test.eof"], eof_count=2, + ignore_messages=[]) + self._emit_after(0.05, Message("test.eof")) + self._emit_after(0.10, Message("test.mid")) + self._emit_after(0.15, Message("test.eof")) + cs2.capture(Message("test.trigger2"), timeout=3) + types = [m.msg_type for m in cs2.finish()] + self.assertIn("test.mid", types) + def test_capture_timeout_returns_partial_results(self): """If the EOF never fires, capture() must return after the timeout and finish() must still return whatever was captured.""" diff --git a/test/unittests/test_end2end_extended.py b/test/unittests/test_end2end_extended.py index 8462d81..7a26417 100644 --- a/test/unittests/test_end2end_extended.py +++ b/test/unittests/test_end2end_extended.py @@ -49,6 +49,22 @@ def handle_async(self, message: Message): self.bus.emit(Message("ovos.utterance.handled", context=message.context)) +class TwoLifecycleSkill(OVOSSkill): + """Emits two lifecycles tagged with distinct context skill_ids, to exercise + the End2EndTest ``skill_id`` filter. Each lifecycle ends on the shared + ``ovos.utterance.handled`` topic (so eof_count=2 spans both).""" + + def initialize(self): + self.add_event("unittest.two_lifecycles", self.handle_two) + + def handle_two(self, message: Message): + for sid in ("life.a", "life.b"): + ctx = dict(message.context) + ctx["skill_id"] = sid + self.bus.emit(Message(f"{sid}.step", context=ctx)) + self.bus.emit(Message("ovos.utterance.handled", context=ctx)) + + def _session(sid="ext-test", pipeline=None): s = Session(sid) s.lang = "en-US" @@ -983,5 +999,77 @@ def test_count_mismatch_prints_first_differing(self): test.execute(timeout=10) +class TestSkillIdFilter(unittest.TestCase): + """The skill_id filter isolates one dispatch lifecycle from concurrent ones.""" + + def setUp(self): + LOG.set_level("ERROR") + self.mc = get_minicroft([SKILL_ID], + extra_skills={SKILL_ID: TwoLifecycleSkill}) + + def tearDown(self): + self.mc.stop() + LOG.set_level("CRITICAL") + + def _common_kwargs(self): + return dict( + minicroft=self.mc, + skill_ids=[SKILL_ID], + eof_msgs=["ovos.utterance.handled"], + eof_count=2, # both lifecycles terminate on ovos.utterance.handled + test_routing=False, + test_active_skills=False, + test_final_session=False, + ignore_messages=DEFAULT_IGNORED + HANDLER_LIFECYCLE, + verbose=False, + ) + + def test_filter_isolates_one_lifecycle(self): + """Only messages whose context skill_id matches are asserted.""" + src = _make_custom("unittest.two_lifecycles") + test = End2EndTest( + source_message=src, + skill_id="life.a", + expected_messages=[ + Message("life.a.step", {}, {"skill_id": "life.a"}), + Message("ovos.utterance.handled", {}, {"skill_id": "life.a"}), + ], + **self._common_kwargs(), + ) + # passes only if life.b.* and the source (no skill_id) are filtered out + test.execute(timeout=10) + + def test_filter_the_other_lifecycle(self): + """The same scenario, filtered to the other skill_id.""" + src = _make_custom("unittest.two_lifecycles") + test = End2EndTest( + source_message=src, + skill_id="life.b", + expected_messages=[ + Message("life.b.step", {}, {"skill_id": "life.b"}), + Message("ovos.utterance.handled", {}, {"skill_id": "life.b"}), + ], + **self._common_kwargs(), + ) + test.execute(timeout=10) + + def test_unfiltered_sees_both_lifecycles(self): + """Without the filter, eof_count=2 captures both lifecycles' messages.""" + src = _make_custom("unittest.two_lifecycles") + test = End2EndTest( + source_message=src, + expected_messages=[src], + test_message_number=False, + test_msg_type=False, + test_msg_data=False, + test_msg_context=False, + **self._common_kwargs(), + ) + result = test.execute(timeout=10) + types = [m.msg_type for m in result] + self.assertIn("life.a.step", types) + self.assertIn("life.b.step", types) + + if __name__ == "__main__": unittest.main() From 70fee9ceb05ea3451ff585e106ee3f9585bca10f Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Mon, 29 Jun 2026 22:47:25 +0000 Subject: [PATCH 81/82] Increment Version to 1.4.0a1 --- ovoscope/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovoscope/version.py b/ovoscope/version.py index efeaeb9..f6c30c5 100644 --- a/ovoscope/version.py +++ b/ovoscope/version.py @@ -1,6 +1,6 @@ # START_VERSION_BLOCK VERSION_MAJOR = 1 -VERSION_MINOR = 3 +VERSION_MINOR = 4 VERSION_BUILD = 0 VERSION_ALPHA = 1 # END_VERSION_BLOCK From 5377e967821f171b956ec0012fb081df817aa505 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Mon, 29 Jun 2026 22:47:57 +0000 Subject: [PATCH 82/82] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f90deb..15d81aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [1.4.0a1](https://github.com/OpenVoiceOS/ovoscope/tree/1.4.0a1) (2026-06-29) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/1.3.0a1...1.4.0a1) + +**Merged pull requests:** + +- feat: skill\_id lifecycle filter + eof\_count for End2EndTest [\#110](https://github.com/OpenVoiceOS/ovoscope/pull/110) ([JarbasAl](https://github.com/JarbasAl)) + ## [1.3.0a1](https://github.com/OpenVoiceOS/ovoscope/tree/1.3.0a1) (2026-06-29) [Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/1.2.0a1...1.3.0a1)