diff --git a/.github/workflows/build-tests.yml b/.github/workflows/build-tests.yml index e1fc761..6af0db7 100644 --- a/.github/workflows/build-tests.yml +++ b/.github/workflows/build-tests.yml @@ -11,5 +11,5 @@ jobs: secrets: inherit with: python_versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]' - install_extras: 'audio,pydantic' + install_extras: 'audio,pydantic,media,listener' test_path: 'test/unittests/' diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 22cc219..eb94e5f 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -13,4 +13,5 @@ jobs: python_version: '3.11' coverage_source: 'ovoscope' test_path: 'test/unittests/' + test_extras: 'audio,pydantic,media,listener' min_coverage: 0 diff --git a/AUDIT.md b/AUDIT.md deleted file mode 100644 index d4dd424..0000000 --- a/AUDIT.md +++ /dev/null @@ -1,184 +0,0 @@ -# ovoscope — Audit Report -## Documentation Status -- [x] AGENTS.md Header Format -- [x] QUICK_FACTS.md -- [x] FAQ.md -- [x] MAINTENANCE_REPORT.md -- [x] AUDIT.md -- [x] SUGGESTIONS.md -- [x] docs/index.md -- [x] docs/usage-guide.md -- [x] docs/ci-integration.md ---- -## Technical Debt & Issues -### ~~[CRITICAL] No unit tests for ovoscope itself~~ ✅ FIXED -62 unit tests across `test/unittests/test_capture_session.py`, `test/unittests/test_end2end.py`, -`test/unittests/test_minicroft.py`, `test/unittests/test_pydantic_helpers.py`, and -`test/unittests/test_audio_harness.py`. -All pass (62 passed, 0 failed). ---- -### ~~[MAJOR] Missing LICENSE file~~ ✅ FIXED -LICENSE file (Apache-2.0) added to repo root. `pyproject.toml` correctly references it. ---- -### ~~[MAJOR] `End2EndTest.execute()` returns `None`~~ ✅ FIXED -`execute()` now returns `List[Message]`. Signature changed to -`def execute(self, timeout: int = 30) -> List[Message]`. `assert_spoke()` builds on this. ---- -### ~~[MAJOR] No type annotations on any public method~~ ✅ FIXED -All public methods in `ovoscope/__init__.py` and new modules now have PEP 484 annotations. ---- -### [MODERATE] `CaptureSession.capture()` returns `None` — captured messages inaccessible inline -**Evidence**: `ovoscope/__init__.py` — `capture()` returns nothing. Captured -messages are only accessible via `self.responses` after `finish()` is called. The pattern -`capture.capture(msg); messages = capture.finish()` is functional but forces a two-step flow. -**Impact**: Cannot chain captures or do inline assertions without accessing the attribute -directly. Slightly awkward for advanced multi-turn test composition. -**Recommended fix**: Return `List[Message]` (the current `self.responses` snapshot) from -`capture()`, or add a `messages` property. No breaking change required. ---- -### ~~[MODERATE] `get_minicroft()` busy-waits with no timeout~~ ✅ FIXED -`get_minicroft()` now accepts `max_wait: float = 60` and raises `TimeoutError` if the deadline -is exceeded. Signature: `def get_minicroft(skill_ids, *args, max_wait=60, **kwargs) -> MiniCroft`. ---- -### ~~[MINOR] `setup.py` should migrate to `pyproject.toml`~~ ✅ FIXED -`setup.py` removed. `pyproject.toml` uses `build-backend = "setuptools.build_meta"`, -`dynamic = ["version"]`, and `[tool.setuptools.dynamic] version = {attr = "ovoscope.version.__version__"}`. -`version.py` now exports `__version__`. Optional pydantic extras added: -`pip install ovoscope[pydantic]`. ---- -### [MINOR] CI action pins at `@master` -**Evidence**: GitHub Actions workflows use `pypa/gh-action-pypi-publish@master` and -`ad-m/github-push-action@master`. The `@master` ref is not pinned to a specific SHA, making -builds non-reproducible if the upstream action changes. -**Recommended fix**: Pin to `pypa/gh-action-pypi-publish@release/v1` and a specific SHA for -`ad-m/github-push-action`. ---- -## Next Steps (Priority Order) -1. Address CaptureSession.capture() return value — usability -2. Pin CI action refs — reproducibility - ---- - -## Audit [2026-03-12] — Correctness Bugs, Coverage Gaps, Docs - -### ~~[CRITICAL] `pipeline.py` race condition in `match()`~~ ✅ FIXED - -**Evidence**: `ovoscope/pipeline.py:159–181` — a single `threading.Event` was -shared for both success (`intent.service.skills.activated`) and failure -(`intent_failure`, `mycroft.skill.handler.start`) handlers. If `intent_failure` -fired first, `captured[0]` would be a failure message returned as success. -Additionally, `done.wait()` return value was not checked; a timeout silently -returned `captured[0]` (or raised `IndexError` on empty list). - -**Fix**: Separate `_matched` and `_failed` events; only the success handler -populates `captured`. Timeout and failure both return `None`. -Source: `ovoscope/pipeline.py:149`. - -### ~~[MAJOR] `diff.py` — subset comparison silently ignores extra keys~~ ✅ FIXED - -**Evidence**: `ovoscope/diff.py:121–138` — `_dict_diff()` only iterated keys -in `expected`, so unexpected keys in `actual` were never flagged. - -**Fix**: Added `strict: bool = False` parameter to `_dict_diff()` and -`diff_fixtures()`. When `strict=True`, extra keys in `actual` are included in -the diff detail. Default `False` preserves existing behaviour. -Source: `ovoscope/diff.py:121`. - -### ~~[MINOR] `bus_coverage.py` — dead method `_skill_id_for_handler()`~~ ✅ FIXED - -**Evidence**: `ovoscope/bus_coverage.py:745–763` — `_skill_id_for_handler()` -was never called anywhere in the codebase (verified by grep). Its logic is -a subset of `_skill_id_for_closure()` which is the method actually used. - -**Fix**: Method deleted. -Source: formerly `ovoscope/bus_coverage.py:745`. - -### ~~[MAJOR] No unit tests for `media.py` (`MockOCPBackend`, `OCPCaptureSession`, `OCPPlayerHarness`)~~ ✅ FIXED - -**Evidence**: `ovoscope/media.py` had no corresponding test file. - -**Fix**: Created `test/unittests/test_media.py` — 20 tests covering -`MockOCPBackend` state transitions, `OCPCaptureSession` message accumulation, -and assertion helpers. - -### ~~[MAJOR] No unit tests for `remote_recorder.py`~~ ✅ FIXED - -**Evidence**: `ovoscope/remote_recorder.py` had no corresponding test file. - -**Fix**: Created `test/unittests/test_remote_recorder.py` — 15 tests covering -constructor defaults, `_parse_url`, connect/disconnect lifecycle, `record()` with -mocked bus client, timeout handling, and fixture serialization. - -### ~~[MINOR] Deprecated `ovos_utils.messagebus.Message` import in `test_phal.py`~~ ✅ FIXED - -**Evidence**: `test/unittests/test_phal.py:23` — imported `Message` from -deprecated `ovos_utils.messagebus` instead of canonical `ovos_bus_client.message`. - -**Fix**: Import changed to `from ovos_bus_client.message import Message`. -Source: `test/unittests/test_phal.py:23`. - -### ~~[MINOR] `SUGGESTIONS.md` missing~~ ✅ FIXED - -**Evidence**: `SUGGESTIONS.md` was absent despite being required by `AGENTS.md`. - -**Fix**: `SUGGESTIONS.md` created with 10 structured proposals. - ---- -## Bus Coverage Module — Full Audit [2026-03-12] - -### ~~[CRITICAL] async_responses excluded from emitter coverage~~ ✅ FIXED -`execute()` now passes `messages + list(capture.async_responses)` to `record_session()`. Source: `ovoscope/__init__.py:666`. - -### ~~[CRITICAL] Unattributed expected_messages silently disappear~~ ✅ FIXED -Both observed and expected messages with no `skill_id` in context now fall back to the `"__core__"` sentinel bucket. Source: `BusCoverageTracker.record_session` — `ovoscope/bus_coverage.py:510`. - -### [CRITICAL] Registration-time handlers always show NOT TESTED — misleading -**Evidence**: `ovoscope/bus_coverage.py:368` / `ovoscope/__init__.py:647` — `snapshot_listeners()` is called after `MiniCroft` reaches READY. Handlers invoked *during* skill loading were called *before* the snapshot and will always show 0 invocations. -**Impact**: Users see `register_intent: NOT TESTED` and conclude their intent registration failed. -**Status**: Documented in `docs/bus-coverage.md` Limitations as a known structural constraint. A `LOAD_TIME` tag or pre-READY tracking remains a future enhancement. - -### ~~[CRITICAL] Non-skill messages silently excluded from emitter coverage~~ ✅ FIXED -Messages with no `skill_id` in context are now attributed to the `"__core__"` bucket in `record_session()`. Source: `ovoscope/bus_coverage.py:510`. - -### ~~[MAJOR] Unused `skill_map` variable in `record_session()`~~ ✅ FIXED -Dead `skill_map = self._skill_instance_map()` call removed from `record_session()`. - -### ~~[MAJOR] `_get_bus_events()` called three times in `snapshot_listeners()`~~ ✅ FIXED -`bus_events = self._get_bus_events()` is now called once and reused across all three passes. Source: `ovoscope/bus_coverage.py:432`. - -### ~~[MAJOR] Double-stop risk in `cmd_bus_coverage()`~~ ✅ FIXED -`test.managed = False` is now set explicitly before `test.execute()`. The `finally` block is the sole owner of `mc.stop()`. The redundant `mc.stop()` in the `except Exception` branch was also removed. Source: `ovoscope/cli.py:349`. - -### ~~[MAJOR] `pytest_terminal_summary` hook uses private pytest internals~~ ✅ FIXED -`bus_coverage_session` fixture now stores merged reports on `request.config._bus_coverage_reports` in its teardown. `pytest_terminal_summary` reads that list — no private attrs. Source: `ovoscope/pytest_plugin.py:191`. - -### [MAJOR] `once()` handlers invisible after firing -**Evidence**: `bus_coverage.py:368` — one-shot handlers fired during skill loading are de-registered before the snapshot. -**Status**: Documented in `docs/bus-coverage.md` Limitations. Pre-READY tracking is a future enhancement. - -### [MAJOR] `ignore_messages` list silently excludes messages from emitter coverage -**Evidence**: `ovoscope/__init__.py:504-505` — messages in `ignore_messages` never reach `responses`. -**Status**: Documented in `docs/bus-coverage.md` Limitations. Passing ignored messages as a separate bucket is a future enhancement. - -### ~~[MAJOR] No JSON schema version field~~ ✅ FIXED -`to_json()` now includes `"schema_version": "1"` as the first key. Source: `ovoscope/bus_coverage.py:297`. - -### ~~[MINOR] Column widths hardcoded in `print_report()`~~ ✅ FIXED -`col_w = max(len(s.skill_id) for s in self.skills) + 2` now drives column width dynamically. Source: `ovoscope/bus_coverage.py:237`. - -### ~~[MINOR] Coverage summary printed before test assertions~~ ✅ FIXED -`print_bus_coverage` block moved to after all assertions, just before `managed` teardown. Source: `ovoscope/__init__.py:795`. - -### ~~[MINOR] `bus_coverage_report` type hint uses `Optional[Any]`~~ ✅ FIXED -Type hint changed to `Optional["BusCoverageReport"]`. Source: `ovoscope/__init__.py:589`. - -### ~~[NITPICK] `_SKIP` set has fragile `"type"` entry~~ ✅ FIXED -Pass 2 now uses `isinstance(owner, type)` check to skip class objects, and the `"type"` string entry removed from `_SKIP`. Source: `ovoscope/bus_coverage.py:450`. - -### ~~Docs gaps (bus-coverage.md)~~ ✅ FIXED -All five missing Limitations items added to `docs/bus-coverage.md`: -- Registration-time handlers always show NOT TESTED -- Pipeline matching is not bus-driven -- `async_responses` now included (fix note) -- `ignore_messages` types excluded -- Core services use `__core__` bucket (not 0/0 — fix note) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a2ec47..15d81aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,220 @@ # Changelog -## [0.13.1a1](https://github.com/TigreGotico/ovoscope/tree/0.13.1a1) (2026-03-14) +## [1.4.0a1](https://github.com/OpenVoiceOS/ovoscope/tree/1.4.0a1) (2026-06-29) -[Full Changelog](https://github.com/TigreGotico/ovoscope/compare/0.13.0...0.13.1a1) +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/1.3.0a1...1.4.0a1) **Merged pull requests:** -- fix: thread names in bus coverage report [\#52](https://github.com/TigreGotico/ovoscope/pull/52) ([JarbasAl](https://github.com/JarbasAl)) +- feat: skill\_id lifecycle filter + eof\_count for End2EndTest [\#110](https://github.com/OpenVoiceOS/ovoscope/pull/110) ([JarbasAl](https://github.com/JarbasAl)) + +## [1.3.0a1](https://github.com/OpenVoiceOS/ovoscope/tree/1.3.0a1) (2026-06-29) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/1.2.0a1...1.3.0a1) + +**Merged pull requests:** + +- feat: emit recognizer\_loop:audio\_output\_start in \_mock\_tts alongside audio\_output\_end [\#108](https://github.com/OpenVoiceOS/ovoscope/pull/108) ([JarbasAl](https://github.com/JarbasAl)) + +## [1.2.0a1](https://github.com/OpenVoiceOS/ovoscope/tree/1.2.0a1) (2026-06-29) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/1.1.0a2...1.2.0a1) + +**Merged pull requests:** + +- feat: MockTTS publishes audio\_output\_end via the full bus \(faithful\) [\#106](https://github.com/OpenVoiceOS/ovoscope/pull/106) ([JarbasAl](https://github.com/JarbasAl)) + +## [1.1.0a2](https://github.com/OpenVoiceOS/ovoscope/tree/1.1.0a2) (2026-06-29) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/1.1.0a1...1.1.0a2) + +**Merged pull requests:** + +- docs: clarify MockTTS bus.ee.emit rationale [\#104](https://github.com/OpenVoiceOS/ovoscope/pull/104) ([JarbasAl](https://github.com/JarbasAl)) + +## [1.1.0a1](https://github.com/OpenVoiceOS/ovoscope/tree/1.1.0a1) (2026-06-29) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/1.0.2a1...1.1.0a1) + +**Merged pull requests:** + +- feat: MockTTS — emit audio\_output\_end on delay for speak\_dialog\(wait=True\) [\#102](https://github.com/OpenVoiceOS/ovoscope/pull/102) ([JarbasAl](https://github.com/JarbasAl)) + +## [1.0.2a1](https://github.com/OpenVoiceOS/ovoscope/tree/1.0.2a1) (2026-06-27) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/1.0.1a1...1.0.2a1) + +**Merged pull requests:** + +- fix: MockTTS destructor must not stop the shared playback thread [\#100](https://github.com/OpenVoiceOS/ovoscope/pull/100) ([JarbasAl](https://github.com/JarbasAl)) + +## [1.0.1a1](https://github.com/OpenVoiceOS/ovoscope/tree/1.0.1a1) (2026-06-27) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/1.0.0a1...1.0.1a1) + +**Merged pull requests:** + +- fix: guard None blacklisted\_skills/intents in final-session check [\#98](https://github.com/OpenVoiceOS/ovoscope/pull/98) ([JarbasAl](https://github.com/JarbasAl)) + +## [1.0.0a1](https://github.com/OpenVoiceOS/ovoscope/tree/1.0.0a1) (2026-06-25) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.22.1a1...1.0.0a1) + +**Breaking changes:** + +- feat!: audio harness on OVOS spec bus namespace [\#92](https://github.com/OpenVoiceOS/ovoscope/pull/92) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.22.1a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.22.1a1) (2026-06-25) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.22.0a1...0.22.1a1) + +**Merged pull requests:** + +- fix: pytest 9 compatibility for the pytest11 plugin [\#88](https://github.com/OpenVoiceOS/ovoscope/pull/88) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.22.0a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.22.0a1) (2026-06-25) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.21.1a1...0.22.0a1) + +**Merged pull requests:** + +- feat: stream audio frames through MiniListener for multi-frame decoders [\#86](https://github.com/OpenVoiceOS/ovoscope/pull/86) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.21.1a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.21.1a1) (2026-06-25) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.21.0a1...0.21.1a1) + +**Merged pull requests:** + +- fix: repair ovoscope record in-process path \(default\_pipeline kwarg + from\_message skill\_ids\) [\#85](https://github.com/OpenVoiceOS/ovoscope/pull/85) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.21.0a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.21.0a1) (2026-06-25) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.20.0a1...0.21.0a1) + +**Merged pull requests:** + +- feat: export ovos-media OCP harness from the package + add \[media\] extra [\#89](https://github.com/OpenVoiceOS/ovoscope/pull/89) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.20.0a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.20.0a1) (2026-06-24) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.19.4a1...0.20.0a1) + +**Merged pull requests:** + +- feat: assert\_template\_shown for SYSTEM\_\* GUI templates [\#83](https://github.com/OpenVoiceOS/ovoscope/pull/83) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.19.4a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.19.4a1) (2026-06-17) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.19.3a1...0.19.4a1) + +**Merged pull requests:** + +- fix\(tts-intelligibility\): normalise rendered audio to 16kHz mono before STT [\#81](https://github.com/OpenVoiceOS/ovoscope/pull/81) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.19.3a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.19.3a1) (2026-06-17) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.19.2a1...0.19.3a1) + +**Merged pull requests:** + +- fix\(tts-intelligibility\): transcode non-WAV engine output before scoring [\#79](https://github.com/OpenVoiceOS/ovoscope/pull/79) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.19.2a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.19.2a1) (2026-06-16) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.19.1a2...0.19.2a1) + +**Merged pull requests:** + +- fix\(tts-intelligibility\): score synthesis failures as total miss, not abort [\#77](https://github.com/OpenVoiceOS/ovoscope/pull/77) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.19.1a2](https://github.com/OpenVoiceOS/ovoscope/tree/0.19.1a2) (2026-06-15) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.19.1a1...0.19.1a2) + +**Merged pull requests:** + +- feat: TTS end-to-end intelligibility harness [\#75](https://github.com/OpenVoiceOS/ovoscope/pull/75) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.19.1a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.19.1a1) (2026-06-14) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.19.0a3...0.19.1a1) + +**Merged pull requests:** + +- fix: drop removed 'path' arg from pytest\_pycollect\_makemodule hook \(pytest\>=8 compat\) [\#73](https://github.com/OpenVoiceOS/ovoscope/pull/73) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.19.0a3](https://github.com/OpenVoiceOS/ovoscope/tree/0.19.0a3) (2026-06-13) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.19.0a2...0.19.0a3) + +**Merged pull requests:** + +- chore: remove agent-audit scratch files [\#71](https://github.com/OpenVoiceOS/ovoscope/pull/71) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.19.0a2](https://github.com/OpenVoiceOS/ovoscope/tree/0.19.0a2) (2026-06-13) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.19.0a1...0.19.0a2) + +**Merged pull requests:** + +- docs: standardize NGI0 Commons Fund attribution [\#69](https://github.com/OpenVoiceOS/ovoscope/pull/69) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.19.0a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.19.0a1) (2026-06-12) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.18.0a1...0.19.0a1) + +**Merged pull requests:** + +- feat: MiniVoiceLoop + simple/classic listener bus-sequence harnesses [\#67](https://github.com/OpenVoiceOS/ovoscope/pull/67) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.18.0a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.18.0a1) (2026-06-10) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.17.1a1...0.18.0a1) + +**Merged pull requests:** + +- feat\(phal\): plugin\_factories for MiniPHAL and PHALTest [\#65](https://github.com/OpenVoiceOS/ovoscope/pull/65) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.17.1a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.17.1a1) (2026-05-20) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.17.0a1...0.17.1a1) + +**Merged pull requests:** + +- fix\(pipeline-harness\): default \_SinkSkill bus to FakeBus [\#62](https://github.com/OpenVoiceOS/ovoscope/pull/62) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.17.0a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.17.0a1) (2026-05-14) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.16.0a1...0.17.0a1) + +**Merged pull requests:** + +- feat\(intent-cases\): markdown reporter, baseline diff, auto-discovery, deterministic m2v warmup [\#60](https://github.com/OpenVoiceOS/ovoscope/pull/60) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.16.0a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.16.0a1) (2026-05-14) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.15.0a1...0.16.0a1) + +**Merged pull requests:** + +- feat\(intent-cases\): file-based intent test layout + pytest accuracy gate [\#58](https://github.com/OpenVoiceOS/ovoscope/pull/58) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.15.0a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.15.0a1) (2026-05-14) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.14.0a1...0.15.0a1) + +**Merged pull requests:** + +- feat\(e2e\): reusable harness, bus helpers, and intent-registration shims [\#55](https://github.com/OpenVoiceOS/ovoscope/pull/55) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.14.0a1](https://github.com/OpenVoiceOS/ovoscope/tree/0.14.0a1) (2026-05-14) + +[Full Changelog](https://github.com/OpenVoiceOS/ovoscope/compare/0.13.1...0.14.0a1) + +**Merged pull requests:** + +- feat: add NEBULENTO\_PIPELINE and PALAVREADO\_PIPELINE stage groups [\#54](https://github.com/OpenVoiceOS/ovoscope/pull/54) ([JarbasAl](https://github.com/JarbasAl)) diff --git a/FAQ.md b/FAQ.md deleted file mode 100644 index 4c92f15..0000000 --- a/FAQ.md +++ /dev/null @@ -1,445 +0,0 @@ -# FAQ — `ovoscope` -## How do I measure which bus message handlers my tests actually exercise? - -Use bus coverage: set `track_bus_coverage=True` on `End2EndTest`. After -`execute()`, `test.bus_coverage_report` contains a `BusCoverageReport` with -per-skill listener coverage (which `bus.on()` registrations were triggered) -and emitter coverage (which message types were observed / asserted). - -```python -test = End2EndTest( - skill_ids=["my-skill.author"], - source_message=message, - expected_messages=[...], - track_bus_coverage=True, - print_bus_coverage=True, # print inline summary -) -test.execute() -print(test.bus_coverage_report.to_json()) -``` - -See [docs/bus-coverage.md](docs/bus-coverage.md) for the full reference. -`BusCoverageTracker` — `ovoscope/bus_coverage.py:242`. - -## How do I get an aggregate bus coverage report across an entire test suite? - -Use the `bus_coverage_session` pytest fixture. Each test calls -`bus_coverage_session.add(test.bus_coverage_report)` after `execute()`. A -merged table is printed automatically at session end. See -[docs/bus-coverage.md](docs/bus-coverage.md). - -## How do I run bus coverage from the command line without writing pytest tests? - -Use the `ovoscope bus-coverage` subcommand: - -```bash -ovoscope bus-coverage path/to/fixtures/ # table report -ovoscope bus-coverage path/to/fixtures/ --format json -ovoscope bus-coverage path/to/fixtures/ --verbose # per-msg detail -``` - -`cmd_bus_coverage` — `ovoscope/cli.py`. - - -## How do I test AudioService or PlaybackService without real audio hardware? -Use `AudioServiceHarness` or `PlaybackServiceHarness` from `ovoscope.audio`. Both run on a -`FakeBus` with `MockAudioBackend`/`MockTTS` respectively — no real audio device, TTS engine, -or network required. See [docs/audio-testing.md](docs/audio-testing.md) for the full API -reference. Requires `pip install ovoscope[audio]` (or `ovos-audio` installed separately). - -## Why does AudioService.stop() silently ignore my stop() call in tests? -`AudioService._stop()` — `ovos-audio/ovos_audio/audio.py` — has a 1-second stop guard: -it does nothing if called within 1 second of `play()`. Tests must `time.sleep(1.1)` after -`play()` before calling `stop()`. - -## Why doesn't FakeBus.wait_for_response() work in audio harness tests? -`FakeBus.wait_for_response()` does not work for synchronous in-process handlers because the -reply is emitted before the internal listener is registered. Use subscribe-emit-wait with a -`threading.Event` instead. `AudioServiceHarness.get_track_info()` and `list_backends()` -implement this pattern — `ovoscope/audio.py`. - -## How do I test VAD (Voice Activity Detection) without a real microphone? - -Use `MockVADEngine` from `ovoscope.listener`. It classifies all-zero bytes as silence and -any non-zero byte as speech. Inject it into `MiniListener(config, vad_instance=MockVADEngine())` -or use the declarative `VADTest` dataclass. No microphone, audio driver, or OPM plugin required. - -```python -from ovoscope.listener import MockVADEngine, VADTest -VADTest(vad_instance=MockVADEngine(), audio_input=b"\\x01" * 512, expect_silence=False).execute() -``` - -## How do I test Wake Word detection without loading a real model? - -Use `MockHotWordEngine(trigger_after=N)` from `ovoscope.listener`. It fires after exactly N -`update()` calls and auto-resets. Inject via `MiniListener(config, ww_instances={"hey_mycroft": engine})` -or use the declarative `WakeWordTest` dataclass. - -```python -from ovoscope.listener import MockHotWordEngine, WakeWordTest -WakeWordTest( - ww_instances={"hey_mycroft": MockHotWordEngine(trigger_after=2)}, - audio_chunks=[b"\\x00" * 512] * 4, - expect_detected=True, - expected_detection_frame=1, -).execute() -``` - -## What is `ovoscope`? -`ovoscope` is End-to-end test framework for OpenVoiceOS skills. -## How do I install it? -```bash -pip install ovoscope -``` -Or for development: -```bash -uv pip install -e ovoscope/ -``` -## Where do I report bugs? -Open an issue on the GitHub repository. Ensure you are targeting the `dev` branch for fixes. -## How do I run tests? -```bash -uv run pytest ovoscope/test/ --cov=ovoscope -``` -## How do I contribute? -1. Fork the repository and create a feature branch from `dev`. -2. Write tests for your changes. -3. Open a PR targeting the `dev` branch. -4. Ensure CI passes before requesting review. -## What CI workflows does ovoscope run? -Seven workflows: `unit_tests.yml` (pytest + coverage on PRs), `build_tests.yml` (sdist/wheel matrix build), `license_tests.yml` (dependency license audit via gh-automations), `pipaudit.yml` (CVE scanning), `release_workflow.yml` (test-gated alpha release), `publish_stable.yml` (stable release), and `conventional-label.yaml` (PR label automation). -## Does the release workflow run tests before publishing? -Yes. The `release_workflow.yml` has a `build_tests` job that runs the full test suite. The `publish_alpha` job depends on it via `needs: build_tests`, so a failing test blocks the alpha release. -## How does ovoscope's coverage reporting work in CI? -The `unit_tests.yml` workflow runs `pytest --cov=ovoscope --cov-report xml` and uses `py-cov-action/python-coverage-comment-action@v3` to post a coverage summary as a PR comment. -## What test coverage does ovoscope have? -104 tests across 6 test files achieving 89% overall coverage. Key areas tested: End2EndTest execute/assertions/serialization/routing/active skills/boot sequence/final session/from_message recording, CaptureSession lifecycle, MiniCroft config isolation/lang/pipeline, pytest_plugin fixture logic, pydantic_helpers bridge. -## What Python versions are supported? -See `QUICK_FACTS.md` — currently `>=3.10`. -## My tests pass locally but fail on CI — why? -Usually one of three causes: -1. **Different pipeline plugins installed** — The default session pipeline includes whatever - pipeline plugins happen to be installed. On CI, Gemma/Ollama/persona plugins may not be - installed (or vice versa), changing which plugin handles the utterance. - **Fix**: always pass an explicit `default_pipeline` to `get_minicroft()` (or use the default - `DEFAULT_TEST_PIPELINE` by leaving `isolate_config=True`). -2. **User locale affecting intent matching** — `isolate_config=True` (the default) removes the - user's `~/.config/mycroft/mycroft.conf` from the config chain so the test environment locale - does not affect results. Always leave this enabled. -3. **Skill plugin not discoverable** — The skill must be registered under the `opm.skill` entry - point group. Old-style `ovos.plugin.skill` entries are warned but not loaded by - `find_skill_plugins()`. Use `extra_skills={SKILL_ID: SkillClass}` to inject skills that - lack a proper entry point. -## A persona / AI plugin is intercepting my test utterances -This happens because `SessionManager.default_session` is initialized at import time from the -full system config (which may include persona pipeline stages). -`MiniCroft` solves this with `default_pipeline` (default: `DEFAULT_TEST_PIPELINE`): -```python -from ovoscope import get_minicroft, DEFAULT_TEST_PIPELINE, ADAPT_PIPELINE -# Default — all standard stages, no AI/persona/OCP (recommended) -mc = get_minicroft([]) -# Adapt-only for fast unit-style end2end tests -mc = get_minicroft([SKILL_ID], default_pipeline=ADAPT_PIPELINE) -# Opt in to persona explicitly -from ovoscope import PERSONA_PIPELINE -mc = get_minicroft([SKILL_ID], default_pipeline=DEFAULT_TEST_PIPELINE + PERSONA_PIPELINE) -``` -`DEFAULT_TEST_PIPELINE` excludes persona, Ollama, OCP, and m2v stages. -The original pipeline is restored when `mc.stop()` is called. -## Why does `SessionManager.default_session.pipeline` matter? -When a message is emitted without an explicit `session` in its context, ovos-core creates a -session by copying `SessionManager.default_session`. That copy inherits its `pipeline` list, -which controls which pipeline plugins are consulted for intent matching. -`MiniCroft.run()` overrides `SessionManager.default_session.pipeline` to `default_pipeline` -(set after FakeBus init messages are processed, just before `READY`), and restores it on -`stop()`. -## How is `isolate_config` different from `default_pipeline`? -- `isolate_config=True` — clears `Configuration.xdg_configs` so `~/.config/mycroft/mycroft.conf` - is not read. Prevents user locale, custom wake words, and user-level pipeline *config* from - affecting tests. -- `default_pipeline` — overrides `SessionManager.default_session.pipeline` directly. Necessary - because the default session is initialized at module import time (before config isolation takes - effect) and may already have the user's pipeline. -Both are enabled by default. They are complementary. -## How do I know whether to use ADAPT_PIPELINE or PADATIOUS_PIPELINE for my test? -It depends on how the skill registers its intent: -| Decorator | Pipeline | -|-----------|----------| -| `@intent_handler(IntentBuilder(...))` | `ADAPT_PIPELINE` | -| `@intent_handler("my.intent")` (string ending in `.intent`) | `PADATIOUS_PIPELINE` | -| `@fallback_handler(priority=N)` | `FALLBACK_PIPELINE` | -| `@converse_handler` | `CONVERSE_PIPELINE` | -When in doubt, look at the intent files in `locale/en-us/` — if there is a file named `*.intent`, -it is Padatious. If there is a file named `*.voc` or `*.rx`, it is Adapt. -## My skill emits extra messages (enclosure.eyes.*, add_context, configuration.patch) — how do I handle them? -Some skills emit low-level hardware events or internal context messages that are not part of the -utterance handling protocol. Add them to `ignore_messages`: -```python -test = End2EndTest( - ... - ignore_messages=[ - "enclosure.eyes.level", # Mark 1 LED animation - "enclosure.eyes.look", - "add_context", # set_context() calls - "configuration.patch", # disable_confirm_listening() etc. - ], - ... -) -``` -## A skill emits a raw message (no source/dest) like recognizer_loop:sleep — how do I test it? -Some skills call `self.bus.emit(Message("some.message"))` without inheriting source/dest. -These messages have `source=None` and will fail source-checking in the framework. -Use `async_messages` to assert they were received without checking order or source: -```python -test = End2EndTest( - ... - ignore_messages=["recognizer_loop:sleep"], # exclude from ordered sequence - async_messages=["recognizer_loop:sleep"], # assert it was received somewhere - ... -) -``` -## My test passes locally but the user's blacklisted skills cause failures on CI -Users may have skills like `skill-ovos-stop.openvoiceos` in `blacklisted_skills` in their -`~/.config/mycroft/mycroft.conf`. `Session.__init__` reads this from the live -`Configuration()` singleton dict cache (not invalidated by `reload()`). -`MiniCroft` solves this: when `isolate_config=True` (the default), it patches -`Configuration()["skills"]["blacklisted_skills"] = []` and -`Configuration()["intents"]["blacklisted_intents"] = []` in `run()`, and restores them in `stop()`. -This is complementary to the `xdg_configs = []` isolation applied in `__init__`. -## Can I use typed pydantic models instead of raw Message objects? -Yes. Install the optional pydantic extras: -```bash -pip install ovoscope[pydantic] -``` -Then use the bridge in `ovoscope.pydantic_helpers`: -```python -from ovoscope.pydantic_helpers import to_bus_message, from_bus_message -from ovos_pydantic_models import RecognizerLoopUtteranceMessage, RecognizerLoopUtteranceData, SpeakMessage -# Build a typed source message — validated at construction -utterance = to_bus_message(RecognizerLoopUtteranceMessage( - data=RecognizerLoopUtteranceData(utterances=["hello"], lang="en-us") -)) -# Parse a received message into a typed model for richer assertions -messages = test.execute() -speak = from_bus_message(messages[0], SpeakMessage) -assert "hello" in speak.data.utterance.lower() -``` -A typo in a field name (`"utterance"` vs `"utterances"`) raises `ValidationError` at -construction time instead of silently producing a wrong test. -## How do I validate a JSON fixture file before loading it? -Use `validate_fixture()` from `ovoscope.pydantic_helpers` (requires `ovoscope[pydantic]`): -```python -from ovoscope.pydantic_helpers import validate_fixture -from ovoscope import End2EndTest -test = End2EndTest.deserialize(validate_fixture("test/fixtures/hello_world.json")) -test.execute() -``` -If any message in the fixture is malformed, a clear `ValidationError` is raised pointing -to the offending field — instead of a cryptic `KeyError` inside `deserialize()`. -## How do I trigger non-utterance events during a test? -Use `MiniCroft.inject_message(msg)`: -```python -from ovos_bus_client.message import Message -mc.inject_message(Message("mycroft.gui.connected", {"connected": True})) -``` -This emits an arbitrary message on the FakeBus during a test without going through the -utterance pipeline — useful for timer events, GUI events, or skill API calls. -## How do I assert a skill spoke a specific phrase without checking the full message sequence? -Use `End2EndTest.assert_spoke(text, lang)`: -```python -test = End2EndTest( - skill_ids=["my-skill.author"], - source_message=Message("recognizer_loop:utterance", {"utterances": ["hello"], "lang": "en-US"}, {}), - expected_messages=[], # not used by assert_spoke -) -test.assert_spoke("Hello, world!", lang="en-US") -``` -`assert_spoke()` calls `execute()` internally and scans captured messages for a `speak` -message with the matching utterance and lang. -## `get_minicroft()` hangs forever — what do I do? -Pass `max_wait` to set a timeout: -```python -mc = get_minicroft(["my-skill.author"], max_wait=30) -``` -If `MiniCroft` does not reach `READY` within `max_wait` seconds, a `TimeoutError` is raised -with the skill IDs — pointing you at the skill startup logs. The default is 60 seconds. -## How do I use the `minicroft` pytest fixture? -The fixture is registered automatically when ovoscope is installed (via the `pytest11` entry -point). Just declare `skill_ids` on your test class: -```python -class TestMySkill: - skill_ids = ["my-skill.author"] - def test_something(self, minicroft): - from ovoscope import End2EndTest - from ovos_bus_client.message import Message - test = End2EndTest( - minicroft=minicroft, - skill_ids=self.skill_ids, - source_message=Message( - "recognizer_loop:utterance", - {"utterances": ["hello"], "lang": "en-US"}, - {}, - ), - expected_messages=[...], - ) - test.execute() -``` -The `MiniCroft` is started once per class and stopped in teardown — no `setUp`/`tearDown` -boilerplate needed. -## How do I test a pipeline plugin (not a skill) like PersonaService? -Pipeline plugins are loaded by `MiniCroft` automatically via `IntentService`. Access them via: -```python -mc = get_minicroft([], default_pipeline=PERSONA_PIPELINE) -persona_svc = mc.intents.pipeline_plugins["ovos-persona-pipeline-plugin"] -``` -Inject mocks directly into the plugin's state before each test: -```python -def setUp(self): - persona_svc.personas.clear() # remove real solvers (Gemma, Ollama, etc.) - persona_svc.active_persona = None # reset pipeline state - persona_svc.personas["TestBot"] = MockPersona("TestBot", "forty two") -``` -The `skill_ids=[]` parameter tells MiniCroft to load no skills — only pipeline plugins. -See `ovos-persona/test/end2end/test_persona.py` for a full working example. ---- -## How do I test skills in non-English languages? -Pass `secondary_langs` to `get_minicroft()`: -```python -croft = get_minicroft( - [SKILL_ID], - secondary_langs=["pt-PT", "de-DE", "es-ES"], -) -``` -This patches `Configuration()["secondary_langs"]` before Adapt/Padatious initialize, so they create per-language engines and register vocab for all specified languages. Without this, only the system's default language has vocab registered. -## Why does `End2EndTest.from_message()` crash with `TypeError: argument of type 'NoneType' is not iterable`? -This was a bug where `async_messages` defaulted to `None` and was passed to `CaptureSession`, which tried `msg.msg_type in None`. Fixed by defaulting to `[]`. -## Why do JSON fixture replays fail on session context? -Session context includes timestamps (e.g., `active_skills` activation time) that differ between recording and replay. Set `test_msg_context=False` on fixture tests. For skills with random dialog rendering (like quote pools), also set `test_msg_data=False`. -## Does `from_message()` filter GUI messages during recording? -Yes — `from_message()` now accepts `ignore_gui=True` (default), which adds `GUI_IGNORED` messages to the capture filter. This prevents GUI namespace messages from appearing in recorded fixtures. -## How do I override pipeline plugin config in a test (e.g. M2V model path)? -Pass `pipeline_config` to `get_minicroft()`. It is a `dict` keyed by the plugin's config key under `Configuration()["intents"]`: -```python -croft = get_minicroft( - [SKILL_ID], - default_pipeline=M2V_PIPELINE, - pipeline_config={ - "ovos_m2v_pipeline": { - "model": "Jarbas/ovos-model2vec-intents-distiluse-base-multilingual-cased-v2" - } - }, -) -``` -The override is patched into `Configuration()["intents"]` before `super().__init__()`, so the pipeline plugin reads the test value in its `__init__`. It is restored in `stop()`. This is useful for forcing a specific model regardless of what `mycroft.conf` says locally. -## Why do M2V tests skip when the multilingual model is not cached? -`ovos-m2v-pipeline` classifies utterances using a pre-trained model whose `classes_` are fixed intent labels. A language-specific model (e.g. Portuguese-only) won't contain English intent names and will always return no match. The multilingual model (`Jarbas/ovos-model2vec-intents-distiluse-base-multilingual-cased-v2`) covers all OVOS skill intent names. Tests that use M2V should skip when this model is not cached locally (to avoid downloading a large model in CI). Download it once with: -```bash -python -c "from model2vec.inference import StaticModelPipeline; StaticModelPipeline.from_pretrained('Jarbas/ovos-model2vec-intents-distiluse-base-multilingual-cased-v2')" -``` - ---- - -## Bus Coverage - -### Why do I see `Thread-1`, `Thread-2` entries in my bus coverage report? - -**This was fixed in ovoscope 0.14.0.** Previously, OVOS components that inherit -from `Thread` (such as `SkillManager`, `PlaybackService`, `OVOSDinkumVoiceService`, -and `MediaService`) had their handlers attributed to generic names like `Thread-1`, -`Thread-2`, etc. - -The fix uses Python's Method Resolution Order (MRO) to automatically resolve -Thread subclasses to their actual class names. The `_get_component_name()` method -walks the MRO chain and skips the `Thread` class, returning the first non-Thread -class name. - -**Special case:** `MiniCroft` (the test harness) is automatically renamed to -`SkillManager` in reports for clarity, since MiniCroft is just a test wrapper -around SkillManager. - -This approach is automatic and requires no manual maintenance of message type patterns. -Any future Thread-based components will be correctly attributed without code changes. - -See [docs/bus-coverage.md](docs/bus-coverage.md) for the full reference. - ---- - -## CLI - -### How do I record a fixture from the command line? -```bash -ovoscope record --skill-id ovos-skill-hello-world.openvoiceos \ - --utterance "hello" --output fixture.json -``` - -### How do I replay a fixture? -```bash -ovoscope run fixture.json --verbose -``` - -### How do I compare two fixture files? -```bash -ovoscope diff expected.json actual.json -``` -Exit code 0 = identical, 1 = differences found. - -### How do I scan my workspace for E2E coverage gaps? -```bash -ovoscope coverage "OpenVoiceOS Workspace/" --format table -``` - ---- - -## PHAL Testing - -### Can I test PHAL plugins with ovoscope? -Yes — any PHAL plugin that communicates only via the MessageBus (no physical -hardware) is testable with `MiniPHAL` or `PHALTest` from `ovoscope.phal`. - -### Which PHAL plugins require real hardware? -`ovos-PHAL-plugin-alsa`, `ovos-PHAL-plugin-mk1`, `ovos-PHAL-plugin-dotstar`. -These should use hardware-in-the-loop integration tests instead. - ---- - -## OCP Testing - -### How do I test an OCP skill without a real HTTP server? -Use `OCPTest` with `mock_responses` — keys are URL substrings matched -against actual requests, values are the JSON bodies returned. - -### What message flow does OCP testing drive? -`recognizer_loop:utterance` → `ovos.common_play.query` → `ovos.common_play.query.response` → `ovos.common_play.start` - ---- - -## GUI Assertions - -### How do I assert that a skill showed a GUI page? -```python -from ovoscope import GUICaptureSession -with GUICaptureSession(mc.bus) as gui: - # ... trigger interaction ... - gui.assert_page_shown("my_skill", "main.qml") -``` - -### How do I assert that a skill set a specific session data key in a namespace? -Use `assert_namespace_has_key()`: -```python -with GUICaptureSession(mc.bus) as gui: - # ... trigger interaction ... - gui.assert_namespace_has_key("my_skill", "temperature") -``` -This checks that a `mycroft.session.set` message was captured containing the given key in the specified namespace. See [docs/gui-testing.md](docs/gui-testing.md). - ---- - -## Coverage Scanner - -### What entry-point groups does the scanner detect? -`opm.skill`, `opm.pipeline`, `opm.phal`, `opm.plugin.tts`, `opm.plugin.stt`, -`opm.plugin.audio`, `opm.common_play`, `opm.solver`. - -### How is "covered" defined? -A repo is considered covered when `test/end2end/` (or `tests/end2end/`) -exists and contains at least one `.py` file (excluding `__init__.py`). diff --git a/MAINTENANCE_REPORT.md b/MAINTENANCE_REPORT.md deleted file mode 100644 index f17aadd..0000000 --- a/MAINTENANCE_REPORT.md +++ /dev/null @@ -1,464 +0,0 @@ -# Maintenance Report — `ovoscope` - -## [2026-03-13] — GUICaptureSession: assert_namespace_has_key + unit tests - -- **AI Model**: Claude Opus 4.6 -- **Actions Taken**: - - Added `assert_namespace_has_key()` method to `GUICaptureSession` for asserting that a skill set a specific session data key in a GUI namespace - - Updated `docs/gui-testing.md` with documentation for the new method - - Created `test/unittests/test_gui_capture.py` with 10 unit tests covering the new assertion method - - Updated `FAQ.md` with Q&A for `assert_namespace_has_key()` -- **Oversight**: Human-reviewed plan execution - ---- - -## [2026-03-12] — Full Audit Improvements (Correctness, Coverage, Docs, Packaging) - -- **AI Model**: claude-sonnet-4-6 -- **Actions Taken**: - - **P1.1** Fixed `pipeline.py` race condition: split success/failure into - separate `threading.Event` objects; `match()` now returns `None` on timeout - or failure instead of returning a failure message as success. - Source: `ovoscope/pipeline.py:149–200`. - - **P1.2** Added `strict: bool = False` to `diff.py` `_dict_diff()` and - `diff_fixtures()`. When `strict=True`, extra keys in `actual` not in - `expected` are flagged. Default `False` preserves existing behaviour. - Source: `ovoscope/diff.py:121`. - - **P1.3** Deleted dead `_skill_id_for_handler()` from `bus_coverage.py` - (lines 745–763, never called anywhere in the codebase). - - **P2.1** Created `test/unittests/test_media.py` — 20 unit tests for - `MockOCPBackend` state transitions and `OCPCaptureSession` accumulation. - - **P2.2** Created `test/unittests/test_remote_recorder.py` — 15 unit tests - for `RemoteRecorder` using mocked `MessageBusClient`. - - **P2.3** Fixed deprecated `ovos_utils.messagebus.Message` import in - `test/unittests/test_phal.py` → `ovos_bus_client.message.Message`. - - **P3.1** Created `SUGGESTIONS.md` with 10 structured proposals. - - **P3.2** Updated `QUICK_FACTS.md` test count: 243 → 306. - - **P3.3** Expanded `docs/pipeline.md` to full API reference with `pipeline.py:LINE` - citations, `_SinkSkill` explanation, Adapt/Padatious examples, and - pipeline success/failure signal documentation. - - **P3.4** Fixed `docs/ocp.md` to correctly reference `OCPTest` (the class - in `ocp.py`) and add cross-reference to `OCPPlayerHarness` in `media.py`. - - **P3.5** Updated `AUDIT.md` with 7 new findings (5 fixed, 2 pre-existing). - - **P4.1** Updated `pyproject.toml`: added Documentation and Issue Tracker - URLs, `[tool.setuptools.package-data]`, `timeout = 60` in pytest options, - and comment explaining the `ovos-core>=2.0.4a2` alpha pin. - - **P5.1** Made `_count_fixtures()` in `coverage.py` use `Path.rglob("*.json")` - for recursive fixture counting instead of `os.listdir()`. - - **P5.2** Added `TYPE_CHECKING` guard and proper `List["BusCoverageReport"]` - type annotation to `BusCoverageCollector._reports` in `pytest_plugin.py`. -- **Oversight**: Human review pending. 348 unit tests pass locally (was 301; +35 new, +12 from new files). - ---- - -## [2026-03-12] — Bus Coverage Report Feature - -- **AI Model**: Claude Sonnet 4.6 -- **Actions Taken**: - - Created `ovoscope/bus_coverage.py` — `BusCoverageTracker`, `BusCoverageReport`, `SkillBusCoverage`, `HandlerEntry`, `EmitterEntry` dataclasses. Tracks listener invocations (via `bus.emit` monkey-patch) and emitter observed/asserted counts per skill_id. Handler attribution via `handler.__self__` → `minicroft.plugin_skills`. Handles pyee v9 `OrderedDict` storage format. - - Modified `ovoscope/__init__.py`: added `track_bus_coverage`, `print_bus_coverage`, `bus_coverage_report` fields to `End2EndTest`; hooked `BusCoverageTracker` into `execute()` around the capture block. - - Modified `ovoscope/pytest_plugin.py`: added `BusCoverageCollector`, `bus_coverage_session` session fixture, `pytest_terminal_summary` hook for merged end-of-session report. - - Modified `ovoscope/cli.py`: added `cmd_bus_coverage` subcommand and `bus-coverage` parser entry. - - Created `docs/bus-coverage.md` — full API reference with source citations. - - Updated `FAQ.md` with three new Q&A entries. - - Created `test/unittests/test_bus_coverage.py` — 32 unit tests, all passing. -- **Oversight**: 301 unit tests pass locally. `bus_coverage.py` at 97% coverage. - - -## [2026-03-11] — Add ovoscope-setup entrypoint for AI assistant skill installation - -- **AI Model**: Claude Sonnet 4.6 -- **Actions Taken**: - - Created `ovoscope/setup_skill.py` — `ovoscope-setup` CLI with install/uninstall for Claude, Gemini, OpenCode; auto-detect mode; `--list`, `--path`, `--uninstall` flags. - - Created `ovoscope/skill_data/` package with bundled skill definitions: - - `claude/SKILL.md` + `claude/scripts/ovoscope.sh` + `claude/assets/docs/` + `claude/assets/FAQ.md|QUICK_FACTS.md` - - `gemini/` — identical structure (Gemini uses same SKILL.md format, project-level install) - - `opencode/ovoscope.md` — YAML frontmatter agent definition for OpenCode - - Updated `pyproject.toml`: added `ovoscope-setup` script entrypoint, `[tool.setuptools.packages.find]`, and `[tool.setuptools.package-data]` to bundle `skill_data/`. - - Added 26 unit tests in `test/unittests/test_setup_skill.py` — all passing. -- **Oversight**: 269 unit tests pass locally. - -## [2026-03-11] — Docs Gap Review and Fixes - -- **AI Model**: Claude Sonnet 4.6 -- **Actions Taken**: - - `docs/ocp.md`: Documented `execute()` return type (`List[Message]`), clarified `patch_targets` format (dotted Python path where symbol is used), added aiohttp example. - - `docs/pipeline.md`: Documented `assert_matches(intent_type=...)` as substring check with example; added `ovoscope/pipeline.py:LINE` citations to all methods. - - `docs/cli.md`: Corrected `--ignore-context` → `--include-context`, explained when/why to use it; clarified `validate` pydantic fallback trigger. - - `docs/end2end-test.md`, `docs/minicroft.md`, `docs/capture-session.md`: Added `ovoscope/__init__.py:LINE` source citations to class and key method definitions. - - `docs/capture-session.md`: Documented `finish()` idempotency. - - `docs/listener.md`: Added full VAD/WakeWord API section (`MockVADEngine`, `MockHotWordEngine`, `is_silence`, `extract_speech`, `detect_wakeword`, `scan_for_wakeword`, `VADTest`, `WakeWordTest`) with examples and `ovoscope/listener.py:LINE` citations. Updated constructor parameter table. Fixed stale line references. - - `docs/index.md`: Added `gui-testing.md` link; updated Public API section with `GUICaptureSession`, VAD/WW helpers; fixed "Does NOT Do" section for VAD/WW. - - `QUICK_FACTS.md`: Added entry-point groups table; updated test count (243) and coverage note. -- **Oversight**: No new code changes — docs only. - -## [2026-03-11] — Add VAD and WakeWord Support to MiniListener - -- **AI Model**: Claude Haiku 4.5 -- **Actions Taken**: - - Extended `ovoscope/listener.py` with `MockVADEngine`, `MockHotWordEngine`, `VADTest`, `WakeWordTest`. - - Extended `MiniListener` with `vad_instance` / `ww_instances` constructor params. - - Added `is_silence()`, `extract_speech()`, `detect_wakeword()`, `scan_for_wakeword()` methods to `MiniListener` — `listener.py:466–600`. - - Extended `get_mini_listener()` factory with `vad_plugin`, `vad_instance`, `ww_plugin`, `ww_instances` params. - - Made `ovos_dinkum_listener` import lazy (graceful `ImportError`) so VAD/WW tests work without the full listener stack installed. - - Added 41 unit tests in `test/unittests/test_listener_vad_ww.py`. - - Updated `FAQ.md` with VAD and WakeWord testing Q&A. -- **Oversight**: 243 unit tests pass locally. - -## [2026-03-11] — Enhance Audio Testing Robustness and CI - -- **AI Model**: Claude Sonnet 4.6 -- **Actions Taken**: - - Added `LIGHT_TEST_PIPELINE` as a lightweight fallback when Adapt/Padatious are missing. - - Updated `MiniCroft` to auto-fallback to `LIGHT_TEST_PIPELINE` if stages are missing. - - Refactored `PlaybackServiceHarness` for better robustness (proper patch cleanup, timeout handling). - - Added skip guard to audio harness tests to prevent failures when `ovos-audio` is not installed. - - Fixed documentation path references and added prerequisites. - - Added missing `LICENSE` (Apache-2.0) file. - - Updated CI workflows to include `audio` extra for unit tests. -- **Oversight**: All 147 unit tests pass locally. - -## [2026-03-10] — Add Audio Testing Harnesses - -- **AI Model**: Claude Sonnet 4.6 -- **Actions Taken**: - - Created `ovoscope/audio.py` — 5 new classes: - - `MockAudioBackend` (inherits `AudioBackend`) — no-op backend tracking state - - `AudioServiceHarness` — context manager wrapping `AudioService` with `MockAudioBackend` - - `MockTTS` (inherits `TTS`) — writes 44-byte silent WAV, records spoken utterances - - `PlaybackServiceHarness` — context manager wrapping `PlaybackService` with `MockTTS` - - `AudioCaptureSession` — records bus messages matching configurable prefix list - - Updated `ovoscope/__init__.py` — guarded import of audio harness classes - - Updated `pyproject.toml` — added `[audio]` optional dependency - - Created `test/unittests/test_audio_harness.py` — 38 unit tests (all passing) - - Created `ovos-audio/test/end2end/__init__.py` — empty marker - - Created `ovos-audio/test/end2end/test_audio_service_e2e.py` — 11 E2E tests (all passing) - - Created `ovos-audio/test/end2end/test_playback_service_e2e.py` — 7 E2E tests (all passing) - - Created `docs/audio-testing.md` — full API reference with source citations - - Updated `docs/index.md` — link to audio-testing.md - - Updated `FAQ.md` — 3 new Q&As for audio testing - - Updated `QUICK_FACTS.md` — new audio harness classes, updated test count -- **Key design decisions**: - - `AudioServiceHarness` uses `autoload=False` then manually injects `MockAudioBackend` - - `PlaybackServiceHarness` patches `ovos_audio.playback.play_audio` to prevent real audio - - `TTS.queue` is class-level; harness drains it before each `PlaybackService` construction - - `stop()` MUST return `True` to trigger `mycroft.stop.handled` in `AudioService` - - `FakeBus.wait_for_response()` does not work in-process; subscribe-emit-wait pattern used -- **Oversight**: All 38 ovoscope unit tests + 18 ovos-audio E2E tests pass - -## [2026-03-10] — Add `pipeline_config` parameter to `MiniCroft` -- **AI Model**: Claude Sonnet 4.6 -- **Actions Taken**: - - Added `pipeline_config: Optional[Dict[str, Dict]] = None` parameter to `MiniCroft.__init__` — `ovoscope/__init__.py` - - Patches `Configuration()["intents"][plugin_key]` before `super().__init__()` so pipeline plugins read overridden config in their own `__init__` - - Restores all overrides in `MiniCroft.stop()` — `ovoscope/__init__.py` - - Updated `docs/minicroft.md`: added `pipeline_config` to constructor table and added "Pipeline Plugin Config Overrides" section with usage example - - Updated `FAQ.md`: added Q&A for `pipeline_config` and M2V multilingual model skip behaviour - - Added 5 unit tests in `test/unittests/test_minicroft.py::TestMiniCroftPipelineConfig`: patch active, restore after stop, existing key preserved, None is no-op, multiple keys -- **Oversight**: 18/18 minicroft unit tests pass; confucius e2e suite: 20 passed, 2 skipped. -- **Motivation**: Needed to force the M2V multilingual model in `TestConfuciusM2VEN` regardless of what `mycroft.conf` says locally. Language-specific models (e.g. Portuguese) don't contain English intent labels and always return no match. -- **Oversight**: All ovoscope unit tests pass; confucius e2e suite: 20 passed, 2 skipped (M2V — multilingual model not cached locally). - -## [2026-03-10] — Test coverage improvement (78% → 89%) -### Changes -- Created `test/unittests/test_end2end_extended.py` — 46 new tests covering: - - **Routing internals**: flip_points, entry_points, keep_original_src assertions - - **Active skills**: inject_active, activation_points, deactivation_points, disallow_extra_active_skills - - **Boot sequence**: correct/incorrect boot message assertions - - **Final session**: lang mismatch raises, matching session passes - - **Async messages**: captured separately, missing raises, count mismatch raises - - **Context assertions**: wrong context raises - - **GUI filtering**: ignore_gui=True/False behavior - - **Serialization**: JSON string input, flip_points/flags preservation, anonymize_message - - **from_message recording**: captures sequence, wraps single message - - **Pipeline constants**: composition validation - - **MiniCroft lang config**: override and restore - - **Verbose output**: exercises all print branches - - **Message count verbose**: first differing message output -- Created `test/unittests/test_pytest_plugin.py` — 6 new tests for minicroft fixture logic via `__wrapped__` -- Updated `FAQ.md` — added coverage FAQ entry -### AI Transparency Report -- **AI Model**: Claude Opus 4.6 -- **Actions Taken**: Created 2 new test files with 52 total new tests -- **Oversight**: All tests verified passing. Coverage: 78% → 89% overall, `__init__.py` 54% → 68%, `pytest_plugin.py` 0% → 64% ---- -## [2026-03-09] — CI workflows and test-gated releases -### Changes -- Created `.github/workflows/unit_tests.yml` — runs 58 unit tests with `pytest --cov=ovoscope` on PRs/pushes to `dev`, posts coverage comment via `py-cov-action/python-coverage-comment-action@v3` -- Created `.github/workflows/build_tests.yml` — matrix build (Python 3.10, 3.11) with `python -m build`, tests sdist/wheel creation and package install -- Created `.github/workflows/license_tests.yml` — calls `OpenVoiceOS/gh-automations/.github/workflows/license-check.yml@dev` reusable workflow -- Created `.github/workflows/pipaudit.yml` — CVE scanning via `pypa/gh-action-pip-audit@v1.0.0` on Python 3.10/3.11 matrix -- Updated `.github/workflows/release_workflow.yml` — added `build_tests` job that runs full test suite; `publish_alpha` now depends on `build_tests` via `needs:`, gating alpha releases on test success -- Updated `docs/ci-integration.md` — documented ovoscope's own CI workflow table -- Updated `FAQ.md` — added 3 new CI-related Q&A entries -### AI Transparency Report -- **AI Model**: Claude Opus 4.6 -- **Actions Taken**: Created 4 new workflow files, updated 1 existing workflow, updated docs and FAQ -- **Oversight**: All workflows follow established OVOS conventions (actions/checkout@v4, actions/setup-python@v5, python-version 3.11, python -m build). 58 existing tests verified passing. ---- -## [2026-03-09] — pytest_plugin: safe teardown guard -### Changes -- `ovoscope/pytest_plugin.py` — `minicroft` fixture: initialise `mc = None` before calling - `get_minicroft()`, then wrap `yield mc` in `try/finally` with `if mc is not None: mc.stop()`. - Previously, if `get_minicroft()` raised (e.g. `TimeoutError`), teardown would hit a - `NameError: name 'mc' is not defined`, masking the original exception in pytest output. -### AI Transparency Report -- **AI Model**: Claude Sonnet 4.6 -- **Actions Taken**: Applied targeted edit to `pytest_plugin.py` lines 57–62. -- **Oversight**: No logic change — only teardown safety. Existing tests unaffected. ---- -## [2026-03-09] — pydantic_helpers: typing, docstrings, tests, bug fix, pyproject.toml migration -### Changes -**`ovoscope/pydantic_helpers.py`** (new module, renamed from the initial `pydantic.py`): -- Full module docstring with install instructions and usage example. -- `TYPE_CHECKING`-guarded import of `OpenVoiceOSMessage` — type checkers see full annotations; - no `ImportError` at runtime when `ovos-pydantic-models` is absent. -- All three public functions have complete Google-style docstrings with `Args`, `Returns`, - `Raises`, and `Example` blocks; all parameter and return types are annotated. -- **Bug fixed** in `validate_fixture()`: was constructing `raise ValidationError(str, ...)` which - pydantic v2 does not support (it is not user-constructible). Changed to `raise ValueError(...) from exc`. -- **Bug fixed** in `validate_fixture()`: normalisation fallback changed from `""` to `None` so - messages missing both `"type"` and `"message_type"` keys are correctly rejected by pydantic - (an empty string passes `message_type: str = Field(...)` validation silently). -**`test/unittests/test_pydantic_helpers.py`** (new, 20 tests): -| Class | Tests | What is covered | -|-------|-------|----------------| -| `TestToBusMessage` | 6 | `msg_type`, data fields, return type, utterance msg, empty context, roundtrip | -| `TestFromBusMessage` | 5 | valid speak, valid utterance, return type, invalid raises `ValidationError`, base model leniency | -| `TestValidateFixture` | 9 | valid fixture, source/expected preserved, missing file, malformed source, malformed expected, error chains `ValidationError`, `message_type` key accepted, empty lists | -All 20 tests pass. Full suite now 58 tests (38 pre-existing + 20 new), all passing. -**`pyproject.toml`** — completed migration: -- `build-backend` changed from `setuptools.backends.legacy:build` to `setuptools.build_meta`. -- `dynamic = ["version"]` added; `[tool.setuptools.dynamic] version = {attr = "ovoscope.version.__version__"}`. -- `[project.optional-dependencies] pydantic = ["ovos-pydantic-models>=0.1.0"]` added. -- `setup.py` removed. -**`ovoscope/version.py`**: -- Added `__version__` computed from `VERSION_MAJOR`, `VERSION_MINOR`, `VERSION_BUILD`, `VERSION_ALPHA` - so `pyproject.toml` dynamic versioning works without `setup.py`. -**`AUDIT.md`**, **`SUGGESTIONS.md`**, **`FAQ.md`**: -- All module references updated from `ovoscope.pydantic` → `ovoscope.pydantic_helpers`. -- AUDIT unit-test count updated to 58; setup.py fix marked fully complete. -- SUGGESTIONS.md item 6 file path corrected. -- FAQ.md pydantic section import paths corrected. -### AI Transparency Report -- **AI Model**: Claude Sonnet 4.6 -- **Actions Taken**: Read existing module and all test files; identified two bugs in `validate_fixture()` - via live `python -c` verification; wrote 20 tests and iterated until all pass; migrated pyproject.toml. -- **Oversight**: `validate_fixture()` validates top-level message structure only (`message_type`, - `data`, `context` shape) via `OpenVoiceOSMessage` — it does not validate data-field-level schemas - (e.g. `utterances` type). Use `from_bus_message(msg, SpecificModel)` for field-level validation. ---- -## [2026-03-09] — End2end tests written for ovos-persona (11 tests) -### Tests Created -New `ovos-persona/test/end2end/test_persona.py` — 4 test classes, 11 tests: -| Class | Tests | Intents/triggers | -|-------|-------|-----------------| -| `TestPersonaList` | 2 | `list_personas.intent` (no personas / 2 personas) | -| `TestPersonaCheck` | 2 | `active_persona.intent` (no active / active) | -| `TestPersonaSummon` | 2 | `summon.intent` (known / unknown persona) | -| `TestPersonaRelease` | 1 | `Release.voc` via `voc_match()` | -| `TestPersonaQuery` | 4 | `ask.intent` explicit / active fallback / error / no-match | -### Key Patterns Discovered -- Pipeline plugins accessed via `mc.intents.pipeline_plugins["ovos-persona-pipeline-plugin"]` -- Inject mock personas into `persona_svc.personas` dict — bypasses real solver loading -- `setUp()` clears real personas (Gemma etc.) and resets `active_persona = None` -- `skill_ids=[]` — no skills needed; pipeline plugin loads automatically -- `ovos.utterance.handled` data is `{"name": "PersonaService.handle_persona_*"}` — not empty -- `speak` with dialog template checked only by `context={"skill_id": SKILL_ID}` (text varies) -- Direct speaks from query answers checked with `data={"utterance": "forty two"}` -### AI Transparency Report -- **AI Model**: Claude Sonnet 4.6 -- **Actions Taken**: Traced all message sequences via live FakeBus capture; wrote tests iteratively; all 11 pass. -- **Oversight**: Dialog text assertions omitted — locale-dependent. `test_list_two_personas` relies on dict insertion order (Python 3.7+). ---- -## [2026-03-09] — End2end tests written for 6 skills (18 tests) -### Skills Covered -New `test/end2end/` directories and test files created for: -| Skill | Test file | Tests | Intents tested | -|-------|-----------|-------|----------------| -| `ovos-skill-hello-world` | `test_hello_world.py` | 4 | HelloWorldIntent (Adapt), Greetings.intent (Padatious), no-match cases | -| `ovos-skill-fallback-unknown` | `test_fallback_unknown.py` | 2 | fallback-low match, Adapt no-match | -| `ovos-skill-naptime` | `test_naptime.py` | 2 | naptime.intent (Padatious), no-match | -| `ovos-skill-volume` | `test_volume.py` | 4 | volume.max.intent, volume.mute.intent, volume.unmute.intent (Padatious), no-match | -| `ovos-skill-count` | `test_count.py` | 3 | count_to_N.intent (Padatious), no-match | -| `ovos-skill-parrot` | `test_parrot.py` | 3 | speak.intent, repeat.tts.intent, no-match | -### Key Patterns Discovered -- Intents registered with string `"name.intent"` → Padatious; `IntentBuilder(...)` → Adapt -- Skills emitting raw `Message(...)` without `forward(...)`/`reply()` have `source=None` — use `async_messages` + `ignore_messages` -- Enclosure/LED messages and `add_context`/`configuration.patch` must be in `ignore_messages` -- `message.forward(...)` inherits the post-flip source/dest — do NOT add these to `keep_original_src` -### AI Transparency Report -- **AI Model**: Claude Sonnet 4.6 -- **Actions Taken**: Read each skill's `__init__.py` and locale files; wrote tests iteratively with test runs to fix pipeline selection, ignore_messages, meta dict content. All 18 tests pass. -- **Oversight**: naptime test skips dialog content check (varies with `listener.wake_word` config). ---- -## [2026-03-09] — Config isolation extended: blacklisted_skills + blacklisted_intents -### Problem Addressed -After pipeline isolation, `test_stop.py` (6 tests) and `test_cancel_plugin.py` (1 test) still failed. -Root cause: `Session.__init__` reads `Configuration()["skills"]["blacklisted_skills"]` and -`Configuration()["intents"]["blacklisted_intents"]` from the live singleton dict cache, same problem -as the pipeline. The user had `skill-ovos-stop.openvoiceos` blacklisted in `~/.config/mycroft/mycroft.conf`. -Additionally, `ovos-skill-count` and `ovos-utterance-plugin-cancel` were not installed in the workspace venv. -### Changes -- `ovoscope/__init__.py` — `MiniCroft.__init__`: added `_original_blacklisted_skills` and - `_original_blacklisted_intents` state variables. -- `ovoscope/__init__.py` — `MiniCroft.run()`: when `isolate_config=True`, patches - `Configuration()["skills"]["blacklisted_skills"] = []` and - `Configuration()["intents"]["blacklisted_intents"] = []` in the live singleton dict cache. -- `ovoscope/__init__.py` — `MiniCroft.stop()`: restores both lists from saved originals. -- `ovos-core/test/end2end/test_stop.py` — Added `"ovos-hivemind-pipeline-plugin.stop.response"` to - `ignore_messages` in both `TestStopNoSkills` and `TestCountSkills` (hivemind responds to `mycroft.stop`). -- Installed `Skills/ovos-skill-count` and `Transformer plugins/ovos-utterance-plugin-cancel` with `uv pip install --no-deps -e`. -### Result -27/27 ovos-core end2end tests pass (was 20/27). -### AI Transparency Report -- **AI Model**: Claude Sonnet 4.6 -- **Actions Taken**: Diagnosed `blacklisted_skills` stale cache issue; extended `run()`/`stop()` patch pattern; - identified hivemind stop response as uncaught message; installed missing plugins. -- **Oversight**: `blacklisted_skills` patch only runs when `isolate_config=True` (same condition as xdg isolation). ---- -## [2026-03-09] — Pipeline isolation and reproducible test pipelines -### Problem Addressed -`MiniCroft.isolate_config=True` cleared `Configuration.xdg_configs` to remove the user's -`~/.config/mycroft/mycroft.conf`, but `SessionManager.default_session` is a singleton -initialised at module import time — it already held the user's pipeline (which could include -`ovos-persona-pipeline-plugin-high`, Ollama, OCP, etc.). Any utterance emitted without an -explicit session in its context would inherit this pipeline and be intercepted by AI plugins -non-deterministically, making tests environment-dependent. -### Changes -- `ovoscope/__init__.py` — Added pipeline stage constants: - `STOP_PIPELINE`, `CONVERSE_PIPELINE`, `ADAPT_PIPELINE`, `PADATIOUS_PIPELINE`, - `FALLBACK_PIPELINE`, `COMMON_QUERY_PIPELINE`, `PERSONA_PIPELINE`. -- `ovoscope/__init__.py` — Added `DEFAULT_TEST_PIPELINE`: all standard built-in pipeline stages - (stop/converse/adapt/padatious/fallback/common-query), **no** AI/LLM/persona/OCP stages. -- `ovoscope/__init__.py` — `MiniCroft.__init__`: added `default_pipeline: Optional[List[str]]` - parameter (default `DEFAULT_TEST_PIPELINE`). Stored as `_default_pipeline`; original - pipeline stored for restoration. -- `ovoscope/__init__.py` — `MiniCroft.run()`: after `load_plugin_skills()` and before - `set_ready()`, sets `SessionManager.default_session.pipeline = self._default_pipeline` when - not `None`. Setting it here (post-FakeBus-sync) is the correct point — after all init bus - messages have been processed. -- `ovoscope/__init__.py` — `MiniCroft.stop()`: restores `SessionManager.default_session.pipeline` - to its pre-test value, enabling test isolation within a single process. -- `test/unittests/test_minicroft.py` — Added `TestMiniCroftPipelineIsolation` (5 tests): - pipeline overrides default session, restored after stop, `isolate_config=True` uses - `DEFAULT_TEST_PIPELINE`, persona/ollama/m2v absent from `DEFAULT_TEST_PIPELINE`, - `default_pipeline=None` leaves session unchanged. -### Use Cases Unblocked -- `get_minicroft([])` → `complete_intent_failure` tests now pass without Gemma/persona intercepting. -- `get_minicroft([SKILL_ID], default_pipeline=ADAPT_PIPELINE)` — Adapt-only testing. -- `get_minicroft([SKILL_ID], default_pipeline=ADAPT_PIPELINE + FALLBACK_PIPELINE)` — intent+fallback. -- `get_minicroft([SKILL_ID], default_pipeline=PERSONA_PIPELINE)` — explicitly test persona behaviour. -- `get_minicroft([SKILL_ID], default_pipeline=None)` — use system default (includes all installed plugins). -### AI Transparency Report -- **AI Model**: Claude Sonnet 4.6 -- **Actions Taken**: Diagnosed root cause (singleton default_session pre-initialised from user config); - added constants + `default_pipeline` param; updated `run()` and `stop()`; added 5 unit tests. -- **Oversight**: `DEFAULT_TEST_PIPELINE` does not include OCP or m2v stages — repos that test - media skills should pass an explicit pipeline including those stages. ---- -## [2026-03-08] — Code improvements from SUGGESTIONS.md -### Changes -- `ovoscope/__init__.py` — `get_minicroft()`: added `max_wait: float = 60` parameter; raises - `TimeoutError` if MiniCroft does not reach READY within the deadline. Return type annotated - as `-> MiniCroft`. -- `ovoscope/__init__.py` — `MiniCroft.inject_message(msg: Message) -> None`: new helper method - for emitting arbitrary messages during a test without going through the utterance pipeline. -- `ovoscope/__init__.py` — `End2EndTest.execute()`: now returns `List[Message]` (was `None`). - Return type annotated. Enables test composition and `assert_spoke()`. -- `ovoscope/__init__.py` — `End2EndTest.assert_spoke(text, lang, timeout)`: new sugar method; - calls `execute()` and asserts a matching `speak` message was emitted. -- `ovoscope/__init__.py` — `End2EndTest.save()`: return type annotated as `-> None`. -- `ovoscope/pytest_plugin.py` (NEW): class-scoped `minicroft` pytest fixture; reads `skill_ids` - from the test class attribute; handles startup and teardown automatically. -- `setup.py`: registered `pytest11` entry point so the fixture is auto-discovered. -- `pyproject.toml` (NEW): `[build-system]`, `[project]`, `[project.entry-points."pytest11"]`, - and `[tool.pytest.ini_options]` tables. `setup.py` retained for dynamic version reading. -- `AUDIT.md`: marked 3 issues as FIXED; updated Next Steps. -### AI Transparency Report -- **AI Model**: Claude Sonnet 4.6 -- **Actions Taken**: Read `__init__.py` fully; made targeted edits; created `pytest_plugin.py` - and `pyproject.toml`. No logic was changed — only additions and the `return messages` fix. -- **Oversight**: `assert_spoke()` depends on `execute()` returning messages — verify against - a live install. `pyproject.toml` dynamic version section has a TODO comment for when - `version.py` exports `__version__`. ---- -## [2026-03-08] — Documentation enrichment and audit deepening -### Changes -- Created `docs/usage-guide.md` — full tutorial from install to 8 test patterns; references - hello-world canonical examples and real class/method signatures from `ovoscope/__init__.py`. -- Created `docs/ci-integration.md` — directory layout, pytest config, GitHub Actions job - template, fixture management, and CI gotchas. -- Updated `docs/index.md` — added `usage-guide.md` and `ci-integration.md` to navigation table; - added "Who Uses ovoscope" section; added gh-automations cross-reference. -- Replaced `AUDIT.md` — shallow CI-pin findings replaced with 7 evidence-based issues - (CRITICAL/MAJOR/MODERATE/MINOR) all traced to specific lines in `__init__.py`. -- Replaced `SUGGESTIONS.md` — 4 generic stubs replaced with 7 concrete, repo-specific proposals - with code snippets pointing to specific lines. -### Rationale -The previous docs scaffold was boilerplate with no practical value. This pass enriches docs to the -level where every OVOS repo can adopt ovoscope end-to-end testing without reading source code. -### Verification -- `ls ovoscope/docs/` shows 7 files (5 pre-existing + `usage-guide.md` + `ci-integration.md`). -- All code examples in `usage-guide.md` use real imports and class names verified from source. -- All `AUDIT.md` findings reference specific file:line evidence. -### AI Transparency Report -- **AI Model**: Claude Sonnet 4.6 -- **Actions Taken**: Read `ovoscope/__init__.py` (485 lines), `test/test_helloworld.py`, - `ovos-core/test/end2end/test_adapt.py`, and all existing docs; then generated enriched content. -- **Oversight**: Code examples are illustrative but not executed. Verify against live skill install before treating as runnable. - ---- - -## 2026-03-11 — Phase 1–3 Feature Additions - -**AI Model**: claude-sonnet-4-6 -**Oversight**: Human review pending - -### Actions Taken - -Added CLI, PHAL harness, fixture differ, OCP harness, pipeline harness, -ecosystem coverage scanner, GUI capture session, and remote recorder. - -**New modules:** -- `ovoscope/cli.py` — `ovoscope` CLI with `record`, `run`, `diff`, `validate`, `coverage` -- `ovoscope/diff.py` — `MessageDiff`, `FixtureDiffResult`, `diff_fixtures` -- `ovoscope/phal.py` — `MiniPHAL`, `PHALTest` -- `ovoscope/ocp.py` — `OCPTest`, `assert_ocp_query_response` -- `ovoscope/pipeline.py` — `PipelineHarness` -- `ovoscope/coverage.py` — `RepoCoverage`, `EcosystemCoverageReport`, `scan_workspace` -- `ovoscope/remote_recorder.py` — `RemoteRecorder` - -**Extended modules:** -- `ovoscope/__init__.py` — added `GUICaptureSession` - -**New docs:** -- `docs/cli.md`, `docs/phal.md`, `docs/ocp.md`, `docs/pipeline.md` -- `docs/usage-guide.md` — Patterns 9–12 appended - -**New tests:** -- `test/unittests/test_diff.py` — 7 test methods -- `test/unittests/test_phal.py` — 8 test methods -- `test/unittests/test_coverage.py` — 11 test methods -- `test/unittests/test_cli.py` — 14 test methods - -**pyproject.toml changes:** -- Added `[project.scripts] ovoscope = "ovoscope.cli:main"` - -All 202 tests pass. No regressions introduced. - ---- - -## [2026-03-08] — Initial compliance scaffold -### Changes -- Created `QUICK_FACTS.md` with machine-readable package metadata. -- Created `FAQ.md` with common Q&A. -- Created `MAINTENANCE_REPORT.md` (this file) as the change log. -- Created `SUGGESTIONS.md` with initial improvement proposals. -- Created `docs/index.md` as the documentation entry point (if missing). -### Rationale -Establishing the required file set mandated by `AGENTS.md` for all active workspace repositories. -### AI Transparency Report -- **AI Model**: Claude Sonnet 4.6 -- **Actions Taken**: Generated boilerplate compliance scaffold (QUICK_FACTS, FAQ, MAINTENANCE_REPORT, SUGGESTIONS, docs/index). -- **Oversight**: Files were stubs — enriched in the 2026-03-08 documentation pass above. diff --git a/QUICK_FACTS.md b/QUICK_FACTS.md deleted file mode 100644 index 7a9a2c6..0000000 --- a/QUICK_FACTS.md +++ /dev/null @@ -1,55 +0,0 @@ -# Quick Facts — `ovoscope` -End-to-end test framework for OpenVoiceOS skills -## Core Information -| Feature | Details | -|---------|---------| -| Package Name | `ovoscope` | -| Version | `0.7.2` | -| License | Apache-2.0 | -| Repository | [https://github.com/TigreGotico/ovoscope](https://github.com/TigreGotico/ovoscope) | -| Python Support | >=3.10 | -| Status | Active development | -## Entry Points -| Group | Value | Description | -|-------|-------|-------------| -| `console_scripts` | `ovoscope = ovoscope.cli:main` | CLI entry point | -| `pytest11` | `ovoscope = ovoscope.pytest_plugin` | pytest plugin (auto-loaded by pytest) | - -## Testing & CI -| Feature | Details | -|---------|---------| -| Unit Tests | 348 tests across `test/unittests/` (all passing) | -| Coverage | 53% overall (transformer/remote code excluded — requires optional deps) | -| Test Framework | pytest with custom fixtures | -| Coverage Reporter | py-cov-action/python-coverage-comment-action@v3 | -## CI Workflows -| Workflow | Trigger | Status | -|----------|---------|--------| -| `unit_tests.yml` | Push to dev | Uses coverage.yml@dev | -| `build_tests.yml` | Push to master, PR to dev | Uses build-tests.yml@dev | -| `license_check.yml` | Push to master/dev, PR | Uses license-check.yml@dev | -| `pip_audit.yml` | Push to master/dev, PR | Uses pip-audit.yml@dev | -| `release_workflow.yml` | PR merge to dev | Gates on build_tests, calls publish-alpha.yml@dev | -| `publish_stable.yml` | Push to master | Calls publish-stable.yml@dev | -| `release_preview.yml` | PR to dev | Uses release-preview.yml@dev | -| `repo_health.yml` | PR to dev | Uses repo-health.yml@dev | -## Key Features -- **End-to-end testing framework** for OpenVoiceOS skills -- **MiniCroft fixture** — pytest integration with class-scoped skill testing -- **Message capture** — CaptureSession for recording skill responses -- **Assertions** — End2EndTest with assertions (assert_spoke, etc.) -- **Audio harnesses** — AudioServiceHarness, PlaybackServiceHarness, MockAudioBackend, MockTTS, AudioCaptureSession (optional `[audio]` extra) -- **Pydantic integration** — Optional typed bridge with ovos-pydantic-models -- **Version from pyproject.toml** — Full migration from setup.py - -## Audio Harness Classes (ovoscope.audio) -| Class | Description | -|---|---| -| `MockAudioBackend` | No-op AudioBackend tracking state (is_playing, is_paused, played_tracks, ducking counters) | -| `AudioServiceHarness` | Context manager: AudioService + MockAudioBackend on FakeBus | -| `MockTTS` | No-op TTS writing silent WAV, recording spoken_utterances | -| `PlaybackServiceHarness` | Context manager: PlaybackService + MockTTS on FakeBus | -| `AudioCaptureSession` | Records bus messages matching prefix list for sequence assertions | -## Test-Gated Releases -✅ Alpha releases gate on `build_tests` passing (100+ unit tests) -✅ Stable releases gate on master push (must pass alpha CI first) diff --git a/README.md b/README.md index c781d9b..4c16bb6 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,24 @@ stages and deliberately excludes persona, Ollama, OCP, and m2v plugins. | [docs/pydantic-integration.md](docs/pydantic-integration.md) | Typed message models with `ovos-pydantic-models` | | [FAQ.md](FAQ.md) | Common questions and gotchas | --- + +--- + +## Credits + +Developed by [TigreGótico](https://tigregotico.pt) for +[OpenVoiceOS](https://openvoiceos.org). + +[![NGI0 Commons Fund](./ngi.png)](https://nlnet.nl/project/OpenVoiceOS) + +This project was funded through the [NGI0 Commons Fund](https://nlnet.nl/commonsfund), +a fund established by [NLnet](https://nlnet.nl) with financial support from the +European Commission's [Next Generation Internet](https://ngi.eu) programme, under +the aegis of [DG Communications Networks, Content and Technology](https://commission.europa.eu/about-european-commission/departments-and-executive-agencies/communications-networks-content-and-technology_en) +under grant agreement No [101135429](https://cordis.europa.eu/project/id/101135429). + +--- + ## License [Apache 2.0](LICENSE) --- diff --git a/SUGGESTIONS.md b/SUGGESTIONS.md deleted file mode 100644 index 19ff107..0000000 --- a/SUGGESTIONS.md +++ /dev/null @@ -1,153 +0,0 @@ -# Suggestions — `ovoscope` - -Agent-generated proposals for refactors and enhancements. -Each item includes a rationale, affected file, and implementation sketch. - ---- - -## 1. Share MiniCroft Across Fixtures in `cmd_bus_coverage()` [PERFORMANCE] - -**File**: `ovoscope/cli.py` — `cmd_bus_coverage()` - -**Problem**: The current implementation creates a new `MiniCroft` for every -fixture file it runs. When a workspace has many fixtures for the same skill, -this means repeated skill loading, plugin initialisation, and READY-wait -overhead for each fixture — typically 5–20 seconds per fixture. - -**Suggestion**: Group fixture files by their `skill_ids` list, create a single -`MiniCroft` per unique skill set, then replay all fixtures against that shared -instance. Expected speedup: 10–50× for typical skill test suites. - -**Sketch**: -```python -from itertools import groupby -fixtures_by_skills = groupby(sorted(fixtures, key=lambda f: f.skill_ids_key), ...) -for skill_key, group in fixtures_by_skills: - mc = get_minicroft(skill_key) - for fixture in group: - fixture.execute(minicroft=mc) - mc.stop() -``` - ---- - -## 2. Add `PipelineHarness.assert_no_match()` Convenience Method [DONE] - -**File**: `ovoscope/pipeline.py` - -**Status**: Already implemented — `assert_no_match(utterance, timeout=2.0)` is -present at `pipeline.py:213`. No further action needed. - ---- - -## 3. Add `LOAD_TIME` Tag for Registration-Time Handlers [BUS COVERAGE] - -**File**: `ovoscope/bus_coverage.py` - -**Problem**: Handlers invoked during skill loading (before the snapshot) always -show `0 invocations` and `NOT TESTED` in bus coverage reports. This is -misleading because intent registration handlers *are* exercised — just before -the snapshot window. - -**Suggestion**: Capture a pre-READY handler snapshot in `MiniCroft.__init__` -(before `super().__init__()`), then tag any handler present in both the -pre-READY and post-READY snapshots with a `LOAD_TIME` label in the report. -These handlers should be excluded from the `NOT TESTED` count. - ---- - -## 4. `diff.py` — Strict Mode for Extra Keys [DONE] - -**File**: `ovoscope/diff.py` - -**Status**: Implemented in this audit cycle. `_dict_diff()` now accepts -`strict: bool = False`; when `True`, keys present in `actual` but not in -`expected` are flagged as unexpected extras. `diff_fixtures()` exposes the -same `strict` parameter. Default `False` preserves existing behaviour. - ---- - -## 5. Make Coverage Fixture Search Recursive [DONE] - -**File**: `ovoscope/coverage.py` — `_count_fixtures()` - -**Status**: Implemented in this audit cycle. The search now uses -`Path.rglob("*.json")` instead of `os.listdir()`, so fixtures in -sub-directories are counted correctly. - ---- - -## 6. Add Noise-Floor Tolerance to `MockVADEngine.is_silence()` [LISTENER] - -**File**: `ovoscope/listener.py` - -**Problem**: `MockVADEngine.is_silence()` currently returns a fixed value -configured at construction time. Real VAD engines apply a noise-floor -threshold; tests that simulate borderline audio may need to replicate this -behaviour. - -**Suggestion**: Add `noise_floor: float = 0.0` parameter to -`MockVADEngine.__init__()`. When `noise_floor > 0`, `is_silence()` returns -`True` only if the sample RMS is below `noise_floor`. Default `0.0` -preserves existing behaviour (fixed return value). - ---- - -## 7. Make OCP HTTP Patch Targets Configurable [OCP] - -**File**: `ovoscope/ocp.py` - -**Problem**: The default patch targets (`requests.Session.get` and -`requests.get`) are hardcoded in `_apply_patches`. Skills that use -`httpx`, `aiohttp`, or a custom HTTP wrapper cannot be mocked without -specifying `patch_targets`. - -**Suggestion**: Expose `default_patch_targets: List[str]` as a class-level -constant on `OCPTest` so subclasses can override it without rewriting each -test instance: - -```python -class OCPTest: - default_patch_targets: List[str] = ["requests.Session.get", "requests.get"] -``` - ---- - -## 8. Support Env Var for GitHub URL in `setup_skill.py` [SETUP] - -**File**: `ovoscope/setup_skill.py` - -**Problem**: The GitHub URL for skill assets is hardcoded. CI environments or -forks may need to point to a different repository. - -**Suggestion**: Read `OVOSCOPE_SKILL_URL` environment variable as an override: - -```python -import os -SKILL_URL = os.environ.get("OVOSCOPE_SKILL_URL", DEFAULT_SKILL_URL) -``` - ---- - -## 9. Add `RemoteRecorder` Usage to Docs [DOCUMENTATION] - -**File**: `docs/index.md`, `docs/usage-guide.md` - -**Problem**: `RemoteRecorder` — `ovoscope/remote_recorder.py:46` — is not -documented in any public-facing doc file. Users who want to capture fixtures -from a live OVOS instance have no guide. - -**Suggestion**: Add a "Pattern 13: Recording from a Live OVOS Instance" section -to `docs/usage-guide.md` showing the `connect()`/`record()`/`disconnect()` -workflow and the `--live` CLI flag. - ---- - -## 10. Expand `docs/pipeline.md` to Full API Reference [DONE] - -**File**: `docs/pipeline.md` - -**Status**: Expanded in this audit cycle. The file now includes the full -`PipelineHarness` API table with `pipeline.py:LINE` citations, examples for -Adapt and Padatious pipelines, an explanation of `_SinkSkill`, and notes on -pipeline stage ordering and success/failure signals. diff --git a/docs/gui-testing.md b/docs/gui-testing.md index 7a04675..cf1ae47 100644 --- a/docs/gui-testing.md +++ b/docs/gui-testing.md @@ -225,6 +225,30 @@ Setting `ignore_gui=True` (the default on `End2EndTest`) keeps the ordered message sequence clean while `GUICaptureSession` captures the GUI events independently. +## Template-Based GUI (`SYSTEM_*` templates) + +Skills that use the typed template API (`self.gui.show_weather(...)`, +`show_text(...)`, `show_list(...)`, …) emit a `gui.page.show` for a built-in +`SYSTEM_*` template plus `gui.value.set` for its data keys. `assert_template_shown` +checks both in one call: + +```python +with GUICaptureSession(mc.bus) as gui: + mc.bus.emit(Message("recognizer_loop:utterance", + {"utterances": ["what's the weather"], "lang": "en-US"})) + import time; time.sleep(2) + # the SYSTEM_ prefix is optional ("weather" == "SYSTEM_weather") + gui.assert_template_shown( + "ovos-skill-weather.openvoiceos", + "weather", + values={"current_temp": 22, "condition": "Sunny"}, + ) +``` + +This is the recommended assertion for the template-based GUI: it does not care +which display backend (Qt, pyhtmx, …) renders the template — only that the skill +requested the right `SYSTEM_*` template with the right session data. + ## What `GUICaptureSession` Does NOT Cover - Full GUI rendering — only bus messages are captured; no QML engine is run. diff --git a/docs/index.md b/docs/index.md index 0c113f5..2d23772 100644 --- a/docs/index.md +++ b/docs/index.md @@ -10,7 +10,11 @@ | [end2end-test.md](end2end-test.md) | `End2EndTest` — full test runner reference | | [pydantic-integration.md](pydantic-integration.md) | Using `ovos-pydantic-models` with OvoScope | | [audio-testing.md](audio-testing.md) | `AudioServiceHarness`, `PlaybackServiceHarness` — testing audio services | +| [media-testing.md](media-testing.md) | `OCPPlayerHarness`, `OCPCaptureSession`, `MockOCPBackend` — testing the `ovos-media` OCP player (and driving a real OCP backend) | +| [media-provider-testing.md](media-provider-testing.md) | `MediaProviderHarness` — testing `opm.media.provider` catalog/search plugins | +| [ocp.md](ocp.md) | `OCPTest` — testing legacy OCP search skills (`@ocp_search`) | | [listener.md](listener.md) | `MiniListener`, `get_mini_listener`, `ListenerTest`, `MockVADEngine`, `MockHotWordEngine`, `VADTest`, `WakeWordTest` — testing audio transformer plugins, STT pipeline, VAD, and wake-word | +| [voice-loop.md](voice-loop.md) | `MiniVoiceLoop` / `MiniSimpleListener` / `MiniClassicListener` — file-driven bus-sequence testing for the ovos-dinkum, ovos-simple, and mycroft-classic listener services (wake-word → record-begin → utterance), with verifier-chain gating | | [gui-testing.md](gui-testing.md) | `GUICaptureSession` — asserting GUI page navigation and namespace values | | [bus-coverage.md](bus-coverage.md) | `BusCoverageTracker`, `BusCoverageReport` — measuring handler and emitter coverage per skill | ## Conceptual Model @@ -84,6 +88,17 @@ from ovoscope.listener import ( VADTest, # declarative VAD test runner WakeWordTest, # declarative WakeWord test runner ) +# Listener-service bus-sequence testing (dinkum / simple / classic) +from ovoscope import ( + MiniVoiceLoop, # ovos-dinkum-listener: feed PCM chunks or an audio file + MiniSimpleListener, # ovos-simple-listener: drive the loop over an audio file + MiniClassicListener, # mycroft-classic-listener: best-effort file drive + bridge + get_mini_voice_loop, # factory: create MiniVoiceLoop + VoiceLoopTest, # declarative wake-word → bus-sequence test runner + MiniHotwordContainer, # controllable hotword container with a verifier chain + MockFileMicrophone, # file-backed mic plugin shared across listener harnesses + MockStreamingSTT, # configurable transcript STT mock +) ``` Type aliases also exported: ```python @@ -115,12 +130,37 @@ msgs = listener.feed_audio(b"\x00" * 1024) listener.shutdown() ``` +## Listener-Service Bus-Sequence Testing + +OVOS has several listener **services** — ovos-dinkum-listener, ovos-simple-listener, +and mycroft-classic-listener — each emitting the same `recognizer_loop:*` bus +events. `MiniVoiceLoop`, `MiniSimpleListener`, and `MiniClassicListener` each +wire their real service to a `FakeBus` with mock mic/VAD/STT/wake-word plugins, +drive it over an arbitrary audio file (or PCM frames), and capture the emitted +sequence — sharing one set of assertion helpers. `MiniVoiceLoop` also exercises +the dinkum verifier-chain gate that decides whether a detection survives. + +See [voice-loop.md](voice-loop.md) for full API reference and usage patterns. + +```python +from unittest.mock import Mock +from ovoscope.voice_loop import MiniVoiceLoop, MockHotWordEngine + +ww = MockHotWordEngine("hey_mycroft", trigger_after=3) +accepting = Mock(); accepting.verify.return_value = True + +with MiniVoiceLoop(ww_instances={"hey_mycroft": ww}, + verifiers=[accepting]) as vl: + msgs = vl.feed_chunks([b"\x00" * 512] * 5) + vl.assert_record_begin_emitted(msgs) +``` + ## What OvoScope Does NOT Do - Does not start a real WebSocket MessageBus server — uses `FakeBus` (in-process pub/sub). - Does not load PHAL plugins or the audio service — only skills and the intent pipeline. - Does not test GUI rendering — GUI namespace messages are ignored by default (`ignore_gui=True`). - Does not test TTS — operates at the `recognizer_loop:utterance` level (see [audio-testing.md](audio-testing.md) for TTS lifecycle testing). -- `MiniListener` covers `AudioTransformersService`, the STT pipeline, and mock VAD/WakeWord engines — not the full `DinkumVoiceLoop` state machine. +- `MiniListener` covers `AudioTransformersService`, the STT pipeline, and mock VAD/WakeWord engines. `MiniVoiceLoop` / `MiniSimpleListener` / `MiniClassicListener` drive the dinkum, simple, and classic listener **services** from an audio file and capture the `recognizer_loop:*` bus sequence; the classic file drive is best-effort (energy-based pipeline). ## Quick Links | Resource | Path | |---|---| diff --git a/docs/listener.md b/docs/listener.md index 3fe008e..c19749b 100644 --- a/docs/listener.md +++ b/docs/listener.md @@ -68,6 +68,32 @@ assert any(m.msg_type == "recognizer_loop:utterance" for m in msgs) listener.shutdown() ``` +**Streaming real ggwave audio** — the ggwave decoder only fires after it has +accumulated enough frames, so feed the whole waveform with `feed_audio_stream`, +which keeps every message emitted across the stream (unlike `feed_audio`, which +clears its buffer on each call): + +```python +import ggwave, numpy as np +from ovos_audio_transformer_plugin_ggwave import GGWavePlugin +from ovoscope.listener import get_mini_listener + +# real ggwave waveform (48kHz float32 → 16kHz int16, as the mic would deliver it) +wf = np.frombuffer(ggwave.encode("UTT:turn on the lights", protocolId=1, volume=20), + dtype=np.float32) +mic = np.interp(np.linspace(0, len(wf) - 1, int(len(wf) * 16000 / 48000)), + np.arange(len(wf)), wf) +audio = (np.clip(mic, -1, 1) * 32767).astype(np.int16).tobytes() + +plugin = GGWavePlugin(config={"start_enabled": True, "sample_rate": 16000}) +listener = get_mini_listener( + plugin_instances={"ovos-audio-transformer-plugin-ggwave": plugin} +) +msgs = listener.feed_audio_stream(audio, chunk_size=2048) +assert any(m.msg_type == "recognizer_loop:utterance" for m in msgs) +listener.shutdown() +``` + **Full pipeline testing** (STT with real WAV): ```python @@ -105,6 +131,7 @@ listener.shutdown() |--------|-----------|-------------| | `feed_audio(chunk)` — `ovoscope/listener.py:351` | `(bytes) → List[Message]` | Calls `AudioTransformersService.feed_audio()`. Requires `ovos-dinkum-listener`. | | `feed_speech(chunk)` — `ovoscope/listener.py:371` | `(bytes) → List[Message]` | Calls `AudioTransformersService.feed_speech()`. Requires `ovos-dinkum-listener`. | +| `feed_audio_stream(chunks, feed, chunk_size)` | `(bytes\|list[bytes], str, int) → List[Message]` | Streams frames in order **without** clearing between them; aggregates all emitted messages. Use for decoders that fire after many frames (ggwave). | | `transform(chunk)` — `ovoscope/listener.py:390` | `(bytes) → tuple[bytes, dict, List[Message]]` | Full transform pipeline; returns `(audio, ctx, messages)`. Requires `ovos-dinkum-listener`. | | `listen(audio, ...)` — `ovoscope/listener.py:410` | `(audio, language, stt_instance, ...) → List[Message]` | Full pipeline: audio → transformers → STT → utterance message. Requires `ovos-dinkum-listener`. | diff --git a/docs/media-provider-testing.md b/docs/media-provider-testing.md new file mode 100644 index 0000000..4e84a98 --- /dev/null +++ b/docs/media-provider-testing.md @@ -0,0 +1,97 @@ +# MediaProvider (Search) Testing with ovoscope + +`MediaProviderHarness` (`ovoscope.media_provider`) tests `opm.media.provider` +plugins — the in-process **catalog/search** providers introduced by the +ovos-media sprint to replace OCP search skills. It is the search-side counterpart +to [`OCPPlayerHarness`](media-testing.md), which drives the *player*. + +> **No extra required.** Unlike the player harness, `MediaProviderHarness` is +> **duck-typed**: it imports neither `mediavocab` nor `ovos-plugin-manager`'s +> `MediaProvider` / `QueryContext`. Your *test* supplies the `Signals` / +> `QueryContext` objects, so ovoscope itself stays dependency-free. The provider +> package and `mediavocab` only need to be installed for the test that uses it. + +## Why a dedicated harness + +There is no released loader for the `opm.media.provider` entry-point group, and +the OCP pipeline does not yet load providers in-process. So the harness models the +pipeline's *intended* path and removes the boilerplate every provider e2e would +otherwise repeat: + +``` +discover the entry-point -> serves(signals, context) gate -> search_safe() +``` + +## Basic usage + +```python +from ovoscope import MediaProviderHarness +from mediavocab import MediaType, Signals +from ovos_plugin_manager.templates.media_provider import QueryContext + +h = MediaProviderHarness.from_entrypoint( + "music_assistant", # opm.media.provider entry-point name + config={"url": "http://mass.local:8095"}, + mock_api=my_mock_client, # injected onto provider._api +) + +# packaging registered the plugin +h.assert_entrypoint_registered() + +# three-axis + context routing gate +h.assert_routes(Signals(medium=MediaType.MUSIC), + QueryContext(supported_playback_types={"audio"})) +h.assert_not_routes(Signals(medium=MediaType.MUSIC), + QueryContext(supported_playback_types={"video"})) # audio-only provider +h.assert_not_routes(Signals(medium=MediaType.MOVIE)) + +# the never-raising search the pipeline calls — ranked, playable results +releases = h.assert_returns_playables(Signals(title="worms")) +assert all(r.uri.startswith("library://") for r in releases) +``` + +## Constructing the harness + +| Constructor | Use | +|---|---| +| `MediaProviderHarness.from_entrypoint(name, config=None, group="opm.media.provider", mock_api=None, api_attr="_api")` | Discover the provider through its installed entry-point (the real e2e). Raises `AssertionError` if the entry-point is missing or ambiguous. | +| `MediaProviderHarness.from_class(provider_cls, config=None, mock_api=None, api_attr="_api")` | Wrap a class you already hold (no packaging needed) — handy for unit tests. | + +`mock_api` is set onto `provider.` (default `_api`) to bypass the lazy, +network-backed client a provider builds on first use. + +## Drivers + +Thin pass-throughs mirroring the `MediaProvider` contract: + +| Method | Delegates to | +|---|---| +| `is_available()` | `provider.is_available()` | +| `serves(signals, context=None)` | `provider.serves(...)` | +| `search(signals, lang="en-us")` | `provider.search(...)` (may raise) | +| `search_safe(signals, context=None, lang="en-us")` | `provider.search_safe(...)` (never raises) | +| `featured_media(lang="en-us")` | `provider.featured_media(...)` | + +## Assertions + +| Method | Checks | +|---|---| +| `assert_entrypoint_registered(name=None, group=None)` | the provider is discoverable under its entry-point group | +| `assert_routes(signals, context=None)` | `serves(...)` is `True` | +| `assert_not_routes(signals, context=None)` | `serves(...)` is `False` | +| `assert_returns_playables(signals, context=None, lang="en-us")` | `search_safe` returns results, each with a truthy `uri`, a `match_confidence` in `[0,1]`, and a `work`; returns the results | + +## Exposed attributes + +| Attribute | Description | +|---|---| +| `provider` | the wrapped provider instance | +| `api` | the injected mock client (for custom `assert_called_with` checks) | +| `entrypoint_name` / `entrypoint_group` | set when built via `from_entrypoint` | + +## Cross-references + +- `MediaProvider` / `QueryContext` — `ovos_plugin_manager.templates.media_provider` +- `Signals`, `Release`, `MediaType` — `mediavocab` +- Player-side harness — [media-testing.md](media-testing.md) +- OCP *search skill* testing (legacy stack) — [ocp.md](ocp.md) diff --git a/docs/media-testing.md b/docs/media-testing.md index 3516676..5ab02b3 100644 --- a/docs/media-testing.md +++ b/docs/media-testing.md @@ -190,6 +190,45 @@ with OCPPlayerHarness() as h: # Player auto-advances or stops depending on queue + autoplay config ``` +### Driving a Real OCP Backend + +By default `OCPPlayerHarness` injects a `MockOCPBackend` and mocks out +`AudioService`, so it exercises the **player state machine** but never the real +backend routing. To test a **real** OCP audio backend end-to-end — e.g. assert +that playing a uri makes a Music Assistant backend call its server — pass a +`backend_factory`: a `bus -> AudioBackend` callable. The harness then wires a +*real* `AudioService` (no autoload) with your backend as its sole service, so the +player's `play -> load_track -> LOADED_MEDIA -> backend.play()` path actually +reaches it. + +```python +from ovoscope.media import OCPPlayerHarness +from ovos_utils.ocp import MediaEntry, PlaybackType + +def make_backend(bus): + backend = MAssOCPAudioService(config={"url": "http://mass.local:8095"}, bus=bus) + backend.api = mock_client # mock the network client the backend reaches + backend.player_state = {"available": True} + return backend + +with OCPPlayerHarness(backend_factory=make_backend) as h: + h.play(MediaEntry(uri="library://track/42", playback=PlaybackType.AUDIO)) + h.backend.api.play_media.assert_called_once_with(queue_id, "library://track/42") +``` + +Notes: + +- The factory **owns mocking** any network client the real backend would reach. +- Deferred uris (`library://`, `{sei}//…`) are resolved by the OCP pipeline's + stream extractors *before* the player in production; the harness loads no + extractor plugins, so it bypasses the player's stream validation when a backend + factory is used. +- `name`/`namespace` are supplied by the harness if the backend lacks them + (normally set by `BaseMediaService.load_services()`, which the harness bypasses). +- The mock-only helpers (`assert_backend_paused`, `backend.played_uris`) assume a + `MockOCPBackend` and may not apply to a real backend — assert on the backend's + own state/spies instead. + ## OCPCaptureSession `OCPCaptureSession` — `ovoscope/media.py` @@ -245,6 +284,12 @@ with OCPPlayerHarness() as h: `OCPPlayerHarness` — `ovoscope/media.py` +**Constructor:** `OCPPlayerHarness(backend_namespace="audio", backend_factory=None)`. +`backend_factory` is an optional `bus -> AudioBackend` callable; when given, the +harness drives that real backend through a real `AudioService` (see +[Driving a Real OCP Backend](#driving-a-real-ocp-backend)) instead of the default +`MockOCPBackend`. + **Control methods** (each emits the corresponding bus message + `time.sleep(0.05)`): | Method | Bus message emitted | diff --git a/docs/voice-loop.md b/docs/voice-loop.md new file mode 100644 index 0000000..aee94df --- /dev/null +++ b/docs/voice-loop.md @@ -0,0 +1,203 @@ +# Listener Service Bus-Sequence Testing + +OVOS ships more than one listener **service**, and each emits the same +``recognizer_loop:*`` bus events as audio flows through it. ovoscope provides an +in-process harness for each, sharing one capture bus and one set of assertion +helpers (:class:`ovoscope.voice_loop.ListenerHarness`): + +| Service | Harness | Drive | +|---|---|---| +| ovos-dinkum-listener | `MiniVoiceLoop` | `feed_chunks` (wake-word / verifier gate) + `feed_file` (full `DinkumVoiceLoop.run()`) | +| ovos-simple-listener | `MiniSimpleListener` | `feed_file` (full `SimpleListener` loop) | +| mycroft-classic-listener | `MiniClassicListener` | `feed_file` (best-effort) + `bridge_recognizer_loop_to_bus` | + +Each harness wires its real listener to a `FakeBus` with mock mic / VAD / STT / +wake-word plugins, runs it over an arbitrary audio file (or PCM frames), and +captures the emitted bus sequence. The mocks live in +`ovoscope.voice_loop`: `MockFileMicrophone`, `MockStreamingSTT`, +`MockVADEngine`, `MockHotWordEngine`. + +## Shared assertion helpers + +Every harness inherits these (each takes an optional message list, defaulting to +the last feed result, and returns the checked list): + +- `assert_record_begin_emitted()` — `recognizer_loop:record_begin` present. +- `assert_wakeword_detected()` — both `recognizer_loop:wakeword` and `…:record_begin`. +- `assert_wakeword_suppressed()` — neither wake-word nor record-begin present. +- `assert_utterance_emitted(utterance=None)` — a `recognizer_loop:utterance` (optionally with the given text). + +--- + +## ovos-dinkum-listener — `MiniVoiceLoop` + +### Wake-word / verifier gate (`feed_chunks`) + +Feeds PCM frames straight through `DinkumVoiceLoop._detect_ww` to assert the +wake-word detection and the verifier chain in isolation. + +```python +from unittest.mock import Mock +from ovoscope.voice_loop import MiniVoiceLoop, MockHotWordEngine + +SILENT = b"\x00" * 512 + +def loop(verifiers): + ww = MockHotWordEngine("hey_mycroft", trigger_after=3) + return MiniVoiceLoop(ww_instances={"hey_mycroft": ww}, verifiers=verifiers) + +acc = Mock(); acc.verify.return_value = True +with loop([acc]) as vl: + vl.assert_wakeword_detected(vl.feed_chunks([SILENT] * 5)) + +rej = Mock(); rej.verify.return_value = False +with loop([rej]) as vl: + vl.assert_wakeword_suppressed(vl.feed_chunks([SILENT] * 5)) + +boom = Mock(); boom.verify.side_effect = RuntimeError("boom") +with loop([boom]) as vl: # fail-open + vl.assert_record_begin_emitted(vl.feed_chunks([SILENT] * 5)) +``` + +| Sequence | Expected bus events | +|---|---| +| WW detected + all verifiers accept | `recognizer_loop:wakeword` + `…:record_begin` | +| WW detected + a verifier rejects | suppressed — no `recognizer_loop:*` | +| WW detected + a verifier raises (fail-open) | `…:record_begin` emitted | +| No WW detected | no `recognizer_loop:*` | + +The verifier gate lives inside `DinkumVoiceLoop._detect_ww` and is only present +in ovos-dinkum-listener builds that ship the hotword-verifier feature +(`HotwordContainer.verify`). On a build without it the gate is absent and a +detection is never suppressed — assert accordingly for the version under test. + +### Full loop from an audio file (`feed_file`) + +Runs the whole `DinkumVoiceLoop.run()` state machine over an audio file through a +`MockFileMicrophone`, emitting the full record-begin → record-end → utterance +sequence. + +```python +from ovoscope.voice_loop import MiniVoiceLoop, MockStreamingSTT + +stt = MockStreamingSTT(transcript="what time is it") +with MiniVoiceLoop(stt_instance=stt) as vl: + msgs = vl.feed_file("command.wav") + vl.assert_record_begin_emitted(msgs) + vl.assert_utterance_emitted("what time is it", msgs) +``` + +An empty `MockStreamingSTT` transcript yields +`recognizer_loop:speech.recognition.unknown` instead of an utterance. + +`MiniVoiceLoop` builds a real `DinkumVoiceLoop`; it raises `RuntimeError` when +ovos-dinkum-listener is not installed. + +--- + +## ovos-simple-listener — `MiniSimpleListener` + +Drives the real `SimpleListener` thread with the canonical bus callbacks (a +per-instance mirror of `OVOSCallbacks`). + +```python +from ovoscope.simple_listener import MiniSimpleListener +from ovoscope.voice_loop import MockHotWordEngine, MockStreamingSTT + +with MiniSimpleListener( + wakeword=MockHotWordEngine("hey_mycroft", trigger_after=2), + stt_instance=MockStreamingSTT(transcript="turn on the lights"), +) as sl: + msgs = sl.feed_file("command.wav") + sl.assert_record_begin_emitted(msgs) + sl.assert_utterance_emitted("turn on the lights", msgs) +``` + +The default timing is tightened (`min_speech_seconds=0`, +`max_silence_seconds=0.1`) so a file-driven command ends promptly; raise them to +mimic production. Raises `RuntimeError` when ovos-simple-listener is absent. + +--- + +## mycroft-classic-listener — `MiniClassicListener` + +The classic listener is a threaded, energy-based pipeline. Two entry points: + +### Event bridge (always available) + +`bridge_recognizer_loop_to_bus(loop, bus)` forwards a `RecognizerLoop`'s internal +EventEmitter events onto a `FakeBus`, exactly as the classic listener's +`service.py` does. Drive the loop with real audio and assert on the bus: + +```python +from ovoscope.classic_listener import bridge_recognizer_loop_to_bus +from ovos_utils.fakebus import FakeBus + +bus = FakeBus() +bridge_recognizer_loop_to_bus(loop, bus) # loop = a RecognizerLoop +``` + +### Best-effort file drive + +Injects a file-backed audio source and mock wake-word / STT into a fresh +`RecognizerLoop` and runs it to completion: + +```python +from ovoscope.classic_listener import MiniClassicListener +from ovoscope.voice_loop import MockHotWordEngine, MockStreamingSTT + +with MiniClassicListener( + wakeword=MockHotWordEngine("hey_mycroft", trigger_after=1), + stt_instance=MockStreamingSTT(transcript="hello world"), +) as cl: + msgs = cl.feed_file("command.wav", tail_silence_seconds=3.0) + cl.assert_record_begin_emitted(msgs) + cl.assert_utterance_emitted("hello world", msgs) +``` + +The file drive depends on the energy-based recogniser, so assertions are +presence-based (a busy pipeline may emit more than one cycle before it is +stopped). Use `classic_listener_available()` to gate tests on the environment; +`MiniClassicListener(...)` (built mode) raises `RuntimeError` when the package is +absent. + +--- + +## Declarative helper — `VoiceLoopTest` + +For the dinkum backend, `VoiceLoopTest` runs a scenario and asserts in one call — +via `feed_chunks` by default, or `feed_file` when `audio_file` is set: + +```python +from unittest.mock import Mock +from ovoscope.voice_loop import VoiceLoopTest, MockHotWordEngine, MockStreamingSTT + +# verifier gate +accepting = Mock(); accepting.verify.return_value = True +VoiceLoopTest( + ww_instances={"hey_mycroft": MockHotWordEngine(trigger_after=3)}, + verifiers=[accepting], + audio_chunks=[b"\x00" * 512] * 5, + expect_record_begin=True, +).execute() + +# full loop from a file +VoiceLoopTest( + audio_file="command.wav", + stt_instance=MockStreamingSTT(transcript="what time is it"), + expect_utterance="what time is it", +).execute() +``` + +## API surface + +| Symbol | Description | +|---|---| +| `ListenerHarness` | Base: FakeBus capture + assertion helpers + file-mic. | +| `MiniVoiceLoop` / `get_mini_voice_loop` | ovos-dinkum-listener harness + factory. | +| `MiniHotwordContainer` | Controllable hotword container with a fail-open verifier chain. | +| `MiniSimpleListener` / `get_mini_simple_listener` | ovos-simple-listener harness + factory. | +| `MiniClassicListener` | mycroft-classic-listener harness (best-effort). | +| `bridge_recognizer_loop_to_bus` / `classic_listener_available` | Classic event-bridge + capability probe. | +| `MockFileMicrophone`, `MockStreamingSTT`, `MockVADEngine`, `MockHotWordEngine` | Mock plugins shared across backends. | +| `VoiceLoopTest` | Declarative dinkum scenario runner. | diff --git a/downstream_report.txt b/downstream_report.txt deleted file mode 100644 index 85cec10..0000000 --- a/downstream_report.txt +++ /dev/null @@ -1 +0,0 @@ -ovoscope==0.7.2 diff --git a/ngi.png b/ngi.png new file mode 100644 index 0000000..bfd401a Binary files /dev/null and b/ngi.png differ diff --git a/ovoscope/__init__.py b/ovoscope/__init__.py index 3d88df6..9dd6688 100644 --- a/ovoscope/__init__.py +++ b/ovoscope/__init__.py @@ -15,6 +15,7 @@ from ovos_utils.fakebus import FakeBus from ovos_utils.log import LOG from ovos_utils.process_utils import ProcessState +from ovos_spec_tools import SpecMessage from ovos_workshop.skills.ovos import OVOSSkill SerializedMessage = Dict[str, Union[str, Dict[str, Any]]] @@ -75,6 +76,11 @@ "ovos-m2v-pipeline-medium", "ovos-m2v-pipeline-low", ] +# Nebulento — fuzzy intent matching (ConfidenceMatcherPipeline). Single OPM +# entry point; the pipeline manager handles confidence-tier routing. +NEBULENTO_PIPELINE = ["ovos-nebulento-pipeline-plugin"] +# Palavreado — keyword/slot intent parser (ConfidenceMatcherPipeline). +PALAVREADO_PIPELINE = ["palavreado"] # Standard test pipeline — all standard built-in stages. # This requires ovos-adapt-pipeline-plugin and ovos-padatious-pipeline-plugin. @@ -296,7 +302,19 @@ def __init__(self, skill_ids, lang: Optional[str] = None, secondary_langs: Optional[List[str]] = None, pipeline_config: Optional[Dict[str, Dict]] = None, + modernize: bool = True, + emit_legacy: bool = True, *args, **kwargs): + # Namespace-migration flags forwarded to the harness FakeBus so callers + # can choose which bus namespace(s) to exercise: + # modernize=True emitting a legacy topic ALSO emits the ovos.* spec + # topic (legacy producer -> spec listener) + # emit_legacy=True emitting an ovos.* spec topic ALSO emits the legacy + # topic (spec producer -> legacy listener) + # Both default on (mirrors MessageBusClient). Set BOTH False to isolate a + # single namespace and assert no cross-namespace bridging occurs. + self._modernize = modernize + self._emit_legacy = emit_legacy self._isolated_config = isolate_config self._original_xdg_configs: Optional[List[LocalConf]] = None @@ -371,8 +389,27 @@ def __init__(self, skill_ids, LOG.debug(f"ovoscope: pipeline_config patched '{plugin_key}'") self.boot_messages: List[Message] = [] - bus = FakeBus() + bus = FakeBus(modernize=self._modernize, + emit_legacy=self._emit_legacy) bus.on("message", self.handle_boot_message) + + # TTS mock: speak_dialog(…, wait=True) blocks in wait_while_speaking on + # recognizer_loop:audio_output_end. With no real TTS that event never + # arrives, so the handler stalls until the dispatcher's §8.3 timeout. We + # emit audio_output_start synchronously (duck) and schedule a short-delay + # audio_output_end (unduck) to simulate the full TTS playback lifecycle. + def _mock_tts(message): + # TTS playback begins — duck immediately. + # message.forward copies source/destination/session from the speak, + # matching what the real audio service would do. + bus.emit(message.forward("recognizer_loop:audio_output_start")) + # TTS playback ends after a short delay — unduck + threading.Timer(0.1, lambda: bus.emit( + message.forward("recognizer_loop:audio_output_end") + )).start() + + bus.on(SpecMessage.SPEAK, _mock_tts) + self.skill_ids = skill_ids self.extra_skills = extra_skills or {} @@ -619,9 +656,16 @@ class CaptureSession: async_responses: List[Message] = dataclasses.field(default_factory=list) eof_msgs: List[str] = dataclasses.field(default_factory=lambda: DEFAULT_EOF) + # end capture only after an eof message has been seen this many times. Use >1 + # when the scenario produces N concurrent lifecycles that each terminate on the + # same eof topic (e.g. two ovos.utterance.handled — one per utterance — when a + # stop interrupts a running skill), so capture spans all of them. + eof_count: int = 1 ignore_messages: List[str] = dataclasses.field(default_factory=lambda: DEFAULT_IGNORED) async_messages: List[str] = dataclasses.field(default_factory=list) # these come from an external thread and might come in any order done: threading.Event = dataclasses.field(default_factory=lambda: threading.Event()) + _eof_lock: threading.Lock = dataclasses.field(default_factory=lambda: threading.Lock()) + _eof_seen: int = 0 def handle_message(self, msg: str): if self.done.is_set(): @@ -633,7 +677,10 @@ def handle_message(self, msg: str): self.responses.append(msg) def handle_end_of_test(self, msg: Message): - self.done.set() + with self._eof_lock: + self._eof_seen += 1 + if self._eof_seen >= self.eof_count: + self.done.set() def __post_init__(self): self.minicroft.bus.on("message", self.handle_message) @@ -643,6 +690,8 @@ def __post_init__(self): def capture(self, source_message: Message, timeout=20): test_message = deepcopy(source_message) # ensure object not mutated by ovos-core self.done.clear() + with self._eof_lock: + self._eof_seen = 0 self.minicroft.bus.emit(test_message) self.done.wait(timeout) @@ -672,7 +721,16 @@ class End2EndTest: # message type runtime modifiers ############################## eof_msgs: List[str] = dataclasses.field(default_factory=lambda: DEFAULT_EOF) # if received, end message capture + eof_count: int = 1 # end capture only after an eof message has been seen this many times (one per concurrent lifecycle terminating on the same topic) ignore_messages: List[str] = dataclasses.field(default_factory=lambda: DEFAULT_IGNORED) # pretend any message in this list was not emitted for testing purposes + # Assert only the messages belonging to a single dispatch lifecycle, identified + # by message.context["skill_id"]. When set, captured messages whose skill_id does + # not match are dropped before assertion. This isolates one lifecycle's §8 trio + + # §9 terminals from a CONCURRENT lifecycle whose messages interleave + # non-deterministically (e.g. stopping a skill that is mid-dispatch: the stop + # dispatch and the interrupted skill's own completion race). Run the same + # scenario once per skill_id to assert each lifecycle deterministically. + skill_id: Optional[str] = None ignore_gui: bool = True # ignore the gui namespace bus messages, usually unwanted unless explicitly testing gui integration async_messages: List[str] = dataclasses.field(default_factory=list) # these come from an external thread and might come in any order, validate they are received outside the main test @@ -728,8 +786,12 @@ def __post_init__(self): if GLOBAL_BUS_COVERAGE: self.track_bus_coverage = True - # standardize to be a list - if isinstance(self.source_message, Message): + # standardize to be a list. Use "not a list" rather than an + # isinstance(Message) check: depending on installed versions the + # message class may come from ovos_bus_client / ovos_spec_tools / + # ovos_utils.fakebus, and a cross-class isinstance can be False — which + # would leave a single (non-iterable) Message and break later iteration. + if not isinstance(self.source_message, list): self.source_message = [self.source_message] if self.ignore_gui: # ensure we don't mutate a shared default list @@ -782,6 +844,7 @@ def execute(self, timeout: int = 30) -> List[Message]: # the capture session will store all messages until capture.finish() # even if multiple messages are emitted capture = CaptureSession(self.minicroft, eof_msgs=self.eof_msgs, + eof_count=self.eof_count, ignore_messages=self.ignore_messages, async_messages=self.async_messages) for idx, source_message in enumerate(self.source_message): @@ -793,6 +856,14 @@ def execute(self, timeout: int = 30) -> List[Message]: # final message list messages = capture.finish() + # isolate a single dispatch lifecycle by skill_id — drop messages from a + # concurrent (interleaving) lifecycle so the assertion is deterministic. + if self.skill_id is not None: + messages = [m for m in messages + if (m.context or {}).get("skill_id") == self.skill_id] + if self.verbose: + print(f"💡 filtered to skill_id='{self.skill_id}': {len(messages)} messages") + if _bus_tracker is not None: _bus_tracker.stop_tracking() all_responses = messages + list(getattr(capture, "async_responses", [])) @@ -866,7 +937,7 @@ def execute(self, timeout: int = 30) -> List[Message]: assert received.context[k] == v, f"❌ message context mismatch for key '{k}' - expected '{v}' | got '{received.context[k]}'" if self.verbose: print(f"✅ got expected message context '{k}: '{v}'") - if self.test_routing: + if self.test_routing and self.skill_id is None: r_src = received.context.get("source") r_dst = received.context.get("destination") if expected.msg_type in self.keep_original_src: @@ -919,8 +990,8 @@ def execute(self, timeout: int = 30) -> List[Message]: assert sess.time_format == expected_sess.time_format, f"❌ final session time_format doesn't match" assert sess.site_id == expected_sess.site_id, f"❌ final session site_id doesn't match" assert sess.session_id == expected_sess.session_id, f"❌ final session session_id doesn't match" - assert set(sess.blacklisted_skills) == set(expected_sess.blacklisted_skills), f"❌ final session blacklisted_skills doesn't match" - assert set(sess.blacklisted_intents) == set(expected_sess.blacklisted_intents), f"❌ final session blacklisted_intents doesn't match" + assert set(sess.blacklisted_skills or []) == set(expected_sess.blacklisted_skills or []), f"❌ final session blacklisted_skills doesn't match" + assert set(sess.blacklisted_intents or []) == set(expected_sess.blacklisted_intents or []), f"❌ final session blacklisted_intents doesn't match" if self.verbose: print(f"✅ final session matches: {expected_sess.serialize()}") @@ -1020,7 +1091,7 @@ def from_message(cls, message: Union[Message, List[Message]], async_messages=async_messages) for idx, source_message in enumerate(message): - if "session" not in source_message.context: + if "session" not in source_message.context and len(capture.responses): # propagate session updates as a client would do source_message.context["session"] = capture.responses[-1].context["session"] capture.capture(source_message, timeout) @@ -1090,6 +1161,47 @@ def assert_spoke(self, text: str, lang: str = "en-US", timeout: int = 30) -> Non else: raise +try: + from ovoscope.media import ( # noqa: F401 + MockOCPBackend, + OCPPlayerHarness, + OCPCaptureSession, + ) +except ImportError as e: + # Optional [media] extra (ovos-media). Silence only when the missing module + # is ovos-media itself; a logic error in a present lib must re-raise. + if isinstance(e, ModuleNotFoundError) and e.name in ("ovos_media", "ovos_media.player"): + pass + else: + raise + +# MediaProvider (catalog/search) harness — duck-typed, stdlib-only at import time, +# so it needs no optional dependency guard (the provider package + mediavocab are +# only needed by the *test* that uses it, not by ovoscope itself). +from ovoscope.media_provider import MediaProviderHarness # noqa: F401,E402 + +try: + from ovoscope.tts_intelligibility import ( # noqa: F401 + TTSIntelligibilityHarness, + IntelligibilityReport, + UtteranceScore, + score_tts_intelligibility, + ) +except ImportError as e: + # Optional [tts] extra. Silence only when the missing module is one of the + # optional TTS-scoring deps; a logic error in a present lib must re-raise. + _TTS_OPTIONAL_MODULES = ( + "jiwer", + "ovos_audio", "ovos_audio.audio", + "ovos_utterance_normalizer", + "ovos_stt_plugin_fasterwhisper", + "faster_whisper", + ) + if isinstance(e, ModuleNotFoundError) and e.name in _TTS_OPTIONAL_MODULES: + pass + else: + raise + try: from ovoscope.listener import ( # noqa: F401 MiniListener, @@ -1102,6 +1214,25 @@ def assert_spoke(self, text: str, lang: str = "en-US", timeout: int = 30) -> Non else: raise +from ovoscope.voice_loop import ( # noqa: F401 + ListenerHarness, + MiniVoiceLoop, + MiniHotwordContainer, + MockFileMicrophone, + MockStreamingSTT, + get_mini_voice_loop, + VoiceLoopTest, +) +from ovoscope.simple_listener import ( # noqa: F401 + MiniSimpleListener, + get_mini_simple_listener, +) +from ovoscope.classic_listener import ( # noqa: F401 + MiniClassicListener, + bridge_recognizer_loop_to_bus, + classic_listener_available, +) + @dataclasses.dataclass class GUICaptureSession: @@ -1203,6 +1334,33 @@ def assert_page_shown(self, namespace: str, page: str, timeout: float = 2.0) -> f"but no matching gui.page.show message was captured.\nGot: {captured}" ) + def assert_template_shown(self, namespace: str, template: str, + values: Optional[Dict[str, Any]] = None, + timeout: float = 2.0) -> None: + """Assert that a built-in ``SYSTEM_*`` template was shown. + + Ergonomic helper for the template-based GUI: a skill calling a typed + method such as ``self.gui.show_weather(...)`` emits a + ``gui.page.show`` for the ``SYSTEM_weather`` template plus + ``gui.value.set`` for its data keys. This asserts both in one call. + + Args: + namespace: GUI namespace (typically the skill ID). + template: Template name, with or without the ``SYSTEM_`` prefix + (``"weather"`` and ``"SYSTEM_weather"`` are equivalent). + values: Optional mapping of session-data keys to expected values; + each is checked via :meth:`assert_namespace_value`. + timeout: Maximum seconds to wait for the page-show message. + + Raises: + AssertionError: If the template was not shown, or a listed value + was not set. + """ + name = template if template.startswith("SYSTEM_") else f"SYSTEM_{template}" + self.assert_page_shown(namespace, name, timeout=timeout) + for key, value in (values or {}).items(): + self.assert_namespace_value(namespace, key, value) + def assert_namespace_value(self, namespace: str, key: str, value: Any) -> None: """Assert that a namespace key was set to a specific value. @@ -1276,3 +1434,29 @@ def assert_namespace_cleared(self, namespace: str) -> None: f"Expected namespace {namespace!r} to be cleared, " f"but no matching message was captured." ) + + +# --------------------------------------------------------------------------- +# Public re-exports — see ovoscope/e2e.py for full docs +# --------------------------------------------------------------------------- +from ovoscope.intent_cases import ( # noqa: E402,F401 + DEFAULT_IGNORE_MESSAGES, + DEFAULT_PIPELINE_FAMILIES, + IntentCase, + assert_intent_case, + load_intent_cases, + register_intent_case_tests, +) +from ovoscope.e2e import ( # noqa: E402,F401 + E2EPipelineHarness, + detach_intent, + detach_skill, + make_session, + make_utterance_message, + register_adapt_intent, + register_adapt_vocab, + register_padatious_entity, + register_padatious_intent, + wait_for_failure, + wait_for_match, +) diff --git a/ovoscope/audio.py b/ovoscope/audio.py index d7e7e81..bd504c1 100644 --- a/ovoscope/audio.py +++ b/ovoscope/audio.py @@ -33,6 +33,7 @@ from unittest.mock import MagicMock, patch from ovos_bus_client.message import Message +from ovos_spec_tools import SpecMessage from ovos_plugin_manager.templates.audio import AudioBackend from ovos_plugin_manager.templates.tts import TTS from ovos_utils.fakebus import FakeBus @@ -227,17 +228,28 @@ class AudioServiceHarness: def __init__(self, backend_name: str = "mock", validate_source: bool = False, - disable_ocp: bool = True) -> None: + disable_ocp: bool = True, + modernize: bool = True, + emit_legacy: bool = True) -> None: """Initialise harness parameters. Args: backend_name: Name for the MockAudioBackend instance. validate_source: Enable source-session validation in AudioService. disable_ocp: Disable OCP plugin during tests. + modernize: FakeBus also emits the ovos.* spec topic when a legacy + topic is emitted (legacy producer -> spec listener). ovos-audio + emits legacy audio_output_start/end; the harness subscribes on + the spec topics, so this bridge is what connects them. + emit_legacy: FakeBus also emits the legacy topic when an ovos.* spec + topic is emitted (spec producer -> legacy listener). Set both + False to exercise a single namespace with no bridging. """ self.backend_name: str = backend_name self.validate_source: bool = validate_source self.disable_ocp: bool = disable_ocp + self.modernize: bool = modernize + self.emit_legacy: bool = emit_legacy self.bus: Optional[FakeBus] = None self.service = None # AudioService instance self.backend: Optional[MockAudioBackend] = None @@ -250,7 +262,8 @@ def __enter__(self) -> "AudioServiceHarness": """ from ovos_audio.audio import AudioService - self.bus = FakeBus() + self.bus = FakeBus(modernize=self.modernize, + emit_legacy=self.emit_legacy) self.backend = MockAudioBackend(config={}, bus=self.bus, name=self.backend_name) @@ -282,9 +295,9 @@ def __enter__(self) -> "AudioServiceHarness": self.service._get_track_length) self.bus.on("mycroft.audio.service.seek_forward", self.service._seek_forward) self.bus.on("mycroft.audio.service.seek_backward", self.service._seek_backward) - self.bus.on("recognizer_loop:audio_output_start", + self.bus.on(SpecMessage.AUDIO_OUTPUT_STARTED, self.service._lower_volume_on_speak) - self.bus.on("recognizer_loop:audio_output_end", + self.bus.on(SpecMessage.AUDIO_OUTPUT_ENDED, self.service._restore_volume_on_speak) self.bus.on("recognizer_loop:record_begin", self.service._lower_volume_on_record) @@ -499,6 +512,26 @@ def reset(self) -> None: """Clear the list of recorded spoken utterances.""" self.spoken_utterances.clear() + def __del__(self) -> None: + """No-op destructor. + + ``TTS.__del__`` chains into ``TTS.shutdown() -> TTS.stop() -> + TTS.playback.stop()``. ``TTS.playback`` is a **class-level** attribute + shared by every TTS instance in the process, so when an earlier + harness's MockTTS is garbage-collected its inherited destructor stops + whatever PlaybackThread is *currently* registered there — which, by + then, belongs to a later, still-running harness. The victim thread sets + ``_terminated`` and exits mid-run, so its queued speak never plays and + ``ovos.audio.output.ended`` is never emitted, hanging the next + ``speak()`` wait. + + GC timing is nondeterministic, so the failure surfaces as a flaky + ``TimeoutError`` only after several harness instances have been created + and collected. The harness already manages thread lifecycle explicitly + via ``PlaybackService.shutdown()`` on context exit, so a MockTTS + instance must never tear down the shared playback thread on collection. + """ + # --------------------------------------------------------------------------- # PlaybackServiceHarness @@ -508,8 +541,8 @@ class PlaybackServiceHarness: """Context manager wrapping PlaybackService with a MockTTS on a FakeBus. PlaybackService is a ``Thread``; this harness starts it and wires it to the - provided FakeBus so tests can emit ``speak`` messages and observe the - resulting ``recognizer_loop:audio_output_start/end`` events. + provided FakeBus so tests can emit ``ovos.utterance.speak`` messages and + observe the resulting ``ovos.audio.output.started/ended`` events. The harness patches ``ovos_utils.sound.play_audio`` so no actual audio device is accessed. It also drains ``TTS.queue`` before construction to @@ -518,21 +551,43 @@ class PlaybackServiceHarness: Args: validate_source: Enable session-source validation in the service. disable_ocp: Disable legacy OCP in the encapsulated AudioService. + tts: TTS instance to drive the PlaybackService with. Defaults to a + fresh ``MockTTS()`` (backward compatible). Pass a real TTS plugin + to synthesise actual audio — the rendered WAV path of each + utterance is captured in :attr:`captured_wavs`. """ def __init__(self, validate_source: bool = False, - disable_ocp: bool = True) -> None: + disable_ocp: bool = True, + tts: Optional[TTS] = None, + modernize: bool = True, + emit_legacy: bool = True) -> None: """Initialise harness parameters. Args: validate_source: Enable session-source validation. disable_ocp: Disable OCP audio plugin. + tts: TTS instance to inject. Defaults to ``MockTTS()`` when None. + modernize: FakeBus also emits the ovos.* spec topic when a legacy + topic is emitted (legacy producer -> spec listener). PlaybackService + emits legacy audio_output_start/end and mic.listen; the harness + subscribes on the spec topics, so this bridge connects them. + emit_legacy: FakeBus also emits the legacy topic when an ovos.* spec + topic is emitted (spec producer -> legacy listener). Set both + False to exercise a single namespace with no bridging. """ self.validate_source: bool = validate_source self.disable_ocp: bool = disable_ocp + self.modernize: bool = modernize + self.emit_legacy: bool = emit_legacy self.bus: Optional[FakeBus] = None self.svc = None # PlaybackService instance - self.mock_tts: Optional[MockTTS] = None + # ``mock_tts`` keeps its historic name for backward compatibility but + # holds whatever TTS was injected (real plugin or MockTTS). + self.tts: Optional[TTS] = tts + self.mock_tts: Optional[TTS] = None + # Paths captured from the ``play_audio`` side_effect, in playback order. + self.captured_wavs: List[str] = [] self._play_audio_patcher = None self._audio_enabled_patcher = None self._audio_output_start = threading.Event() @@ -557,16 +612,27 @@ def __enter__(self) -> "PlaybackServiceHarness": break TTS.queue = Queue() - self.bus = FakeBus() - self.mock_tts = MockTTS() + self.bus = FakeBus(modernize=self.modernize, + emit_legacy=self.emit_legacy) + # Inject the provided TTS (real plugin) or fall back to MockTTS. + self.mock_tts = self.tts if self.tts is not None else MockTTS() - # Patch play_audio so no real audio device is accessed + # Patch play_audio so no real audio device is accessed. The side_effect + # records the first positional arg — the rendered WAV path + # (ovos_audio/playback.py: ``self.p = play_audio(data)``) — so callers + # can round-trip the synthesised audio through a reference STT. mock_proc = MagicMock() mock_proc.communicate.return_value = (b"", b"") mock_proc.wait.return_value = 0 + self.captured_wavs = [] + + def _capture_play_audio(data, *args, **kwargs): + self.captured_wavs.append(data) + return mock_proc + self._play_audio_patcher = patch( - "ovos_audio.playback.play_audio", return_value=mock_proc + "ovos_audio.playback.play_audio", side_effect=_capture_play_audio ) self._play_audio_patcher.start() @@ -582,11 +648,11 @@ def __enter__(self) -> "PlaybackServiceHarness": self.mock_tts.init(self.bus, self.svc.playback_thread) # Subscribe lifecycle events for synchronisation - self.bus.on("recognizer_loop:audio_output_start", + self.bus.on(SpecMessage.AUDIO_OUTPUT_STARTED, lambda m: self._audio_output_start.set()) - self.bus.on("recognizer_loop:audio_output_end", + self.bus.on(SpecMessage.AUDIO_OUTPUT_ENDED, lambda m: self._audio_output_end.set()) - self.bus.on("mycroft.mic.listen", + self.bus.on(SpecMessage.MIC_LISTEN, lambda m: self._mic_listen.set()) except Exception: @@ -639,7 +705,7 @@ def speak(self, utterance: str, expect_response: bool = False, self._audio_output_end.clear() self._mic_listen.clear() - self.bus.emit(Message("speak", { + self.bus.emit(Message(SpecMessage.SPEAK, { "utterance": utterance, "lang": "en-US", "expect_response": expect_response, @@ -671,31 +737,31 @@ def assert_spoke(self, text: str) -> None: ) def assert_audio_output_started(self, timeout: float = 3.0) -> None: - """Assert that recognizer_loop:audio_output_start was emitted. + """Assert that ovos.audio.output.started was emitted. Args: timeout: Seconds to wait for the event. """ assert self._audio_output_start.wait(timeout), \ - "recognizer_loop:audio_output_start was not emitted" + "ovos.audio.output.started was not emitted" def assert_audio_output_ended(self, timeout: float = 3.0) -> None: - """Assert that recognizer_loop:audio_output_end was emitted. + """Assert that ovos.audio.output.ended was emitted. Args: timeout: Seconds to wait for the event. """ assert self._audio_output_end.wait(timeout), \ - "recognizer_loop:audio_output_end was not emitted" + "ovos.audio.output.ended was not emitted" def assert_mic_listen(self, timeout: float = 3.0) -> None: - """Assert that mycroft.mic.listen was emitted after speech. + """Assert that ovos.mic.listen was emitted after speech. Args: timeout: Seconds to wait for the event. """ assert self._mic_listen.wait(timeout), \ - "mycroft.mic.listen was not emitted" + "ovos.mic.listen was not emitted" # --------------------------------------------------------------------------- @@ -719,9 +785,18 @@ class AudioCaptureSession: """ bus: FakeBus + # capture BOTH the legacy and the ovos.* spec topics of the migrating audio + # messages. The capture session observes the raw "message" wire stream, which + # carries the producer's ORIGINAL topic only (FakeBus' namespace bridging + # re-dispatches the counterpart as a typed event, not a second "message" + # event). Listing both namespaces lets the session record the sequence + # whether the producer emits legacy or spec, so harness users can assert on + # either namespace. track_prefixes: List[str] = dataclasses.field(default_factory=lambda: [ "mycroft.audio.", + "ovos.audio.output", "recognizer_loop:audio_output", + "ovos.mic.listen", "mycroft.mic.listen", ]) messages: List[Message] = dataclasses.field(default_factory=list) diff --git a/ovoscope/classic_listener.py b/ovoscope/classic_listener.py new file mode 100644 index 0000000..6eed52d --- /dev/null +++ b/ovoscope/classic_listener.py @@ -0,0 +1,406 @@ +# Copyright 2024 OpenVoiceOS +# Licensed under the Apache License, Version 2.0 +"""MiniClassicListener — drive mycroft-classic-listener for bus-sequence testing. + +``mycroft-classic-listener`` is an alternative OVOS listener service built around +a ``RecognizerLoop`` (a ``pyee`` ``EventEmitter``) running an energy-based +``ResponsiveRecognizer`` across producer/consumer threads. Its service layer +bridges the loop's internal events (``recognizer_loop:record_begin``, +``…:wakeword``, ``…:record_end``, ``…:utterance``, +``…:speech.recognition.unknown``) onto the OVOS messagebus. + +Two pieces are provided: + +* :func:`bridge_recognizer_loop_to_bus` — the reusable event→bus bridge (mirrors + the classic listener's ``service.py``). Wire any ``RecognizerLoop`` to a + ``FakeBus`` and assert on the captured sequence. +* :class:`MiniClassicListener` — a best-effort, file-driven harness that injects + a :class:`FileAudioSource` and mock wake-word/STT into a ``RecognizerLoop`` and + runs it to completion, sharing the assertion helpers of + :class:`ovoscope.voice_loop.ListenerHarness`. + +The classic pipeline is energy-threshold based; the file drive is best-effort. +Use :func:`classic_listener_available` to gate tests on the environment. + +Example — event bridge:: + + from ovoscope.classic_listener import bridge_recognizer_loop_to_bus + from ovos_utils.fakebus import FakeBus + + bus = FakeBus() + bridge_recognizer_loop_to_bus(loop, bus) # loop = a RecognizerLoop + # ... drive `loop` with real audio; assert on `bus` +""" + +from __future__ import annotations + +import time +from pathlib import Path +from typing import Any, List, Optional, Union + +from ovos_bus_client.message import Message +from ovos_utils.fakebus import FakeBus + +from ovoscope.voice_loop import ( + ListenerHarness, + MockHotWordEngine, + MockStreamingSTT, + _read_audio, +) + + +# --------------------------------------------------------------------------- +# Event → bus bridge (mirrors mycroft_classic_listener/service.py) +# --------------------------------------------------------------------------- + +def bridge_recognizer_loop_to_bus(loop: Any, bus: FakeBus) -> Any: + """Forward a ``RecognizerLoop``'s internal events onto a ``FakeBus``. + + Registers handlers on *loop* (any object with an ``on(event, handler)`` + EventEmitter API) that re-emit the listener events as bus :class:`Message` + objects, exactly as the classic listener's service layer does. + + Args: + loop: A ``RecognizerLoop`` (or compatible ``EventEmitter``). + bus: The :class:`FakeBus` to emit translated messages on. + + Returns: + *loop*, for chaining. + """ + ctx = {"client_name": "mycroft_listener", "source": "audio"} + + loop.on( + "recognizer_loop:record_begin", + lambda *a: bus.emit(Message("recognizer_loop:record_begin", context=ctx)), + ) + loop.on( + "recognizer_loop:record_end", + lambda *a: bus.emit(Message("recognizer_loop:record_end", context=ctx)), + ) + loop.on( + "recognizer_loop:wakeword", + lambda event=None, *a: bus.emit( + Message("recognizer_loop:wakeword", event or {}) + ), + ) + loop.on( + "recognizer_loop:utterance", + lambda event=None, *a: bus.emit(Message( + "recognizer_loop:utterance", + event or {}, + {**ctx, "destination": ["skills"]}, + )), + ) + loop.on( + "recognizer_loop:speech.recognition.unknown", + lambda *a: bus.emit( + Message("recognizer_loop:speech.recognition.unknown", context=ctx) + ), + ) + loop.on( + "recognizer_loop:awoken", + lambda *a: bus.emit(Message("mycroft.awoken", context=ctx)), + ) + return loop + + +# --------------------------------------------------------------------------- +# File-backed audio source (classic Microphone subclass) +# --------------------------------------------------------------------------- + +class _FileStream: + """Minimal stream serving file PCM then silence, frame by frame. + + Mirrors the read contract the classic ``ResponsiveRecognizer`` expects from + a microphone stream: ``read(num_frames, of_exc)`` returns + ``num_frames * sample_width`` bytes. + + Args: + pcm: Raw PCM bytes of the audio. + sample_width: Bytes per sample. + tail_silence_bytes: Trailing silence appended after the audio. + """ + + def __init__(self, pcm: bytes, sample_width: int, tail_silence_bytes: int) -> None: + self._data: bytes = pcm + (b"\x00" * tail_silence_bytes) + self._sample_width: int = sample_width + self._pos: int = 0 + + def read(self, num_frames: int, of_exc: bool = False) -> bytes: + """Return ``num_frames`` of audio, padding with silence past EOF.""" + nbytes = num_frames * self._sample_width + chunk = self._data[self._pos:self._pos + nbytes] + self._pos += len(chunk) + if len(chunk) < nbytes: + chunk = chunk + b"\x00" * (nbytes - len(chunk)) + return chunk + + def close(self) -> None: + """No-op close.""" + + def stop_stream(self) -> None: + """No-op stop.""" + + def is_stopped(self) -> bool: + return False + + +def _make_file_audio_source( + audio: Union[bytes, str, Path], + chunk_size: int, + tail_silence_seconds: float, +) -> Any: + """Build a classic ``Microphone`` whose stream reads from *audio*. + + Bypasses ``Microphone.__init__`` (which opens PyAudio and needs a real input + device) while still satisfying the ``isinstance(source, Microphone)`` check + in ``ResponsiveRecognizer.listen``. + """ + from mycroft_classic_listener.mic import Microphone + + pcm, sr, sw, _ch = _read_audio(audio, 16000, 2, 1) + + class FileAudioSource(Microphone): + """File-backed classic microphone (no PyAudio).""" + + def __init__(self) -> None: + self.device_index = None + self.format = 8 # pyaudio.paInt16 + self.SAMPLE_WIDTH = sw + self.SAMPLE_RATE = sr + self.CHUNK = chunk_size + self.muted = False + self.audio = None + self.stream = None + self._pcm = pcm + self._tail = int(tail_silence_seconds * sr * sw) + + def __enter__(self): + self.stream = _FileStream(self._pcm, self.SAMPLE_WIDTH, self._tail) + return self + + def __exit__(self, *_): + self.stream = None + + def restart(self): + self.stream = _FileStream(self._pcm, self.SAMPLE_WIDTH, self._tail) + + def duration_to_bytes(self, sec): + return int(sec * self.SAMPLE_RATE * self.SAMPLE_WIDTH) + + def mute(self): + self.muted = True + + def unmute(self): + self.muted = False + + def is_muted(self): + return self.muted + + return FileAudioSource() + + +def _build_recognizer_loop(file_source: Any, wakeword: Any, stt: Any) -> Any: + """Build a ``RecognizerLoop`` with mocks injected, bypassing plugin/hardware. + + Subclasses ``RecognizerLoop`` to replace ``_load_config`` (which would open + PyAudio and load real wake-word plugins) and ``start_async`` (which would + create a real STT) with the injected file source, wake-word engine, and STT. + """ + from mycroft_classic_listener.listener import ( + AudioConsumer, + AudioProducer, + RecognizerLoop, + RecognizerLoopState, + ResponsiveRecognizer, + ) + try: + from queue import Queue + except ImportError: # pragma: no cover + from Queue import Queue # type: ignore + + class _InjectedRecognizerLoop(RecognizerLoop): + def __init__(self) -> None: + super(RecognizerLoop, self).__init__() # EventEmitter.__init__ + self._watchdog = lambda: None + self.mute_calls = 0 + self.lang = "en-us" + self.config = {} + self.microphone = file_source + self.wakeword_recognizer = wakeword + self.wakeup_recognizer = MockHotWordEngine("wake_up", trigger_after=10**9) + self.responsive_recognizer = ResponsiveRecognizer( + self.wakeword_recognizer, self._watchdog + ) + self.state = RecognizerLoopState() + self._config_hash = None + + def start_async(self) -> None: + self.state.running = True + self.producer = AudioProducer( + self.state, Queue(), self.microphone, + self.responsive_recognizer, self, None, + ) + # share one queue between producer and consumer + queue = self.producer.queue + self.producer.start() + self.consumer = AudioConsumer( + self.state, queue, self, stt, + self.wakeup_recognizer, self.wakeword_recognizer, + ) + self.consumer.start() + + def stop(self) -> None: + self.state.running = False + try: + self.producer.stop() + self.producer.join(timeout=2.0) + self.consumer.join(timeout=2.0) + except Exception: + pass + + return _InjectedRecognizerLoop() + + +def classic_listener_available() -> bool: + """Return ``True`` if mycroft-classic-listener can be imported here.""" + import importlib.util + + try: + return all( + importlib.util.find_spec(mod) is not None + for mod in ( + "mycroft_classic_listener.listener", + "mycroft_classic_listener.mic", + ) + ) + except ModuleNotFoundError: + return False + + +# --------------------------------------------------------------------------- +# MiniClassicListener +# --------------------------------------------------------------------------- + +class MiniClassicListener(ListenerHarness): + """Best-effort in-process mycroft-classic-listener harness. + + Wires a ``RecognizerLoop`` (built with a file audio source + mock wake-word / + STT, or one supplied by the caller) to a ``FakeBus`` via + :func:`bridge_recognizer_loop_to_bus`, then drives it over an audio file. + + Args: + recognizer_loop: A pre-built ``RecognizerLoop`` (or compatible + EventEmitter). When provided, only the bus bridge is wired and the + caller drives the loop; :meth:`feed_file` is unavailable. + wakeword: Wake-word engine for the built loop (defaults to a + :class:`MockHotWordEngine`). + stt_instance: STT engine for the built loop (defaults to + :class:`MockStreamingSTT`). + bus: Optional :class:`FakeBus` to capture on. When ``None`` a fresh + :class:`FakeBus` is built using *modernize* / *emit_legacy*. + modernize: When a fresh bus is built, have :class:`FakeBus` also emit the + ovos.* spec topic whenever a legacy topic is emitted (legacy -> + spec bridging). Ignored when *bus* is supplied. Defaults to True. + emit_legacy: When a fresh bus is built, have :class:`FakeBus` also emit + the legacy topic whenever an ovos.* spec topic is emitted (spec -> + legacy bridging). Ignored when *bus* is supplied. Defaults to True. + + Raises: + RuntimeError: If *recognizer_loop* is ``None`` and + mycroft-classic-listener is not importable. + """ + + def __init__( + self, + recognizer_loop: Optional[Any] = None, + *, + wakeword: Optional[Any] = None, + stt_instance: Optional[Any] = None, + bus: Optional[FakeBus] = None, + modernize: bool = True, + emit_legacy: bool = True, + ) -> None: + if bus is None: + bus = FakeBus(modernize=modernize, emit_legacy=emit_legacy) + self.modernize: bool = modernize + self.emit_legacy: bool = emit_legacy + super().__init__(bus) + self._built = recognizer_loop is None + self._wakeword = wakeword + self._stt = stt_instance + + if recognizer_loop is not None: + self.loop: Any = recognizer_loop + bridge_recognizer_loop_to_bus(self.loop, self.bus) + else: + if not classic_listener_available(): + raise RuntimeError( + "mycroft-classic-listener is required to build a " + "MiniClassicListener. Install it with: " + "pip install mycroft-classic-listener" + ) + self.loop = None # built per-run in feed_file (needs the audio) + + def feed_file( + self, + audio: Union[bytes, str, Path], + *, + tail_silence_seconds: float = 2.0, + chunk_size: int = 1024, + timeout: float = 15.0, + ) -> List[Message]: + """Run the classic loop over an audio file and capture bus events. + + Builds a fresh ``RecognizerLoop`` with a :class:`FileAudioSource`, bridges + it to the bus, runs it until a command finishes + (``recognizer_loop:record_end``) or *timeout* elapses, then stops it. + + Args: + audio: Path to a ``.wav`` file, WAV bytes, or raw PCM bytes. + tail_silence_seconds: Trailing silence appended after the audio so + the energy recogniser can end the command. + chunk_size: Frames per microphone read. + timeout: Maximum seconds to wait for the command to finish. + + Returns: + The list of :class:`Message` objects emitted during the run. + + Raises: + RuntimeError: If this harness was constructed with an external loop. + """ + if not self._built: + raise RuntimeError( + "feed_file is only available when MiniClassicListener builds the " + "RecognizerLoop. Drive the supplied loop yourself and assert on " + ".bus instead." + ) + + wakeword = self._wakeword or MockHotWordEngine("hey_mycroft", trigger_after=1) + stt = self._stt or MockStreamingSTT() + source = _make_file_audio_source(audio, chunk_size, tail_silence_seconds) + + self.loop = _build_recognizer_loop(source, wakeword, stt) + bridge_recognizer_loop_to_bus(self.loop, self.bus) + + self._messages.clear() + self.loop.start_async() + try: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if self._has(self._messages, "recognizer_loop:record_end"): + break + time.sleep(0.02) + finally: + self.loop.stop() + + self._last_messages = list(self._messages) + return list(self._messages) + + def shutdown(self) -> None: + """Stop the loop if the harness owns it.""" + if self._built and self.loop is not None: + try: + self.loop.stop() + except Exception: + pass diff --git a/ovoscope/cli.py b/ovoscope/cli.py index 722d91d..5548b86 100644 --- a/ovoscope/cli.py +++ b/ovoscope/cli.py @@ -78,7 +78,7 @@ def _record_inprocess(args: argparse.Namespace) -> int: Exit code (0 = success, 1 = failure). """ try: - from ovoscope import End2EndTest, get_minicroft + from ovoscope import End2EndTest from ovos_utils.messagebus import Message except ImportError as exc: _die(f"ovoscope import failed: {exc}") @@ -88,26 +88,29 @@ def _record_inprocess(args: argparse.Namespace) -> int: pipeline: Optional[List[str]] = args.pipeline.split(",") if args.pipeline else None timeout: float = args.timeout + src_msg = Message( + "recognizer_loop:utterance", + data={"utterances": [args.utterance], "lang": lang}, + ) + + # from_message owns the MiniCroft lifecycle: it loads the skills, emits the + # source utterance, captures the response sequence, and stops MiniCroft. + # Loading a MiniCroft here as well would double-load the skill plugins. print(f"[record] Loading skills: {skill_ids}") + print(f"[record] Sending utterance: {args.utterance!r}") try: - mc = get_minicroft(skill_ids, lang=lang, pipeline=pipeline, max_wait=60) + from_message_kwargs = {"lang": lang, "timeout": timeout} + if pipeline is not None: + # MiniCroft's pipeline override kwarg is `default_pipeline`; it is + # forwarded through from_message -> get_minicroft -> MiniCroft. + from_message_kwargs["default_pipeline"] = pipeline + test = End2EndTest.from_message(src_msg, skill_ids, **from_message_kwargs) except TimeoutError: _die("MiniCroft did not reach READY state in time.") - try: - src_msg = Message( - "recognizer_loop:utterance", - data={"utterances": [args.utterance], "lang": lang}, - ) - - print(f"[record] Sending utterance: {args.utterance!r}") - test = End2EndTest.from_message(src_msg, mc, timeout=timeout) - - test.save(args.output) - print(f"[record] Fixture saved to {args.output}") - return 0 - finally: - mc.stop() + test.save(args.output) + print(f"[record] Fixture saved to {args.output}") + return 0 def _record_live(args: argparse.Namespace) -> int: diff --git a/ovoscope/e2e.py b/ovoscope/e2e.py new file mode 100644 index 0000000..6de66b1 --- /dev/null +++ b/ovoscope/e2e.py @@ -0,0 +1,371 @@ +"""End-to-end test scaffolding for ConfidenceMatcherPipeline plugins. + +Most pipeline plugins (Adapt, Padatious, Padacioso, Nebulento, Palavreado, …) +need the same end-to-end shape: + +1. Mutate ``Configuration()["intents"][]`` with a per-test config. +2. Spin up a ``MiniCroft`` pinned to that one pipeline. +3. Drive the bus with utterances and capture the dispatched intent message + (or the ``complete_intent_failure`` signal). +4. Tear everything down without leaking state into the next test class. + +This module factors out that shape so a plugin only has to subclass +``E2EPipelineHarness`` and declare a handful of class attributes. It also +exposes the standalone bus helpers (``wait_for_match``, +``make_utterance_message``, …) and engine-family registration shims +(``register_padatious_intent``, ``register_adapt_intent``, …) for the cases +where pytest-style tests are preferred over ``unittest``. +""" +from __future__ import annotations + +import threading +import time +import unittest +from typing import Any, ClassVar, Dict, List, Optional + +from ovos_bus_client.message import Message +from ovos_bus_client.session import Session +from ovos_config.config import Configuration + + +# --------------------------------------------------------------------------- +# Standalone bus helpers (work with any FakeBus / MessageBusClient) +# --------------------------------------------------------------------------- + +def make_session( + session_id: str = "ovoscope-test", + *, + pipeline: Optional[List[str]] = None, + blacklisted_intents: Optional[List[str]] = None, + blacklisted_skills: Optional[List[str]] = None, + lang: str = "en-US", +) -> Session: + """Build a ``Session`` with the most common overrides preset.""" + kwargs: Dict[str, Any] = {"session_id": session_id, "lang": lang} + if pipeline is not None: + kwargs["pipeline"] = pipeline + if blacklisted_intents is not None: + kwargs["blacklisted_intents"] = blacklisted_intents + if blacklisted_skills is not None: + kwargs["blacklisted_skills"] = blacklisted_skills + return Session(**kwargs) + + +def make_utterance_message( + utterance: str, + *, + lang: str = "en-US", + session: Optional[Session] = None, +) -> Message: + """Build a ``recognizer_loop:utterance`` Message, optional Session override.""" + ctx: Dict[str, Any] = {} + if session is not None: + ctx["session"] = session.serialize() + return Message( + "recognizer_loop:utterance", + data={"utterances": [utterance], "lang": lang}, + context=ctx, + ) + + +def wait_for_match( + bus, + expected_types: List[str], + *, + timeout: float = 5.0, +) -> Optional[Message]: + """Subscribe to ``expected_types`` and ``complete_intent_failure``; return + the first match Message, or ``None`` on failure / timeout. + + The caller is responsible for emitting the utterance *after* calling this + helper if used in a pytest style — for the ``unittest`` style use + :meth:`E2EPipelineHarness.send_and_capture` which emits internally. + """ + got: List[Message] = [] + done = threading.Event() + failed = threading.Event() + + def _on_match(msg: Message) -> None: + got.append(msg) + done.set() + + def _on_fail(_msg: Message) -> None: + failed.set() + done.set() + + for t in expected_types: + bus.on(t, _on_match) + bus.on("complete_intent_failure", _on_fail) + try: + done.wait(timeout=timeout) + finally: + for t in expected_types: + bus.remove(t, _on_match) + bus.remove("complete_intent_failure", _on_fail) + if failed.is_set() and not got: + return None + return got[0] if got else None + + +def wait_for_failure(bus, *, timeout: float = 2.0) -> bool: + """Wait for a ``complete_intent_failure`` Message; return whether one fired.""" + failed = threading.Event() + + def _on_fail(_msg: Message) -> None: + failed.set() + + bus.on("complete_intent_failure", _on_fail) + try: + failed.wait(timeout=timeout) + finally: + bus.remove("complete_intent_failure", _on_fail) + return failed.is_set() + + +# --------------------------------------------------------------------------- +# Intent-registration shims — emit the bus event a given engine family expects +# --------------------------------------------------------------------------- +# +# Padatious family (padatious, padacioso, nebulento, …) — registers intents by +# emitting ``padatious:register_intent`` with inline ``samples``. +# Adapt family (adapt, palavreado, …) — registers vocab + IntentBuilder. + +def register_padatious_intent( + bus, name: str, samples: List[str], *, lang: str = "en-US", + settle: float = 0.1, +) -> None: + bus.emit(Message("padatious:register_intent", { + "name": name, "samples": samples, "lang": lang, + })) + if settle: + time.sleep(settle) + + +def register_padatious_entity( + bus, name: str, samples: List[str], *, lang: str = "en-US", + settle: float = 0.1, +) -> None: + bus.emit(Message("padatious:register_entity", { + "name": name, "samples": samples, "lang": lang, + })) + if settle: + time.sleep(settle) + + +def register_adapt_vocab( + bus, entity_type: str, words: List[str], *, lang: str = "en-US", + settle: float = 0.1, +) -> None: + for word in words: + bus.emit(Message("register_vocab", { + "entity_value": word, "entity_type": entity_type, "lang": lang, + })) + if settle: + time.sleep(settle) + + +def register_adapt_intent(bus, builder, *, lang: str = "en-US", + settle: float = 0.1) -> None: + """Register an Adapt intent. + + ``builder`` may be an ``IntentBuilder`` (will be ``.build()``-ed) or an + already-built intent with a ``__dict__`` payload. + """ + intent = builder.build() if hasattr(builder, "build") else builder + msg = Message("register_intent", intent.__dict__) + msg.context["lang"] = lang + bus.emit(msg) + if settle: + time.sleep(settle) + + +def detach_intent(bus, intent_name: str, *, settle: float = 0.1) -> None: + bus.emit(Message("detach_intent", {"intent_name": intent_name})) + if settle: + time.sleep(settle) + + +def detach_skill(bus, skill_id: str, *, settle: float = 0.1) -> None: + bus.emit(Message("detach_skill", {"skill_id": skill_id})) + if settle: + time.sleep(settle) + + +# --------------------------------------------------------------------------- +# unittest.TestCase harness +# --------------------------------------------------------------------------- + +class E2EPipelineHarness(unittest.TestCase): + """Base class for end-to-end tests of a single pipeline plugin. + + Subclass and set the four class attributes: + + ``PIPELINE_ID`` + OPM ``opm.pipeline`` entry-point name to pin MiniCroft to + (e.g. ``"ovos-nebulento-pipeline-plugin"``). + ``CONFIG_KEY`` + Key under ``Configuration()["intents"]`` for plugin config. + ``PLUGIN_CONFIG`` + Dict merged into ``Configuration()["intents"][CONFIG_KEY]`` before + MiniCroft starts. Restored on teardown. + ``SKILL_ID`` + Skill id used by helpers when registering intents. Detached in + ``setUp`` to keep tests isolated. + + The harness then exposes: + + - ``self.mc`` — the running ``MiniCroft`` + - ``self.bus`` — shortcut to ``self.mc.bus`` + - ``self.pipeline`` — the loaded pipeline plugin instance + - ``self.send_and_capture(utterance, expected_types, …)`` + - ``self.expect_no_match(utterance, …)`` + - ``self.make_utterance(utterance, session=…)`` + """ + + PIPELINE_ID: ClassVar[str] = "" + CONFIG_KEY: ClassVar[str] = "" + PLUGIN_CONFIG: ClassVar[Dict[str, Any]] = {} + SKILL_ID: ClassVar[str] = "test_skill_ovoscope" + DEFAULT_LANG: ClassVar[str] = "en-US" + STARTUP_MAX_WAIT: ClassVar[float] = 60.0 + # Namespace-migration flags forwarded to the harness MiniCroft / FakeBus. + # MODERNIZE on: a legacy emit (recognizer_loop:utterance) also dispatches its + # ovos.* spec counterpart (ovos.utterance.handle); EMIT_LEGACY on: a spec emit + # also dispatches the legacy topic. Both default on. Subclasses set BOTH False + # to drive a single isolated namespace. + MODERNIZE: ClassVar[bool] = True + EMIT_LEGACY: ClassVar[bool] = True + + mc: ClassVar[Any] + pipeline: ClassVar[Any] + _orig_intents_cfg: ClassVar[Any] = None + + @classmethod + def setUpClass(cls) -> None: + if not cls.PIPELINE_ID or not cls.CONFIG_KEY: + raise unittest.SkipTest( + f"{cls.__name__} must set PIPELINE_ID and CONFIG_KEY" + ) + # Import here so importing this module does not pull in MiniCroft. + from ovoscope import get_minicroft + + cfg = Configuration() + intents_cfg = cfg.setdefault("intents", {}) + cls._orig_intents_cfg = intents_cfg.get(cls.CONFIG_KEY) + intents_cfg[cls.CONFIG_KEY] = dict(cls.PLUGIN_CONFIG or {}) + + cls.mc = get_minicroft( + skill_ids=[], + lang=cls.DEFAULT_LANG, + default_pipeline=[cls.PIPELINE_ID], + max_wait=cls.STARTUP_MAX_WAIT, + modernize=cls.MODERNIZE, + emit_legacy=cls.EMIT_LEGACY, + ) + cls.pipeline = cls.mc.intents.pipeline_plugins[cls.PIPELINE_ID] + + @classmethod + def tearDownClass(cls) -> None: + try: + cls.mc.stop() + finally: + cfg = Configuration() + intents_cfg = cfg.get("intents", {}) + if cls._orig_intents_cfg is None: + intents_cfg.pop(cls.CONFIG_KEY, None) + else: + intents_cfg[cls.CONFIG_KEY] = cls._orig_intents_cfg + + @property + def bus(self): + return self.mc.bus + + def setUp(self) -> None: + # Isolate tests by detaching this skill_id's registrations. + detach_skill(self.bus, self.SKILL_ID) + + # -- helpers -------------------------------------------------------- + + def make_utterance(self, utterance: str, *, + session: Optional[Session] = None) -> Message: + return make_utterance_message( + utterance, lang=self.DEFAULT_LANG, session=session + ) + + def send_and_capture( + self, + utterance: str, + expected_types: List[str], + *, + timeout: float = 5.0, + session: Optional[Session] = None, + ) -> Optional[Message]: + """Emit ``utterance`` and return the first match Message (or None).""" + got: List[Message] = [] + done = threading.Event() + failed = threading.Event() + + def _on_match(msg: Message) -> None: + got.append(msg) + done.set() + + def _on_fail(_msg: Message) -> None: + failed.set() + done.set() + + for t in expected_types: + self.bus.on(t, _on_match) + self.bus.on("complete_intent_failure", _on_fail) + try: + self.bus.emit(self.make_utterance(utterance, session=session)) + done.wait(timeout=timeout) + finally: + for t in expected_types: + self.bus.remove(t, _on_match) + self.bus.remove("complete_intent_failure", _on_fail) + if failed.is_set() and not got: + return None + return got[0] if got else None + + def expect_no_match( + self, + utterance: str, + *, + timeout: float = 2.0, + session: Optional[Session] = None, + ) -> None: + """Assert that emitting ``utterance`` produces a ``complete_intent_failure``.""" + failed = threading.Event() + + def _on_fail(_msg: Message) -> None: + failed.set() + + self.bus.on("complete_intent_failure", _on_fail) + try: + self.bus.emit(self.make_utterance(utterance, session=session)) + failed.wait(timeout=timeout) + finally: + self.bus.remove("complete_intent_failure", _on_fail) + self.assertTrue( + failed.is_set(), + f"Expected no match for {utterance!r} but no " + f"complete_intent_failure was emitted.", + ) + + +__all__ = [ + # bus helpers + "make_session", + "make_utterance_message", + "wait_for_match", + "wait_for_failure", + # registration shims + "register_padatious_intent", + "register_padatious_entity", + "register_adapt_vocab", + "register_adapt_intent", + "detach_intent", + "detach_skill", + # harness + "E2EPipelineHarness", +] diff --git a/ovoscope/intent_cases.py b/ovoscope/intent_cases.py new file mode 100644 index 0000000..a6fa0b5 --- /dev/null +++ b/ovoscope/intent_cases.py @@ -0,0 +1,507 @@ +"""File-based intent test cases for OVOS skills. + +Lets skill authors describe expected intent routing as plain-text files +under ``test/end2end/cases//`` — adding a phrase, intent, or whole +new language is a pure text edit; no Python required. + +Layout +------ + +:: + + test/end2end/cases/ + / + .intent.test # one utterance per line, expected + # to match + no_match.test # utterances expected to match + # NO intent of this skill + +``#`` comments and blank lines are ignored. + +Usage (one call, in a test module owned by the skill) +----------------------------------------------------- + +:: + + # test/end2end/test_intents.py + from pathlib import Path + from ovoscope.intent_cases import register_intent_case_tests + + register_intent_case_tests( + globals(), + skill_id="ovos-skill-personal.openvoiceos", + handlers={ + "WhoAreYou.intent": "PersonalSkill.handle_who_are_you_intent", + "WhatAreYou.intent": "PersonalSkill.handle_what_are_you_intent", + # ... + }, + cases_dir=Path(__file__).parent / "cases", + ) + +The call creates four ``unittest.TestCase`` classes in the caller's +module — one per pipeline family (Padatious, Padacioso, Model2Vec) plus +one for the full default OVOS stack — each containing one ``test_*`` +method per (lang, utterance) pair. A test passes if any tier of its +pipeline family routes the utterance to the expected intent, which +matches realistic production cascade behaviour. + +Override the generated set with ``pipelines={"name": [stage, ...]}`` if +you only want a subset, or to test against custom pipeline stages. +""" +from __future__ import annotations + +import dataclasses +import time +from copy import deepcopy +from pathlib import Path +from typing import Dict, Iterable, Iterator, List, Optional, Union +from unittest import TestCase + +from ovos_bus_client.message import Message +from ovos_bus_client.session import Session +from ovos_utils.log import LOG + +# Imports from the top-level package — guarded to avoid a circular import at +# module load. The runtime calls happen well after package init. +from ovoscope import (DEFAULT_TEST_PIPELINE, End2EndTest, M2V_PIPELINE, + PADACIOSO_PIPELINE, PADATIOUS_PIPELINE, get_minicroft) + +__all__ = [ + "IntentCase", + "load_intent_cases", + "assert_intent_case", + "register_intent_case_tests", + "DEFAULT_IGNORE_MESSAGES", + "DEFAULT_PIPELINE_FAMILIES", +] + +DEFAULT_IGNORE_MESSAGES: List[str] = [ + "speak", + "mycroft.audio.play_sound", + "ovos.common_play.stop.response", +] + +# Named pipeline families generated by default. +DEFAULT_PIPELINE_FAMILIES: Dict[str, List[str]] = { + "Padatious": PADATIOUS_PIPELINE, + "Padacioso": PADACIOSO_PIPELINE, + "M2V": M2V_PIPELINE, + "DefaultPipeline": DEFAULT_TEST_PIPELINE, +} + + +@dataclasses.dataclass(frozen=True) +class IntentCase: + """A single expectation: ``utterance`` in ``lang`` should match ``intent``. + + ``intent`` is ``None`` to assert no intent of the skill under test + fires (the utterance falls through to ``complete_intent_failure``). + """ + lang: str + utterance: str + intent: Optional[str] + source: Path + + +# --------------------------------------------------------------------------- +# Case-file discovery +# --------------------------------------------------------------------------- +def _read_lines(path: Path) -> List[str]: + out: List[str] = [] + for raw in path.read_text(encoding="utf-8").splitlines(): + line = raw.strip() + if line and not line.startswith("#"): + out.append(line) + return out + + +def load_intent_cases(cases_dir: Union[str, Path], + known_intents: Optional[Iterable[str]] = None + ) -> List[IntentCase]: + """Discover every ``IntentCase`` under ``cases_dir``. + + Args: + cases_dir: directory containing ``/.intent.test`` and + optionally ``/no_match.test`` files. + known_intents: if given, every ``.intent`` filename found + is validated against this set — a typo will raise + ``AssertionError`` instead of silently being skipped. + + Returns: + List of cases in stable (lang, file, line) order. + """ + base = Path(cases_dir) + if not base.is_dir(): + return [] + known = set(known_intents) if known_intents is not None else None + cases: List[IntentCase] = [] + for lang_dir in sorted(base.iterdir()): + if not lang_dir.is_dir() or lang_dir.name.startswith("_"): + continue + lang = lang_dir.name + for case_file in sorted(lang_dir.glob("*.test")): + if case_file.name == "no_match.test": + expected: Optional[str] = None + elif case_file.stem.endswith(".intent"): + expected = case_file.stem # ".intent" + if known is not None and expected not in known: + raise AssertionError( + f"{case_file} targets unknown intent " + f"{expected!r}; expected one of {sorted(known)}") + else: + continue + for utt in _read_lines(case_file): + cases.append(IntentCase(lang=lang, utterance=utt, + intent=expected, source=case_file)) + return cases + + +# --------------------------------------------------------------------------- +# Single-case assertion +# --------------------------------------------------------------------------- +def assert_intent_case(minicroft, skill_id: str, handlers: Dict[str, str], + case: IntentCase, pipeline: List[str], + *, + ignore_messages: Optional[List[str]] = None, + timeout: float = 30) -> None: + """Fire ``case.utterance`` through ``pipeline`` and assert routing. + + Asserts that an ``IntentCase`` with ``intent=None`` falls through to + ``complete_intent_failure``; otherwise asserts the expected intent + and handler-lifecycle messages fire in order. + + Args: + minicroft: a running ``MiniCroft`` instance with ``skill_id`` loaded. + skill_id: the full skill id under test (e.g. ``"my-skill.author"``). + handlers: ``{intent_name: handler_method_name}`` for the skill. + case: the case to run. + pipeline: list of pipeline stage ids to populate + ``session.pipeline`` with. + ignore_messages: extra non-deterministic / noisy message types to + filter out of the comparison. + timeout: per-case execution timeout, in seconds. + """ + ignored = list(ignore_messages or DEFAULT_IGNORE_MESSAGES) + + session = Session(f"intent-case-{case.lang}-{case.utterance[:24]}") + session.lang = case.lang + session.pipeline = list(pipeline) + + source = Message( + "recognizer_loop:utterance", + {"utterances": [case.utterance], "lang": case.lang}, + {"session": session.serialize()}, + ) + + if case.intent is None: + final_session = deepcopy(session) + expected_messages = [ + source, + Message("complete_intent_failure", + {"utterances": [case.utterance], "lang": case.lang}, {}), + Message("ovos.utterance.handled", {}, {}), + ] + test = End2EndTest( + minicroft=minicroft, + skill_ids=[skill_id], + eof_msgs=["ovos.utterance.handled"], + flip_points=["recognizer_loop:utterance"], + ignore_messages=ignored, + source_message=source, + final_session=final_session, + expected_messages=expected_messages, + test_msg_data=False, + test_msg_context=False, + ) + else: + if case.intent not in handlers: + raise AssertionError( + f"No handler mapping for intent {case.intent!r} " + f"(case from {case.source}). Add it to ``handlers``.") + handler = handlers[case.intent] + final_session = deepcopy(session) + final_session.active_skills = [(skill_id, 0.0)] + expected_messages = [ + source, + Message(f"{skill_id}.activate", {}, {"skill_id": skill_id}), + Message(f"{skill_id}:{case.intent}", + {"utterance": case.utterance, "lang": case.lang}, + {"skill_id": skill_id}), + Message("mycroft.skill.handler.start", + {"name": handler}, {"skill_id": skill_id}), + Message("mycroft.skill.handler.complete", + {"name": handler}, {"skill_id": skill_id}), + Message("ovos.utterance.handled", {}, {"skill_id": skill_id}), + ] + test = End2EndTest( + minicroft=minicroft, + skill_ids=[skill_id], + eof_msgs=["ovos.utterance.handled"], + flip_points=["recognizer_loop:utterance"], + ignore_messages=ignored, + source_message=source, + final_session=final_session, + activation_points=[f"{skill_id}:{case.intent}"], + expected_messages=expected_messages, + ) + test.execute(timeout=timeout) + + +# --------------------------------------------------------------------------- +# Shared minicroft helper +# --------------------------------------------------------------------------- +_SHARED_MINICROFT_KEY = "_ovoscope_shared_minicroft" + + +def _wait_for_m2v_sync(mc, max_wait: float = 15.0, + quiet_window: float = 0.5) -> float: + """Wait deterministically for the m2v pipeline to finish its + ``handle_sync_intents`` debounce after ``mycroft.ready``. + + The m2v plugin (`ovos_m2v_pipeline/__init__.py:handle_sync_intents`) + sleeps 3 s then re-queries the adapt + padatious intent manifests. + Until that returns, the pipeline matches against an empty label set + and every utterance falls through. We avoid relying on a fixed + ``time.sleep`` by: + + 1. Subscribing to every ``padatious:register_intent`` / + ``register_intent`` event on the bus. + 2. Emitting ``mycroft.ready`` (which triggers + ``handle_sync_intents``). + 3. Waiting until no new register events have arrived for + ``quiet_window`` seconds AND at least one register event has + actually been observed (or ``max_wait`` is exhausted). + 4. Adding a small constant grace period for the in-plugin + ``time.sleep(3)`` debounce to actually return. + + Returns the seconds slept (for diagnostics). + """ + seen = {"count": 0, "last_t": 0.0} + + def on_register(_serialized): + seen["count"] += 1 + seen["last_t"] = time.monotonic() + + mc.bus.ee.on("padatious:register_intent", + lambda _: on_register(None)) + mc.bus.ee.on("register_intent", lambda _: on_register(None)) + + # Wildcard "message" listener catches the serialized form on FakeBus. + def on_any(serialized): + try: + t = serialized.get("type") if isinstance(serialized, dict) else None + except Exception: + return + if not t: + return + if t == "padatious:register_intent" or t == "register_intent": + on_register(None) + + mc.bus.ee.on("message", on_any) + + t0 = time.monotonic() + mc.bus.emit(Message("mycroft.ready", {}, {})) + + # Wait for the burst of register events to settle. + while time.monotonic() - t0 < max_wait: + time.sleep(0.1) + if seen["count"] > 0 and (time.monotonic() - seen["last_t"]) > quiet_window: + break + # m2v's handle_sync_intents does an internal time.sleep(3) before the + # actual intent set update. Pad for that ceiling. + time.sleep(3.5) + return time.monotonic() - t0 + + +def _shared_minicroft(skill_id: str, langs: List[str], m2v_warmup: float): + """Lazily create one MiniCroft and warm m2v's label index. + + Caches the instance on a process-global so every generated class + shares the same boot. ``m2v_warmup`` is now an *upper bound*: if the + deterministic event-based wait finishes faster, we return early. + """ + cache = globals().setdefault(_SHARED_MINICROFT_KEY, {}) + key = (skill_id, tuple(langs)) + if key not in cache: + LOG.set_level("CRITICAL") + secondary = [l for l in langs if l != "en-US"] + mc = get_minicroft([skill_id], secondary_langs=secondary or None) + if m2v_warmup > 0: + try: + _wait_for_m2v_sync(mc, max_wait=max(m2v_warmup, 5.0)) + except Exception: + # If the deterministic wait fails for any reason + # (e.g. FakeBus internals change), fall back to a sleep + # so the suite still runs. + mc.bus.emit(Message("mycroft.ready", {}, {})) + time.sleep(m2v_warmup) + cache[key] = mc + return cache[key] + + +# --------------------------------------------------------------------------- +# Test-class generator +# --------------------------------------------------------------------------- +def _slug(s: str) -> str: + out = [] + for ch in s: + if ch.isalnum(): + out.append(ch) + elif ch in " -_/": + out.append("_") + return "".join(out).strip("_") or "x" + + +def _build_test_class(name: str, pipeline: List[str], cases: List[IntentCase], + skill_id: str, handlers: Dict[str, str], + langs: List[str], + ignore_messages: Optional[List[str]], + timeout: float, + m2v_warmup: float, + doc: str) -> type: + def _make(case: IntentCase): + def _test(self): + mc = _shared_minicroft(skill_id, langs, m2v_warmup) + assert_intent_case(mc, skill_id, handlers, case, pipeline, + ignore_messages=ignore_messages, + timeout=timeout) + label = case.intent.split(".")[0] if case.intent else "no_match" + _test.__doc__ = (f"[{case.lang}] {case.utterance!r} -> " + f"{case.intent if case.intent else 'no match'} ({name})") + # Attach metadata for the pytest accuracy/coverage reporter. + _test._intent_case = case # type: ignore[attr-defined] + _test._intent_case_pipeline = name # type: ignore[attr-defined] + _test._intent_case_skill_id = skill_id # type: ignore[attr-defined] + return _test, label + + body: Dict[str, object] = {"__doc__": doc} + for case in cases: + method, label = _make(case) + slug = _slug(case.utterance) + attr = f"test_{case.lang.replace('-', '_')}__{label}__{slug}" + # avoid clobbers if two utterances slug-collide + base = attr + n = 2 + while attr in body: + attr = f"{base}_{n}" + n += 1 + body[attr] = method + return type(name, (TestCase,), body) + + +def register_intent_case_tests( + target_globals: dict, + *, + skill_id: str, + handlers: Dict[str, str], + cases_dir: Union[str, Path], + pipelines: Optional[Dict[str, List[str]]] = None, + ignore_messages: Optional[List[str]] = None, + timeout: float = 30, + m2v_warmup: float = 10.0, + ) -> Dict[str, type]: + """Create per-pipeline ``TestCase`` classes in ``target_globals``. + + Args: + target_globals: pass ``globals()`` from the caller test module — + generated classes are inserted here so pytest collects them. + skill_id: full skill plugin id (e.g. ``"my-skill.author"``). + handlers: mapping ``{".intent": ""}`` + covering every intent referenced by case files. + cases_dir: directory containing ``/.intent.test`` and + optional ``/no_match.test`` files. + pipelines: ``{class_suffix: pipeline_stage_list}`` to override the + default per-family classes. Each entry becomes a + ``Test`` class. Defaults to one class per family + in :data:`DEFAULT_PIPELINE_FAMILIES`. + ignore_messages: extra message types to filter out of comparison. + timeout: per-case execution timeout, in seconds. + m2v_warmup: seconds to sleep after booting the MiniCroft so the + m2v pipeline finishes syncing its label index. Set to 0 if + you're not running m2v cases. + + Returns: + ``{class_name: class_object}`` for every class created. + """ + cases = load_intent_cases(cases_dir, known_intents=handlers.keys()) + if not cases: + # Empty cases dir: skip silently so a freshly-copied template + # doesn't fail collection before any .test files are added. + return {} + + langs = sorted({c.lang for c in cases}) + families = pipelines if pipelines is not None else DEFAULT_PIPELINE_FAMILIES + + created: Dict[str, type] = {} + for suffix, pipeline in families.items(): + cls_name = f"Test{suffix}" + doc = (f"ovoscope intent-case tests for {skill_id} on pipeline " + f"{pipeline!r}.") + cls = _build_test_class(cls_name, pipeline, cases, skill_id, handlers, + langs, ignore_messages, timeout, m2v_warmup, + doc) + target_globals[cls_name] = cls + created[cls_name] = cls + # Mark generated classes so the pytest auto-discovery hook can skip a + # cases_dir whose tests are already registered by an explicit call. + target_globals["_ovoscope_intent_cases_registered"] = True + return created + + +# --------------------------------------------------------------------------- +# Pytest auto-discovery (no Python boilerplate required in the skill). +# +# A skill can opt in by adding a ``conftest.py`` next to its +# ``cases/`` directory containing: +# +# ovoscope_intent_cases = dict( +# skill_id="my-skill.author", +# handlers={"DoX.intent": "MySkill.handle_do_x", ...}, +# ) +# +# The ``pytest_collect_directory`` hook on the ovoscope pytest plugin +# discovers the conftest, walks ``/cases/`` and generates the same +# TestCase classes ``register_intent_case_tests`` would have created — +# zero boilerplate in the test module. +# --------------------------------------------------------------------------- +def autodiscover_from_conftest(conftest_dir: Union[str, Path], + target_globals: dict) -> Dict[str, type]: + """Look for ``ovoscope_intent_cases`` config in a conftest namespace + and call :func:`register_intent_case_tests` accordingly. + + The conftest must expose a dict like:: + + ovoscope_intent_cases = { + "skill_id": "my-skill.author", + "handlers": {"WhoAreYou.intent": "MySkill.handle_who", ...}, + # optional overrides: + "cases_dir": "cases", + "pipelines": {"Custom": [...]}, + "ignore_messages": [...], + "timeout": 30, + "m2v_warmup": 10.0, + } + + Returns ``{}`` if the conftest has no ``ovoscope_intent_cases`` or + the cases directory does not exist. + """ + cfg = target_globals.get("ovoscope_intent_cases") + if not cfg: + return {} + base = Path(conftest_dir) + cases_dir = Path(cfg.get("cases_dir", "cases")) + if not cases_dir.is_absolute(): + cases_dir = base / cases_dir + if not cases_dir.is_dir(): + return {} + return register_intent_case_tests( + target_globals, + skill_id=cfg["skill_id"], + handlers=cfg["handlers"], + cases_dir=cases_dir, + pipelines=cfg.get("pipelines"), + ignore_messages=cfg.get("ignore_messages"), + timeout=cfg.get("timeout", 30), + m2v_warmup=cfg.get("m2v_warmup", 10.0), + ) + return created diff --git a/ovoscope/listener.py b/ovoscope/listener.py index 4fe93fb..a5ec659 100644 --- a/ovoscope/listener.py +++ b/ovoscope/listener.py @@ -283,6 +283,13 @@ class MiniListener: ww_instances: Optional mapping of hotword name → :class:`HotWordEngine` (or mock) instance. Multiple wake-word engines can be registered simultaneously. + modernize: FakeBus also emits the ovos.* spec topic when a legacy + topic is emitted (legacy producer -> spec listener). The listener + pipeline emits legacy ``recognizer_loop:*`` topics; this bridge is + what lets a spec-topic subscriber observe them. + emit_legacy: FakeBus also emits the legacy topic when an ovos.* spec + topic is emitted (spec producer -> legacy listener). Set both False + to exercise a single namespace with no bridging. Example:: @@ -308,8 +315,11 @@ def __init__( stt_instance: Optional[Any] = None, vad_instance: Optional[Any] = None, ww_instances: Optional[Dict[str, Any]] = None, + modernize: bool = True, + emit_legacy: bool = True, ) -> None: - self.bus: FakeBus = FakeBus() + self.bus: FakeBus = FakeBus(modernize=modernize, + emit_legacy=emit_legacy) self._messages: List[Message] = [] self._stt_instance: Optional[Any] = stt_instance self._vad: Optional[Any] = vad_instance @@ -402,6 +412,59 @@ def transform(self, chunk: bytes) -> Tuple[bytes, dict, List[Message]]: audio, ctx = self.transformers.transform(chunk) return audio, ctx, list(self._messages) + def feed_audio_stream( + self, + chunks: Union[bytes, List[bytes]], + feed: str = "feed_audio", + chunk_size: int = 2048, + ) -> List[Message]: + """Stream a sequence of audio frames and aggregate emitted messages. + + Unlike :meth:`feed_audio` / :meth:`feed_speech`, which clear the + capture buffer on every call, this feeds each frame in order and keeps + every message emitted across the **whole** stream. This is required + for transformers whose decoder only fires after accumulating many + frames of audio (e.g. ggwave data-over-sound). + + Args: + chunks: Either a flat ``bytes`` object (split into *chunk_size* + frames internally) or a pre-segmented ``List[bytes]`` of frames. + feed: Which transformer feed to drive per frame — + ``"feed_audio"`` (non-speech) or ``"feed_speech"``. + chunk_size: Bytes per frame when *chunks* is a flat ``bytes`` + object (ignored when *chunks* is already a list). + + Returns: + All ``Message`` objects emitted on the bus across every frame. + + Raises: + RuntimeError: If ``ovos-dinkum-listener`` is not installed. + ValueError: If *feed* is not a recognised feed method. + """ + if self.transformers is None: + raise RuntimeError( + "ovos-dinkum-listener is required for feed_audio_stream. " + "Install it with: pip install ovos-dinkum-listener" + ) + if feed not in ("feed_audio", "feed_speech"): + raise ValueError( + f"feed must be 'feed_audio' or 'feed_speech', got {feed!r}" + ) + + if isinstance(chunks, bytes): + frames = [ + chunks[i:i + chunk_size] + for i in range(0, len(chunks), chunk_size) + ] + else: + frames = list(chunks) + + feeder = getattr(self.transformers, feed) + self._messages.clear() + for frame in frames: + feeder(frame) + return list(self._messages) + def listen( self, audio: Union[bytes, str, Path], @@ -635,6 +698,8 @@ def get_mini_listener( vad_instance: Optional[Any] = None, ww_plugin: Optional[str] = None, ww_instances: Optional[Dict[str, Any]] = None, + modernize: bool = True, + emit_legacy: bool = True, ) -> MiniListener: """Factory: create a ready-to-use :class:`MiniListener`. @@ -664,6 +729,11 @@ def get_mini_listener( ww_instances: Mapping of hotword name → engine instance. Supports multiple simultaneous wake-word engines. Takes precedence over *ww_plugin*. + modernize: FakeBus also emits the ovos.* spec topic when a legacy topic + is emitted (legacy producer -> spec listener). + emit_legacy: FakeBus also emits the legacy topic when an ovos.* spec + topic is emitted (spec producer -> legacy listener). Set both False + to exercise a single namespace with no bridging. Returns: A fully initialised :class:`MiniListener` ready to receive audio. @@ -717,6 +787,8 @@ def get_mini_listener( stt_instance=stt_instance, vad_instance=resolved_vad, ww_instances=resolved_ww, + modernize=modernize, + emit_legacy=emit_legacy, ) @@ -758,8 +830,17 @@ class ListenerTest: """Raw audio bytes to inject into the pipeline.""" feed_method: str = "feed_audio" - """Which feed method to call: ``"feed_audio"``, ``"feed_speech"``, or - ``"transform"``.""" + """Which feed method to call: ``"feed_audio"``, ``"feed_speech"``, + ``"feed_audio_stream"``, ``"transform"``, or ``"listen"``. + + Use ``"feed_audio_stream"`` for transformers that decode only after + accumulating many frames (e.g. ggwave): *audio_input* is split into + *chunk_size* frames, fed in order, and all emitted messages are + aggregated across the whole stream.""" + + chunk_size: int = 2048 + """Frame size (bytes) used to split *audio_input* when *feed_method* is + ``"feed_audio_stream"``.""" expected_types: List[str] = field(default_factory=list) """Message types that MUST appear in the captured output.""" @@ -787,11 +868,14 @@ def execute(self) -> List[Message]: stt_instance=self.stt_instance, ) try: - method = getattr(listener, self.feed_method) if self.feed_method == "listen": messages = listener.listen(self.audio_input) + elif self.feed_method == "feed_audio_stream": + messages = listener.feed_audio_stream( + self.audio_input, chunk_size=self.chunk_size + ) else: - result = method(self.audio_input) + result = getattr(listener, self.feed_method)(self.audio_input) if self.feed_method == "transform": messages: List[Message] = result[2] else: diff --git a/ovoscope/media.py b/ovoscope/media.py index 0c07ee1..36b3683 100644 --- a/ovoscope/media.py +++ b/ovoscope/media.py @@ -26,7 +26,7 @@ import dataclasses import time -from typing import List, Optional +from typing import Callable, List, Optional from unittest.mock import MagicMock, patch from ovos_bus_client.message import Message @@ -268,16 +268,43 @@ class OCPPlayerHarness: backend_namespace: Namespace for ``MockOCPBackend``; default ``"audio"``. """ - def __init__(self, backend_namespace: str = "audio") -> None: + def __init__(self, backend_namespace: str = "audio", + backend_factory: Optional[Callable[[FakeBus], AudioBackend]] = None, + modernize: bool = True, + emit_legacy: bool = True) -> None: """Initialise harness parameters. Args: backend_namespace: Namespace prefix passed to ``MockOCPBackend``. + backend_factory: Optional ``bus -> AudioBackend`` callable used to build + the injected backend instead of the default :class:`MockOCPBackend`. + The harness owns the ``FakeBus``, so a *factory* (not a pre-built + instance) is taken: it is called with the harness bus inside + ``__enter__``. Use it to drive a **real** OCP backend (e.g. a + Music Assistant audio backend) through the real ``OCPMediaPlayer``; + the factory is responsible for mocking any network client the real + backend would otherwise reach. Note the mock-only assertion helpers + (:meth:`assert_backend_paused`, ``backend.played_uris``) assume a + :class:`MockOCPBackend` and may not apply to a real backend. + modernize: FakeBus also emits the ovos.* spec topic when a legacy + topic is emitted (legacy producer -> spec listener). OCPMediaPlayer + subscribes to the LEGACY duck/cork topics + (recognizer_loop:audio_output_start/end, record_begin/end); this + bridge lets a spec-namespace producer's ovos.audio.output.* / + ovos.listener.record.* reach those legacy handlers. + emit_legacy: FakeBus also emits the legacy topic when an ovos.* spec + topic is emitted (spec producer -> legacy listener). Because the + player subscribes on the legacy topics, this is the bridge that + connects a spec producer to the player. Set both False to exercise + a single namespace with no bridging. """ self.backend_namespace: str = backend_namespace + self.backend_factory = backend_factory + self.modernize: bool = modernize + self.emit_legacy: bool = emit_legacy self.bus: Optional[FakeBus] = None self.player = None # OCPMediaPlayer instance - self.backend: Optional[MockOCPBackend] = None + self.backend: Optional[AudioBackend] = None self.gui: Optional[MagicMock] = None self._patches: list = [] @@ -289,10 +316,21 @@ def __enter__(self) -> "OCPPlayerHarness": """ from ovos_media.player import OCPMediaPlayer - self.bus = FakeBus() - self.backend = MockOCPBackend( - config={}, bus=self.bus, namespace=self.backend_namespace - ) + self.bus = FakeBus(modernize=self.modernize, + emit_legacy=self.emit_legacy) + if self.backend_factory is not None: + self.backend = self.backend_factory(self.bus) + # A config-loaded backend gets name/namespace from + # BaseMediaService.load_services(), which the harness bypasses — supply + # sane defaults so the service's bookkeeping (shutdown, routing) works. + if not getattr(self.backend, "name", None): + self.backend.name = "test-backend" + if not getattr(self.backend, "namespace", None): + self.backend.namespace = self.backend_namespace + else: + self.backend = MockOCPBackend( + config={}, bus=self.bus, namespace=self.backend_namespace + ) # Build patch targets gui_mock = MagicMock() @@ -341,22 +379,50 @@ def __init__(self, *args, **kwargs): # Instantiate the real player (all heavy deps are now mocked) self.player = OCPMediaPlayer(self.bus, config={}) - # Inject MockOCPBackend as the sole audio backend - audio_svc = self.player.audio_service - audio_svc.services = [self.backend] - audio_svc.default = self.backend - self.backend.set_track_start_callback(audio_svc.track_start) - - # Register the audio service bus handlers manually - # (normally done inside BaseMediaService.load_services) ns = self.backend_namespace - self.bus.on(f"ovos.{ns}.service.play", audio_svc.handle_play) - self.bus.on(f"ovos.{ns}.service.pause", audio_svc.pause) - self.bus.on(f"ovos.{ns}.service.resume", audio_svc.resume) - self.bus.on(f"ovos.{ns}.service.stop", audio_svc.stop) - self.bus.on("ovos.common_play.media.state", - audio_svc.handle_media_state_change) - audio_svc._loaded.set() + if self.backend_factory is not None: + # Real-backend mode: the mocked AudioService (a MagicMock) never routes + # play()->load_track()->backend.play(), so swap in a *real* AudioService + # with autoload off and the injected backend as its sole service. Now the + # player's playback path actually drives the real backend (e.g. asserting + # a Music Assistant client's play_media() call). + from ovos_media.media_backends.audio import AudioService as _RealAudioService + audio_svc = _RealAudioService(self.bus, config={"audio_players": {}}, + autoload=False, validate_source=False) + self.player.audio_service = audio_svc + # Deferred uris (e.g. library://, {sei}//) are resolved by the OCP + # pipeline's stream extractors *before* the player sees them; this + # harness drives the backend directly, so bypass the player's + # stream-extraction validation (no extractor plugins are loaded). + self.player.validate_stream = lambda: True + audio_svc.services = [self.backend] + audio_svc.default = self.backend + self.backend.set_track_start_callback(audio_svc.track_start) + # load_services() (skipped with autoload=False) would register these. + self.bus.on(f"ovos.{ns}.service.play", audio_svc.handle_play) + self.bus.on(f"ovos.{ns}.service.pause", audio_svc.pause) + self.bus.on(f"ovos.{ns}.service.resume", audio_svc.resume) + self.bus.on(f"ovos.{ns}.service.stop", audio_svc.stop) + # NB: BaseMediaService.__init__ already wired ovos.common_play.media.state + # -> handle_media_state_change; re-registering it would fire backend.play() + # twice, so it is deliberately omitted here. + audio_svc._loaded.set() + else: + # Mock-backend mode: drive the player state-machine against the MagicMock + # AudioService; the backend is exposed for manual simulate_*/state asserts. + audio_svc = self.player.audio_service + audio_svc.services = [self.backend] + audio_svc.default = self.backend + self.backend.set_track_start_callback(audio_svc.track_start) + # Register the audio service bus handlers manually + # (normally done inside BaseMediaService.load_services) + self.bus.on(f"ovos.{ns}.service.play", audio_svc.handle_play) + self.bus.on(f"ovos.{ns}.service.pause", audio_svc.pause) + self.bus.on(f"ovos.{ns}.service.resume", audio_svc.resume) + self.bus.on(f"ovos.{ns}.service.stop", audio_svc.stop) + self.bus.on("ovos.common_play.media.state", + audio_svc.handle_media_state_change) + audio_svc._loaded.set() return self @@ -574,9 +640,18 @@ class OCPCaptureSession: """ bus: FakeBus + # capture BOTH the legacy and the ovos.* spec topics of the duck/cork + # messages the player consumes. The session observes the raw "message" wire + # stream, which carries the producer's ORIGINAL topic only (FakeBus' namespace + # bridging re-dispatches the counterpart as a typed event, not a second + # "message" event). Listing both namespaces lets the session record the + # sequence whether the producer emits legacy or spec. track_prefixes: List[str] = dataclasses.field(default_factory=lambda: [ "ovos.common_play.", "ovos.audio.", + "recognizer_loop:audio_output", + "ovos.listener.record", + "recognizer_loop:record", ]) messages: List[Message] = dataclasses.field(default_factory=list) diff --git a/ovoscope/media_provider.py b/ovoscope/media_provider.py new file mode 100644 index 0000000..c70f476 --- /dev/null +++ b/ovoscope/media_provider.py @@ -0,0 +1,194 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""MediaProvider (catalog/search) test harness for ovoscope. + +``ovoscope.media`` drives the OCP *player* state-machine; this module is its +catalog/search counterpart — a harness for ``opm.media.provider`` plugins +(``MediaProvider`` subclasses introduced by the ovos-media sprint). + +There is no released loader for the ``opm.media.provider`` entry-point group, and +the OCP pipeline does not yet load providers in-process, so this harness models +the pipeline's *intended* path: + + discover the provider -> gate it with ``serves(signals, context)`` + -> call the never-raising ``search_safe`` + +It is deliberately **duck-typed**: it imports neither mediavocab (``Signals`` / +``Release``) nor ovos-plugin-manager's ``MediaProvider`` / ``QueryContext``. The +test supplies whatever ``signals`` / ``context`` objects the provider expects, so +ovoscope stays dependency-free and usable even without the (currently branch-only) +``opm.media.provider`` plugin type installed. + +Usage:: + + from ovoscope import MediaProviderHarness + + h = MediaProviderHarness.from_entrypoint( + "music_assistant", + config={"url": "http://mass.local:8095"}, + mock_api=my_mock_client, # injected onto provider._api + ) + h.assert_entrypoint_registered() + h.assert_routes(Signals(medium=MediaType.MUSIC), + QueryContext(supported_playback_types={"audio"})) + h.assert_not_routes(Signals(medium=MediaType.MOVIE)) + releases = h.assert_returns_playables(Signals(title="worms")) +""" +from __future__ import annotations + +from importlib.metadata import entry_points +from typing import Any, List, Optional + +DEFAULT_GROUP = "opm.media.provider" + + +class MediaProviderHarness: + """Wrap a ``MediaProvider`` instance and drive it the way the OCP pipeline does. + + Build one with :meth:`from_entrypoint` (real plugin discovery) or + :meth:`from_class` (a class you already hold — used by ovoscope's own tests). + The wrapped instance is exposed as :attr:`provider` and the injected mock + client as :attr:`api` for custom assertions. + """ + + def __init__(self, provider: Any, api: Any = None, + entrypoint_name: Optional[str] = None, + entrypoint_group: str = DEFAULT_GROUP) -> None: + self.provider = provider + self.api = api + self.entrypoint_name = entrypoint_name + self.entrypoint_group = entrypoint_group + + # ------------------------------------------------------------------ + # Constructors + # ------------------------------------------------------------------ + + @classmethod + def from_class(cls, provider_cls: Any, config: Optional[dict] = None, + mock_api: Any = None, api_attr: str = "_api") -> "MediaProviderHarness": + """Instantiate ``provider_cls(config)`` and (optionally) inject ``mock_api``. + + Args: + provider_cls: a ``MediaProvider`` subclass (duck-typed — anything with + ``is_available`` / ``serves`` / ``search`` / ``search_safe`` / + ``featured_media``). + config: config dict passed to the provider constructor. + mock_api: object set onto ``provider.`` to bypass the real + (network) client built lazily by the provider. + api_attr: attribute name the provider reads its client from + (default ``"_api"``). + """ + provider = provider_cls(config or {}) + if mock_api is not None: + setattr(provider, api_attr, mock_api) + return cls(provider, api=mock_api) + + @classmethod + def from_entrypoint(cls, name: str, config: Optional[dict] = None, + group: str = DEFAULT_GROUP, mock_api: Any = None, + api_attr: str = "_api") -> "MediaProviderHarness": + """Discover the provider through its installed entry-point and wrap it. + + Resolves ``name`` in the ``group`` entry-point group (default + ``opm.media.provider``), loads the class, and delegates to + :meth:`from_class`. Raises ``AssertionError`` if the entry-point is not + installed (or is ambiguous) — that *is* the e2e signal that the plugin's + packaging registered it correctly. + """ + matches = [ep for ep in entry_points(group=group) if ep.name == name] + if not matches: + raise AssertionError( + f"no {group!r} entry-point named {name!r} is installed — " + f"is the provider package installed (pip install -e .)?" + ) + if len(matches) > 1: + raise AssertionError( + f"multiple {group!r} entry-points named {name!r}: {matches!r}" + ) + provider_cls = matches[0].load() + harness = cls.from_class(provider_cls, config=config, + mock_api=mock_api, api_attr=api_attr) + harness.entrypoint_name = name + harness.entrypoint_group = group + return harness + + # ------------------------------------------------------------------ + # Drivers — mirror the pipeline's discover -> gate -> search path + # ------------------------------------------------------------------ + + def is_available(self) -> bool: + """Provider self-check (server reachable / keys present).""" + return self.provider.is_available() + + def serves(self, signals: Any, context: Any = None) -> bool: + """Context-aware routing gate (three-axis ``matches`` + device/policy).""" + return self.provider.serves(signals, context) + + def search(self, signals: Any, lang: str = "en-us") -> List[Any]: + """Raw search — may raise, mirroring a direct provider call.""" + return self.provider.search(signals, lang=lang) + + def search_safe(self, signals: Any, context: Any = None, + lang: str = "en-us") -> List[Any]: + """The never-raising entry the pipeline's thread-pool dispatch calls.""" + return self.provider.search_safe(signals, context=context, lang=lang) + + def featured_media(self, lang: str = "en-us") -> List[Any]: + """Curated/home content (recently-played, recommendations, …).""" + return self.provider.featured_media(lang=lang) + + # ------------------------------------------------------------------ + # Assertions + # ------------------------------------------------------------------ + + def assert_entrypoint_registered(self, name: Optional[str] = None, + group: Optional[str] = None) -> None: + """Assert the provider is discoverable under its entry-point group.""" + name = name or self.entrypoint_name + group = group or self.entrypoint_group + assert name, ("no entry-point name to check — pass name=, or build the " + "harness with from_entrypoint()") + names = [ep.name for ep in entry_points(group=group)] + assert name in names, ( + f"{name!r} is not registered under {group!r}; found {names!r}" + ) + + def assert_routes(self, signals: Any, context: Any = None) -> None: + """Assert the provider serves ``signals`` under ``context``.""" + assert self.serves(signals, context), ( + f"provider does not serve {signals!r} (context={context!r})" + ) + + def assert_not_routes(self, signals: Any, context: Any = None) -> None: + """Assert the provider is gated out for ``signals`` under ``context``.""" + assert not self.serves(signals, context), ( + f"provider unexpectedly serves {signals!r} (context={context!r})" + ) + + def assert_returns_playables(self, signals: Any, context: Any = None, + lang: str = "en-us") -> List[Any]: + """Assert ``search_safe`` returns ranked, playable results and return them. + + Each result must carry a truthy ``uri``, a ``match_confidence`` in + ``[0.0, 1.0]``, and a ``work``. + """ + results = self.search_safe(signals, context=context, lang=lang) + assert results, f"provider returned no results for {signals!r}" + for r in results: + assert getattr(r, "uri", None), f"result has no uri: {r!r}" + mc = getattr(r, "match_confidence", None) + assert mc is not None and 0.0 <= mc <= 1.0, ( + f"result match_confidence out of [0,1]: {mc!r} ({r!r})" + ) + assert getattr(r, "work", None) is not None, f"result has no work: {r!r}" + return results diff --git a/ovoscope/ocp.py b/ovoscope/ocp.py index 9d5d8b5..826bfd5 100644 --- a/ovoscope/ocp.py +++ b/ovoscope/ocp.py @@ -67,6 +67,15 @@ class OCPTest: patch_targets: Additional ``requests``-like module paths to patch (e.g. ``["my_skill.http_client.requests"]``). The default target is ``"requests.Session.get"``. + modernize: Forwarded to the harness ``MiniCroft`` / ``FakeBus``. When + on (default), emitting a LEGACY topic also dispatches its ovos.* + spec counterpart (legacy producer -> spec listener). The OCP flow + is driven by ``recognizer_loop:utterance``; bridging lets it be + observed on / driven from ``ovos.utterance.handle`` too. + emit_legacy: Forwarded to the harness. When on (default), emitting an + ovos.* spec topic also dispatches the legacy one (spec producer -> + legacy listener). Set BOTH False to exercise a single namespace + with no cross-namespace bridging. Example:: @@ -86,6 +95,8 @@ class OCPTest: lang: str = "en-US" timeout: float = 20.0 patch_targets: List[str] = field(default_factory=list) + modernize: bool = True + emit_legacy: bool = True def execute(self) -> List[Message]: """Run the OCP test with optional HTTP mocking. @@ -100,7 +111,9 @@ def execute(self) -> List[Message]: captured: List[Message] = [] - mc = get_minicroft(self.skill_ids, lang=self.lang, max_wait=60) + mc = get_minicroft(self.skill_ids, lang=self.lang, max_wait=60, + modernize=self.modernize, + emit_legacy=self.emit_legacy) mc.bus.on("message", lambda m: captured.append( Message.deserialize(m) if isinstance(m, str) else m )) diff --git a/ovoscope/phal.py b/ovoscope/phal.py index 6349574..156ac90 100644 --- a/ovoscope/phal.py +++ b/ovoscope/phal.py @@ -34,10 +34,10 @@ import threading import time from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional from ovos_utils.fakebus import FakeBus -from ovos_utils.messagebus import Message +from ovos_bus_client.message import Message class MiniPHAL: @@ -50,7 +50,22 @@ class MiniPHAL: plugin_ids: OPM entry-point IDs of the PHAL plugins to load. plugin_instances: Pre-built or mocked plugin instances keyed by plugin_id. When provided the corresponding entry in *plugin_ids* is skipped. + Note: instances must have been constructed with the same ``FakeBus`` + that ``MiniPHAL`` provides — use *plugin_factories* instead when + the plugin must be built inside the harness context. + plugin_factories: Callables ``(bus: FakeBus) -> plugin`` keyed by + plugin_id. The factory is called during ``__enter__`` so the plugin + is always wired to the harness ``FakeBus``. Takes precedence over + *plugin_instances* for the same plugin_id. config: Per-plugin configuration overrides keyed by plugin_id. + modernize: FakeBus also emits the ovos.* spec topic when a legacy topic + is emitted (legacy producer -> spec listener). PHAL plugins do not use + any of the migrated audio/listener topics, so this only matters for a + plugin that happens to consume/produce a migrated topic; it is threaded + for consistency with the audio/media harnesses. + emit_legacy: FakeBus also emits the legacy topic when an ovos.* spec topic + is emitted (spec producer -> legacy listener). Set both False to + exercise a single namespace with no bridging. Example:: @@ -60,18 +75,35 @@ class MiniPHAL: ) as phal: phal.emit(Message("system.reboot")) phal.assert_emitted("system.reboot.confirmed") + + Factory example (plugin must be built with the harness bus):: + + from ovos_phal_plugin_tools import OVOSToolsPHALPlugin + + with MiniPHAL( + plugin_ids=["ovos-phal-plugin-tools"], + plugin_factories={"ovos-phal-plugin-tools": lambda bus: OVOSToolsPHALPlugin(bus=bus)}, + ) as phal: + phal.emit(Message("ovos.tools.list", {})) + phal.assert_emitted("ovos.tools.list.response") """ def __init__( self, plugin_ids: Optional[List[str]] = None, plugin_instances: Optional[Dict[str, Any]] = None, + plugin_factories: Optional[Dict[str, Callable[[FakeBus], Any]]] = None, config: Optional[Dict[str, Dict[str, Any]]] = None, + modernize: bool = True, + emit_legacy: bool = True, ) -> None: self.plugin_ids: List[str] = plugin_ids or [] self.plugin_instances: Dict[str, Any] = plugin_instances or {} + self.plugin_factories: Dict[str, Callable[[FakeBus], Any]] = plugin_factories or {} self.config: Dict[str, Dict[str, Any]] = config or {} - self._bus: FakeBus = FakeBus() + self.modernize: bool = modernize + self.emit_legacy: bool = emit_legacy + self._bus: FakeBus = FakeBus(modernize=modernize, emit_legacy=emit_legacy) self._captured: List[Message] = [] self._loaded: Dict[str, Any] = {} @@ -108,9 +140,19 @@ def _capture(self, message: Any) -> None: self._captured.append(message) def _load_plugins(self) -> None: - """Load PHAL plugins via OPM or use pre-built instances.""" + """Load PHAL plugins via factories, pre-built instances, or OPM.""" for plugin_id in self.plugin_ids: - if plugin_id in self.plugin_instances: + if plugin_id in self.plugin_factories: + try: + instance = self.plugin_factories[plugin_id](self._bus) + except Exception as exc: + import warnings + warnings.warn( + f"Factory for PHAL plugin {plugin_id!r} raised: {exc}", + stacklevel=2, + ) + instance = None + elif plugin_id in self.plugin_instances: instance = self.plugin_instances[plugin_id] else: instance = self._instantiate_plugin(plugin_id) @@ -215,8 +257,13 @@ class PHALTest: expected_types: Message types that MUST appear in the capture. forbidden_types: Message types that MUST NOT appear. plugin_instances: Pre-built plugin instances (keyed by plugin_id). + plugin_factories: Callables ``(bus) -> plugin`` keyed by plugin_id. + Use this when the plugin must be constructed with the harness bus. config: Per-plugin config overrides. timeout: Maximum seconds to wait for expected messages (default 5.0). + modernize: Forwarded to :class:`MiniPHAL` — FakeBus bridges legacy->spec. + emit_legacy: Forwarded to :class:`MiniPHAL` — FakeBus bridges spec->legacy. + Set both False to exercise a single namespace with no bridging. Example:: @@ -236,8 +283,11 @@ class PHALTest: expected_types: List[str] = field(default_factory=list) forbidden_types: List[str] = field(default_factory=list) plugin_instances: Dict[str, Any] = field(default_factory=dict) + plugin_factories: Dict[str, Callable[[FakeBus], Any]] = field(default_factory=dict) config: Dict[str, Dict[str, Any]] = field(default_factory=dict) timeout: float = 5.0 + modernize: bool = True + emit_legacy: bool = True def execute(self) -> List[Message]: """Run the test: load plugins, emit trigger, assert expectations. @@ -251,7 +301,10 @@ def execute(self) -> List[Message]: with MiniPHAL( plugin_ids=self.plugin_ids, plugin_instances=self.plugin_instances, + plugin_factories=self.plugin_factories, config=self.config, + modernize=self.modernize, + emit_legacy=self.emit_legacy, ) as phal: phal.emit(self.trigger_message, wait=0.1) diff --git a/ovoscope/pipeline.py b/ovoscope/pipeline.py index e8ccbd2..de21cb2 100644 --- a/ovoscope/pipeline.py +++ b/ovoscope/pipeline.py @@ -42,12 +42,32 @@ class _SinkSkill: return it from :meth:`match`. """ - def __init__(self, bus: Any, skill_id: str = "__ovoscope_sink__") -> None: - self.bus = bus + def __init__(self, bus: Optional[Any] = None, skill_id: str = "__ovoscope_sink__") -> None: + from ovos_utils.fakebus import FakeBus + self.skill_id = skill_id self._last_match: Optional[Message] = None - bus.on("intent.service.skills.activated", self._handle) - bus.on("intent_failure", self._handle_failure) + self._bus: Any = bus if bus is not None else FakeBus() + self._bus.on("intent.service.skills.activated", self._handle) + self._bus.on("intent_failure", self._handle_failure) + + @property + def bus(self) -> Any: + return self._bus + + @bus.setter + def bus(self, new_bus: Any) -> None: + if new_bus is None: + raise ValueError("_SinkSkill.bus cannot be None; pass a real bus or omit to default to FakeBus.") + # Detach handlers from the previous bus before rebinding. + try: + self._bus.remove("intent.service.skills.activated", self._handle) + self._bus.remove("intent_failure", self._handle_failure) + except Exception: + pass + self._bus = new_bus + new_bus.on("intent.service.skills.activated", self._handle) + new_bus.on("intent_failure", self._handle_failure) def _handle(self, message: Any) -> None: """Capture matched intent messages.""" @@ -75,6 +95,15 @@ class PipelineHarness: pipeline: List of OPM pipeline stage IDs to load. pipeline_config: Per-stage config overrides keyed by stage ID. lang: Language tag (default ``"en-US"``). + modernize: Forwarded to the harness ``MiniCroft`` / ``FakeBus``. When + on (default), emitting a LEGACY topic also dispatches its ovos.* + spec counterpart (legacy producer -> spec listener). Utterances are + injected via ``recognizer_loop:utterance``; bridging lets them also + drive / be observed on ``ovos.utterance.handle``. + emit_legacy: Forwarded to the harness. When on (default), emitting an + ovos.* spec topic also dispatches the legacy one (spec producer -> + legacy listener). Set BOTH False to exercise a single namespace + with no cross-namespace bridging. Example:: @@ -91,10 +120,14 @@ def __init__( pipeline: Optional[List[str]] = None, pipeline_config: Optional[Dict[str, Dict[str, Any]]] = None, lang: str = "en-US", + modernize: bool = True, + emit_legacy: bool = True, ) -> None: self.pipeline: List[str] = pipeline or [] self.pipeline_config: Dict[str, Dict[str, Any]] = pipeline_config or {} self.lang: str = lang + self.modernize: bool = modernize + self.emit_legacy: bool = emit_legacy self._mc: Any = None # ------------------------------------------------------------------ @@ -105,8 +138,9 @@ def __enter__(self) -> "PipelineHarness": """Start MiniCroft with the specified pipeline and no skills.""" from ovoscope import get_minicroft - # Inject internal sink skill to capture matched intents - sink_skill = _SinkSkill(bus=None) # bus set after MiniCroft creation + # Inject internal sink skill to capture matched intents. + # Constructed with a default FakeBus; rebound to MiniCroft's real bus below. + sink_skill = _SinkSkill() self._mc = get_minicroft( skill_ids=[], @@ -114,6 +148,8 @@ def __enter__(self) -> "PipelineHarness": default_pipeline=self.pipeline or None, extra_skills={"__ovoscope_sink__": sink_skill}, max_wait=60, + modernize=self.modernize, + emit_legacy=self.emit_legacy, ) # Update sink skill's bus reference now that MiniCroft is created diff --git a/ovoscope/pytest_plugin.py b/ovoscope/pytest_plugin.py index 5745d87..7f51d1a 100644 --- a/ovoscope/pytest_plugin.py +++ b/ovoscope/pytest_plugin.py @@ -52,6 +52,8 @@ def test_something(self, minicroft, bus_coverage_session): if TYPE_CHECKING: from ovoscope.bus_coverage import BusCoverageReport +from pathlib import Path + import pytest from ovoscope import MiniCroft, get_minicroft, End2EndTest @@ -96,6 +98,59 @@ def pytest_addoption(parser): metavar="PATTERN", help="Exclude skills/components matching this regex from the coverage report.", ) + group.addoption( + "--ovoscope-accuracy-report", + action="store", + default=None, + metavar="PATH", + help="Write per-(pipeline, lang, intent) accuracy report as JSON.", + ) + group.addoption( + "--ovoscope-accuracy-min", + action="store", + type=float, + default=None, + metavar="RATIO", + help=("Minimum overall intent-case pass rate (0.0-1.0). If set and " + "the rate falls below, the session exits non-zero — useful " + "as a CI gate on regression in intent routing accuracy."), + ) + group.addoption( + "--ovoscope-accuracy-baseline", + action="store", + default=None, + metavar="PATH", + help=("Path to a previous --ovoscope-accuracy-report JSON. The " + "session fails if overall accuracy is lower than the " + "baseline (helpful for blocking PRs that lower accuracy)."), + ) + group.addoption( + "--ovoscope-accuracy-tolerant", + action="store_true", + default=False, + help=("Downgrade individual intent-case failures to xfail so they " + "don't fail the build; only the aggregate accuracy gate " + "(--ovoscope-accuracy-min / --ovoscope-accuracy-baseline) " + "can block the session."), + ) + group.addoption( + "--ovoscope-accuracy-md", + action="store", + default=None, + metavar="PATH", + help=("Write a Markdown intent-case accuracy report (the format " + "consumed by the OpenVoiceOS gh-automations PR-comment " + "workflow). Pairs naturally with --ovoscope-accuracy-report."), + ) + group.addoption( + "--ovoscope-accuracy-top-n", + action="store", + type=int, + default=10, + metavar="N", + help=("Show the N hardest utterances (lowest cross-pipeline pass " + "rate) in the Markdown report. Default: 10."), + ) @pytest.fixture(scope="class") @@ -307,12 +362,9 @@ def patched_execute(self, *args, **kwargs): print(f"\nERROR: Failed to save bus coverage report to {cov_file}: {exc}") -def pytest_terminal_summary(terminalreporter, exitstatus, config): # noqa: ARG001 - """Print the merged bus coverage report at the end of the pytest session. - - Only runs if at least one test used the ``bus_coverage_session`` fixture - (or ``--ovoscope-bus-cov`` was used) and reports were collected. - """ +def _bus_coverage_summary(terminalreporter, config): + """Print the merged bus coverage report (factored out so the combined + ``pytest_terminal_summary`` below can call both reporters).""" reports = getattr(config, "_bus_coverage_reports", None) if not reports: return @@ -321,3 +373,478 @@ def pytest_terminal_summary(terminalreporter, exitstatus, config): # noqa: ARG0 terminalreporter.write_sep("=", "Bus Coverage Report") report.print_report(verbose=verbose) terminalreporter.write_line("") + + +# --------------------------------------------------------------------------- +# Intent-case accuracy reporter +# +# Aggregates pass/fail per (pipeline, lang, intent) for tests generated by +# :func:`ovoscope.intent_cases.register_intent_case_tests`. Each generated +# method carries an ``_intent_case`` attribute; we read it during the +# ``pytest_runtest_logreport`` hook, accumulate, then emit a report and an +# optional pass/fail gate during ``pytest_terminal_summary``. +# --------------------------------------------------------------------------- +def _resolve_intent_case_meta(item): + """Return (IntentCase, pipeline_name, skill_id) or None if not generated.""" + func = getattr(item, "function", None) + if func is None: + return None + case = getattr(func, "_intent_case", None) + if case is None: + return None + return (case, + getattr(func, "_intent_case_pipeline", "Unknown"), + getattr(func, "_intent_case_skill_id", "")) + + +def _autodiscover_intent_cases(config): + """Walk pytest's loaded test modules and trigger auto-discovery. + + Pytest collects tests from files matching ``test_*.py`` (or + ``*_test.py``) — *not* from ``conftest.py``. So the auto-discovery + target is a thin shim module like ``test_intent_cases.py`` that + declares:: + + ovoscope_intent_cases = dict(skill_id=..., handlers=...) + + On the very first ``pytest_pycollect_makemodule`` we resolve the + module, scan it for that declaration, and inject the generated + TestCase classes into its namespace before pytest collects items + from it. The user writes one variable assignment, no + ``register_intent_case_tests`` call, no ``globals()`` argument. + + Skills already calling :func:`register_intent_case_tests` explicitly + are skipped via the ``_ovoscope_intent_cases_registered`` marker. + """ + # Tracked in pytest_pycollect_makemodule; this helper kept for symmetry + # / future use (e.g. a CLI hook). + return None + + +@pytest.hookimpl(wrapper=True) +def pytest_pycollect_makemodule(module_path, parent): + """Auto-register intent-case tests on shim modules that declare + ``ovoscope_intent_cases = {...}``. + + The hook wrapper imports the module first, lets pytest build the + collector, then injects the generated TestCase classes into the + module's namespace so the standard Python-class collector finds them. + + Uses the modern ``wrapper=True`` hook-wrapper protocol (pluggy >=1.2 / + pytest >=7.2): the downstream result arrives from ``yield`` and is + returned unchanged. The legacy ``hookwrapper=True`` / + ``outcome.get_result()`` protocol is removed in pytest 10, so we adopt + the new one to stay loadable under pytest 8 and 9. + """ + from ovoscope.intent_cases import autodiscover_from_conftest + + collector = yield + if collector is None: + return collector + try: + mod = collector.obj # imports the module if not already loaded + except Exception: + return collector + if not hasattr(mod, "ovoscope_intent_cases"): + return collector + if getattr(mod, "_ovoscope_intent_cases_registered", False): + return collector + try: + autodiscover_from_conftest(Path(mod.__file__).parent, mod.__dict__) + except Exception as exc: # noqa: BLE001 + print(f"ovoscope auto-discovery skipped for {mod.__file__}: {exc}") + return collector + + +def pytest_collection_modifyitems(config, items): + """In tolerant mode, mark every intent-case test as ``xfail(strict=False)``. + + That lets the suite measure per-case routing accuracy without forcing + every CI run to be green only when 100% of cases match. The aggregate + gate (``--ovoscope-accuracy-min`` / ``--ovoscope-accuracy-baseline``) + is the real blocker. + """ + if not config.getoption("--ovoscope-accuracy-tolerant"): + return + for item in items: + if _resolve_intent_case_meta(item) is not None: + item.add_marker(pytest.mark.xfail(reason="intent-case (tolerant)", + strict=False, run=True)) + + +def pytest_runtest_logreport(report): + """Record intent-case outcomes on the session config.""" + if report.when != "call": + return + item_func = getattr(report, "_intent_case_func", None) + # We can't get the item directly from a logreport; fall back to nodeid. + # The accumulator is keyed by nodeid -> (case, pipeline) which we set in + # ``pytest_runtest_setup``. + accum = getattr(pytest_runtest_logreport, "_accum", None) + if accum is None: + return + meta = accum["meta"].get(report.nodeid) + if meta is None: + return + case, pipeline, skill_id = meta + passed = report.outcome == "passed" or ( + report.outcome == "skipped" + and isinstance(report.longrepr, tuple) + and "XPASS" in str(report.longrepr)) + # xfail/xpass handling: in tolerant mode a "failure" surfaces as xfail. + if report.outcome == "skipped" and hasattr(report, "wasxfail"): + passed = False + accum["results"].append({ + "nodeid": report.nodeid, + "skill_id": skill_id, + "pipeline": pipeline, + "lang": case.lang, + "intent": case.intent or "no_match", + "utterance": case.utterance, + "source": str(case.source), + "passed": passed, + }) + + +def pytest_runtest_setup(item): + """Index intent-case metadata by nodeid for the logreport hook.""" + meta = _resolve_intent_case_meta(item) + if meta is None: + return + accum = getattr(pytest_runtest_logreport, "_accum", None) + if accum is None: + accum = {"results": [], "meta": {}} + pytest_runtest_logreport._accum = accum + accum["meta"][item.nodeid] = meta + + +def _accuracy_summary(results): + """Aggregate results into pivot tables for reporting. + + Adds, on top of the previous pivots, a per-utterance roll-up that + feeds the "hardest utterances" section of the Markdown report. + """ + by_pipeline: Dict[str, Dict[str, int]] = {} + by_pipeline_lang: Dict[tuple, Dict[str, int]] = {} + by_pipeline_intent: Dict[tuple, Dict[str, int]] = {} + # Cross-pipeline rollup: (lang, intent, utterance) -> {pass, total, + # failing_pipelines}. Lets us surface "which exact phrasing routes + # poorly across the whole stack" in one place. + by_utterance: Dict[tuple, Dict[str, object]] = {} + total_pass = total = 0 + for r in results: + total += 1 + if r["passed"]: + total_pass += 1 + for bucket, key in ( + (by_pipeline, r["pipeline"]), + (by_pipeline_lang, (r["pipeline"], r["lang"])), + (by_pipeline_intent, (r["pipeline"], r["intent"])), + ): + d = bucket.setdefault(key, {"pass": 0, "total": 0}) + d["total"] += 1 + if r["passed"]: + d["pass"] += 1 + utt_key = (r["lang"], r["intent"], r["utterance"]) + u = by_utterance.setdefault(utt_key, { + "lang": r["lang"], "intent": r["intent"], + "utterance": r["utterance"], + "pass": 0, "total": 0, "failing_pipelines": []}) + u["total"] += 1 + if r["passed"]: + u["pass"] += 1 + else: + u["failing_pipelines"].append(r["pipeline"]) + overall = (total_pass / total) if total else 0.0 + return { + "overall_accuracy": overall, + "passed": total_pass, + "total": total, + "by_pipeline": by_pipeline, + "by_pipeline_lang": {f"{p}|{l}": v for (p, l), v in by_pipeline_lang.items()}, + "by_pipeline_intent": {f"{p}|{i}": v for (p, i), v in by_pipeline_intent.items()}, + "by_utterance": list(by_utterance.values()), + } + + +# --------------------------------------------------------------------------- +# Baseline diff +# --------------------------------------------------------------------------- +def _result_key(r: dict) -> tuple: + """Stable identity for a single (pipeline, lang, intent, utterance) case.""" + return (r.get("pipeline", ""), r.get("lang", ""), + r.get("intent", ""), r.get("utterance", "")) + + +def _baseline_diff(baseline_results, current_results): + """Compare two result lists by case key, returning a structural diff. + + Returns ``{"regressed": [...], "recovered": [...], "added": [...], + "removed": [...], "baseline_accuracy": float}``. Each item in + ``regressed`` / ``recovered`` is the matching *current* result dict + so callers can quote the offending utterance verbatim. + """ + base_by_key = {_result_key(r): r for r in baseline_results or []} + cur_by_key = {_result_key(r): r for r in current_results or []} + + regressed, recovered, added, removed = [], [], [], [] + for k, cur in cur_by_key.items(): + if k not in base_by_key: + added.append(cur) + continue + base = base_by_key[k] + if base.get("passed") and not cur.get("passed"): + regressed.append(cur) + elif (not base.get("passed")) and cur.get("passed"): + recovered.append(cur) + for k, base in base_by_key.items(): + if k not in cur_by_key: + removed.append(base) + + base_total = len(baseline_results or []) + base_pass = sum(1 for r in (baseline_results or []) if r.get("passed")) + base_acc = (base_pass / base_total) if base_total else 0.0 + + return { + "regressed": regressed, "recovered": recovered, + "added": added, "removed": removed, + "baseline_accuracy": base_acc, "baseline_passed": base_pass, + "baseline_total": base_total, + } + + +# --------------------------------------------------------------------------- +# Markdown report +# --------------------------------------------------------------------------- +def _accuracy_markdown(summary, results, baseline_diff=None, top_n=10): + """Render the intent-case accuracy summary as Markdown. + + Mirrors the table-and-collapsible-section style used by the existing + bus-coverage section in the gh-automations PR-comment workflow: + headline line, summary table, per-(pipeline,lang) and + per-(pipeline,intent) breakdowns in ``
`` blocks, then a + "hardest utterances" call-out and, when present, a baseline diff. + """ + overall = summary["overall_accuracy"] + icon = "✅" if overall >= 0.9 else ("⚠️" if overall >= 0.7 else "❌") + lines = [ + f"{icon} **{summary['passed']}/{summary['total']}** intent-case " + f"checks passed — overall accuracy **{overall:.1%}**.", + "", + ] + + # --- Per-pipeline summary table (always visible) --- + lines.append("| Pipeline | Pass / Total | Accuracy |") + lines.append("|---|---:|---:|") + for pipe, d in sorted(summary["by_pipeline"].items()): + ratio = d["pass"] / d["total"] if d["total"] else 0.0 + lines.append(f"| `{pipe}` | {d['pass']} / {d['total']} | {ratio:.1%} |") + lines.append("") + + # --- Per-(pipeline, lang) --- + lines.append("
Per-pipeline × language") + lines.append("") + lines.append("| Pipeline | Lang | Pass / Total | Accuracy |") + lines.append("|---|---|---:|---:|") + for key in sorted(summary["by_pipeline_lang"]): + pipe, lang = key.split("|", 1) + d = summary["by_pipeline_lang"][key] + ratio = d["pass"] / d["total"] if d["total"] else 0.0 + lines.append(f"| `{pipe}` | `{lang}` | {d['pass']} / {d['total']} | {ratio:.1%} |") + lines.append("") + lines.append("
") + lines.append("") + + # --- Per-(pipeline, intent) --- + lines.append("
Per-pipeline × intent") + lines.append("") + lines.append("| Pipeline | Intent | Pass / Total | Accuracy |") + lines.append("|---|---|---:|---:|") + for key in sorted(summary["by_pipeline_intent"]): + pipe, intent = key.split("|", 1) + d = summary["by_pipeline_intent"][key] + ratio = d["pass"] / d["total"] if d["total"] else 0.0 + lines.append(f"| `{pipe}` | `{intent}` | {d['pass']} / {d['total']} | {ratio:.1%} |") + lines.append("") + lines.append("
") + lines.append("") + + # --- Hardest utterances (lowest cross-pipeline pass rate) --- + utts = list(summary.get("by_utterance", [])) + # Only utterances that failed at least once and aren't no_match + hard = [u for u in utts + if u["total"] > 0 and u["pass"] < u["total"] + and u["intent"] != "no_match"] + hard.sort(key=lambda u: (u["pass"] / u["total"], -u["total"])) + if hard: + lines.append(f"
Hardest utterances " + f"(top {min(top_n, len(hard))})") + lines.append("") + lines.append("| Lang | Intent | Utterance | Pass / Total | Failing pipelines |") + lines.append("|---|---|---|---:|---|") + for u in hard[:top_n]: + fail_pipes = ", ".join(f"`{p}`" for p in sorted(set(u["failing_pipelines"]))) + lines.append(f"| `{u['lang']}` | `{u['intent']}` " + f"| {u['utterance']} " + f"| {u['pass']} / {u['total']} | {fail_pipes} |") + lines.append("") + lines.append("
") + lines.append("") + + # --- Baseline diff (when supplied) --- + if baseline_diff is not None: + regressed = baseline_diff["regressed"] + recovered = baseline_diff["recovered"] + delta = overall - baseline_diff["baseline_accuracy"] + delta_str = f"{delta:+.1%}" + head = (f"vs baseline ({baseline_diff['baseline_passed']}/" + f"{baseline_diff['baseline_total']} = " + f"{baseline_diff['baseline_accuracy']:.1%}): " + f"{len(regressed)} regressed, {len(recovered)} recovered, " + f"Δ {delta_str}") + lines.append(f"**Baseline diff** — {head}") + lines.append("") + if regressed: + lines.append("
❌ Regressed " + f"({len(regressed)})") + lines.append("") + lines.append("| Pipeline | Lang | Intent | Utterance |") + lines.append("|---|---|---|---|") + for r in regressed[:50]: + lines.append(f"| `{r['pipeline']}` | `{r['lang']}` | " + f"`{r['intent']}` | {r['utterance']} |") + if len(regressed) > 50: + lines.append(f"| … | … | … | _+{len(regressed)-50} more_ |") + lines.append("") + lines.append("
") + lines.append("") + if recovered: + lines.append("
✅ Recovered " + f"({len(recovered)})") + lines.append("") + lines.append("| Pipeline | Lang | Intent | Utterance |") + lines.append("|---|---|---|---|") + for r in recovered[:50]: + lines.append(f"| `{r['pipeline']}` | `{r['lang']}` | " + f"`{r['intent']}` | {r['utterance']} |") + if len(recovered) > 50: + lines.append(f"| … | … | … | _+{len(recovered)-50} more_ |") + lines.append("") + lines.append("
") + lines.append("") + + return "\n".join(lines).rstrip() + "\n" + + +def pytest_terminal_summary(terminalreporter, exitstatus, config): # noqa: ARG001 + """Combined session summary: bus coverage + intent-case accuracy.""" + _bus_coverage_summary(terminalreporter, config) + accum = getattr(pytest_runtest_logreport, "_accum", None) + if not accum or not accum["results"]: + return + summary = _accuracy_summary(accum["results"]) + + tr = terminalreporter + tr.write_sep("=", "ovoscope Intent-Case Accuracy") + tr.write_line( + f"Overall: {summary['passed']}/{summary['total']} " + f"= {summary['overall_accuracy']:.1%}") + tr.write_line("") + tr.write_line("By pipeline:") + for pipe, d in sorted(summary["by_pipeline"].items()): + ratio = d["pass"] / d["total"] if d["total"] else 0.0 + tr.write_line(f" {pipe:24s} {d['pass']:4d}/{d['total']:<4d} {ratio:>6.1%}") + tr.write_line("") + tr.write_line("By pipeline x lang:") + for key in sorted(summary["by_pipeline_lang"]): + d = summary["by_pipeline_lang"][key] + ratio = d["pass"] / d["total"] if d["total"] else 0.0 + tr.write_line(f" {key:36s} {d['pass']:4d}/{d['total']:<4d} {ratio:>6.1%}") + tr.write_line("") + tr.write_line("By pipeline x intent:") + for key in sorted(summary["by_pipeline_intent"]): + d = summary["by_pipeline_intent"][key] + ratio = d["pass"] / d["total"] if d["total"] else 0.0 + tr.write_line(f" {key:48s} {d['pass']:4d}/{d['total']:<4d} {ratio:>6.1%}") + + # Load baseline (if any) and compute structural diff up-front so both + # the JSON / Markdown outputs and the gate can reuse it. + baseline_path = config.getoption("--ovoscope-accuracy-baseline") + baseline_diff = None + baseline_warning = None + if baseline_path: + try: + import json as _json + with open(baseline_path, "r", encoding="utf-8") as fh: + baseline_doc = _json.load(fh) + baseline_diff = _baseline_diff( + baseline_doc.get("results") or [], + accum["results"]) + except Exception as exc: + baseline_warning = (f"could not read baseline " + f"{baseline_path}: {exc}") + tr.write_line(f"\nWARNING: {baseline_warning}") + + # Persist JSON. + report_path = config.getoption("--ovoscope-accuracy-report") + if report_path: + import json + import os + os.makedirs(os.path.dirname(os.path.abspath(report_path)) or ".", + exist_ok=True) + doc = {"summary": summary, "results": accum["results"]} + if baseline_diff is not None: + doc["baseline_diff"] = { + "regressed": baseline_diff["regressed"], + "recovered": baseline_diff["recovered"], + "added": baseline_diff["added"], + "removed": baseline_diff["removed"], + "baseline_accuracy": baseline_diff["baseline_accuracy"], + "baseline_passed": baseline_diff["baseline_passed"], + "baseline_total": baseline_diff["baseline_total"], + } + with open(report_path, "w", encoding="utf-8") as fh: + json.dump(doc, fh, indent=2) + tr.write_line(f"\nWrote accuracy report -> {report_path}") + + # Persist Markdown (for PR-comment ingestion). + md_path = config.getoption("--ovoscope-accuracy-md") + if md_path: + import os + os.makedirs(os.path.dirname(os.path.abspath(md_path)) or ".", + exist_ok=True) + top_n = config.getoption("--ovoscope-accuracy-top-n") + md = _accuracy_markdown(summary, accum["results"], + baseline_diff=baseline_diff, top_n=top_n) + with open(md_path, "w", encoding="utf-8") as fh: + fh.write(md) + tr.write_line(f"Wrote accuracy markdown -> {md_path}") + + # Gate the session. + min_acc = config.getoption("--ovoscope-accuracy-min") + failures = [] + if min_acc is not None and summary["overall_accuracy"] < min_acc: + failures.append( + f"overall accuracy {summary['overall_accuracy']:.1%} < " + f"required {min_acc:.1%}") + if baseline_diff is not None: + if baseline_diff["regressed"]: + top_regression = baseline_diff["regressed"][0] + failures.append( + f"{len(baseline_diff['regressed'])} cases regressed vs " + f"baseline (first: `{top_regression['pipeline']}` / " + f"`{top_regression['lang']}` / " + f"`{top_regression['intent']}` / " + f"{top_regression['utterance']!r})") + if failures: + tr.write_sep("!", "ovoscope accuracy gate FAILED") + for f in failures: + tr.write_line(f" - {f}") + config._ovoscope_accuracy_gate_failed = True + + +def pytest_sessionfinish(session, exitstatus): # noqa: ARG001 + """Propagate accuracy-gate failure as a non-zero exit status.""" + if getattr(session.config, "_ovoscope_accuracy_gate_failed", False): + if session.exitstatus == 0: + session.exitstatus = 1 diff --git a/ovoscope/simple_listener.py b/ovoscope/simple_listener.py new file mode 100644 index 0000000..3ea2d34 --- /dev/null +++ b/ovoscope/simple_listener.py @@ -0,0 +1,243 @@ +# Copyright 2024 OpenVoiceOS +# Licensed under the Apache License, Version 2.0 +"""MiniSimpleListener — drive ovos-simple-listener for bus-sequence testing. + +``ovos-simple-listener`` is an alternative OVOS listener service: a single +``SimpleListener`` thread that reads microphone chunks, detects a wake word (or +VAD activation), records the command, and dispatches a small set of callbacks. +Its canonical bus integration (``OVOSCallbacks`` in +``ovos_simple_listener.__main__``) emits ``recognizer_loop:wakeword`` / +``…:record_begin`` on activation, ``…:utterance`` / +``…:speech.recognition.unknown`` after STT, and ``…:record_end`` when the command +finishes. + +:class:`MiniSimpleListener` wires a ``SimpleListener`` to a ``FakeBus`` with mock +wake-word / VAD / STT plugins and a :class:`MockFileMicrophone`, runs the loop +over an arbitrary audio file, and captures the emitted bus sequence — sharing the +assertion helpers of :class:`ovoscope.voice_loop.ListenerHarness`. + +Example:: + + from ovoscope.simple_listener import MiniSimpleListener + from ovoscope.voice_loop import MockHotWordEngine, MockStreamingSTT + + with MiniSimpleListener( + wakeword=MockHotWordEngine("hey_mycroft", trigger_after=2), + stt_instance=MockStreamingSTT(transcript="turn on the lights"), + ) as sl: + msgs = sl.feed_file("command.wav") + sl.assert_record_begin_emitted(msgs) + sl.assert_utterance_emitted("turn on the lights", msgs) +""" + +from __future__ import annotations + +import time +from pathlib import Path +from typing import Any, List, Optional, Union + +from ovos_bus_client.message import Message +from ovos_utils.fakebus import FakeBus + +from ovoscope.voice_loop import ( + ListenerHarness, + MockHotWordEngine, + MockStreamingSTT, + MockVADEngine, +) + + +class _SimpleBusCallbacks: + """Per-instance ``ListenerCallbacks`` that emit on a ``FakeBus``. + + Mirrors the canonical ``OVOSCallbacks`` from + ``ovos_simple_listener.__main__`` (minus the listen-sound playback), but + binds the bus per instance instead of on the class so concurrent harnesses + do not clobber one another. + + Args: + bus: The :class:`FakeBus` to emit listener events on. + """ + + def __init__(self, bus: FakeBus) -> None: + self.bus: FakeBus = bus + + def listen_callback(self) -> None: + """Activation: emit wakeword + record-begin.""" + self.bus.emit(Message("recognizer_loop:wakeword")) + self.bus.emit(Message("recognizer_loop:record_begin")) + + def end_listen_callback(self) -> None: + """Command finished: emit record-end.""" + self.bus.emit(Message("recognizer_loop:record_end")) + + def audio_callback(self, audio: Any) -> None: + """Recorded command audio is available (no-op).""" + + def error_callback(self, audio: Any) -> None: + """Empty transcription: emit the recognition-unknown event.""" + self.bus.emit(Message("recognizer_loop:speech.recognition.unknown")) + + def text_callback(self, utterance: str, lang: str) -> None: + """Transcription succeeded: emit the utterance.""" + self.bus.emit(Message( + "recognizer_loop:utterance", + {"utterances": [utterance], "lang": lang}, + )) + + +class MiniSimpleListener(ListenerHarness): + """In-process ovos-simple-listener harness for bus-sequence testing. + + Args: + wakeword: Wake-word engine (``update`` + ``found_wake_word``). Defaults + to a :class:`MockHotWordEngine` keyed ``"hey_mycroft"``. Pass + ``None`` to use VAD-only activation. + vad_instance: VAD engine (defaults to :class:`MockVADEngine`). + stt_instance: Streaming STT engine (defaults to + :class:`MockStreamingSTT` returning no transcript). + min_speech_seconds: Minimum command speech before a silence can end it. + max_silence_seconds: Trailing silence that ends a command. + max_speech_seconds: Hard cap on command length. + bus: Optional :class:`FakeBus` to capture on. + modernize: When a fresh bus is created, also emit the ovos.* spec topic + whenever a legacy topic is emitted (legacy producer -> spec + listener). The ``_SimpleBusCallbacks`` emit legacy + ``recognizer_loop:*`` topics; this bridge lets a spec-topic + subscriber observe them. Ignored when *bus* is supplied. + emit_legacy: When a fresh bus is created, also emit the legacy topic + whenever an ovos.* spec topic is emitted. Set both False to exercise + a single namespace with no bridging. Ignored when *bus* is supplied. + + Raises: + RuntimeError: If ovos-simple-listener is not installed. + + The default timing is tightened (``min_speech_seconds=0`` / + ``max_silence_seconds=0.1``) so a file-driven command ends promptly and + deterministically; raise them to mimic production behaviour. + """ + + def __init__( + self, + *, + wakeword: Optional[Any] = "__default__", + vad_instance: Optional[Any] = None, + stt_instance: Optional[Any] = None, + min_speech_seconds: float = 0.0, + max_silence_seconds: float = 0.1, + max_speech_seconds: float = 8.0, + bus: Optional[FakeBus] = None, + modernize: bool = True, + emit_legacy: bool = True, + ) -> None: + super().__init__(bus, modernize=modernize, emit_legacy=emit_legacy) + + try: + from ovos_simple_listener import SimpleListener + except ImportError as e: + raise RuntimeError( + "ovos-simple-listener is required to use MiniSimpleListener. " + "Install it with: pip install ovos-simple-listener" + ) from e + + if wakeword == "__default__": + wakeword = MockHotWordEngine("hey_mycroft", trigger_after=2) + + self.callbacks = _SimpleBusCallbacks(self.bus) + self.listener = SimpleListener( + wakeword=wakeword, + mic=None, # supplied per-run by feed_file + vad=vad_instance if vad_instance is not None else MockVADEngine(), + stt=stt_instance if stt_instance is not None else MockStreamingSTT(), + min_speech_seconds=min_speech_seconds, + max_silence_seconds=max_silence_seconds, + max_speech_seconds=max_speech_seconds, + callbacks=self.callbacks, + ) + + def feed_file( + self, + audio: Union[bytes, str, Path], + *, + silence_tail_chunks: int = 25, + chunk_size: int = 2048, + timeout: float = 10.0, + ) -> List[Message]: + """Run the simple listener over an audio file and capture bus events. + + Streams *audio* through a :class:`MockFileMicrophone`, runs the listener + thread until the command completes (``recognizer_loop:record_end``) or + *timeout* elapses, then stops it. + + Args: + audio: Path to a ``.wav`` file, WAV bytes, or raw PCM bytes. + silence_tail_chunks: Trailing silent frames appended after the audio + so the command can end. + chunk_size: Bytes per microphone read. + timeout: Maximum seconds to wait for the command to finish. + + Returns: + The list of :class:`Message` objects emitted during the run. + """ + mic = self._build_file_mic(audio, silence_tail_chunks, chunk_size) + self.listener.mic = mic + + self._messages.clear() + self.listener.start() # Thread.start → runs the loop in the background + try: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if self._has(self._messages, "recognizer_loop:record_end"): + break + time.sleep(0.02) + finally: + self.listener.stop() + self.listener.join(timeout=2.0) + + self._last_messages = list(self._messages) + return list(self._messages) + + def shutdown(self) -> None: + """Stop the listener thread if still running.""" + try: + self.listener.stop() + if self.listener.is_alive(): + self.listener.join(timeout=2.0) + except Exception: + pass + + +def get_mini_simple_listener( + wakeword: Optional[Any] = "__default__", + vad_instance: Optional[Any] = None, + stt_instance: Optional[Any] = None, + bus: Optional[FakeBus] = None, + modernize: bool = True, + emit_legacy: bool = True, +) -> MiniSimpleListener: + """Factory: create a ready-to-feed :class:`MiniSimpleListener`. + + Args: + wakeword: Wake-word engine (defaults to a :class:`MockHotWordEngine`). + vad_instance: VAD engine (defaults to :class:`MockVADEngine`). + stt_instance: Streaming STT engine (defaults to + :class:`MockStreamingSTT`). + bus: Optional :class:`FakeBus` to capture on. + modernize: When a fresh bus is created, also emit the ovos.* spec topic + whenever a legacy topic is emitted (legacy producer -> spec + listener). Ignored when *bus* is supplied. + emit_legacy: When a fresh bus is created, also emit the legacy topic + whenever an ovos.* spec topic is emitted. Set both False to exercise + a single namespace with no bridging. Ignored when *bus* is supplied. + + Returns: + A fully initialised :class:`MiniSimpleListener`. + """ + return MiniSimpleListener( + wakeword=wakeword, + vad_instance=vad_instance, + stt_instance=stt_instance, + bus=bus, + modernize=modernize, + emit_legacy=emit_legacy, + ) diff --git a/ovoscope/tts_intelligibility.py b/ovoscope/tts_intelligibility.py new file mode 100644 index 0000000..826b6f9 --- /dev/null +++ b/ovoscope/tts_intelligibility.py @@ -0,0 +1,486 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""End-to-end TTS intelligibility scoring for ovoscope. + +Synthesises speech with a TTS plugin under test, transcribes the rendered +audio back with a reference STT, and scores the round-trip with word- and +character-error-rate (WER/CER). This catches regressions that file-existence +unit tests miss — garbled audio, wrong sample rate, broken transforms, silent +output — and gives every TTS plugin a comparable intelligibility number. + +Two synthesis modes: + +* ``"playback"`` (default) drives the full ovos-audio stack + (``speak`` -> PlaybackService -> tts.execute -> get_tts -> tts_transform -> + play_audio) via :class:`ovoscope.audio.PlaybackServiceHarness`, with the real + plugin injected. The rendered WAV is captured from the patched ``play_audio``. +* ``"direct"`` calls ``tts.get_tts(utterance, wav_path, ...)`` directly with no + bus — a fallback for engines that hang under the playback thread or when + ``ovos_audio`` is unavailable. + +Public API: + score_tts_intelligibility -- convenience function returning an IntelligibilityReport + TTSIntelligibilityHarness -- context manager form of the above + IntelligibilityReport -- aggregate report with mean WER/CER + serialisation + UtteranceScore -- per-utterance result +""" + +import dataclasses +import os +import re +import shutil +import string +import tempfile +import threading +from typing import Any, List, Optional + +import jiwer + +from ovos_plugin_manager.utils.audio import AudioFile +from ovos_utils.log import LOG + +# Module-level singleton for the reference STT — model load is expensive. +_REFERENCE_STT: Optional[Any] = None +_REFERENCE_STT_LOCK = threading.Lock() + +# Module-level singleton for the utterance normaliser (cheap, but shared). +_NORMALIZER: Optional[Any] = None + + +def get_reference_stt() -> Any: + """Return a lazily-instantiated faster-whisper ``tiny`` reference STT. + + The model is loaded once per process and reused. ``beam_size=1`` and + ``compute_type="int8"`` keep it deterministic and light enough for CI. + + Returns: + A ready-to-use ``FasterWhisperSTT`` instance. + """ + global _REFERENCE_STT + if _REFERENCE_STT is None: + with _REFERENCE_STT_LOCK: + if _REFERENCE_STT is None: + from ovos_stt_plugin_fasterwhisper import FasterWhisperSTT + _REFERENCE_STT = FasterWhisperSTT({ + "model": "tiny", + "compute_type": "int8", + "beam_size": 1, + }) + return _REFERENCE_STT + + +def _normalize(text: str, lang: str) -> str: + """Normalise a transcript for fair WER/CER scoring. + + Uses ``ovos_utterance_normalizer`` (lowercase, number expansion, + contraction expansion, punctuation strip) so cosmetic differences between + reference and hypothesis don't inflate the error rate. The normaliser + yields several variants per input; the last one is the fully-normalised + form, which is what we score against. + + Args: + text: The raw text to normalise. + lang: BCP-47 language tag (e.g. ``"en-US"``). + + Returns: + A single normalised string (whitespace-collapsed, lowercased). + """ + global _NORMALIZER + if _NORMALIZER is None: + from ovos_utterance_normalizer import UtteranceNormalizerPlugin + _NORMALIZER = UtteranceNormalizerPlugin() + if not text: + return "" + variants, _ = _NORMALIZER.transform([text], {"lang": lang}) + # transform() emits [contraction-expanded, original, number-normalized] + # per utterance, deduplicated and order-preserving. The first variant + # (contractions expanded, punctuation stripped, words kept as words) is the + # stable choice — the number-collapsed variant ("two" -> "2") would create + # spurious mismatches against a word-emitting STT. Lowercasing/whitespace + # collapse are applied here so both reference and hypothesis align. + normalized = variants[0] if variants else text + # The normaliser only strips leading/trailing punctuation of the whole + # string; interior punctuation (e.g. a tokenised comma) survives and would + # inflate WER. Strip all punctuation characters here, then collapse space. + normalized = re.sub(rf"[{re.escape(string.punctuation)}]", " ", normalized) + return " ".join(normalized.lower().split()) + + +def _score_pair(reference: str, hypothesis: str) -> "tuple[float, float]": + """Compute (WER, CER) for a reference/hypothesis pair. + + jiwer raises on an empty reference, so empty references are handled + explicitly: a non-empty hypothesis against an empty reference scores 1.0 + (fully wrong), two empties score 0.0 (trivially correct). + + Args: + reference: Normalised ground-truth string. + hypothesis: Normalised transcript from the reference STT. + + Returns: + Tuple of ``(wer, cer)`` as floats. + """ + if not reference: + return (0.0, 0.0) if not hypothesis else (1.0, 1.0) + wer = float(jiwer.wer(reference, hypothesis)) + cer = float(jiwer.cer(reference, hypothesis)) + return wer, cer + + +@dataclasses.dataclass +class UtteranceScore: + """Per-utterance intelligibility result. + + Attributes: + utterance: The text that was synthesised (the ground truth). + transcript: What the reference STT heard back. + wer: Word error rate of transcript vs utterance (0.0 = perfect). + cer: Character error rate of transcript vs utterance. + wav_path: Path to the captured rendered WAV (may be None on failure). + lang: BCP-47 language tag used for synthesis and scoring. + voice: Voice identifier used, if any. + """ + + utterance: str + transcript: str + wer: float + cer: float + wav_path: Optional[str] = None + lang: str = "en-US" + voice: Optional[str] = None + + def to_dict(self) -> dict: + """Return a JSON-serialisable dict of this score.""" + return { + "utterance": self.utterance, + "transcript": self.transcript, + "wer": round(self.wer, 4), + "cer": round(self.cer, 4), + "wav_path": self.wav_path, + "lang": self.lang, + "voice": self.voice, + } + + +@dataclasses.dataclass +class IntelligibilityReport: + """Aggregate intelligibility report over a set of utterances. + + Attributes: + scores: Per-utterance :class:`UtteranceScore` results. + lang: BCP-47 language tag the run used. + voice: Voice identifier the run used, if any. + mode: Synthesis mode used (``"playback"`` or ``"direct"``). + """ + + scores: List[UtteranceScore] = dataclasses.field(default_factory=list) + lang: str = "en-US" + voice: Optional[str] = None + mode: str = "playback" + + @property + def mean_wer(self) -> float: + """Mean word error rate across all scored utterances (0.0 if empty).""" + if not self.scores: + return 0.0 + return sum(s.wer for s in self.scores) / len(self.scores) + + @property + def mean_cer(self) -> float: + """Mean character error rate across all scored utterances (0.0 if empty).""" + if not self.scores: + return 0.0 + return sum(s.cer for s in self.scores) / len(self.scores) + + def to_dict(self) -> dict: + """Return a JSON-serialisable dict of the full report.""" + return { + "lang": self.lang, + "voice": self.voice, + "mode": self.mode, + "mean_wer": round(self.mean_wer, 4), + "mean_cer": round(self.mean_cer, 4), + "n_utterances": len(self.scores), + "scores": [s.to_dict() for s in self.scores], + } + + def to_markdown_row(self) -> str: + """Return a single markdown table row: ``| voice | lang | mean_wer | mean_cer | n |``.""" + voice = self.voice or "default" + return ( + f"| {voice} | {self.lang} | " + f"{self.mean_wer:.3f} | {self.mean_cer:.3f} | {len(self.scores)} |" + ) + + +class TTSIntelligibilityHarness: + """Context manager that scores TTS intelligibility end-to-end. + + Usage:: + + with TTSIntelligibilityHarness(tts, lang="en-US") as h: + report = h.score(["hello world", "what time is it"]) + print(report.mean_wer) + + In ``mode="playback"`` the harness owns a :class:`PlaybackServiceHarness` + for its lifetime; in ``mode="direct"`` no bus is started. A temp directory + holds copies of the rendered WAVs (the TTS cache may delete originals); it + is cleaned up on exit. + + Args: + tts: The TTS plugin under test. + lang: BCP-47 language tag for synthesis and scoring. + voice: Optional voice identifier passed to ``get_tts``. + reference_stt: STT used to transcribe. Defaults to the lazy + faster-whisper ``tiny`` singleton. + mode: ``"playback"`` (full ovos-audio stack) or ``"direct"`` + (``tts.get_tts`` only). + speak_timeout: Per-utterance timeout for playback mode. + """ + + def __init__(self, tts: Any, *, lang: str = "en-US", + voice: Optional[str] = None, + reference_stt: Optional[Any] = None, + mode: str = "playback", + speak_timeout: float = 30.0) -> None: + if mode not in ("playback", "direct"): + raise ValueError(f"mode must be 'playback' or 'direct', got {mode!r}") + self.tts = tts + self.lang = lang + self.voice = voice + self._reference_stt = reference_stt + self.mode = mode + self.speak_timeout = speak_timeout + self._tmpdir: Optional[str] = None + self._playback = None # PlaybackServiceHarness in playback mode + + @property + def reference_stt(self) -> Any: + """The reference STT, lazily resolved to the faster-whisper singleton.""" + if self._reference_stt is None: + self._reference_stt = get_reference_stt() + return self._reference_stt + + def __enter__(self) -> "TTSIntelligibilityHarness": + self._tmpdir = tempfile.mkdtemp(prefix="ovoscope-tts-") + if self.mode == "playback": + from ovoscope.audio import PlaybackServiceHarness + self._playback = PlaybackServiceHarness(tts=self.tts) + self._playback.__enter__() + return self + + def __exit__(self, *args) -> None: + if self._playback is not None: + try: + self._playback.__exit__(*args) + except Exception: + pass + self._playback = None + if self._tmpdir and os.path.isdir(self._tmpdir): + shutil.rmtree(self._tmpdir, ignore_errors=True) + self._tmpdir = None + + # ------------------------------------------------------------------ + # Synthesis + # ------------------------------------------------------------------ + + def _render_playback(self, utterance: str) -> Optional[str]: + """Synthesise via the full ovos-audio stack; return a copied WAV path.""" + before = len(self._playback.captured_wavs) + self._playback.speak(utterance, timeout=self.speak_timeout) + captured = self._playback.captured_wavs[before:] + if not captured: + return None + return self._copy_out(captured[-1]) + + def _render_direct(self, utterance: str) -> Optional[str]: + """Synthesise via ``tts.get_tts`` directly; return the rendered audio path. + + The output extension follows the engine's ``audio_ext`` (``wav`` by + default) so engines that natively emit a non-PCM container (e.g. gTTS / + edge-tts write ``mp3``) produce a valid file instead of mp3 bytes in a + ``.wav`` that the WAV reader can't decode. ``_transcribe`` transcodes + non-WAV output to PCM before scoring. + """ + ext = (getattr(self.tts, "audio_ext", "wav") or "wav").lstrip(".") + out_path = os.path.join( + self._tmpdir, f"direct_{abs(hash(utterance)) & 0xffffffff}.{ext}" + ) + self.tts.get_tts(utterance, out_path, lang=self.lang, voice=self.voice) + return out_path if os.path.isfile(out_path) else None + + def _copy_out(self, wav_path: str) -> Optional[str]: + """Copy a rendered WAV into the harness temp dir before the cache prunes it.""" + if not wav_path or not os.path.isfile(wav_path): + return wav_path if wav_path and os.path.isfile(wav_path) else None + dst = os.path.join( + self._tmpdir, f"play_{len(os.listdir(self._tmpdir))}_{os.path.basename(wav_path)}" + ) + try: + shutil.copyfile(wav_path, dst) + return dst + except OSError: + return wav_path + + # ------------------------------------------------------------------ + # Scoring + # ------------------------------------------------------------------ + + def _transcribe(self, audio_path: str) -> str: + """Round-trip rendered audio through the reference STT. + + The rendered file is normalised to a 16 kHz mono PCM WAV first. This + covers two cases the ``AudioFile`` -> STT path otherwise mishandles: + non-WAV containers (mp3 from gTTS / edge-tts, which ``AudioFile`` cannot + decode) and WAVs at a non-16 kHz rate (e.g. mimic's 44.1 kHz ``ap`` + voice, which the STT reads at the wrong rate and hears sped-up). The + normalisation uses PyAV, already a faster-whisper dependency, so no + extra requirement is introduced. + """ + wav_path = self._ensure_wav(audio_path) + with AudioFile(wav_path) as source: + audio = source.read() + return self.reference_stt.execute(audio, language=self.lang) or "" + + def _ensure_wav(self, audio_path: str) -> str: + """Return a 16 kHz mono PCM-WAV path for ``audio_path``. + + Already-conforming WAVs (16 kHz, mono, 16-bit PCM) are returned as-is; + anything else (other container, rate, channel count, or sample format) + is transcoded with PyAV. + """ + if self._is_pcm16k_mono(audio_path): + return audio_path + wav_path = os.path.splitext(audio_path)[0] + ".transcoded.wav" + import av # bundled with faster-whisper + from av.audio.resampler import AudioResampler + + in_container = av.open(audio_path) + out_container = av.open(wav_path, mode="w", format="wav") + out_stream = out_container.add_stream("pcm_s16le", rate=16000) + out_stream.layout = "mono" + resampler = AudioResampler(format="s16", layout="mono", rate=16000) + try: + for frame in in_container.decode(audio=0): + frame.pts = None + for rframe in resampler.resample(frame): + for packet in out_stream.encode(rframe): + out_container.mux(packet) + for rframe in resampler.resample(None): + for packet in out_stream.encode(rframe): + out_container.mux(packet) + for packet in out_stream.encode(None): + out_container.mux(packet) + finally: + out_container.close() + in_container.close() + return wav_path + + @staticmethod + def _is_pcm16k_mono(audio_path: str) -> bool: + """True if ``audio_path`` is already a 16 kHz mono 16-bit PCM WAV.""" + if not audio_path.lower().endswith(".wav"): + return False + import wave + + try: + with wave.open(audio_path, "rb") as w: + return (w.getframerate() == 16000 + and w.getnchannels() == 1 + and w.getsampwidth() == 2) + except (wave.Error, EOFError, OSError): + return False + + def score_one(self, utterance: str) -> UtteranceScore: + """Synthesise, transcribe, and score a single utterance. + + Args: + utterance: The text to synthesise and score. + + Returns: + An :class:`UtteranceScore`. On synthesis/transcription failure the + transcript is empty and WER/CER reflect a total miss. + """ + try: + if self.mode == "playback": + wav_path = self._render_playback(utterance) + else: + wav_path = self._render_direct(utterance) + except Exception as exc: + # A synthesis failure (engine crash, missing model/binary, bad audio) + # is itself an intelligibility failure: score it as a total miss so the + # report is still produced and a marker is still emitted, instead of + # letting the exception abort the run with "no scores reported". + LOG.error(f"TTS synthesis failed for {utterance!r}: {exc}") + wav_path = None + + transcript = "" + if wav_path and os.path.isfile(wav_path): + try: + transcript = self._transcribe(wav_path) + except Exception: + transcript = "" + + ref = _normalize(utterance, self.lang) + hyp = _normalize(transcript, self.lang) + wer, cer = _score_pair(ref, hyp) + return UtteranceScore( + utterance=utterance, + transcript=transcript, + wer=wer, + cer=cer, + wav_path=wav_path, + lang=self.lang, + voice=self.voice, + ) + + def score(self, utterances: List[str]) -> IntelligibilityReport: + """Score a list of utterances and return the aggregate report. + + Args: + utterances: Phrases to synthesise and score. + + Returns: + An :class:`IntelligibilityReport`. + """ + report = IntelligibilityReport(lang=self.lang, voice=self.voice, mode=self.mode) + for utt in utterances: + report.scores.append(self.score_one(utt)) + return report + + +def score_tts_intelligibility(tts: Any, utterances: List[str], *, + lang: str = "en-US", + voice: Optional[str] = None, + reference_stt: Optional[Any] = None, + mode: str = "playback", + speak_timeout: float = 30.0) -> IntelligibilityReport: + """Synthesise, transcribe, and score a set of utterances in one call. + + Args: + tts: The TTS plugin under test. + utterances: Phrases to synthesise and score. + lang: BCP-47 language tag for synthesis and scoring. + voice: Optional voice identifier passed to ``get_tts``. + reference_stt: STT used to transcribe. Defaults to faster-whisper tiny. + mode: ``"playback"`` (full ovos-audio stack) or ``"direct"``. + speak_timeout: Per-utterance timeout for playback mode. + + Returns: + An :class:`IntelligibilityReport` with per-utterance and mean scores. + """ + with TTSIntelligibilityHarness( + tts, lang=lang, voice=voice, reference_stt=reference_stt, + mode=mode, speak_timeout=speak_timeout, + ) as harness: + return harness.score(utterances) diff --git a/ovoscope/version.py b/ovoscope/version.py index 3b482a3..f6c30c5 100644 --- a/ovoscope/version.py +++ b/ovoscope/version.py @@ -1,7 +1,7 @@ # START_VERSION_BLOCK -VERSION_MAJOR = 0 -VERSION_MINOR = 13 -VERSION_BUILD = 1 +VERSION_MAJOR = 1 +VERSION_MINOR = 4 +VERSION_BUILD = 0 VERSION_ALPHA = 1 # END_VERSION_BLOCK diff --git a/ovoscope/voice_loop.py b/ovoscope/voice_loop.py new file mode 100644 index 0000000..a46406b --- /dev/null +++ b/ovoscope/voice_loop.py @@ -0,0 +1,1085 @@ +# Copyright 2024 OpenVoiceOS +# Licensed under the Apache License, Version 2.0 +"""MiniVoiceLoop — drive ``DinkumVoiceLoop`` bus sequences for ovoscope. + +Where :class:`ovoscope.listener.MiniListener` covers the +``AudioTransformersService``, STT, and mock VAD/WakeWord engines, it does **not** +exercise the full ``DinkumVoiceLoop`` state machine. The bus events that matter +for wake-word handling (``recognizer_loop:wakeword``, +``recognizer_loop:record_begin``, ``recognizer_loop:record_end``, +``recognizer_loop:utterance``) are emitted as side-effects of the voice loop as +PCM chunks flow through it. + +``MiniVoiceLoop`` wires a real ``DinkumVoiceLoop`` to a ``FakeBus`` with no-op +mic/STT/transformer plugins, a controllable hotword container, and an optional +verifier chain. It supports two drive modes: + +* :meth:`MiniVoiceLoop.feed_chunks` — feed PCM frames directly through + ``_detect_ww`` to assert the wake-word detection / verifier gate in isolation. +* :meth:`MiniVoiceLoop.feed_file` — run the **whole** ``DinkumVoiceLoop.run()`` + state machine, reading an arbitrary audio file through a file-backed + microphone plugin, so the full record-begin → wakeword → command → record-end + → utterance sequence is captured. + +Example — verifier gate (``_detect_ww`` only):: + + from unittest.mock import Mock + from ovoscope.voice_loop import MiniVoiceLoop, MockHotWordEngine + + SILENT_CHUNK = b"\\x00" * 512 + ww = MockHotWordEngine(key_phrase="hey_mycroft", trigger_after=3) + accepting = Mock(); accepting.verify.return_value = True + + with MiniVoiceLoop(ww_instances={"hey_mycroft": ww}, + verifiers=[accepting]) as vl: + msgs = vl.feed_chunks([SILENT_CHUNK] * 5) + vl.assert_record_begin_emitted(msgs) + +Example — full loop driven from an audio file:: + + from ovoscope.voice_loop import MiniVoiceLoop, MockStreamingSTT + + stt = MockStreamingSTT(transcript="what time is it") + with MiniVoiceLoop(stt_instance=stt) as vl: + msgs = vl.feed_file("command.wav") + assert any(m.msg_type == "recognizer_loop:utterance" for m in msgs) + +The verifier gate lives inside ``DinkumVoiceLoop._detect_ww`` and is only present +in ovos-dinkum-listener builds that ship the hotword-verifier feature +(``HotwordContainer.verify``). On a build without it the gate is absent and the +detection is never suppressed — assert accordingly for the version under test. +""" + +from __future__ import annotations + +import io +import wave +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple, Union + +from ovos_bus_client.message import Message +from ovos_utils.fakebus import FakeBus + +# Re-export the engine mocks so callers have a single import site for the +# voice-loop harness. +from ovoscope.listener import MockHotWordEngine, MockVADEngine # noqa: F401 + + +# --------------------------------------------------------------------------- +# Internal helpers +# --------------------------------------------------------------------------- + +def _read_audio( + audio: Union[bytes, str, Path], + sample_rate: int, + sample_width: int, + sample_channels: int, +) -> Tuple[bytes, int, int, int]: + """Read raw PCM from a WAV file/bytes (or raw PCM) for the file mic. + + Args: + audio: Path to a ``.wav`` file, WAV bytes, or raw PCM bytes. + sample_rate: Fallback sample rate when no WAV header is present. + sample_width: Fallback sample width (bytes). + sample_channels: Fallback channel count. + + Returns: + ``(pcm_bytes, sample_rate, sample_width, sample_channels)``. + """ + if isinstance(audio, (str, Path)): + with open(audio, "rb") as fh: + raw = fh.read() + else: + raw = audio + + try: + with wave.open(io.BytesIO(raw)) as wf: + sample_rate = wf.getframerate() + sample_width = wf.getsampwidth() + sample_channels = wf.getnchannels() + pcm = wf.readframes(wf.getnframes()) + return pcm, sample_rate, sample_width, sample_channels + except (wave.Error, EOFError): + # not a WAV container — treat input as raw PCM + return raw, sample_rate, sample_width, sample_channels + + +# --------------------------------------------------------------------------- +# Mock plugin stand-ins for the DinkumVoiceLoop dependencies +# --------------------------------------------------------------------------- + +class _MockMicrophone: + """Silent stand-in for the listener ``Microphone`` plugin. + + Used when no audio file is driven; ``_detect_ww`` does not read from the + mic, but the dataclass requires one and a few derived values + (``sample_rate``, ``seconds_per_chunk``) are referenced by adjacent loop + stages. + + Args: + sample_rate: Audio sample rate in Hz. + sample_width: Sample width in bytes. + sample_channels: Channel count. + chunk_size: Bytes per read. + """ + + def __init__( + self, + sample_rate: int = 16000, + sample_width: int = 2, + sample_channels: int = 1, + chunk_size: int = 2048, + ) -> None: + self.sample_rate: int = sample_rate + self.sample_width: int = sample_width + self.sample_channels: int = sample_channels + self.chunk_size: int = chunk_size + + @property + def seconds_per_chunk(self) -> float: + """Duration in seconds of one ``chunk_size`` read.""" + frames = self.chunk_size / max(self.sample_width, 1) + return frames / max(self.sample_rate, 1) + + def read_chunk(self) -> bytes: + """Return a silent chunk (the harness feeds audio explicitly).""" + return b"\x00" * self.chunk_size + + def start(self) -> None: + """No-op start hook.""" + + def stop(self) -> None: + """No-op stop hook.""" + + +class MockFileMicrophone: + """File-backed ``Microphone`` plugin for the voice loop. + + Streams an arbitrary audio file (or raw PCM) into ``DinkumVoiceLoop.run()`` + one ``chunk_size`` frame at a time, then appends a tail of silent frames so + a silence-based VAD can detect the end of the command. When every frame has + been read, :meth:`read_chunk` invokes :attr:`on_exhausted` (used by + :class:`MiniVoiceLoop` to stop the loop) and returns ``None``. + + Args: + audio: Path to a ``.wav`` file, WAV bytes, or raw PCM bytes. + chunk_size: Bytes per :meth:`read_chunk`. + sample_rate: Fallback sample rate when the input has no WAV header. + sample_width: Fallback sample width (bytes). + sample_channels: Fallback channel count. + silence_chunks: Number of trailing silent frames appended after the + file so the command can end and the loop can return to wake-word + detection. + + Attributes: + on_exhausted: Optional zero-arg callback invoked once the audio is fully + consumed (set by :class:`MiniVoiceLoop` to stop the loop). + """ + + def __init__( + self, + audio: Union[bytes, str, Path], + chunk_size: int = 2048, + sample_rate: int = 16000, + sample_width: int = 2, + sample_channels: int = 1, + silence_chunks: int = 25, + ) -> None: + pcm, sr, sw, ch = _read_audio( + audio, sample_rate, sample_width, sample_channels + ) + self.sample_rate: int = sr + self.sample_width: int = sw + self.sample_channels: int = ch + self.chunk_size: int = chunk_size + + self._chunks: List[bytes] = [ + pcm[i:i + chunk_size] for i in range(0, len(pcm), chunk_size) + ] + self._chunks.extend(b"\x00" * chunk_size for _ in range(silence_chunks)) + self._idx: int = 0 + self.on_exhausted: Optional[Any] = None + + @property + def seconds_per_chunk(self) -> float: + """Duration in seconds of one ``chunk_size`` read.""" + frames = self.chunk_size / max(self.sample_width, 1) + return frames / max(self.sample_rate, 1) + + def read_chunk(self) -> Optional[bytes]: + """Return the next audio frame, or ``None`` when exhausted. + + On exhaustion the :attr:`on_exhausted` callback is invoked so the loop + can stop. + """ + if self._idx >= len(self._chunks): + if self.on_exhausted is not None: + self.on_exhausted() + return None + chunk = self._chunks[self._idx] + self._idx += 1 + return chunk + + def start(self) -> None: + """No-op start hook.""" + + def stop(self) -> None: + """No-op stop hook.""" + + +class MockStreamingSTT: + """Configurable streaming STT engine for the voice loop. + + Returns a fixed transcript when the recorded command is finalized. An empty + transcript yields ``recognizer_loop:speech.recognition.unknown`` (matching + the real listener), which is useful for asserting the silent-recording path. + + Args: + transcript: Text returned by :meth:`transcribe`. Empty string means "no + transcription". + confidence: Confidence score paired with the transcript. + lang: Language tag reported by the ``lang`` property. + """ + + can_stream: bool = False + + def __init__( + self, + transcript: str = "", + confidence: float = 1.0, + lang: str = "en-us", + ) -> None: + self.transcript: str = transcript + self.confidence: float = confidence + self._lang: str = lang + self.fed_chunks: int = 0 + + @property + def lang(self) -> str: + """Language tag for transcription.""" + return self._lang + + def stream_start(self, *args: Any, **kwargs: Any) -> None: + """Begin a streaming transcription session (no-op).""" + + def stream_data(self, chunk: bytes) -> None: + """Feed audio to the streaming session.""" + self.fed_chunks += 1 + + def stream_stop(self) -> str: + """End the session and return the transcript text.""" + return self.transcript + + def execute(self, audio: Any = None, language: Optional[str] = None) -> Optional[str]: + """Non-streaming transcription (used by mycroft-classic-listener). + + Args: + audio: Recorded audio clip (ignored). + language: Language override (ignored). + + Returns: + The configured transcript, or ``None`` when empty (so the classic + consumer emits ``recognizer_loop:speech.recognition.unknown``). + """ + return self.transcript or None + + def transcribe( + self, audio: Any = None, lang: Optional[str] = None + ) -> List[Tuple[str, float]]: + """Return the configured transcript as ``[(text, confidence)]``. + + Always returns a single ``(text, confidence)`` pair — the text is the + empty string when no transcript is configured. This matches the + ``transcribe(audio)[0][0]`` access pattern used by ovos-simple-listener + while still letting the dinkum loop's confidence filter run; harness + callbacks treat an empty transcript as "no utterance". + + Args: + audio: Ignored (the loop streams audio via :meth:`stream_data`). + lang: Ignored language override. + + Returns: + ``[(transcript, confidence)]``. + """ + return [(self.transcript, self.confidence)] + + def shutdown(self) -> None: + """Graceful shutdown (no-op).""" + + +class _MockTransformers: + """No-op ``AudioTransformersService`` stand-in. + + Args: + bus: The :class:`FakeBus` the loop is wired to. + """ + + def __init__(self, bus: FakeBus) -> None: + self.bus: FakeBus = bus + self.hotword_chunks: List[bytes] = [] + + def feed_hotword(self, chunk: bytes) -> None: + """Record a chunk forwarded after wake-word detection.""" + self.hotword_chunks.append(chunk) + + def feed_audio(self, chunk: bytes) -> None: + """Consume a non-speech chunk (no-op).""" + + def feed_speech(self, chunk: bytes) -> None: + """Consume a speech chunk (no-op).""" + + def transform(self, chunk: bytes) -> Tuple[bytes, Dict[str, Any]]: + """Return the chunk unchanged with empty context.""" + return chunk, {} + + def shutdown(self) -> None: + """Graceful shutdown (no-op).""" + + +class MiniHotwordContainer: + """Controllable hotword container for :class:`MiniVoiceLoop`. + + Implements the subset of ``ovos_dinkum_listener.voice_loop.HotwordContainer`` + that the voice loop relies on, without requiring the wrapped engines to be + real ``HotWordEngine`` subclasses — so ovoscope's :class:`MockHotWordEngine` + can drive the loop directly. + + Every registered engine is treated as a **listen word** (it triggers the STT + stage). :meth:`update` and :meth:`found` are state-aware so the loop's + separate hotword-detection branch (``_detect_hot``) does not double-count or + mis-fire the listen engines. + + The :meth:`verify` chain mirrors the real container's **fail-open** + semantics: a verifier that returns ``False`` suppresses the detection; a + verifier that raises is skipped (the detection survives). + + Args: + ww_instances: Mapping of wake-word name → engine instance (each engine + implements ``update(chunk)``, ``found_wake_word() -> bool`` and + ``reset()``). + verifiers: Optional list of verifier objects with a + ``verify(ww_audio: bytes) -> bool`` method. + + Attributes: + state: Listening state, assigned by the voice loop during detection. + reload_on_failure: Always ``False`` (no engine reloading in tests). + """ + + def __init__( + self, + ww_instances: Dict[str, Any], + verifiers: Optional[List[Any]] = None, + ) -> None: + self._engines: Dict[str, Any] = dict(ww_instances) + self.verifiers: List[Any] = list(verifiers) if verifiers else [] + self.state: Any = None + self.reload_on_failure: bool = False + + def _active_engines(self) -> Dict[str, Any]: + """Return the engines relevant to the current listening state. + + All registered engines are listen words, so they are active in the + ``LISTEN`` state (and when no state has been set yet, as used by direct + ``_detect_ww`` feeding). In every other state — ``HOTWORD``, + ``WAKEUP``, ``RECORDING`` — no engines are active. + """ + name = getattr(self.state, "name", None) + if name in (None, "LISTEN"): + return self._engines + return {} + + def update(self, chunk: bytes) -> None: + """Feed *chunk* to the engines active in the current state. + + Args: + chunk: Raw PCM audio bytes. + """ + for engine in self._active_engines().values(): + engine.update(chunk) + + def found(self) -> Optional[str]: + """Return the name of the first active engine reporting a detection. + + A detected engine is reset so a stale ``update_count`` does not re-fire + the wake word on subsequent chunks (e.g. during the silence tail of a + file-driven run). + + Returns: + The wake-word name, or ``None`` if no active engine fired. + """ + for name, engine in self._active_engines().items(): + if engine.found_wake_word(): + engine.reset() + return name + return None + + def get_ww(self, ww: str) -> Dict[str, Any]: + """Return metadata for wake word *ww*. + + Mirrors the keys ``DinkumVoiceLoop`` and the listener's hotword callback + read. The wake word is treated as a listen word (it triggers the STT + stage), with no confirmation sound. + + Args: + ww: Wake-word name. + + Returns: + Metadata dict for the wake word. + + Raises: + ValueError: If *ww* is not registered. + """ + if ww not in self._engines: + raise ValueError(f"Requested ww not defined: {ww}") + engine = self._engines[ww] + return { + "key_phrase": ww, + "module": getattr(engine, "key_phrase", ww), + "engine": engine.__class__.__name__, + "sound": None, + "listen": True, + "utterance": None, + "stt_lang": "en-us", + "bus_event": None, + "wakeup": False, + "stopword": False, + } + + def verify(self, ww_audio: bytes) -> bool: + """Run the verifier chain against the wake-word audio. + + Fail-open: only an explicit ``False`` return suppresses the detection; + a verifier that raises is skipped. + + Args: + ww_audio: Raw PCM bytes of the audio that triggered the engine. + + Returns: + ``True`` if every verifier accepts (or none are configured), + ``False`` if any verifier rejects the audio. + """ + for verifier in self.verifiers: + try: + if not verifier.verify(ww_audio): + return False + except Exception: + # fail-open: a raising verifier does not discard the detection + pass + return True + + def reset(self) -> None: + """Reset all wrapped engines (called by the loop after a command).""" + for engine in self._engines.values(): + try: + engine.reset() + except Exception: + pass + + def shutdown(self) -> None: + """Shut down all wrapped engines gracefully.""" + for engine in self._engines.values(): + try: + engine.shutdown() + except Exception: + pass + + +# --------------------------------------------------------------------------- +# Shared harness base +# --------------------------------------------------------------------------- + +class ListenerHarness: + """Base for in-process listener-service harnesses. + + Owns a :class:`FakeBus`, captures every ``Message`` emitted on it, and + provides the common ``recognizer_loop:*`` assertion helpers. Concrete + backends (:class:`MiniVoiceLoop` for ovos-dinkum-listener, + ``MiniSimpleListener`` for ovos-simple-listener, ``MiniClassicListener`` for + mycroft-classic-listener) wire their specific listener to this bus and add a + drive method (``feed_file`` / ``feed_chunks``). + + Args: + bus: Optional :class:`FakeBus` to capture on. Defaults to a fresh bus. + modernize: When a fresh bus is created, also emit the ovos.* spec topic + whenever a legacy topic is emitted (legacy producer -> spec + listener). The listener callbacks emit legacy ``recognizer_loop:*`` + topics; this bridge is what lets a spec-topic subscriber observe + them. Ignored when *bus* is supplied. + emit_legacy: When a fresh bus is created, also emit the legacy topic + whenever an ovos.* spec topic is emitted (spec producer -> legacy + listener). Set both False to exercise a single namespace with no + bridging. Ignored when *bus* is supplied. + """ + + def __init__(self, bus: Optional[FakeBus] = None, + modernize: bool = True, emit_legacy: bool = True) -> None: + self.bus: FakeBus = bus if bus is not None else FakeBus( + modernize=modernize, emit_legacy=emit_legacy) + self._messages: List[Message] = [] + self._last_messages: List[Message] = [] + self.bus.on("message", self._capture) + + def _capture(self, msg: Any) -> None: + if isinstance(msg, str): + try: + msg = Message.deserialize(msg) + except Exception: + return + self._messages.append(msg) + + # ------------------------------------------------------------------ + # File microphone + # ------------------------------------------------------------------ + + @staticmethod + def _build_file_mic( + audio: Union[bytes, str, Path], + silence_chunks: int, + chunk_size: int, + ) -> MockFileMicrophone: + """Build a :class:`MockFileMicrophone` for *audio*.""" + return MockFileMicrophone( + audio, chunk_size=chunk_size, silence_chunks=silence_chunks + ) + + # ------------------------------------------------------------------ + # Assertion helpers + # ------------------------------------------------------------------ + + @staticmethod + def _has(messages: List[Message], msg_type: str) -> bool: + return any(m.msg_type == msg_type for m in messages) + + def _resolve(self, messages: Optional[List[Message]]) -> List[Message]: + return messages if messages is not None else self._last_messages + + def assert_record_begin_emitted( + self, messages: Optional[List[Message]] = None + ) -> List[Message]: + """Assert ``recognizer_loop:record_begin`` was emitted. + + Args: + messages: Messages to check; defaults to the last feed result. + + Returns: + The checked message list. + + Raises: + AssertionError: If no record-begin event is present. + """ + msgs = self._resolve(messages) + assert self._has(msgs, "recognizer_loop:record_begin"), ( + "Expected 'recognizer_loop:record_begin' but it was not emitted. " + f"Captured: {[m.msg_type for m in msgs]}" + ) + return msgs + + def assert_wakeword_detected( + self, messages: Optional[List[Message]] = None + ) -> List[Message]: + """Assert a wake word was detected and recording began. + + Checks for both ``recognizer_loop:wakeword`` and + ``recognizer_loop:record_begin``. + + Args: + messages: Messages to check; defaults to the last feed result. + + Returns: + The checked message list. + + Raises: + AssertionError: If either expected event is missing. + """ + msgs = self._resolve(messages) + captured = [m.msg_type for m in msgs] + assert self._has(msgs, "recognizer_loop:wakeword"), ( + "Expected 'recognizer_loop:wakeword' but it was not emitted. " + f"Captured: {captured}" + ) + assert self._has(msgs, "recognizer_loop:record_begin"), ( + "Expected 'recognizer_loop:record_begin' but it was not emitted. " + f"Captured: {captured}" + ) + return msgs + + def assert_wakeword_suppressed( + self, messages: Optional[List[Message]] = None + ) -> List[Message]: + """Assert no wake-word recording was triggered. + + Verifies that neither ``recognizer_loop:wakeword`` nor + ``recognizer_loop:record_begin`` was emitted. + + Args: + messages: Messages to check; defaults to the last feed result. + + Returns: + The checked message list. + + Raises: + AssertionError: If a wake-word or record-begin event is present. + """ + msgs = self._resolve(messages) + captured = [m.msg_type for m in msgs] + assert not self._has(msgs, "recognizer_loop:record_begin"), ( + "Expected wake word to be suppressed, but " + f"'recognizer_loop:record_begin' was emitted. Captured: {captured}" + ) + assert not self._has(msgs, "recognizer_loop:wakeword"), ( + "Expected wake word to be suppressed, but " + f"'recognizer_loop:wakeword' was emitted. Captured: {captured}" + ) + return msgs + + def assert_utterance_emitted( + self, + utterance: Optional[str] = None, + messages: Optional[List[Message]] = None, + ) -> List[Message]: + """Assert a ``recognizer_loop:utterance`` was emitted. + + Args: + utterance: When given, also assert this exact text is among the + emitted utterances. + messages: Messages to check; defaults to the last feed result. + + Returns: + The checked message list. + + Raises: + AssertionError: If no utterance (or the named one) was emitted. + """ + msgs = self._resolve(messages) + utts: List[str] = [] + for m in msgs: + if m.msg_type == "recognizer_loop:utterance": + utts.extend(m.data.get("utterances", [])) + assert utts, ( + "Expected 'recognizer_loop:utterance' but it was not emitted. " + f"Captured: {[m.msg_type for m in msgs]}" + ) + if utterance is not None: + assert utterance in utts, ( + f"Expected utterance {utterance!r} but got: {utts}" + ) + return msgs + + # ------------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------------ + + def shutdown(self) -> None: + """Release backend resources (overridden by subclasses).""" + + def __enter__(self) -> "ListenerHarness": + return self + + def __exit__(self, *_: Any) -> None: + self.shutdown() + + +# --------------------------------------------------------------------------- +# MiniVoiceLoop +# --------------------------------------------------------------------------- + +class MiniVoiceLoop(ListenerHarness): + """In-process ``DinkumVoiceLoop`` harness for bus-sequence testing. + + Captures every ``Message`` emitted on the ``FakeBus`` as audio flows through + the loop. The voice-loop callbacks are wired to emit the same bus events as + the real listener service (``recognizer_loop:wakeword``, + ``recognizer_loop:record_begin``, ``recognizer_loop:record_end``, + ``recognizer_loop:utterance`` / ``…:speech.recognition.unknown``). + + Args: + voice_loop: A pre-built ``DinkumVoiceLoop`` instance. When provided, the + caller owns its plugins/callbacks, which should emit on this + harness's *bus* to be captured. When ``None``, a loop is built from + the mock arguments below. + ww_instances: Mapping of wake-word name → engine instance. Defaults to a + single :class:`MockHotWordEngine` keyed ``"hey_mycroft"``. + verifiers: Optional verifier objects (``verify(audio) -> bool``) gating + detection. + vad_instance: Optional VAD engine (defaults to :class:`MockVADEngine`). + stt_instance: Optional streaming STT engine (defaults to a + :class:`MockStreamingSTT` returning no transcript). Used by + :meth:`feed_file`. + bus: Optional :class:`FakeBus` to capture on. Defaults to a fresh bus. + modernize: When a fresh bus is created, also emit the ovos.* spec topic + whenever a legacy topic is emitted (legacy producer -> spec + listener). Ignored when *bus* is supplied. + emit_legacy: When a fresh bus is created, also emit the legacy topic + whenever an ovos.* spec topic is emitted. Set both False to exercise + a single namespace with no bridging. Ignored when *bus* is supplied. + + Raises: + RuntimeError: If *voice_loop* is ``None`` and ovos-dinkum-listener is not + installed. + + Example:: + + from ovoscope.voice_loop import MiniVoiceLoop, MockHotWordEngine + + ww = MockHotWordEngine("hey_mycroft", trigger_after=3) + with MiniVoiceLoop(ww_instances={"hey_mycroft": ww}) as vl: + msgs = vl.feed_chunks([b"\\x00" * 512] * 5) + vl.assert_record_begin_emitted(msgs) + """ + + def __init__( + self, + voice_loop: Optional[Any] = None, + *, + ww_instances: Optional[Dict[str, Any]] = None, + verifiers: Optional[List[Any]] = None, + vad_instance: Optional[Any] = None, + stt_instance: Optional[Any] = None, + bus: Optional[FakeBus] = None, + modernize: bool = True, + emit_legacy: bool = True, + ) -> None: + super().__init__(bus, modernize=modernize, emit_legacy=emit_legacy) + + self.hotwords: Optional[MiniHotwordContainer] = None + if voice_loop is not None: + self.voice_loop: Any = voice_loop + hw = getattr(voice_loop, "hotwords", None) + if isinstance(hw, MiniHotwordContainer): + self.hotwords = hw + else: + self.voice_loop = self._build_voice_loop( + ww_instances=ww_instances, + verifiers=verifiers, + vad_instance=vad_instance, + stt_instance=stt_instance, + ) + + # ------------------------------------------------------------------ + # Construction + # ------------------------------------------------------------------ + + def _build_voice_loop( + self, + ww_instances: Optional[Dict[str, Any]], + verifiers: Optional[List[Any]], + vad_instance: Optional[Any], + stt_instance: Optional[Any], + ) -> Any: + """Build a ``DinkumVoiceLoop`` wired to this harness's bus. + + Returns: + A ready-to-feed ``DinkumVoiceLoop`` instance. + + Raises: + RuntimeError: If ovos-dinkum-listener is not installed. + """ + try: + from ovos_dinkum_listener.voice_loop.voice_loop import DinkumVoiceLoop + except ImportError as e: + raise RuntimeError( + "ovos-dinkum-listener is required to build a MiniVoiceLoop. " + "Install it with: pip install ovos-dinkum-listener" + ) from e + + if ww_instances is None: + ww_instances = {"hey_mycroft": MockHotWordEngine("hey_mycroft")} + + self.hotwords = MiniHotwordContainer(ww_instances, verifiers=verifiers) + + return DinkumVoiceLoop( + mic=_MockMicrophone(), + hotwords=self.hotwords, + stt=stt_instance if stt_instance is not None else MockStreamingSTT(), + fallback_stt=None, + vad=vad_instance if vad_instance is not None else MockVADEngine(), + transformers=_MockTransformers(self.bus), + wake_callback=self._emit_record_begin, + record_end_callback=self._emit_record_end, + text_callback=self._emit_stt_text, + listenword_audio_callback=self._emit_wakeword, + hotword_audio_callback=self._emit_wakeword, + ) + + # ------------------------------------------------------------------ + # Bus-emitting callbacks (mirror ovos-dinkum-listener service.py) + # ------------------------------------------------------------------ + + def _emit_record_begin(self) -> None: + """Emit ``recognizer_loop:record_begin`` (the real wake callback).""" + self.bus.emit(Message("recognizer_loop:record_begin")) + + def _emit_record_end(self) -> None: + """Emit ``recognizer_loop:record_end`` (the real record-end callback).""" + self.bus.emit(Message("recognizer_loop:record_end")) + + def _emit_wakeword(self, audio_bytes: bytes, ww_context: Dict[str, Any]) -> None: + """Emit ``recognizer_loop:wakeword`` for a detected listen word. + + Mirrors the listen-word branch of the listener's ``_hotword_audio`` + callback so the captured bus sequence matches the real service. + + Args: + audio_bytes: Raw hotword audio collected by the loop. + ww_context: Wake-word metadata from :meth:`MiniHotwordContainer.get_ww`. + """ + payload = dict(ww_context) + key_phrase = ww_context.get("key_phrase", "") + payload["utterance"] = key_phrase.replace("_", " ").replace("-", " ") + context = { + "client_name": "ovos_dinkum_listener", + "source": "audio", + "destination": ["skills"], + } + self.bus.emit(Message("recognizer_loop:wakeword", payload, context)) + + def _emit_stt_text( + self, transcripts: List[Tuple[str, float]], stt_context: Dict[str, Any] + ) -> None: + """Emit the STT result (mirrors the listener's ``_stt_text`` callback). + + Emits ``recognizer_loop:utterance`` for a non-empty transcript, or + ``recognizer_loop:speech.recognition.unknown`` when transcription is + empty. + + Args: + transcripts: List of ``(text, confidence)`` from the STT engine. + stt_context: Context dict accumulated by the loop. + """ + utts = [t[0] for t in transcripts if t[0]] if transcripts else [] + if utts: + lang = stt_context.get("lang") or "en-us" + self.bus.emit(Message( + "recognizer_loop:utterance", + {"utterances": utts, "lang": lang}, + stt_context, + )) + else: + self.bus.emit(Message( + "recognizer_loop:speech.recognition.unknown", + context=stt_context, + )) + + # ------------------------------------------------------------------ + # Feeding + # ------------------------------------------------------------------ + + def feed_chunks(self, chunks: List[bytes]) -> List[Message]: + """Feed PCM *chunks* through the voice loop's wake-word detection. + + Each chunk is passed to ``DinkumVoiceLoop._detect_ww``; all bus messages + emitted as side-effects are collected and returned. This drives only + the wake-word / verifier stage — use :meth:`feed_file` to run the full + state machine. + + Args: + chunks: Ordered list of raw PCM audio frames. + + Returns: + The list of :class:`Message` objects emitted during this call. + """ + self._messages.clear() + for chunk in chunks: + self.voice_loop._detect_ww(chunk) + self._last_messages = list(self._messages) + return list(self._messages) + + def feed_file( + self, + audio: Union[bytes, str, Path], + *, + silence_tail_chunks: int = 25, + chunk_size: int = 2048, + ) -> List[Message]: + """Run the full ``DinkumVoiceLoop`` state machine over an audio file. + + Swaps in a :class:`MockFileMicrophone` that streams *audio* through the + loop, drives ``start()`` + ``run()`` to completion, and returns every + bus message emitted along the way. A tail of silent frames is appended + so a silence-based VAD can end the command and the loop can finalize the + utterance. + + Args: + audio: Path to a ``.wav`` file, WAV bytes, or raw PCM bytes. + silence_tail_chunks: Trailing silent frames appended after the audio + (gives the VAD a chance to detect end-of-command). + chunk_size: Bytes per microphone read. + + Returns: + The list of :class:`Message` objects emitted during the run. + """ + mic = MockFileMicrophone( + audio, + chunk_size=chunk_size, + silence_chunks=silence_tail_chunks, + ) + mic.on_exhausted = self.voice_loop.stop + self.voice_loop.mic = mic + + self._messages.clear() + self.voice_loop.start() + self.voice_loop.run() + self._last_messages = list(self._messages) + return list(self._messages) + + # ------------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------------ + + def shutdown(self) -> None: + """Shut down the hotword container and wrapped engines.""" + if self.hotwords is not None: + self.hotwords.shutdown() + + +# --------------------------------------------------------------------------- +# Factory +# --------------------------------------------------------------------------- + +def get_mini_voice_loop( + ww_instances: Optional[Dict[str, Any]] = None, + verifiers: Optional[List[Any]] = None, + vad_instance: Optional[Any] = None, + stt_instance: Optional[Any] = None, + bus: Optional[FakeBus] = None, + modernize: bool = True, + emit_legacy: bool = True, +) -> MiniVoiceLoop: + """Factory: create a ready-to-feed :class:`MiniVoiceLoop`. + + Args: + ww_instances: Mapping of wake-word name → engine instance. Defaults to a + single :class:`MockHotWordEngine` keyed ``"hey_mycroft"``. + verifiers: Optional verifier objects gating detection. + vad_instance: Optional VAD engine (defaults to :class:`MockVADEngine`). + stt_instance: Optional streaming STT engine (defaults to + :class:`MockStreamingSTT`). + bus: Optional :class:`FakeBus` to capture on. + modernize: When a fresh bus is created, also emit the ovos.* spec topic + whenever a legacy topic is emitted (legacy producer -> spec + listener). Ignored when *bus* is supplied. + emit_legacy: When a fresh bus is created, also emit the legacy topic + whenever an ovos.* spec topic is emitted. Set both False to exercise + a single namespace with no bridging. Ignored when *bus* is supplied. + + Returns: + A fully initialised :class:`MiniVoiceLoop`. + + Example:: + + from ovoscope.voice_loop import get_mini_voice_loop, MockHotWordEngine + + vl = get_mini_voice_loop( + ww_instances={"hey_mycroft": MockHotWordEngine(trigger_after=3)}, + ) + try: + vl.assert_record_begin_emitted(vl.feed_chunks([b"\\x00" * 512] * 3)) + finally: + vl.shutdown() + """ + return MiniVoiceLoop( + ww_instances=ww_instances, + verifiers=verifiers, + vad_instance=vad_instance, + stt_instance=stt_instance, + bus=bus, + modernize=modernize, + emit_legacy=emit_legacy, + ) + + +# --------------------------------------------------------------------------- +# Declarative test helper +# --------------------------------------------------------------------------- + +@dataclass +class VoiceLoopTest: + """Declarative wake-word → bus-sequence test for the voice loop. + + Drives a :class:`MiniVoiceLoop` and asserts whether the wake-word recording + sequence was triggered. Feeds PCM frames via ``feed_chunks`` by default, or + an audio file via ``feed_file`` when *audio_file* is set. + + Example — verifier gate:: + + from unittest.mock import Mock + from ovoscope.voice_loop import VoiceLoopTest, MockHotWordEngine + + accepting = Mock(); accepting.verify.return_value = True + VoiceLoopTest( + ww_instances={"hey_mycroft": MockHotWordEngine(trigger_after=3)}, + verifiers=[accepting], + audio_chunks=[b"\\x00" * 512] * 5, + expect_record_begin=True, + ).execute() + + Example — full loop from a file:: + + from ovoscope.voice_loop import VoiceLoopTest, MockStreamingSTT + + VoiceLoopTest( + audio_file="command.wav", + stt_instance=MockStreamingSTT(transcript="what time is it"), + expect_utterance="what time is it", + ).execute() + """ + + ww_instances: Optional[Dict[str, Any]] = None + """Mapping of wake-word name → engine instance.""" + + verifiers: Optional[List[Any]] = None + """Verifier objects (``verify(audio) -> bool``) gating detection.""" + + vad_instance: Optional[Any] = None + """Optional VAD engine (defaults to :class:`MockVADEngine`).""" + + stt_instance: Optional[Any] = None + """Optional streaming STT engine (defaults to :class:`MockStreamingSTT`).""" + + audio_chunks: List[bytes] = field( + default_factory=lambda: [b"\x00" * 512] * 5 + ) + """PCM frames fed via ``feed_chunks`` (ignored when *audio_file* is set).""" + + audio_file: Optional[Union[bytes, str, Path]] = None + """When set, run the full loop over this audio via ``feed_file``.""" + + expect_record_begin: bool = True + """Assert ``recognizer_loop:record_begin`` was emitted (``True``) or + suppressed (``False``).""" + + expect_utterance: Optional[str] = None + """When set, assert this utterance text was emitted (implies full-loop).""" + + def execute(self) -> List[Message]: + """Run the test and assert the configured expectations. + + Returns: + The captured :class:`Message` list. + + Raises: + AssertionError: If an expectation is not met. + """ + vl = MiniVoiceLoop( + ww_instances=self.ww_instances, + verifiers=self.verifiers, + vad_instance=self.vad_instance, + stt_instance=self.stt_instance, + ) + try: + if self.audio_file is not None: + messages = vl.feed_file(self.audio_file) + else: + messages = vl.feed_chunks(self.audio_chunks) + + if self.expect_record_begin: + vl.assert_record_begin_emitted(messages) + else: + vl.assert_wakeword_suppressed(messages) + + if self.expect_utterance is not None: + vl.assert_utterance_emitted(self.expect_utterance, messages) + return messages + finally: + vl.shutdown() diff --git a/pyproject.toml b/pyproject.toml index e061f24..6a12195 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,16 @@ dependencies = [ # Alpha pin: 2.0.4 stable not yet released; 2.0.4a2 is the minimum that # includes the FakeBus-compatible SkillManager changes ovoscope depends on. "ovos-core>=2.0.4a2", + # The audio harness emits/observes the ovos.* spec topics while ovos-audio + # still produces the legacy topics. >=0.12.0a1 is the first FakeBus that + # mirrors MessageBusClient's legacy<->ovos.* namespace migration (ovos-utils + # #381), so a legacy producer reaches a spec listener (and vice-versa) on the + # test double. Prerelease floor pin: resolves without --pre. + "ovos-utils>=0.12.0a1", + # ovoscope ships a pytest plugin (pytest11 entry point) -> pytest is a runtime + # dependency. >=8 is required: the pytest_pycollect_makemodule hook dropped the + # 'path' argument in pytest 8. + "pytest>=8", ] classifiers = [ "Programming Language :: Python :: 3", @@ -27,11 +37,33 @@ classifiers = [ [project.optional-dependencies] pydantic = ["ovos-pydantic-models>=0.1.0"] -audio = ["ovos-audio>=1.2.0"] +# The audio harness emits/asserts on the OVOS spec bus namespace via SpecMessage, +# and the FakeBus bridging (ovos-utils) imports NamespaceTranslator — new in +# ovos-spec-tools 0.10.0a1, so that is the real floor. +audio = ["ovos-audio>=1.3.0a1", "ovos-spec-tools>=0.10.0a1"] +# OCP / ovos-media player harnesses (ovoscope.media: OCPPlayerHarness, ...). +media = ["ovos-media>=0.0.2a3"] +# MiniListener plugin_instances path (feed_audio_stream) needs the dinkum +# AudioTransformersService. >=0.7.2a1 is the first release that allows +# ovos-bus-client 2.x (older pins cap it <2.0.0 and conflict with ovos-core). +listener = ["ovos-dinkum-listener>=0.7.2a1"] +# End-to-end TTS intelligibility scoring (WER/CER round-trip via reference STT). +# faster-whisper itself is pulled by the plugin — don't list it here to avoid +# version skew. +tts = [ + "ovos-audio>=1.3.0a1", + "jiwer", + "ovos-utterance-normalizer", + "ovos-stt-plugin-fasterwhisper", +] dev = [ - "ovos-audio>=1.2.0", + "ovos-audio>=1.3.0a1", + "ovos-media>=0.0.2a3", + "ovos-dinkum-listener>=0.7.2a1", "ovos-pydantic-models>=0.1.0", - "pytest", + "jiwer", + "ovos-utterance-normalizer", + "ovos-stt-plugin-fasterwhisper", "pytest-cov", ] diff --git a/test/unittests/test_audio_harness.py b/test/unittests/test_audio_harness.py index 354459a..009c05c 100644 --- a/test/unittests/test_audio_harness.py +++ b/test/unittests/test_audio_harness.py @@ -34,6 +34,7 @@ from ovos_utils.fakebus import FakeBus if AUDIO_AVAILABLE: + from ovos_plugin_manager.templates.tts import TTS from ovoscope.audio import ( AudioCaptureSession, AudioServiceHarness, @@ -439,5 +440,132 @@ def test_assert_sequence_fails_for_missing_type(self) -> None: cap.assert_sequence("recognizer_loop:audio_output_end") +# --------------------------------------------------------------------------- +# TestAudioHarnessNamespaceBridging +# --------------------------------------------------------------------------- + +@unittest.skipUnless(AUDIO_AVAILABLE, "ovos-audio (audio extra) not installed") +class TestAudioHarnessNamespaceBridging(unittest.TestCase): + """The audio harness subscribes on the ovos.* SPEC topics while ovos-audio + emits the LEGACY topics. These tests pin that the FakeBus namespace bridging + is what connects them, and that turning it off isolates a single namespace. + """ + + def test_ducking_works_via_bridging_default(self) -> None: + """Default harness (bridging on): ovos-audio's legacy + recognizer_loop:audio_output_start reaches the spec-subscribed + _lower_volume_on_speak via modernize bridging.""" + with AudioServiceHarness() as h: # modernize/emit_legacy default on + h.play(["http://example.com/song.mp3"]) + h.bus.emit(Message("recognizer_loop:audio_output_start")) + start = time.monotonic() + while h.backend.lower_volume_calls == 0 and time.monotonic() - start < 2.0: + time.sleep(0.01) + h.assert_volume_lowered() + + def test_ducking_via_spec_topic_directly(self) -> None: + """A SPEC producer (ovos.audio.output.started) also reaches the + spec-subscribed ducking handler — the harness exercises the new namespace + natively too.""" + from ovos_spec_tools import SpecMessage + with AudioServiceHarness() as h: + h.play(["http://example.com/song.mp3"]) + h.bus.emit(Message(str(SpecMessage.AUDIO_OUTPUT_STARTED))) + start = time.monotonic() + while h.backend.lower_volume_calls == 0 and time.monotonic() - start < 2.0: + time.sleep(0.01) + h.assert_volume_lowered() + + def test_no_bridging_isolates_legacy_from_spec(self) -> None: + """With bridging OFF, a legacy emit does NOT reach the spec-subscribed + ducking handler — proving the harness can exercise a single namespace.""" + with AudioServiceHarness(modernize=False, emit_legacy=False) as h: + h.play(["http://example.com/song.mp3"]) + h.bus.emit(Message("recognizer_loop:audio_output_start")) + time.sleep(0.3) # give any (incorrect) bridge a chance to fire + self.assertEqual(h.backend.lower_volume_calls, 0) + + def test_speak_lifecycle_via_bridging(self) -> None: + """PlaybackService emits legacy audio_output_start/end; the harness + observes them on the spec topics via bridging (default on).""" + with PlaybackServiceHarness() as h: + h.speak("namespace test") + h.assert_audio_output_started() + h.assert_audio_output_ended() + + +@unittest.skipUnless(AUDIO_AVAILABLE, "ovos-audio (audio extra) not installed") +class TestPlaybackServiceHarnessIsolation(unittest.TestCase): + """Repeated, independent harness instances must not interfere. + + Regression for the shared ``TTS.playback`` class-attribute hazard: a + garbage-collected MockTTS from an earlier harness used to stop the + PlaybackThread of a *later*, still-running harness (via the inherited + ``TTS.__del__`` -> ``TTS.stop`` -> ``TTS.playback.stop()`` chain). The + victim thread terminated mid-run, its queued speak never played, and the + next ``speak()`` hung until timeout. Because GC timing is nondeterministic + this manifested as a flaky ``TimeoutError`` only after several + create/destroy cycles. + """ + + def test_many_sequential_harnesses_each_complete_speaks(self) -> None: + """Boot and tear down many harnesses, forcing GC between them, and + require every speak in every harness to complete deterministically.""" + import gc + + for i in range(12): + with PlaybackServiceHarness() as h: + for tag in ("a", "b", "c"): + # unique sentences so the persistent TTS cache never + # short-circuits synthesis — each must drive real playback + h.speak(f"iter {i} part {tag}", timeout=8.0) + self.assertIn(f"iter {i} part {tag}", + h.mock_tts.spoken_utterances) + # provoke collection of the just-exited MockTTS *now*, while a + # fresh harness will shortly own TTS.playback. Pre-fix, this is + # exactly what killed the next harness's playback thread. + gc.collect() + + def test_stale_mock_destructor_does_not_kill_live_thread(self) -> None: + """A finished harness's MockTTS destructor must not terminate the + playback thread that a *later* harness now owns. + + Deterministic reproduction of the GC race: keep a reference to harness + A's MockTTS so it outlives A, open harness B (which registers its own + thread on the shared ``TTS.playback`` class attribute), then run A's + destructor. Pre-fix, ``MockTTS.__del__`` chained into + ``TTS.playback.stop()`` and terminated B's live thread; B's next speak + would then hang. With the no-op destructor, B is unaffected. + """ + # Harness A — produce a MockTTS that survives the context exit. + with PlaybackServiceHarness() as ha: + ha.speak("harness A warmup", timeout=8.0) + stale_mock = ha.mock_tts + + # Harness B now owns the shared TTS.playback thread. + with PlaybackServiceHarness() as hb: + self.assertIs(TTS.playback, hb.svc.playback_thread) + self.assertTrue(hb.svc.playback_thread.is_alive()) + + # Fire harness A's destructor explicitly (what GC would do). + stale_mock.__del__() + + # The precise invariant: A's destructor must not have flagged B's + # thread for termination. ``_terminated`` is checked at the top of + # the playback loop, so a single in-flight speak can still slip + # through even when set — but the thread would then exit on its next + # iteration, hanging a subsequent speak. Assert the flag directly. + self.assertFalse( + hb.svc.playback_thread._terminated, + "stale MockTTS destructor terminated the live playback thread", + ) + + # And B must keep working across multiple speaks (the loop must not + # have exited). + for n in range(3): + hb.speak(f"harness B speak {n}", timeout=8.0) + self.assertTrue(hb.svc.playback_thread.is_alive()) + + if __name__ == "__main__": unittest.main() diff --git a/test/unittests/test_capture_session.py b/test/unittests/test_capture_session.py index bad5abd..f2ed5c5 100644 --- a/test/unittests/test_capture_session.py +++ b/test/unittests/test_capture_session.py @@ -161,6 +161,50 @@ def test_multiple_eof_types(self): self.assertIn("test.x", types) self.assertNotIn("test.late", types) + def test_eof_count_waits_for_n_occurrences(self): + """With eof_count>1, capture continues until an eof topic is seen that + many times — for scenarios with N concurrent lifecycles each terminating + on the same eof topic.""" + cs = CaptureSession(self.mc, + eof_msgs=["test.eof"], + eof_count=2, + ignore_messages=[]) + self._emit_after(0.05, Message("test.eof")) # 1st eof — must NOT stop + self._emit_after(0.10, Message("test.between")) # captured (after 1st eof) + self._emit_after(0.15, Message("test.eof")) # 2nd eof — stops capture + self._emit_after(0.30, Message("test.after")) # must NOT appear + + cs.capture(Message("test.trigger"), timeout=3) + msgs = cs.finish() + types = [m.msg_type for m in msgs] + + self.assertIn("test.between", types, + "a message between the 1st and 2nd eof must be captured") + self.assertEqual(types.count("test.eof"), 2, + "both eof occurrences are captured") + self.assertNotIn("test.after", types, + "message after the Nth eof must not be captured") + + def test_eof_count_resets_between_captures(self): + """The eof counter resets per capture() call so eof_count applies fresh.""" + cs = CaptureSession(self.mc, + eof_msgs=["test.eof"], + eof_count=2, + ignore_messages=[]) + self._emit_after(0.05, Message("test.eof")) + self._emit_after(0.10, Message("test.eof")) + cs.capture(Message("test.trigger1"), timeout=3) + cs.finish() + # a second capture must again require 2 eofs, not be already-done + cs2 = CaptureSession(self.mc, eof_msgs=["test.eof"], eof_count=2, + ignore_messages=[]) + self._emit_after(0.05, Message("test.eof")) + self._emit_after(0.10, Message("test.mid")) + self._emit_after(0.15, Message("test.eof")) + cs2.capture(Message("test.trigger2"), timeout=3) + types = [m.msg_type for m in cs2.finish()] + self.assertIn("test.mid", types) + def test_capture_timeout_returns_partial_results(self): """If the EOF never fires, capture() must return after the timeout and finish() must still return whatever was captured.""" diff --git a/test/unittests/test_classic_listener.py b/test/unittests/test_classic_listener.py new file mode 100644 index 0000000..3b7607e --- /dev/null +++ b/test/unittests/test_classic_listener.py @@ -0,0 +1,213 @@ +# Copyright 2024 Jarbas AI +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for the MiniClassicListener harness (mycroft-classic-listener). + +The event-bridge test does not need the classic listener installed (it drives a +plain EventEmitter). The file-drive test exercises the real RecognizerLoop and +is gated on the package being importable. +""" +import io +import unittest +import wave + +from ovos_bus_client.message import Message +from ovos_utils.fakebus import FakeBus + +from ovoscope.classic_listener import ( + MiniClassicListener, + bridge_recognizer_loop_to_bus, + classic_listener_available, +) +from ovoscope.voice_loop import MockHotWordEngine, MockStreamingSTT + +try: + from pyee import EventEmitter + HAS_PYEE = True +except ImportError: + HAS_PYEE = False + +try: + from ovos_spec_tools import SpecMessage + HAS_SPEC = True +except ImportError: + HAS_SPEC = False + +HAS_CLASSIC = classic_listener_available() + + +def _wav(speech_seconds=2.0, sample_rate=16000, sample_width=2): + buf = io.BytesIO() + with wave.open(buf, "wb") as w: + w.setframerate(sample_rate) + w.setsampwidth(sample_width) + w.setnchannels(1) + w.writeframes(b"\x10\x20" * int(sample_rate * speech_seconds)) + return buf.getvalue() + + +@unittest.skipUnless(HAS_PYEE, "pyee not installed") +class TestEventBridge(unittest.TestCase): + """The RecognizerLoop event → FakeBus bridge (no classic listener needed).""" + + def test_bridge_translates_events(self): + """Internal loop events become recognizer_loop:* bus messages.""" + loop = EventEmitter() + harness = MiniClassicListener(recognizer_loop=loop) + + loop.emit("recognizer_loop:record_begin") + loop.emit("recognizer_loop:wakeword", {"utterance": "hey mycroft"}) + loop.emit("recognizer_loop:utterance", + {"utterances": ["hello world"], "lang": "en-us"}) + loop.emit("recognizer_loop:record_end") + harness._last_messages = list(harness._messages) + + harness.assert_record_begin_emitted() + harness.assert_wakeword_detected() + harness.assert_utterance_emitted("hello world") + + def test_bridge_unknown_event(self): + """The unknown event is forwarded to the bus.""" + bus = FakeBus() + seen = [] + bus.on("message", lambda m: seen.append( + m if isinstance(m, str) else m.serialize())) + loop = EventEmitter() + bridge_recognizer_loop_to_bus(loop, bus) + loop.emit("recognizer_loop:speech.recognition.unknown") + self.assertTrue( + any("speech.recognition.unknown" in s for s in seen) + ) + + def test_feed_file_requires_built_loop(self): + """feed_file is unavailable when an external loop was supplied.""" + harness = MiniClassicListener(recognizer_loop=EventEmitter()) + with self.assertRaises(RuntimeError): + harness.feed_file(b"\x00" * 1024) + + +@unittest.skipUnless(HAS_CLASSIC, "mycroft-classic-listener not installed") +class TestMiniClassicListenerFileDrive(unittest.TestCase): + """Best-effort file-driven test against a real RecognizerLoop.""" + + def test_full_sequence_with_utterance(self): + """Driving an audio file yields record-begin and the utterance.""" + with MiniClassicListener( + wakeword=MockHotWordEngine("hey_mycroft", trigger_after=1), + stt_instance=MockStreamingSTT(transcript="hello world"), + ) as cl: + msgs = cl.feed_file(_wav(2.0), tail_silence_seconds=3.0, timeout=20) + types = [m.msg_type for m in msgs] + self.assertIn("recognizer_loop:record_begin", types) + self.assertIn("recognizer_loop:record_end", types) + cl.assert_utterance_emitted("hello world", msgs) + + +@unittest.skipUnless(HAS_PYEE, "pyee not installed") +@unittest.skipUnless(HAS_SPEC, "ovos-spec-tools not installed") +class TestClassicListenerNamespaceBridging(unittest.TestCase): + """The classic listener bridge emits the LEGACY recognizer_loop:* / + mycroft.awoken topics, while OVOS migrates them to the ovos.* SPEC namespace. + These tests pin that the harness FakeBus namespace bridging connects the two, + and that turning it off isolates a single namespace. + + Migrated pairs exercised here (legacy -> spec): + recognizer_loop:utterance -> ovos.utterance.handle + recognizer_loop:record_begin -> ovos.listener.record.started + recognizer_loop:record_end -> ovos.listener.record.ended + mycroft.awoken -> ovos.listener.awoken + """ + + def _harness(self, **kwargs): + """Build a MiniClassicListener around a plain EventEmitter loop. + + No classic listener install is needed — the bridge drives any + EventEmitter and re-emits its events onto the harness FakeBus. + """ + return MiniClassicListener(recognizer_loop=EventEmitter(), **kwargs) + + def test_utterance_legacy_reaches_spec_default(self): + """Default harness (bridging on): the bridge's legacy + recognizer_loop:utterance reaches an ovos.utterance.handle subscriber.""" + h = self._harness() # modernize/emit_legacy default on + seen = [] + h.bus.on(str(SpecMessage.UTTERANCE), lambda m: seen.append(m.msg_type)) + h.loop.emit("recognizer_loop:utterance", + {"utterances": ["hello world"], "lang": "en-us"}) + self.assertIn(str(SpecMessage.UTTERANCE), seen) + + def test_utterance_spec_reaches_legacy_default(self): + """Default harness (bridging on): a SPEC producer of + ovos.utterance.handle reaches a legacy recognizer_loop:utterance + subscriber — the new namespace is exercised natively too.""" + h = self._harness() + seen = [] + h.bus.on("recognizer_loop:utterance", lambda m: seen.append(m.msg_type)) + h.bus.emit(Message(str(SpecMessage.UTTERANCE), + {"utterances": ["hello world"]})) + self.assertIn("recognizer_loop:utterance", seen) + + def test_record_begin_end_bridge_to_spec(self): + """The bridge's record_begin/record_end reach their spec counterparts.""" + h = self._harness() + started, ended = [], [] + h.bus.on(str(SpecMessage.LISTENER_RECORD_STARTED), + lambda m: started.append(m.msg_type)) + h.bus.on(str(SpecMessage.LISTENER_RECORD_ENDED), + lambda m: ended.append(m.msg_type)) + h.loop.emit("recognizer_loop:record_begin") + h.loop.emit("recognizer_loop:record_end") + self.assertIn(str(SpecMessage.LISTENER_RECORD_STARTED), started) + self.assertIn(str(SpecMessage.LISTENER_RECORD_ENDED), ended) + + def test_awoken_bridges_to_spec(self): + """The bridge maps recognizer_loop:awoken -> mycroft.awoken, which + migrates to ovos.listener.awoken via modernize bridging.""" + h = self._harness() + seen = [] + h.bus.on(str(SpecMessage.LISTENER_AWOKEN), lambda m: seen.append(m.msg_type)) + h.loop.emit("recognizer_loop:awoken") + self.assertIn(str(SpecMessage.LISTENER_AWOKEN), seen) + + def test_awoken_spec_reaches_legacy_default(self): + """A SPEC producer of ovos.listener.awoken reaches a legacy + mycroft.awoken subscriber via emit_legacy bridging.""" + h = self._harness() + seen = [] + h.bus.on("mycroft.awoken", lambda m: seen.append(m.msg_type)) + h.bus.emit(Message(str(SpecMessage.LISTENER_AWOKEN))) + self.assertIn("mycroft.awoken", seen) + + def test_no_bridging_isolates_legacy_from_spec(self): + """With bridging OFF, the bridge's legacy emits do NOT reach the + spec-only subscribers — proving a single namespace can be exercised.""" + h = self._harness(modernize=False, emit_legacy=False) + utt, started, ended, awoken = [], [], [], [] + h.bus.on(str(SpecMessage.UTTERANCE), lambda m: utt.append(m)) + h.bus.on(str(SpecMessage.LISTENER_RECORD_STARTED), lambda m: started.append(m)) + h.bus.on(str(SpecMessage.LISTENER_RECORD_ENDED), lambda m: ended.append(m)) + h.bus.on(str(SpecMessage.LISTENER_AWOKEN), lambda m: awoken.append(m)) + + h.loop.emit("recognizer_loop:utterance", {"utterances": ["hi"]}) + h.loop.emit("recognizer_loop:record_begin") + h.loop.emit("recognizer_loop:record_end") + h.loop.emit("recognizer_loop:awoken") + + self.assertEqual(utt, []) + self.assertEqual(started, []) + self.assertEqual(ended, []) + self.assertEqual(awoken, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/unittests/test_e2e_helpers.py b/test/unittests/test_e2e_helpers.py new file mode 100644 index 0000000..776bd57 --- /dev/null +++ b/test/unittests/test_e2e_helpers.py @@ -0,0 +1,171 @@ +"""Fast unit tests for the engine-agnostic helpers in ``ovoscope.e2e``. + +These do not spin up MiniCroft — they exercise the standalone helpers +against a plain ``FakeBus`` so they run in well under a second and can +catch regressions in the helper logic itself. +""" +import threading +import unittest +from unittest.mock import patch + +from ovos_bus_client.message import Message +from ovos_utils.fakebus import FakeBus + +from ovoscope.e2e import ( + detach_intent, + detach_skill, + make_session, + make_utterance_message, + register_adapt_vocab, + register_padatious_entity, + register_padatious_intent, + wait_for_failure, + wait_for_match, +) + + +class TestMakeSession(unittest.TestCase): + def test_only_session_id(self): + s = make_session("abc") + d = s.serialize() + self.assertEqual(d["session_id"], "abc") + + def test_pipeline_override(self): + s = make_session("x", pipeline=["foo", "bar"]) + self.assertEqual(s.serialize()["pipeline"], ["foo", "bar"]) + + def test_blacklists_round_trip(self): + s = make_session( + "y", + blacklisted_intents=["sk:hi"], + blacklisted_skills=["sk"], + ) + data = s.serialize() + self.assertEqual(data["blacklisted_intents"], ["sk:hi"]) + self.assertEqual(data["blacklisted_skills"], ["sk"]) + + +class TestMakeUtteranceMessage(unittest.TestCase): + def test_no_session(self): + m = make_utterance_message("hello there") + self.assertEqual(m.msg_type, "recognizer_loop:utterance") + self.assertEqual(m.data["utterances"], ["hello there"]) + self.assertEqual(m.data["lang"], "en-US") + self.assertNotIn("session", m.context) + + def test_with_session(self): + s = make_session("sid", pipeline=["pipeline-x"]) + m = make_utterance_message("hi", session=s) + self.assertIn("session", m.context) + self.assertEqual(m.context["session"]["pipeline"], ["pipeline-x"]) + + def test_lang_propagates(self): + m = make_utterance_message("oi", lang="pt-PT") + self.assertEqual(m.data["lang"], "pt-PT") + + +class TestRegistrationShims(unittest.TestCase): + def setUp(self): + self.bus = FakeBus() + self.captured = [] + # Wildcard-ish: subscribe to each known type and append. + for t in ( + "padatious:register_intent", + "padatious:register_entity", + "register_vocab", + "detach_intent", + "detach_skill", + ): + self.bus.on(t, self._append) + + def _append(self, msg): + self.captured.append(msg) + + def _types(self): + return [m.msg_type for m in self.captured] + + def test_register_padatious_intent_emits_event(self): + register_padatious_intent( + self.bus, "sk:hi", ["hi", "hello"], settle=0.0 + ) + self.assertEqual(self._types(), ["padatious:register_intent"]) + d = self.captured[0].data + self.assertEqual(d["name"], "sk:hi") + self.assertEqual(d["samples"], ["hi", "hello"]) + self.assertEqual(d["lang"], "en-US") + + def test_register_padatious_entity_emits_event(self): + register_padatious_entity( + self.bus, "item", ["milk", "bread"], settle=0.0 + ) + self.assertEqual(self._types(), ["padatious:register_entity"]) + self.assertEqual(self.captured[0].data["samples"], ["milk", "bread"]) + + def test_register_adapt_vocab_emits_one_event_per_word(self): + register_adapt_vocab( + self.bus, "sk:Light", ["light", "lamp", "bulb"], settle=0.0 + ) + self.assertEqual(self._types(), ["register_vocab"] * 3) + values = [m.data["entity_value"] for m in self.captured] + self.assertEqual(values, ["light", "lamp", "bulb"]) + for m in self.captured: + self.assertEqual(m.data["entity_type"], "sk:Light") + + def test_detach_helpers(self): + detach_intent(self.bus, "sk:hi", settle=0.0) + detach_skill(self.bus, "sk", settle=0.0) + self.assertEqual(self._types(), ["detach_intent", "detach_skill"]) + self.assertEqual(self.captured[0].data["intent_name"], "sk:hi") + self.assertEqual(self.captured[1].data["skill_id"], "sk") + + +class TestWaitHelpers(unittest.TestCase): + def setUp(self): + self.bus = FakeBus() + + def _emit_after(self, delay, msg): + timer = threading.Timer(delay, lambda: self.bus.emit(msg)) + timer.daemon = True + timer.start() + return timer + + def test_wait_for_match_returns_message(self): + self._emit_after(0.05, Message("sk:hello", {"x": 1})) + msg = wait_for_match(self.bus, ["sk:hello"], timeout=1.0) + self.assertIsNotNone(msg) + self.assertEqual(msg.msg_type, "sk:hello") + + def test_wait_for_match_returns_none_on_failure(self): + self._emit_after(0.05, Message("complete_intent_failure", {})) + msg = wait_for_match(self.bus, ["sk:hello"], timeout=1.0) + self.assertIsNone(msg) + + def test_wait_for_match_returns_none_on_timeout(self): + msg = wait_for_match(self.bus, ["sk:nope"], timeout=0.1) + self.assertIsNone(msg) + + def test_wait_for_failure_true_when_event_fires(self): + self._emit_after(0.05, Message("complete_intent_failure", {})) + self.assertTrue(wait_for_failure(self.bus, timeout=1.0)) + + def test_wait_for_failure_false_on_timeout(self): + self.assertFalse(wait_for_failure(self.bus, timeout=0.1)) + + +class TestHarnessClassValidation(unittest.TestCase): + """Without spinning MiniCroft, ensure missing PIPELINE_ID/CONFIG_KEY skips + rather than crashing. This protects against accidental abstract instantiation. + """ + + def test_missing_ids_raises_skip(self): + from ovoscope.e2e import E2EPipelineHarness + + class _Bad(E2EPipelineHarness): + pass + + with self.assertRaises(unittest.SkipTest): + _Bad.setUpClass() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/unittests/test_end2end.py b/test/unittests/test_end2end.py index b3696f5..0fd8e38 100644 --- a/test/unittests/test_end2end.py +++ b/test/unittests/test_end2end.py @@ -19,7 +19,9 @@ # Handler lifecycle messages emitted by add_event() wrappers. # Tests that don't care about lifecycle use this to filter noise. HANDLER_LIFECYCLE = ["mycroft.skill.handler.start", - "mycroft.skill.handler.complete"] + "mycroft.skill.handler.complete", + "recognizer_loop:audio_output_start", + "recognizer_loop:audio_output_end"] # Minimal pipeline: only Adapt-high so we get predictable no-match / match ADAPT_ONLY = ["ovos-adapt-pipeline-plugin-high"] diff --git a/test/unittests/test_end2end_extended.py b/test/unittests/test_end2end_extended.py index de388e2..7a26417 100644 --- a/test/unittests/test_end2end_extended.py +++ b/test/unittests/test_end2end_extended.py @@ -21,7 +21,9 @@ SKILL_ID = "ovoscope-extended-test.test" HANDLER_LIFECYCLE = ["mycroft.skill.handler.start", - "mycroft.skill.handler.complete"] + "mycroft.skill.handler.complete", + "recognizer_loop:audio_output_start", + "recognizer_loop:audio_output_end"] ADAPT_ONLY = ["ovos-adapt-pipeline-plugin-high"] @@ -47,6 +49,22 @@ def handle_async(self, message: Message): self.bus.emit(Message("ovos.utterance.handled", context=message.context)) +class TwoLifecycleSkill(OVOSSkill): + """Emits two lifecycles tagged with distinct context skill_ids, to exercise + the End2EndTest ``skill_id`` filter. Each lifecycle ends on the shared + ``ovos.utterance.handled`` topic (so eof_count=2 spans both).""" + + def initialize(self): + self.add_event("unittest.two_lifecycles", self.handle_two) + + def handle_two(self, message: Message): + for sid in ("life.a", "life.b"): + ctx = dict(message.context) + ctx["skill_id"] = sid + self.bus.emit(Message(f"{sid}.step", context=ctx)) + self.bus.emit(Message("ovos.utterance.handled", context=ctx)) + + def _session(sid="ext-test", pipeline=None): s = Session(sid) s.lang = "en-US" @@ -981,5 +999,77 @@ def test_count_mismatch_prints_first_differing(self): test.execute(timeout=10) +class TestSkillIdFilter(unittest.TestCase): + """The skill_id filter isolates one dispatch lifecycle from concurrent ones.""" + + def setUp(self): + LOG.set_level("ERROR") + self.mc = get_minicroft([SKILL_ID], + extra_skills={SKILL_ID: TwoLifecycleSkill}) + + def tearDown(self): + self.mc.stop() + LOG.set_level("CRITICAL") + + def _common_kwargs(self): + return dict( + minicroft=self.mc, + skill_ids=[SKILL_ID], + eof_msgs=["ovos.utterance.handled"], + eof_count=2, # both lifecycles terminate on ovos.utterance.handled + test_routing=False, + test_active_skills=False, + test_final_session=False, + ignore_messages=DEFAULT_IGNORED + HANDLER_LIFECYCLE, + verbose=False, + ) + + def test_filter_isolates_one_lifecycle(self): + """Only messages whose context skill_id matches are asserted.""" + src = _make_custom("unittest.two_lifecycles") + test = End2EndTest( + source_message=src, + skill_id="life.a", + expected_messages=[ + Message("life.a.step", {}, {"skill_id": "life.a"}), + Message("ovos.utterance.handled", {}, {"skill_id": "life.a"}), + ], + **self._common_kwargs(), + ) + # passes only if life.b.* and the source (no skill_id) are filtered out + test.execute(timeout=10) + + def test_filter_the_other_lifecycle(self): + """The same scenario, filtered to the other skill_id.""" + src = _make_custom("unittest.two_lifecycles") + test = End2EndTest( + source_message=src, + skill_id="life.b", + expected_messages=[ + Message("life.b.step", {}, {"skill_id": "life.b"}), + Message("ovos.utterance.handled", {}, {"skill_id": "life.b"}), + ], + **self._common_kwargs(), + ) + test.execute(timeout=10) + + def test_unfiltered_sees_both_lifecycles(self): + """Without the filter, eof_count=2 captures both lifecycles' messages.""" + src = _make_custom("unittest.two_lifecycles") + test = End2EndTest( + source_message=src, + expected_messages=[src], + test_message_number=False, + test_msg_type=False, + test_msg_data=False, + test_msg_context=False, + **self._common_kwargs(), + ) + result = test.execute(timeout=10) + types = [m.msg_type for m in result] + self.assertIn("life.a.step", types) + self.assertIn("life.b.step", types) + + if __name__ == "__main__": unittest.main() diff --git a/test/unittests/test_gui_capture.py b/test/unittests/test_gui_capture.py index 11cc11d..70fc994 100644 --- a/test/unittests/test_gui_capture.py +++ b/test/unittests/test_gui_capture.py @@ -106,6 +106,50 @@ def test_assert_namespace_has_key_from_field(self) -> None: )] session.assert_namespace_has_key("weather", "current_temp") + # -- assert_template_shown (SYSTEM_* template model) -- + + def test_assert_template_shown_with_prefix(self) -> None: + """Full SYSTEM_ name should match the shown template.""" + session = self._make_session() + session.messages = [self._page_show_msg("weatherskill", "SYSTEM_weather")] + session.assert_template_shown("weatherskill", "SYSTEM_weather") + + def test_assert_template_shown_without_prefix(self) -> None: + """Short name is normalized to the SYSTEM_ prefix.""" + session = self._make_session() + session.messages = [self._page_show_msg("weatherskill", "SYSTEM_weather")] + session.assert_template_shown("weatherskill", "weather") + + def test_assert_template_shown_with_values(self) -> None: + """Template + accompanying session-data values both asserted.""" + session = self._make_session() + session.messages = [ + self._page_show_msg("weatherskill", "SYSTEM_weather"), + self._value_set_msg("weatherskill", {"current_temp": 22, + "condition": "Sunny"}), + ] + session.assert_template_shown("weatherskill", "weather", + values={"current_temp": 22, + "condition": "Sunny"}) + + def test_assert_template_shown_missing_template(self) -> None: + """Template never shown should raise.""" + session = self._make_session() + session.messages = [self._page_show_msg("weatherskill", "SYSTEM_text")] + with self.assertRaises(AssertionError): + session.assert_template_shown("weatherskill", "weather", timeout=0.1) + + def test_assert_template_shown_wrong_value(self) -> None: + """Template shown but a listed value differs should raise.""" + session = self._make_session() + session.messages = [ + self._page_show_msg("weatherskill", "SYSTEM_weather"), + self._value_set_msg("weatherskill", {"current_temp": 22}), + ] + with self.assertRaises(AssertionError): + session.assert_template_shown("weatherskill", "weather", + values={"current_temp": 99}) + # -- assert_namespace_cleared -- def test_assert_namespace_cleared_match(self) -> None: diff --git a/test/unittests/test_intent_cases.py b/test/unittests/test_intent_cases.py new file mode 100644 index 0000000..af15cc8 --- /dev/null +++ b/test/unittests/test_intent_cases.py @@ -0,0 +1,142 @@ +# Copyright 2024 OpenVoiceOS +# Licensed under the Apache License, Version 2.0 +"""Unit tests for ovoscope.intent_cases helpers and the accuracy reporter. + +Tests focus on the parts that don't need a live MiniCroft: the file-layout +loader, the JSON summary roll-ups, the structural baseline diff, and the +Markdown report renderer. The end-to-end execution path is covered by +the consumer skill's e2e suite. +""" +from pathlib import Path +from unittest import TestCase + +from ovoscope.intent_cases import (IntentCase, _read_lines, load_intent_cases) +from ovoscope.pytest_plugin import (_accuracy_markdown, _accuracy_summary, + _baseline_diff) + + +class TestLoadIntentCases(TestCase): + """Discovery of /.intent.test and no_match.test files.""" + + def _write(self, tmp, lang, name, body): + d = tmp / "cases" / lang + d.mkdir(parents=True, exist_ok=True) + (d / name).write_text(body, encoding="utf-8") + + def test_loads_intent_and_no_match_files(self): + import tempfile + with tempfile.TemporaryDirectory() as td: + tmp = Path(td) + self._write(tmp, "en-US", "WhoAreYou.intent.test", + "# header\nwho are you\nwhat is your name\n") + self._write(tmp, "en-US", "no_match.test", "what time is it\n") + cases = load_intent_cases(tmp / "cases", + known_intents=["WhoAreYou.intent"]) + self.assertEqual(len(cases), 3) + self.assertEqual(sum(1 for c in cases if c.intent is None), 1) + self.assertTrue(all(c.lang == "en-US" for c in cases)) + + def test_unknown_intent_raises(self): + import tempfile + with tempfile.TemporaryDirectory() as td: + tmp = Path(td) + self._write(tmp, "en-US", "Mystery.intent.test", "x\n") + with self.assertRaises(AssertionError): + load_intent_cases(tmp / "cases", + known_intents=["WhoAreYou.intent"]) + + def test_skips_comments_and_blank_lines(self): + import tempfile + with tempfile.TemporaryDirectory() as td: + tmp = Path(td) + self._write(tmp, "en-US", "WhoAreYou.intent.test", + "# a\n\n \nfoo\n#bar\n") + cases = load_intent_cases(tmp / "cases", + known_intents=["WhoAreYou.intent"]) + self.assertEqual([c.utterance for c in cases], ["foo"]) + + +class TestAccuracySummary(TestCase): + """Aggregate roll-ups feeding the markdown / terminal pivots.""" + + def _r(self, **kw): + base = {"pipeline": "TestX", "lang": "en-US", + "intent": "WhoAreYou.intent", "utterance": "u", + "passed": True, "source": ""} + base.update(kw) + return base + + def test_overall_and_pivots(self): + results = [ + self._r(), + self._r(passed=False), + self._r(pipeline="TestY"), + ] + s = _accuracy_summary(results) + self.assertEqual(s["total"], 3) + self.assertEqual(s["passed"], 2) + self.assertAlmostEqual(s["overall_accuracy"], 2 / 3) + # Per-pipeline bucket exists for both pipelines. + self.assertEqual(s["by_pipeline"]["TestX"]["total"], 2) + self.assertEqual(s["by_pipeline"]["TestY"]["pass"], 1) + # by_utterance carries failing_pipelines for diagnosis. + by_utt = {(u["lang"], u["intent"], u["utterance"]): u + for u in s["by_utterance"]} + u = by_utt[("en-US", "WhoAreYou.intent", "u")] + self.assertIn("TestX", u["failing_pipelines"]) + + +class TestBaselineDiff(TestCase): + """Structural baseline diff: regressed vs recovered.""" + + def _r(self, pipeline, passed, utterance="u"): + return {"pipeline": pipeline, "lang": "en-US", + "intent": "WhoAreYou.intent", "utterance": utterance, + "passed": passed} + + def test_diff_classifies_flips(self): + baseline = [self._r("TestX", True), self._r("TestY", False)] + current = [self._r("TestX", False), self._r("TestY", True), + self._r("TestZ", True)] + d = _baseline_diff(baseline, current) + self.assertEqual(len(d["regressed"]), 1) + self.assertEqual(d["regressed"][0]["pipeline"], "TestX") + self.assertEqual(len(d["recovered"]), 1) + self.assertEqual(d["recovered"][0]["pipeline"], "TestY") + self.assertEqual(len(d["added"]), 1) + self.assertEqual(d["added"][0]["pipeline"], "TestZ") + self.assertEqual(d["baseline_total"], 2) + + +class TestAccuracyMarkdown(TestCase): + """Markdown report includes every section we need for PR comments.""" + + def test_markdown_contains_expected_sections(self): + results = [ + {"pipeline": "TestA", "lang": "en-US", "intent": "I.intent", + "utterance": "ok phrase", "passed": True}, + {"pipeline": "TestA", "lang": "en-US", "intent": "I.intent", + "utterance": "bad phrase", "passed": False}, + ] + s = _accuracy_summary(results) + md = _accuracy_markdown(s, results) + self.assertIn("intent-case checks passed", md) + self.assertIn("| Pipeline | Pass / Total | Accuracy |", md) + self.assertIn("Per-pipeline × language", md) + self.assertIn("Per-pipeline × intent", md) + self.assertIn("Hardest utterances", md) + # The failing utterance must be quoted in the hardest list. + self.assertIn("bad phrase", md) + + def test_baseline_diff_section_when_supplied(self): + baseline = [{"pipeline": "TestA", "lang": "en-US", + "intent": "I.intent", "utterance": "phrase", + "passed": True}] + current = [{"pipeline": "TestA", "lang": "en-US", + "intent": "I.intent", "utterance": "phrase", + "passed": False}] + s = _accuracy_summary(current) + d = _baseline_diff(baseline, current) + md = _accuracy_markdown(s, current, baseline_diff=d) + self.assertIn("Baseline diff", md) + self.assertIn("Regressed (1)", md) diff --git a/test/unittests/test_listener_stream.py b/test/unittests/test_listener_stream.py new file mode 100644 index 0000000..ef4d491 --- /dev/null +++ b/test/unittests/test_listener_stream.py @@ -0,0 +1,154 @@ +# Copyright 2024 Jarbas AI +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for MiniListener.feed_audio_stream and ListenerTest streaming. + +Uses a self-contained stub transformer (no native ggwave dependency) that only +emits its message after accumulating a threshold number of frames — exactly the +behaviour ``feed_audio_stream`` exists to support. +""" +import unittest + +from ovos_bus_client.message import Message +from ovoscope.listener import ListenerTest, MiniListener, get_mini_listener + + +_BASE_CONFIG = {"listener": {"audio_transformers": {}}} + + +class _AccumulatingTransformer: + """Stub audio transformer that fires only after N fed frames. + + Mirrors a real streaming decoder: a single ``feed_audio`` call never + produces output, so the aggregating ``feed_audio_stream`` is required to + observe the emitted message. + """ + + def __init__(self, trigger_after=3, msg_type="recognizer_loop:utterance"): + self.name = "stub" + self.priority = 10 + self.trigger_after = trigger_after + self.msg_type = msg_type + self.count = 0 + self.bus = None + + def bind(self, bus): + self.bus = bus + + def feed_audio_chunk(self, chunk): + self.count += 1 + if self.count == self.trigger_after: + self.bus.emit(Message(self.msg_type, {"utterances": ["streamed"]})) + + def feed_speech_chunk(self, chunk): + self.feed_audio_chunk(chunk) + + def shutdown(self): + pass + + +def _listener(trigger_after=3): + plugin = _AccumulatingTransformer(trigger_after=trigger_after) + return get_mini_listener( + config=_BASE_CONFIG, + plugin_instances={"stub": plugin}, + ) + + +class TestFeedAudioStream(unittest.TestCase): + """MiniListener.feed_audio_stream aggregation behaviour.""" + + def test_single_feed_audio_misses_late_decode(self): + """A lone feed_audio call before the threshold yields nothing.""" + listener = _listener(trigger_after=3) + try: + self.assertEqual(listener.feed_audio(b"\x00" * 100), []) + finally: + listener.shutdown() + + def test_stream_aggregates_across_frames(self): + """feed_audio_stream keeps the message emitted on a later frame.""" + listener = _listener(trigger_after=3) + try: + msgs = listener.feed_audio_stream([b"\x00" * 100] * 5) + types = [m.msg_type for m in msgs] + self.assertIn("recognizer_loop:utterance", types) + finally: + listener.shutdown() + + def test_stream_splits_flat_bytes_by_chunk_size(self): + """A flat bytes object is split into chunk_size frames.""" + listener = _listener(trigger_after=4) + try: + # 4 frames of 50 bytes -> fourth frame triggers + msgs = listener.feed_audio_stream(b"\x00" * 200, chunk_size=50) + self.assertTrue( + any(m.msg_type == "recognizer_loop:utterance" for m in msgs) + ) + finally: + listener.shutdown() + + def test_stream_feed_speech_path(self): + """feed='feed_speech' drives the speech feed instead of audio.""" + listener = _listener(trigger_after=2) + try: + msgs = listener.feed_audio_stream( + [b"\x00" * 100] * 3, feed="feed_speech" + ) + self.assertTrue( + any(m.msg_type == "recognizer_loop:utterance" for m in msgs) + ) + finally: + listener.shutdown() + + def test_invalid_feed_raises(self): + """An unknown feed method is rejected.""" + listener = _listener() + try: + with self.assertRaises(ValueError): + listener.feed_audio_stream([b"\x00" * 10], feed="nope") + finally: + listener.shutdown() + + +class TestListenerTestStreaming(unittest.TestCase): + """ListenerTest declarative streaming support.""" + + def test_listener_test_feed_audio_stream(self): + """ListenerTest with feed_method='feed_audio_stream' aggregates.""" + plugin = _AccumulatingTransformer(trigger_after=3) + ListenerTest( + config=_BASE_CONFIG, + plugin_instances={"stub": plugin}, + audio_input=b"\x00" * 300, + feed_method="feed_audio_stream", + chunk_size=100, + expected_types=["recognizer_loop:utterance"], + ).execute() + + def test_listener_test_forbidden_absent(self): + """forbidden_types passes when the type never appears.""" + plugin = _AccumulatingTransformer(trigger_after=2) + ListenerTest( + config=_BASE_CONFIG, + plugin_instances={"stub": plugin}, + audio_input=b"\x00" * 200, + feed_method="feed_audio_stream", + chunk_size=100, + expected_types=["recognizer_loop:utterance"], + forbidden_types=["speak"], + ).execute() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/unittests/test_listener_vad_ww.py b/test/unittests/test_listener_vad_ww.py index d6cc398..20851c1 100644 --- a/test/unittests/test_listener_vad_ww.py +++ b/test/unittests/test_listener_vad_ww.py @@ -20,7 +20,13 @@ TestVADTest (5 tests) — VADTest declarative helper TestWakeWordTest (5 tests) — WakeWordTest declarative helper """ +import importlib.util +import time import unittest +from unittest.mock import MagicMock + +from ovos_bus_client.message import Message +from ovos_spec_tools import SpecMessage from ovoscope.listener import ( MockHotWordEngine, @@ -31,6 +37,10 @@ get_mini_listener, ) +# MiniListener.listen() routes audio through AudioTransformersService before +# emitting recognizer_loop:utterance, which requires ovos-dinkum-listener. +DINKUM_AVAILABLE = importlib.util.find_spec("ovos_dinkum_listener") is not None + _BASE_CONFIG = {"listener": {"audio_transformers": {}}} @@ -441,5 +451,73 @@ def test_no_engines_raises(self): WakeWordTest(audio_chunks=[b"\x00" * 512] * 3).execute() +# --------------------------------------------------------------------------- +# TestMiniListenerNamespaceBridging +# --------------------------------------------------------------------------- + +@unittest.skipUnless( + DINKUM_AVAILABLE, "ovos-dinkum-listener not installed" +) +class TestMiniListenerNamespaceBridging(unittest.TestCase): + """MiniListener.listen() emits the LEGACY ``recognizer_loop:utterance`` + (migrated to ``ovos.utterance.handle``). These tests pin that the FakeBus + namespace bridging connects the two namespaces, and that turning it off + isolates a single namespace. + """ + + @staticmethod + def _stt(transcript): + stt = MagicMock() + stt.execute.return_value = transcript + return stt + + def test_utterance_legacy_reaches_spec_via_bridging(self): + """Default harness (bridging on): the legacy utterance emitted by + listen() also reaches a subscriber on the spec topic.""" + listener = get_mini_listener(stt_instance=self._stt("hello world")) + legacy_hits, spec_hits = [], [] + listener.bus.on("recognizer_loop:utterance", lambda m: legacy_hits.append(m)) + listener.bus.on(str(SpecMessage.UTTERANCE), lambda m: spec_hits.append(m)) + try: + listener.listen(b"\x00" * 1024, language="en-us") + time.sleep(0.05) + self.assertTrue(legacy_hits, "legacy recognizer_loop:utterance not seen") + self.assertTrue(spec_hits, "spec ovos.utterance.handle not seen via bridge") + finally: + listener.shutdown() + + def test_utterance_spec_native(self): + """A SPEC producer (ovos.utterance.handle) reaches a spec subscriber + natively — the harness bus exercises the new namespace too.""" + listener = get_mini_listener() + spec_hits = [] + listener.bus.on(str(SpecMessage.UTTERANCE), lambda m: spec_hits.append(m)) + try: + listener.bus.emit(Message(str(SpecMessage.UTTERANCE), + {"utterances": ["hi"], "lang": "en-us"})) + time.sleep(0.05) + self.assertTrue(spec_hits, "spec ovos.utterance.handle not delivered") + finally: + listener.shutdown() + + def test_no_bridging_isolates_legacy_from_spec(self): + """With bridging OFF, the legacy utterance emitted by listen() does NOT + reach a spec-only subscriber.""" + listener = get_mini_listener( + stt_instance=self._stt("hello world"), + modernize=False, emit_legacy=False, + ) + legacy_hits, spec_hits = [], [] + listener.bus.on("recognizer_loop:utterance", lambda m: legacy_hits.append(m)) + listener.bus.on(str(SpecMessage.UTTERANCE), lambda m: spec_hits.append(m)) + try: + listener.listen(b"\x00" * 1024, language="en-us") + time.sleep(0.1) + self.assertTrue(legacy_hits, "legacy utterance should still fire") + self.assertEqual(spec_hits, [], "spec topic must not fire with bridging off") + finally: + listener.shutdown() + + if __name__ == "__main__": unittest.main() diff --git a/test/unittests/test_media.py b/test/unittests/test_media.py index 5c0e405..fb225b2 100644 --- a/test/unittests/test_media.py +++ b/test/unittests/test_media.py @@ -13,6 +13,8 @@ # limitations under the License. """Unit tests for ovoscope.media (MockOCPBackend, OCPCaptureSession, OCPPlayerHarness).""" +import time + import pytest from unittest.mock import MagicMock @@ -20,7 +22,17 @@ from ovos_utils.fakebus import FakeBus from ovos_utils.ocp import MediaState -from ovoscope.media import MockOCPBackend, OCPCaptureSession +from ovoscope.media import MockOCPBackend, OCPCaptureSession, OCPPlayerHarness + + +def test_media_harness_reexported_from_package() -> None: + """The ovos-media harness is reachable from the top-level package, like the + ovos-audio harness. (media.py imports ovos-media lazily, so the export does + not require the [media] extra to be installed.)""" + import ovoscope + assert ovoscope.OCPPlayerHarness is OCPPlayerHarness + assert ovoscope.OCPCaptureSession is OCPCaptureSession + assert ovoscope.MockOCPBackend is MockOCPBackend # --------------------------------------------------------------------------- @@ -202,3 +214,149 @@ def test_custom_prefixes(self) -> None: self.bus.emit(Message("ovos.common_play.play")) # should not be captured session.stop() assert session.message_types == ["custom.prefix.event"] + + +try: + import ovos_media # noqa: F401 + _HAS_OVOS_MEDIA = True +except ImportError: + _HAS_OVOS_MEDIA = False + + +if _HAS_OVOS_MEDIA: + from ovos_plugin_manager.templates.media import AudioPlayerBackend + + class _RecordingBackend(AudioPlayerBackend): + """A real OCP ``MediaBackend`` stand-in built by a factory. + + Subclasses the genuine ``AudioPlayerBackend`` (not ``MockOCPBackend``) so + its ``load_track`` emits the real ``ovos.common_play.media.state`` + ``LOADED_MEDIA`` event the live ``AudioService`` routes on. Records the uri + its ``play()`` is driven with — analogous to a Music Assistant backend + calling ``client.play_media(uri)`` — so a test can assert the player's play + path actually reached the injected backend. + """ + + def __init__(self, bus): + super().__init__(config={}, bus=bus) + self.play_calls = [] + self.is_playing = False + + def supported_uris(self): + return ["library", "http", "https"] + + def play(self, repeat: bool = False): + self.is_playing = True + self.play_calls.append(self._now_playing) + + def stop(self): + self.is_playing = False + return True + + def pause(self): + pass + + def resume(self): + pass + + def lower_volume(self): + pass + + def restore_volume(self): + pass + + def get_track_length(self): + return 0 + + def get_track_position(self): + return 0 + + def set_track_position(self, milliseconds): + pass + + +@pytest.mark.skipif(not _HAS_OVOS_MEDIA, + reason="requires the [media] extra (ovos-media)") +class TestOCPPlayerHarnessBackendInjection: + """OCPPlayerHarness(backend_factory=...) drives a real injected backend.""" + + def test_default_factory_is_mock_backend(self) -> None: + with OCPPlayerHarness() as h: + assert isinstance(h.backend, MockOCPBackend) + assert type(h.backend) is MockOCPBackend + + def test_injected_backend_is_used(self) -> None: + with OCPPlayerHarness(backend_factory=_RecordingBackend) as h: + assert isinstance(h.backend, _RecordingBackend) + # name is supplied by the harness when the backend lacks one + assert getattr(h.backend, "name", None) + + def test_player_drives_injected_backend_play(self) -> None: + from ovos_utils.ocp import MediaEntry, PlaybackType + with OCPPlayerHarness(backend_factory=_RecordingBackend) as h: + h.play(MediaEntry(uri="library://track/42", + playback=PlaybackType.AUDIO)) + assert h.backend.is_playing is True + assert h.backend.play_calls == ["library://track/42"] + + +# --------------------------------------------------------------------------- +# Namespace bridging +# --------------------------------------------------------------------------- + +@pytest.mark.skipif(not _HAS_OVOS_MEDIA, + reason="requires the [media] extra (ovos-media)") +class TestOCPHarnessNamespaceBridging: + """OCPMediaPlayer subscribes to the LEGACY duck/cork topics + (``recognizer_loop:audio_output_start/end``, ``recognizer_loop:record_begin/end``) + which OVOS is migrating to the ``ovos.*`` spec namespace + (``ovos.audio.output.*`` / ``ovos.listener.record.*``). These tests pin that + the harness FakeBus namespace bridging connects a SPEC-namespace producer to + those legacy handlers, and that turning the bridge off isolates a single + namespace. + + The observable behaviour is the *cork* path: while PLAYING, a record-start + event pauses the player (``handle_cork_request``). + """ + + @staticmethod + def _play_then(h): + """Drive the player into PLAYING with an AUDIO MediaEntry.""" + from ovos_utils.ocp import MediaEntry, PlaybackType, PlayerState + h.play(MediaEntry(uri="http://example.com/song.mp3", + playback=PlaybackType.AUDIO)) + h.assert_player_state(PlayerState.PLAYING) + + def test_cork_via_legacy_topic_natively(self) -> None: + """The legacy ``recognizer_loop:record_begin`` corks (pauses) the player — + the namespace the player subscribes on works natively.""" + from ovos_utils.ocp import PlayerState + with OCPPlayerHarness() as h: # bridging default on + self._play_then(h) + h.bus.emit(Message("recognizer_loop:record_begin")) + time.sleep(0.05) + h.assert_player_state(PlayerState.PAUSED) + + def test_cork_via_spec_topic_through_bridging(self) -> None: + """A SPEC producer emitting ``ovos.listener.record.started`` reaches the + legacy-subscribed ``handle_cork_request`` via emit_legacy bridging and + corks (pauses) the player.""" + from ovos_spec_tools import SpecMessage + from ovos_utils.ocp import PlayerState + with OCPPlayerHarness() as h: # bridging default on + self._play_then(h) + h.bus.emit(Message(str(SpecMessage.LISTENER_RECORD_STARTED))) + time.sleep(0.05) + h.assert_player_state(PlayerState.PAUSED) + + def test_no_bridging_isolates_spec_from_legacy(self) -> None: + """With bridging OFF, a SPEC ``ovos.listener.record.started`` emit does + NOT reach the legacy-subscribed cork handler — the player stays PLAYING, + proving the harness can exercise a single namespace.""" + from ovos_spec_tools import SpecMessage + from ovos_utils.ocp import PlayerState + with OCPPlayerHarness(modernize=False, emit_legacy=False) as h: + self._play_then(h) + h.bus.emit(Message(str(SpecMessage.LISTENER_RECORD_STARTED))) + time.sleep(0.2) # give any (incorrect) bridge a chance to fire + h.assert_player_state(PlayerState.PLAYING) diff --git a/test/unittests/test_media_provider.py b/test/unittests/test_media_provider.py new file mode 100644 index 0000000..478955e --- /dev/null +++ b/test/unittests/test_media_provider.py @@ -0,0 +1,217 @@ +"""Tests for ovoscope.media_provider.MediaProviderHarness. + +The harness is duck-typed, so these tests use a dependency-free ``_DummyProvider`` +(no mediavocab / ovos-plugin-manager import) and ``SimpleNamespace`` stand-ins for +``Signals`` / ``QueryContext`` / ``Release``. +""" +import types + +import pytest + +from ovoscope import MediaProviderHarness +from ovoscope.media_provider import DEFAULT_GROUP + + +# -------------------------------------------------------------------------- +# duck-typed fixtures (no real mediavocab / opm types) +# -------------------------------------------------------------------------- + +def _signals(title="x", medium="music", artist=None): + return types.SimpleNamespace(title=title, medium=medium, artist=artist) + + +def _context(supported_playback_types=None, blocked_genres=None): + return types.SimpleNamespace( + supported_playback_types=supported_playback_types or set(), + blocked_genres=blocked_genres or set(), + ) + + +def _release(uri, title="T", mc=0.5): + return types.SimpleNamespace( + uri=uri, match_confidence=mc, + work=types.SimpleNamespace(title=title), + ) + + +class _DummyProvider: + """Minimal MediaProvider look-alike: serves music, returns library:// uris.""" + + name = "dummy" + + def __init__(self, config=None): + self.config = config or {} + self._api = None + self._served = {"music", "radio"} + + def is_available(self): + return self._api is not None + + def serves(self, signals, context=None): + if getattr(signals, "medium", None) not in self._served: + return False + if context is not None: + spt = getattr(context, "supported_playback_types", set()) + if spt and "audio" not in spt: + return False + return True + + def search(self, signals, lang="en-us"): + if self._api is None: + return [] + return [_release("library://track/1", "Hit", 0.9), + _release("library://track/2", "Miss", 0.3)] + + def search_safe(self, signals, context=None, lang="en-us"): + try: + return self.search(signals, lang=lang) + except Exception: + return [] + + def featured_media(self, lang="en-us"): + return [_release("library://track/3", "Featured", 0.0)] + + +class _ExplodingProvider(_DummyProvider): + def search(self, signals, lang="en-us"): + raise RuntimeError("backend exploded") + + +def _harness(mock_api="client", provider_cls=_DummyProvider): + return MediaProviderHarness.from_class( + provider_cls, config={"url": "http://x"}, mock_api=mock_api) + + +# -------------------------------------------------------------------------- +# from_class + injection +# -------------------------------------------------------------------------- + +def test_from_class_instantiates_and_injects_api(): + sentinel = object() + h = _harness(mock_api=sentinel) + assert isinstance(h.provider, _DummyProvider) + assert h.provider._api is sentinel + assert h.api is sentinel + assert h.provider.config == {"url": "http://x"} + + +def test_from_class_custom_api_attr(): + sentinel = object() + h = MediaProviderHarness.from_class(_DummyProvider, mock_api=sentinel, + api_attr="config") + assert h.provider.config is sentinel + + +def test_is_available_reflects_injected_client(): + assert _harness(mock_api="client").is_available() is True + assert MediaProviderHarness.from_class(_DummyProvider).is_available() is False + + +# -------------------------------------------------------------------------- +# routing / serves drivers + asserts +# -------------------------------------------------------------------------- + +def test_serves_and_assert_routes(): + h = _harness() + assert h.serves(_signals(medium="music")) is True + h.assert_routes(_signals(medium="music")) + h.assert_routes(_signals(medium="music"), + _context(supported_playback_types={"audio"})) + + +def test_assert_not_routes_unserved_medium(): + h = _harness() + assert h.serves(_signals(medium="movie")) is False + h.assert_not_routes(_signals(medium="movie")) + + +def test_assert_not_routes_video_only_device(): + h = _harness() + h.assert_not_routes(_signals(medium="music"), + _context(supported_playback_types={"video"})) + + +def test_assert_routes_raises_when_not_served(): + h = _harness() + with pytest.raises(AssertionError): + h.assert_routes(_signals(medium="movie")) + + +# -------------------------------------------------------------------------- +# search / search_safe drivers + playable asserts +# -------------------------------------------------------------------------- + +def test_search_safe_returns_playables(): + h = _harness() + results = h.assert_returns_playables(_signals()) + assert [r.work.title for r in results] == ["Hit", "Miss"] + assert all(r.uri.startswith("library://") for r in results) + + +def test_search_safe_swallows_backend_error(): + h = _harness(provider_cls=_ExplodingProvider) + assert h.search_safe(_signals()) == [] + + +def test_assert_returns_playables_fails_on_empty(): + h = MediaProviderHarness.from_class(_DummyProvider) # no api -> search returns [] + with pytest.raises(AssertionError): + h.assert_returns_playables(_signals()) + + +def test_assert_returns_playables_fails_on_bad_confidence(): + class _BadProvider(_DummyProvider): + def search(self, signals, lang="en-us"): + return [_release("library://track/9", "Bad", mc=5.0)] # out of [0,1] + + h = _harness(provider_cls=_BadProvider) + with pytest.raises(AssertionError): + h.assert_returns_playables(_signals()) + + +def test_featured_media(): + h = _harness() + feats = h.featured_media() + assert [r.work.title for r in feats] == ["Featured"] + + +# -------------------------------------------------------------------------- +# from_entrypoint (monkeypatched discovery) +# -------------------------------------------------------------------------- + +def _fake_entry_points(monkeypatch, eps): + monkeypatch.setattr("ovoscope.media_provider.entry_points", + lambda group=None: eps) + + +def test_from_entrypoint_discovers_and_loads(monkeypatch): + ep = types.SimpleNamespace(name="dummy", load=lambda: _DummyProvider) + _fake_entry_points(monkeypatch, [ep]) + + h = MediaProviderHarness.from_entrypoint("dummy", config={"url": "u"}, + mock_api="client") + assert isinstance(h.provider, _DummyProvider) + assert h.provider._api == "client" + assert h.entrypoint_name == "dummy" + assert h.entrypoint_group == DEFAULT_GROUP + h.assert_entrypoint_registered() # "dummy" present in the patched group + + +def test_from_entrypoint_missing_raises(monkeypatch): + _fake_entry_points(monkeypatch, []) + with pytest.raises(AssertionError): + MediaProviderHarness.from_entrypoint("nope") + + +def test_from_entrypoint_ambiguous_raises(monkeypatch): + ep1 = types.SimpleNamespace(name="dummy", load=lambda: _DummyProvider) + ep2 = types.SimpleNamespace(name="dummy", load=lambda: _DummyProvider) + _fake_entry_points(monkeypatch, [ep1, ep2]) + with pytest.raises(AssertionError): + MediaProviderHarness.from_entrypoint("dummy") + + +def test_assert_entrypoint_registered_without_name_raises(): + h = MediaProviderHarness.from_class(_DummyProvider) # no entrypoint name + with pytest.raises(AssertionError): + h.assert_entrypoint_registered() diff --git a/test/unittests/test_minicroft.py b/test/unittests/test_minicroft.py index c3f448f..f48e4ff 100644 --- a/test/unittests/test_minicroft.py +++ b/test/unittests/test_minicroft.py @@ -3,6 +3,7 @@ import unittest from ovos_bus_client.message import Message +from ovos_spec_tools import SpecMessage from ovos_utils.log import LOG from ovos_workshop.skills.ovos import OVOSSkill @@ -10,6 +11,9 @@ from ovoscope import MiniCroft, get_minicroft, DEFAULT_TEST_PIPELINE, LIGHT_TEST_PIPELINE, ADAPT_PIPELINE +LEGACY_UTTERANCE = "recognizer_loop:utterance" +SPEC_UTTERANCE = str(SpecMessage.UTTERANCE) # ovos.utterance.handle + # --------------------------------------------------------------------------- # Minimal inline skill used across tests @@ -324,5 +328,75 @@ def test_pipeline_config_multiple_keys(self): self.assertIsNone(cfg_after.get("plugin_b")) +class TestMiniCroftNamespaceBridging(unittest.TestCase): + """MiniCroft is the e2e harness bus. Utterances are injected on the LEGACY + topic (``recognizer_loop:utterance``); these tests pin that MiniCroft's + FakeBus bridges that to/from the ovos.* SPEC topic + (``ovos.utterance.handle``) so an e2e test can drive EITHER namespace, and + that disabling the bridge isolates a single namespace. + """ + + def setUp(self): + LOG.set_level("ERROR") + + def tearDown(self): + LOG.set_level("CRITICAL") + + def _emit_and_collect(self, mc, emit_topic, watch_topic, *, timeout=3.0): + seen = [] + got = threading.Event() + + def _on(msg): + if isinstance(msg, str): + msg = Message.deserialize(msg) + seen.append(msg) + got.set() + + mc.bus.on(watch_topic, _on) + try: + mc.bus.emit(Message( + emit_topic, + data={"utterances": ["hello world"], "lang": "en-US"}, + )) + got.wait(timeout) + finally: + mc.bus.remove(watch_topic, _on) + return seen + + def test_legacy_utterance_observed_on_spec_topic(self): + """Default (bridging on): a legacy recognizer_loop:utterance injected + into the e2e harness is observed on ovos.utterance.handle (modernize).""" + mc = get_minicroft([]) # modernize/emit_legacy default on + try: + seen = self._emit_and_collect(mc, LEGACY_UTTERANCE, SPEC_UTTERANCE) + self.assertTrue(seen, "legacy utterance was not bridged to the spec topic") + self.assertEqual(seen[0].data["utterances"], ["hello world"]) + finally: + mc.stop() + + def test_spec_utterance_reaches_legacy_listener(self): + """An utterance injected on the SPEC topic reaches a LEGACY listener + (emit_legacy) — the intent pipeline keys off the legacy topic.""" + mc = get_minicroft([]) + try: + seen = self._emit_and_collect(mc, SPEC_UTTERANCE, LEGACY_UTTERANCE) + self.assertTrue(seen, "spec utterance was not bridged to the legacy topic") + self.assertEqual(seen[0].data["utterances"], ["hello world"]) + finally: + mc.stop() + + def test_no_bridging_isolates_legacy_from_spec(self): + """With bridging OFF, a legacy emit does NOT reach a spec-only + subscriber — the e2e harness exercises a single isolated namespace.""" + mc = get_minicroft([], modernize=False, emit_legacy=False) + try: + seen = self._emit_and_collect(mc, LEGACY_UTTERANCE, SPEC_UTTERANCE, + timeout=0.5) + self.assertEqual(seen, [], + "legacy emit must not reach the spec topic when bridging is off") + finally: + mc.stop() + + if __name__ == "__main__": unittest.main() diff --git a/test/unittests/test_namespace_bridging.py b/test/unittests/test_namespace_bridging.py new file mode 100644 index 0000000..7dd1327 --- /dev/null +++ b/test/unittests/test_namespace_bridging.py @@ -0,0 +1,166 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""End-to-end tests for the FakeBus legacy<->ovos.* namespace migration. + +The ovoscope harness runs components on a :class:`ovos_utils.fakebus.FakeBus` +that mirrors ``MessageBusClient``'s namespace migration (ovos-utils #381): with +``modernize``/``emit_legacy`` on (the defaults), emitting a topic on one +namespace ALSO dispatches its counterpart on the other, and a handler subscribed +to both topics fires once. These tests pin that behaviour through the harness bus +so an ovoscope test can deliberately exercise EITHER namespace, BOTH, or a single +isolated namespace. + +The pairs come from ``ovos_spec_tools.MIGRATION_MAP`` (legacy -> SpecMessage), +e.g. ``speak`` <-> ``ovos.utterance.speak`` and +``recognizer_loop:utterance`` <-> ``ovos.utterance.handle``. +""" + +import threading +import time +import unittest + +from ovos_bus_client.message import Message +from ovos_utils.fakebus import FakeBus +from ovos_spec_tools import SpecMessage +from ovos_spec_tools.messages import MIGRATION_MAP + + +# Representative migrating pairs (legacy topic, spec topic). +LEGACY_SPEAK = "speak" +SPEC_SPEAK = str(SpecMessage.SPEAK) # ovos.utterance.speak +LEGACY_UTTERANCE = "recognizer_loop:utterance" +SPEC_UTTERANCE = str(SpecMessage.UTTERANCE) # ovos.utterance.handle + + +def _collect(bus, topic): + """Subscribe a counting handler on ``topic``; return its list of payloads.""" + received = [] + bus.on(topic, lambda m: received.append(m)) + return received + + +class TestFakeBusNamespaceBridging(unittest.TestCase): + """Both-namespace e2e coverage through the ovoscope harness FakeBus.""" + + def test_migration_map_is_populated(self) -> None: + """Sanity: the spec pairs this suite asserts on are actually migrated.""" + self.assertEqual(MIGRATION_MAP[LEGACY_SPEAK], SpecMessage.SPEAK) + self.assertEqual(MIGRATION_MAP[LEGACY_UTTERANCE], SpecMessage.UTTERANCE) + + # -- modernize: legacy producer reaches a spec listener ----------------- + + def test_legacy_emit_reaches_spec_listener(self) -> None: + """A component emitting a LEGACY topic is received on the ovos.* SPEC + topic (modernize bridging).""" + bus = FakeBus() # both flags default on + spec_seen = _collect(bus, SPEC_SPEAK) + + bus.emit(Message(LEGACY_SPEAK, {"utterance": "hello"})) + + self.assertEqual(len(spec_seen), 1) + self.assertEqual(spec_seen[0].msg_type, SPEC_SPEAK) + self.assertEqual(spec_seen[0].data["utterance"], "hello") + + def test_legacy_utterance_reaches_spec_listener(self) -> None: + """recognizer_loop:utterance is delivered on ovos.utterance.handle.""" + bus = FakeBus() + spec_seen = _collect(bus, SPEC_UTTERANCE) + + bus.emit(Message(LEGACY_UTTERANCE, {"utterances": ["turn on the light"]})) + + self.assertEqual(len(spec_seen), 1) + self.assertEqual(spec_seen[0].data["utterances"], ["turn on the light"]) + + # -- emit_legacy: spec producer reaches a legacy listener --------------- + + def test_spec_emit_reaches_legacy_listener(self) -> None: + """A component emitting the SPEC topic is received by a LEGACY listener + (emit_legacy bridging).""" + bus = FakeBus() + legacy_seen = _collect(bus, LEGACY_SPEAK) + + bus.emit(Message(SPEC_SPEAK, {"utterance": "goodbye"})) + + self.assertEqual(len(legacy_seen), 1) + self.assertEqual(legacy_seen[0].msg_type, LEGACY_SPEAK) + self.assertEqual(legacy_seen[0].data["utterance"], "goodbye") + + # -- dedup: a handler on BOTH topics fires once ------------------------- + + def test_handler_on_both_topics_fires_once(self) -> None: + """A handler subscribed to BOTH the legacy and the spec topic fires once + per real event (the mirror is dropped).""" + bus = FakeBus() + calls = [] + + def handler(message=None): + calls.append(message.msg_type) + + bus.on(LEGACY_SPEAK, handler) + bus.on(SPEC_SPEAK, handler) + + bus.emit(Message(LEGACY_SPEAK, {"utterance": "once"})) + + self.assertEqual(len(calls), 1, + f"dual-subscribed handler fired {len(calls)}x: {calls}") + + def test_two_genuine_events_each_fire(self) -> None: + """Dedup must not swallow two genuine events. A SINGLE handler on both + topics fires exactly once per real event, never zero.""" + bus = FakeBus() + calls = [] + + def handler(message=None): + calls.append(message.data.get("utterance")) + + bus.on(LEGACY_SPEAK, handler) + bus.on(SPEC_SPEAK, handler) + + bus.emit(Message(LEGACY_SPEAK, {"utterance": "first"})) + bus.emit(Message(LEGACY_SPEAK, {"utterance": "second"})) + + # shared mirror-guard dedupes each event's counterpart -> two calls + self.assertEqual(calls, ["first", "second"]) + + # -- single-namespace isolation (no bridging) --------------------------- + + def test_no_bridging_isolates_namespaces(self) -> None: + """FakeBus(modernize=False, emit_legacy=False) keeps each namespace + isolated, proving the harness can exercise ONE namespace explicitly.""" + bus = FakeBus(modernize=False, emit_legacy=False) + spec_seen = _collect(bus, SPEC_SPEAK) + legacy_seen = _collect(bus, LEGACY_SPEAK) + + bus.emit(Message(LEGACY_SPEAK, {"utterance": "legacy only"})) + bus.emit(Message(SPEC_SPEAK, {"utterance": "spec only"})) + + # legacy listener saw only the legacy emit; spec listener only the spec + self.assertEqual([m.data["utterance"] for m in legacy_seen], ["legacy only"]) + self.assertEqual([m.data["utterance"] for m in spec_seen], ["spec only"]) + + def test_modernize_only_does_not_emit_legacy(self) -> None: + """modernize=True, emit_legacy=False: legacy->spec bridges but spec->legacy + does not.""" + bus = FakeBus(modernize=True, emit_legacy=False) + spec_seen = _collect(bus, SPEC_SPEAK) + legacy_seen = _collect(bus, LEGACY_SPEAK) + + bus.emit(Message(LEGACY_SPEAK, {"utterance": "x"})) # bridges -> spec + bus.emit(Message(SPEC_SPEAK, {"utterance": "y"})) # must NOT -> legacy + + self.assertEqual([m.data["utterance"] for m in spec_seen], ["x", "y"]) + self.assertEqual([m.data["utterance"] for m in legacy_seen], ["x"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/unittests/test_ocp_namespace.py b/test/unittests/test_ocp_namespace.py new file mode 100644 index 0000000..85b5822 --- /dev/null +++ b/test/unittests/test_ocp_namespace.py @@ -0,0 +1,125 @@ +# Copyright 2024 Jarbas AI +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Namespace-bridging tests for the OCP harness (ovoscope.ocp.OCPTest). + +OCPTest drives the OCP query flow by emitting the LEGACY +``recognizer_loop:utterance`` topic into a MiniCroft. These tests pin that the +harness FakeBus bridges that topic to/from the ovos.* SPEC topic +(``ovos.utterance.handle``) so an OCP test can deliberately exercise EITHER +namespace, and that disabling the bridge isolates a single namespace. + +The OCP query/response path itself is covered elsewhere; here we assert only the +namespace plumbing OCPTest threads into the harness, driving the SAME real +utterance topic OCPTest emits. +""" + +import importlib.util +import threading +import unittest + +from ovos_bus_client.message import Message +from ovos_spec_tools import SpecMessage + +from ovoscope.ocp import OCPTest + +# OCPTest spins up a full MiniCroft; gate on ovos-core being importable. +CORE_AVAILABLE = importlib.util.find_spec("ovos_core") is not None + +LEGACY_UTTERANCE = "recognizer_loop:utterance" +SPEC_UTTERANCE = str(SpecMessage.UTTERANCE) # ovos.utterance.handle + + +class TestOCPTestNamespaceFields(unittest.TestCase): + """The bridging flags are exposed on OCPTest and default on (no extra deps).""" + + def test_defaults_on(self) -> None: + t = OCPTest(skill_ids=[], utterance="play jazz") + self.assertTrue(t.modernize) + self.assertTrue(t.emit_legacy) + + def test_flags_settable(self) -> None: + t = OCPTest(skill_ids=[], utterance="play jazz", + modernize=False, emit_legacy=False) + self.assertFalse(t.modernize) + self.assertFalse(t.emit_legacy) + + +@unittest.skipUnless(CORE_AVAILABLE, "ovos-core not installed") +class TestOCPTestNamespaceBridging(unittest.TestCase): + """Drive the real OCP utterance topic on a harness MiniCroft and assert the + legacy<->spec bridge OCPTest relies on.""" + + def _make_mc(self, **kwargs): + from ovoscope import get_minicroft + return get_minicroft([], lang="en-US", max_wait=60, **kwargs) + + def _emit_and_collect(self, mc, emit_topic, watch_topic, *, timeout=3.0): + seen = [] + got = threading.Event() + + def _on(msg): + if isinstance(msg, str): + msg = Message.deserialize(msg) + seen.append(msg) + got.set() + + mc.bus.on(watch_topic, _on) + try: + mc.bus.emit(Message( + emit_topic, + data={"utterances": ["play some jazz"], "lang": "en-US"}, + )) + got.wait(timeout) + finally: + mc.bus.remove(watch_topic, _on) + return seen + + def test_legacy_utterance_observed_on_spec_topic(self) -> None: + """Default (bridging on): OCPTest's legacy recognizer_loop:utterance is + observed on the spec ovos.utterance.handle topic (modernize).""" + mc = self._make_mc() # modernize/emit_legacy default on + try: + seen = self._emit_and_collect(mc, LEGACY_UTTERANCE, SPEC_UTTERANCE) + self.assertTrue(seen, "legacy utterance was not bridged to the spec topic") + self.assertEqual(seen[0].data["utterances"], ["play some jazz"]) + finally: + mc.stop() + + def test_spec_utterance_reaches_legacy_listener(self) -> None: + """An utterance emitted on the SPEC topic reaches a LEGACY listener + (emit_legacy) — the OCP query path keys off the legacy topic.""" + mc = self._make_mc() + try: + seen = self._emit_and_collect(mc, SPEC_UTTERANCE, LEGACY_UTTERANCE) + self.assertTrue(seen, "spec utterance was not bridged to the legacy topic") + self.assertEqual(seen[0].data["utterances"], ["play some jazz"]) + finally: + mc.stop() + + def test_no_bridging_isolates_legacy_from_spec(self) -> None: + """With bridging OFF, a legacy emit does NOT reach a spec-only + subscriber — OCPTest(modernize=False, emit_legacy=False) exercises a + single namespace.""" + mc = self._make_mc(modernize=False, emit_legacy=False) + try: + seen = self._emit_and_collect(mc, LEGACY_UTTERANCE, SPEC_UTTERANCE, + timeout=0.5) + self.assertEqual(seen, [], + "legacy emit must not reach the spec topic when bridging is off") + finally: + mc.stop() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/unittests/test_phal.py b/test/unittests/test_phal.py index bccc368..c0e9c89 100644 --- a/test/unittests/test_phal.py +++ b/test/unittests/test_phal.py @@ -141,3 +141,163 @@ def test_phal_test_execute_with_instance(self): ) result = t.execute() assert isinstance(result, list) + + +# --------------------------------------------------------------------------- +# plugin_factories +# --------------------------------------------------------------------------- + + +class TestPluginFactories: + """Tests for the plugin_factories parameter added to MiniPHAL and PHALTest.""" + + def _make_responder_factory(self, trigger_type: str, response_type: str): + """Return a factory callable that wires a trigger→response handler on the bus.""" + + def factory(bus: FakeBus): + plugin = MagicMock() + plugin.shutdown = MagicMock() + bus.on(trigger_type, lambda m: bus.emit(Message(response_type))) + return plugin + + return factory + + def test_factory_called_with_harness_bus(self): + """Factory receives the MiniPHAL internal bus.""" + received_buses = [] + + def factory(bus: FakeBus): + received_buses.append(bus) + return MagicMock() + + with MiniPHAL( + plugin_ids=["test-plugin"], + plugin_factories={"test-plugin": factory}, + ) as phal: + assert len(received_buses) == 1 + assert received_buses[0] is phal._bus + + def test_factory_plugin_receives_emitted_message(self): + """Plugin built by factory can handle messages emitted via phal.emit().""" + factory = self._make_responder_factory("req.type", "resp.type") + with MiniPHAL( + plugin_ids=["echo-plugin"], + plugin_factories={"echo-plugin": factory}, + ) as phal: + phal.emit(Message("req.type"), wait=0.1) + msg = phal.assert_emitted("resp.type", timeout=2.0) + assert msg.msg_type == "resp.type" + + def test_factory_takes_precedence_over_plugin_instances(self): + """When both factory and instance are provided, factory wins.""" + factory_calls = [] + + def factory(bus: FakeBus): + factory_calls.append(True) + return MagicMock() + + stale_instance = MagicMock() + with MiniPHAL( + plugin_ids=["dual-plugin"], + plugin_factories={"dual-plugin": factory}, + plugin_instances={"dual-plugin": stale_instance}, + ) as phal: + assert len(factory_calls) == 1 + assert "dual-plugin" in phal._loaded + # stale_instance was NOT loaded + assert phal._loaded["dual-plugin"] is not stale_instance + + def test_factory_raising_warns_and_skips_plugin(self): + """A factory that raises issues a warning and the plugin is not loaded.""" + import warnings + + def bad_factory(bus: FakeBus): + raise RuntimeError("factory error") + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + with MiniPHAL( + plugin_ids=["bad-plugin"], + plugin_factories={"bad-plugin": bad_factory}, + ) as phal: + assert "bad-plugin" not in phal._loaded + assert any("bad-plugin" in str(warning.message) for warning in w) + + def test_phal_test_plugin_factories_field(self): + """PHALTest.plugin_factories is forwarded to MiniPHAL.""" + factory = self._make_responder_factory("ovos.req", "ovos.resp") + t = PHALTest( + plugin_ids=["my-phal"], + trigger_message=Message("ovos.req"), + expected_types=["ovos.resp"], + plugin_factories={"my-phal": factory}, + timeout=2.0, + ) + captured = t.execute() + assert any(m.msg_type == "ovos.resp" for m in captured) + + +# --------------------------------------------------------------------------- +# Namespace bridging +# --------------------------------------------------------------------------- + +try: + from ovos_spec_tools import SpecMessage + _HAS_SPEC_TOOLS = True +except ImportError: + _HAS_SPEC_TOOLS = False + + +@pytest.mark.skipif(not _HAS_SPEC_TOOLS, + reason="requires ovos-spec-tools (SpecMessage)") +class TestMiniPHALNamespaceBridging: + """PHAL plugins communicate over arbitrary plugin-specific topics and touch + NONE of the legacy<->ovos.* migrated topics, so the harness has no migrated + topic of its own to drive. These tests instead verify that the harness + ``FakeBus`` performs the same namespace bridging as the audio/media harnesses + (so a PHAL plugin that *did* consume/produce a migrated topic would + interoperate across both namespaces), and that the ``modernize``/``emit_legacy`` + flags are threaded through ``MiniPHAL`` to that bus. + """ + + def test_bus_bridges_legacy_to_spec_by_default(self) -> None: + """Default harness (bridging on): a LEGACY emit on the harness bus is + delivered to a SPEC-topic subscriber (modernize bridging).""" + spec_topic = str(SpecMessage.SPEAK) # ovos.utterance.speak + with MiniPHAL() as phal: + seen = [] + phal._bus.on(spec_topic, lambda m: seen.append(m)) + phal._bus.emit(Message("speak", {"utterance": "hi"})) + time.sleep(0.05) + assert [m.data["utterance"] for m in seen] == ["hi"] + + def test_bus_bridges_spec_to_legacy_by_default(self) -> None: + """Default harness (bridging on): a SPEC emit on the harness bus is + delivered to a LEGACY-topic subscriber (emit_legacy bridging).""" + spec_topic = str(SpecMessage.SPEAK) + with MiniPHAL() as phal: + seen = [] + phal._bus.on("speak", lambda m: seen.append(m)) + phal._bus.emit(Message(spec_topic, {"utterance": "bye"})) + time.sleep(0.05) + assert [m.data["utterance"] for m in seen] == ["bye"] + + def test_no_bridging_isolates_namespaces(self) -> None: + """With modernize=False, emit_legacy=False the harness bus keeps each + namespace isolated — a LEGACY emit does NOT reach a SPEC subscriber.""" + spec_topic = str(SpecMessage.SPEAK) + with MiniPHAL(modernize=False, emit_legacy=False) as phal: + seen = [] + phal._bus.on(spec_topic, lambda m: seen.append(m)) + phal._bus.emit(Message("speak", {"utterance": "legacy only"})) + time.sleep(0.1) + assert seen == [] + + def test_phal_test_threads_bridging_flags(self) -> None: + """PHALTest forwards modernize/emit_legacy to MiniPHAL (default True).""" + t = PHALTest( + plugin_ids=[], + trigger_message=Message("harmless.trigger"), + ) + assert t.modernize is True + assert t.emit_legacy is True diff --git a/test/unittests/test_pipeline_harness.py b/test/unittests/test_pipeline_harness.py new file mode 100644 index 0000000..25cfa73 --- /dev/null +++ b/test/unittests/test_pipeline_harness.py @@ -0,0 +1,138 @@ +"""Regression tests for ovoscope.pipeline._SinkSkill.""" + +import importlib.util +import threading +import time +import unittest + +import pytest + +from ovos_bus_client.message import Message +from ovos_spec_tools import SpecMessage +from ovos_utils.fakebus import FakeBus + +from ovoscope.pipeline import PipelineHarness, _SinkSkill + +# PipelineHarness spins up a full MiniCroft pinned to the adapt pipeline. +ADAPT_AVAILABLE = importlib.util.find_spec("ovos_adapt") is not None + +LEGACY_UTTERANCE = "recognizer_loop:utterance" +SPEC_UTTERANCE = str(SpecMessage.UTTERANCE) # ovos.utterance.handle + + +class _RecordingBus: + def __init__(self): + self.handlers = [] + self.removed = [] + + def on(self, event, handler): + self.handlers.append((event, handler)) + + def remove(self, event, handler): + self.removed.append((event, handler)) + + +class TestSinkSkillBusHandling: + def test_default_constructs_with_fakebus(self): + # Regression: previously _SinkSkill(bus=None) crashed and + # PipelineHarness relied on passing None then rebinding. Now bus + # defaults to a FakeBus so construction is always safe and the + # skill is immediately usable. + sink = _SinkSkill() + assert isinstance(sink.bus, FakeBus) + assert sink._last_match is None + + def test_explicit_none_falls_back_to_fakebus(self): + sink = _SinkSkill(bus=None) + assert isinstance(sink.bus, FakeBus) + + def test_constructs_with_supplied_bus(self): + bus = _RecordingBus() + sink = _SinkSkill(bus=bus) + events = [e for e, _ in bus.handlers] + assert "intent.service.skills.activated" in events + assert "intent_failure" in events + + def test_rebinding_bus_detaches_previous(self): + old = _RecordingBus() + new = _RecordingBus() + sink = _SinkSkill(bus=old) + sink.bus = new + old_removed = [e for e, _ in old.removed] + assert "intent.service.skills.activated" in old_removed + assert "intent_failure" in old_removed + new_events = [e for e, _ in new.handlers] + assert "intent.service.skills.activated" in new_events + assert "intent_failure" in new_events + + def test_setting_bus_to_none_raises(self): + sink = _SinkSkill() + with pytest.raises(ValueError): + sink.bus = None + + +# --------------------------------------------------------------------------- +# TestPipelineHarnessNamespaceBridging +# --------------------------------------------------------------------------- + +@unittest.skipUnless(ADAPT_AVAILABLE, "ovos-adapt pipeline plugin not installed") +class TestPipelineHarnessNamespaceBridging(unittest.TestCase): + """PipelineHarness injects utterances on the LEGACY topic + (``recognizer_loop:utterance``). These tests pin that the harness FakeBus + bridges that to/from the ovos.* SPEC topic (``ovos.utterance.handle``) so a + pipeline test can drive EITHER namespace, and that disabling the bridge + isolates a single namespace. + """ + + PIPELINE = ["ovos-adapt-pipeline-plugin-high"] + + def _emit_and_collect(self, harness, emit_topic, watch_topic, *, timeout=3.0): + """Subscribe on ``watch_topic``, emit one utterance on ``emit_topic``, + return the payloads observed on ``watch_topic``.""" + seen = [] + got = threading.Event() + + def _on(msg): + if isinstance(msg, str): + msg = Message.deserialize(msg) + seen.append(msg) + got.set() + + harness._mc.bus.on(watch_topic, _on) + try: + harness._mc.bus.emit(Message( + emit_topic, + data={"utterances": ["turn on the lights"], "lang": "en-US"}, + )) + got.wait(timeout) + finally: + harness._mc.bus.remove(watch_topic, _on) + return seen + + def test_legacy_utterance_observed_on_spec_topic(self): + """Default harness (bridging on): an utterance injected on the legacy + ``recognizer_loop:utterance`` topic is observed on the spec + ``ovos.utterance.handle`` topic (modernize bridging).""" + with PipelineHarness(pipeline=self.PIPELINE) as h: + seen = self._emit_and_collect(h, LEGACY_UTTERANCE, SPEC_UTTERANCE) + self.assertTrue(seen, "legacy utterance was not bridged to the spec topic") + self.assertEqual(seen[0].data["utterances"], ["turn on the lights"]) + + def test_spec_utterance_drives_pipeline_and_legacy_listener(self): + """An utterance injected on the SPEC topic still drives the pipeline + (a match is produced) and reaches a LEGACY listener (emit_legacy).""" + with PipelineHarness(pipeline=self.PIPELINE) as h: + legacy_seen = self._emit_and_collect(h, SPEC_UTTERANCE, LEGACY_UTTERANCE) + self.assertTrue(legacy_seen, + "spec utterance was not bridged to the legacy topic") + self.assertEqual(legacy_seen[0].data["utterances"], ["turn on the lights"]) + + def test_no_bridging_isolates_legacy_from_spec(self): + """With bridging OFF, a legacy utterance emit does NOT reach a + spec-only subscriber — a single namespace is exercised in isolation.""" + with PipelineHarness(pipeline=self.PIPELINE, + modernize=False, emit_legacy=False) as h: + seen = self._emit_and_collect(h, LEGACY_UTTERANCE, SPEC_UTTERANCE, + timeout=0.5) + self.assertEqual(seen, [], + "legacy emit must not reach the spec topic when bridging is off") diff --git a/test/unittests/test_simple_listener.py b/test/unittests/test_simple_listener.py new file mode 100644 index 0000000..c099051 --- /dev/null +++ b/test/unittests/test_simple_listener.py @@ -0,0 +1,168 @@ +# Copyright 2024 Jarbas AI +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for the MiniSimpleListener harness (ovos-simple-listener).""" +import io +import time +import unittest +import wave + +from ovos_bus_client.message import Message +from ovos_spec_tools import SpecMessage + +from ovoscope.simple_listener import MiniSimpleListener +from ovoscope.voice_loop import MockHotWordEngine, MockStreamingSTT + +def _mic_available() -> bool: + """SimpleListener eagerly builds a real microphone in __init__ (mic=None), + so the harness can only be constructed when a microphone plugin is present. + """ + try: + from ovos_plugin_manager.microphone import OVOSMicrophoneFactory + OVOSMicrophoneFactory.create() + return True + except Exception: + return False + + +HAS_MIC = _mic_available() + +try: + import ovos_simple_listener # noqa: F401 + HAS_SIMPLE = True +except ImportError: + HAS_SIMPLE = False + + +def _wav(speech_seconds=0.6, sample_rate=16000, sample_width=2): + buf = io.BytesIO() + with wave.open(buf, "wb") as w: + w.setframerate(sample_rate) + w.setsampwidth(sample_width) + w.setnchannels(1) + w.writeframes(b"\x10\x20" * int(sample_rate * speech_seconds)) + return buf.getvalue() + + +@unittest.skipUnless(HAS_SIMPLE, "ovos-simple-listener not installed") +class TestMiniSimpleListener(unittest.TestCase): + """MiniSimpleListener integration tests against a real SimpleListener.""" + + def test_full_sequence_with_utterance(self): + """A wake word + command yields the full bus sequence + utterance.""" + with MiniSimpleListener( + wakeword=MockHotWordEngine("hey_mycroft", trigger_after=2), + stt_instance=MockStreamingSTT(transcript="turn on the lights"), + ) as sl: + msgs = sl.feed_file(_wav(), silence_tail_chunks=20) + types = [m.msg_type for m in msgs] + self.assertIn("recognizer_loop:wakeword", types) + self.assertIn("recognizer_loop:record_begin", types) + self.assertIn("recognizer_loop:record_end", types) + sl.assert_utterance_emitted("turn on the lights", msgs) + + def test_empty_transcript_unknown(self): + """An empty transcript emits the recognition-unknown event.""" + with MiniSimpleListener( + wakeword=MockHotWordEngine("hey_mycroft", trigger_after=2), + stt_instance=MockStreamingSTT(transcript=""), + ) as sl: + msgs = sl.feed_file(_wav(), silence_tail_chunks=20) + types = [m.msg_type for m in msgs] + self.assertIn("recognizer_loop:speech.recognition.unknown", types) + self.assertNotIn("recognizer_loop:utterance", types) + + def test_record_begin_helper(self): + """The shared assert_record_begin_emitted helper works here too.""" + with MiniSimpleListener( + wakeword=MockHotWordEngine("hey_mycroft", trigger_after=1), + stt_instance=MockStreamingSTT(transcript="hello"), + ) as sl: + sl.feed_file(_wav(), silence_tail_chunks=20) + sl.assert_record_begin_emitted() + + +# --------------------------------------------------------------------------- +# TestMiniSimpleListenerNamespaceBridging +# --------------------------------------------------------------------------- + +@unittest.skipUnless(HAS_SIMPLE, "ovos-simple-listener not installed") +@unittest.skipUnless(HAS_MIC, "no ovos microphone plugin available") +class TestMiniSimpleListenerNamespaceBridging(unittest.TestCase): + """The _SimpleBusCallbacks emit the LEGACY ``recognizer_loop:*`` topics; + ``record_begin``/``record_end``/``utterance`` are migrated to the ovos.* + spec namespace. These tests pin that the FakeBus namespace bridging + connects the two namespaces, and that turning it off isolates one. + """ + + def test_full_sequence_legacy_reaches_spec_via_bridging(self): + """Default harness (bridging on): the legacy record-begin/record-end/ + utterance emitted while running the listener are also delivered to + subscribers on the spec topics.""" + with MiniSimpleListener( + wakeword=MockHotWordEngine("hey_mycroft", trigger_after=2), + stt_instance=MockStreamingSTT(transcript="turn on the lights"), + ) as sl: + spec = {"begin": [], "end": [], "utt": []} + sl.bus.on(str(SpecMessage.LISTENER_RECORD_STARTED), + lambda m: spec["begin"].append(m)) + sl.bus.on(str(SpecMessage.LISTENER_RECORD_ENDED), + lambda m: spec["end"].append(m)) + sl.bus.on(str(SpecMessage.UTTERANCE), + lambda m: spec["utt"].append(m)) + + msgs = sl.feed_file(_wav(), silence_tail_chunks=20) + time.sleep(0.05) + legacy_types = [m.msg_type for m in msgs] + self.assertIn("recognizer_loop:record_begin", legacy_types) + self.assertTrue(spec["begin"], + "ovos.listener.record.started not seen via bridge") + self.assertTrue(spec["end"], + "ovos.listener.record.ended not seen via bridge") + self.assertTrue(spec["utt"], + "ovos.utterance.handle not seen via bridge") + + def test_record_begin_spec_native(self): + """A SPEC producer reaches a spec subscriber natively.""" + with MiniSimpleListener( + wakeword=MockHotWordEngine("hey_mycroft", trigger_after=2), + ) as sl: + spec_hits = [] + sl.bus.on(str(SpecMessage.LISTENER_RECORD_STARTED), + lambda m: spec_hits.append(m)) + sl.bus.emit(Message(str(SpecMessage.LISTENER_RECORD_STARTED))) + time.sleep(0.05) + self.assertTrue(spec_hits, + "ovos.listener.record.started not delivered natively") + + def test_no_bridging_isolates_legacy_from_spec(self): + """With bridging OFF, a legacy record-begin does NOT reach a spec-only + subscriber.""" + with MiniSimpleListener( + wakeword=MockHotWordEngine("hey_mycroft", trigger_after=2), + modernize=False, emit_legacy=False, + ) as sl: + legacy_hits, spec_hits = [], [] + sl.bus.on("recognizer_loop:record_begin", + lambda m: legacy_hits.append(m)) + sl.bus.on(str(SpecMessage.LISTENER_RECORD_STARTED), + lambda m: spec_hits.append(m)) + sl.bus.emit(Message("recognizer_loop:record_begin")) + time.sleep(0.1) + self.assertTrue(legacy_hits, "legacy record_begin should still fire") + self.assertEqual(spec_hits, [], + "spec topic must not fire with bridging off") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/unittests/test_tts_intelligibility.py b/test/unittests/test_tts_intelligibility.py new file mode 100644 index 0000000..7f8f247 --- /dev/null +++ b/test/unittests/test_tts_intelligibility.py @@ -0,0 +1,222 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for ovoscope.tts_intelligibility. + +These tests use a MockTTS (silent WAV) and a MockSTT that echoes a fixed +transcript — no model download, no real audio. They cover WER/CER math, report +aggregation, serialisation, and that playback interception captures a wav path. +""" + +import importlib.util +import subprocess +import sys +import unittest + +TTS_AVAILABLE = ( + importlib.util.find_spec("jiwer") is not None + and importlib.util.find_spec("ovos_audio") is not None + and importlib.util.find_spec("ovos_utterance_normalizer") is not None +) + + +@unittest.skipUnless(TTS_AVAILABLE, "tts extra (jiwer/ovos-audio/normalizer) not installed") +class TestScoring(unittest.TestCase): + """WER/CER math, normalisation and report aggregation — no synthesis.""" + + def setUp(self): + from ovoscope import tts_intelligibility as ti + self.ti = ti + + def test_perfect_match_is_zero(self): + wer, cer = self.ti._score_pair("hello world", "hello world") + self.assertEqual(wer, 0.0) + self.assertEqual(cer, 0.0) + + def test_one_wrong_word_half_wer(self): + wer, _ = self.ti._score_pair("hello world", "hello there") + self.assertAlmostEqual(wer, 0.5, places=3) + + def test_empty_reference_handling(self): + self.assertEqual(self.ti._score_pair("", ""), (0.0, 0.0)) + self.assertEqual(self.ti._score_pair("", "noise"), (1.0, 1.0)) + + def test_normalize_strips_case_and_punctuation(self): + # "Hello, World!" should normalise to "hello world" so it scores 0 + # against the lowercased ground truth. + wer, _ = self.ti._score_pair( + self.ti._normalize("Hello, World!", "en-US"), + self.ti._normalize("hello world", "en-US"), + ) + self.assertEqual(wer, 0.0) + + def test_report_aggregation(self): + UtteranceScore = self.ti.UtteranceScore + IntelligibilityReport = self.ti.IntelligibilityReport + report = IntelligibilityReport(lang="en-US", voice="alan") + report.scores.append(UtteranceScore("a", "a", 0.0, 0.0, lang="en-US")) + report.scores.append(UtteranceScore("b", "x", 1.0, 1.0, lang="en-US")) + self.assertAlmostEqual(report.mean_wer, 0.5) + self.assertAlmostEqual(report.mean_cer, 0.5) + + def test_empty_report_means_zero(self): + report = self.ti.IntelligibilityReport() + self.assertEqual(report.mean_wer, 0.0) + self.assertEqual(report.mean_cer, 0.0) + + def test_to_dict_and_markdown_row(self): + UtteranceScore = self.ti.UtteranceScore + report = self.ti.IntelligibilityReport(lang="en-US", voice="alan") + report.scores.append(UtteranceScore("a", "a", 0.0, 0.0, lang="en-US")) + d = report.to_dict() + self.assertEqual(d["lang"], "en-US") + self.assertEqual(d["voice"], "alan") + self.assertEqual(d["n_utterances"], 1) + self.assertEqual(d["mean_wer"], 0.0) + self.assertIn("scores", d) + row = report.to_markdown_row() + self.assertIn("alan", row) + self.assertIn("en-US", row) + self.assertTrue(row.startswith("|") and row.endswith("|")) + + +class MockSTT: + """Reference STT stub that echoes a fixed transcript regardless of audio.""" + + def __init__(self, transcript="hello world"): + self.transcript = transcript + self.calls = 0 + + def execute(self, audio, language=None): + self.calls += 1 + return self.transcript + + +@unittest.skipUnless(TTS_AVAILABLE, "tts extra (jiwer/ovos-audio/normalizer) not installed") +class TestHarnessDirectMode(unittest.TestCase): + """Direct mode: tts.get_tts only, MockSTT echo — no bus, no model.""" + + def test_direct_mode_perfect_score(self): + from ovoscope.audio import MockTTS + from ovoscope.tts_intelligibility import score_tts_intelligibility + + stt = MockSTT("hello world") + report = score_tts_intelligibility( + MockTTS(), ["hello world"], + reference_stt=stt, mode="direct", + ) + self.assertEqual(len(report.scores), 1) + self.assertEqual(report.mean_wer, 0.0) + self.assertEqual(stt.calls, 1) + self.assertIsNotNone(report.scores[0].wav_path) + + def test_direct_mode_mismatch_scores_high(self): + from ovoscope.audio import MockTTS + from ovoscope.tts_intelligibility import score_tts_intelligibility + + report = score_tts_intelligibility( + MockTTS(), ["completely different text here"], + reference_stt=MockSTT("hello world"), mode="direct", + ) + self.assertGreater(report.mean_wer, 0.0) + + +@unittest.skipUnless(TTS_AVAILABLE, "tts extra (jiwer/ovos-audio/normalizer) not installed") +class TestHarnessPlaybackMode(unittest.TestCase): + """Playback mode: full ovos-audio stack drives MockTTS; wav is captured.""" + + def test_playback_captures_wav_and_scores(self): + from ovoscope.audio import MockTTS + from ovoscope.tts_intelligibility import TTSIntelligibilityHarness + + tts = MockTTS() + stt = MockSTT("hello world") + with TTSIntelligibilityHarness( + tts, reference_stt=stt, mode="playback", speak_timeout=15.0, + ) as h: + report = h.score(["hello world"]) + + self.assertEqual(len(report.scores), 1) + score = report.scores[0] + # Playback interception must have captured a rendered wav path. + self.assertIsNotNone(score.wav_path, "no wav captured from playback") + self.assertIn("hello world", tts.spoken_utterances) + self.assertEqual(report.mean_wer, 0.0) + + +class TestGracefulImport(unittest.TestCase): + """Core ``import ovoscope`` must succeed even without the [tts] extra.""" + + def test_import_without_tts_extra(self): + # Run in a subprocess with the optional tts deps blocked at import time, + # simulating an environment that never installed ovoscope[tts]. + code = ( + "import sys, importlib.abc, importlib.machinery\n" + "BLOCKED = {'jiwer', 'ovos_utterance_normalizer', " + "'ovos_stt_plugin_fasterwhisper', 'faster_whisper'}\n" + "class _Block(importlib.abc.MetaPathFinder):\n" + " def find_spec(self, name, path, target=None):\n" + " if name.split('.')[0] in BLOCKED:\n" + " raise ModuleNotFoundError(name=name.split('.')[0])\n" + " return None\n" + "sys.meta_path.insert(0, _Block())\n" + "for m in list(sys.modules):\n" + " if m.split('.')[0] in BLOCKED:\n" + " del sys.modules[m]\n" + "import ovoscope\n" + "assert not hasattr(ovoscope, 'TTSIntelligibilityHarness'), " + "'harness should be absent without the tts extra'\n" + "print('OK')\n" + ) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, text=True, + ) + self.assertEqual( + result.returncode, 0, + f"core import failed without tts extra:\nstdout={result.stdout}\nstderr={result.stderr}", + ) + self.assertIn("OK", result.stdout) + + +@unittest.skipUnless(TTS_AVAILABLE, "tts extra (jiwer/ovos-audio/normalizer) not installed") +class TestSynthesisFailureResilience(unittest.TestCase): + """A get_tts crash must score as a total miss, not abort the whole run.""" + + def test_synthesis_failure_scores_total_miss(self): + from ovoscope import tts_intelligibility as ti + + class BoomTTS: + def get_tts(self, *args, **kwargs): + raise RuntimeError("synthesis exploded") + + class EchoSTT: + def execute(self, audio, language=None): + return "anything" + + harness = ti.TTSIntelligibilityHarness( + BoomTTS(), lang="en-US", reference_stt=EchoSTT(), mode="direct", + ) + with harness: + report = harness.score(["hello world", "good morning"]) + + # The run completes and every utterance is scored a total miss (WER 1.0) + # rather than the exception propagating and emitting no marker at all. + self.assertEqual(len(report.scores), 2) + self.assertEqual(report.mean_wer, 1.0) + # The report still serialises, so the test's marker can be emitted. + import json + json.loads(json.dumps(report.to_dict())) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/unittests/test_voice_loop.py b/test/unittests/test_voice_loop.py new file mode 100644 index 0000000..888360c --- /dev/null +++ b/test/unittests/test_voice_loop.py @@ -0,0 +1,377 @@ +# Copyright 2024 Jarbas AI +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for the MiniVoiceLoop harness. + +TestMiniHotwordContainer — container update/found/get_ww/verify (no dinkum) +TestMiniVoiceLoop — feed_chunks + helpers (needs ovos-dinkum-listener) +TestVerifierGate — verifier suppression (needs the hotword-verifier gate) +TestVoiceLoopTest — declarative helper +""" +import inspect +import io +import time +import unittest +import wave +from unittest.mock import Mock + +from ovos_bus_client.message import Message +from ovos_spec_tools import SpecMessage + +from ovoscope.voice_loop import ( + MiniHotwordContainer, + MiniVoiceLoop, + MockHotWordEngine, + MockStreamingSTT, + VoiceLoopTest, + get_mini_voice_loop, +) + +_SILENCE = b"\x00" * 512 + + +def _wav(speech_seconds=0.6, sample_rate=16000, sample_width=2): + """Build in-memory WAV bytes of non-zero 'speech'.""" + buf = io.BytesIO() + with wave.open(buf, "wb") as w: + w.setframerate(sample_rate) + w.setsampwidth(sample_width) + w.setnchannels(1) + w.writeframes(b"\x10\x20" * int(sample_rate * speech_seconds)) + return buf.getvalue() + +# ovos-dinkum-listener is an optional test dependency — the harness can build a +# real DinkumVoiceLoop only when it is installed. +try: + from ovos_dinkum_listener.voice_loop.voice_loop import DinkumVoiceLoop + HAS_DINKUM = True + # The verifier gate is only present in builds shipping the hotword-verifier + # feature; suppression assertions require it. + HAS_VERIFY_GATE = "self.hotwords.verify" in inspect.getsource( + DinkumVoiceLoop._detect_ww + ) +except ImportError: + HAS_DINKUM = False + HAS_VERIFY_GATE = False + + +def _accepting(): + v = Mock() + v.verify.return_value = True + return v + + +def _rejecting(): + v = Mock() + v.verify.return_value = False + return v + + +def _raising(): + v = Mock() + v.verify.side_effect = RuntimeError("verifier boom") + return v + + +# --------------------------------------------------------------------------- +# MiniHotwordContainer (no dinkum required) +# --------------------------------------------------------------------------- + +class TestMiniHotwordContainer(unittest.TestCase): + """MiniHotwordContainer unit tests.""" + + def test_found_after_threshold(self): + """found() returns the ww name once the engine fires.""" + c = MiniHotwordContainer( + {"hey_mycroft": MockHotWordEngine(trigger_after=2)} + ) + c.update(_SILENCE) + self.assertIsNone(c.found()) + c.update(_SILENCE) + self.assertEqual(c.found(), "hey_mycroft") + + def test_found_none_when_not_triggered(self): + """found() returns None before the threshold.""" + c = MiniHotwordContainer( + {"hey_mycroft": MockHotWordEngine(trigger_after=10)} + ) + c.update(_SILENCE) + self.assertIsNone(c.found()) + + def test_update_feeds_all_engines(self): + """update() forwards chunks to every registered engine.""" + a = MockHotWordEngine("hey_mycroft", trigger_after=1) + b = MockHotWordEngine("hey_jarbas", trigger_after=1) + c = MiniHotwordContainer({"hey_mycroft": a, "hey_jarbas": b}) + c.update(_SILENCE) + self.assertEqual(a.update_count, 1) + self.assertEqual(b.update_count, 1) + + def test_get_ww_metadata(self): + """get_ww() returns listen-word metadata for a known ww.""" + c = MiniHotwordContainer({"hey_mycroft": MockHotWordEngine()}) + meta = c.get_ww("hey_mycroft") + self.assertEqual(meta["key_phrase"], "hey_mycroft") + self.assertTrue(meta["listen"]) + self.assertIsNone(meta["sound"]) + + def test_get_ww_unknown_raises(self): + """get_ww() raises ValueError for an unregistered ww.""" + c = MiniHotwordContainer({"hey_mycroft": MockHotWordEngine()}) + with self.assertRaises(ValueError): + c.get_ww("unknown") + + def test_verify_accept(self): + """verify() returns True when all verifiers accept.""" + c = MiniHotwordContainer({}, verifiers=[_accepting(), _accepting()]) + self.assertTrue(c.verify(b"audio")) + + def test_verify_reject(self): + """verify() returns False when any verifier rejects.""" + c = MiniHotwordContainer({}, verifiers=[_accepting(), _rejecting()]) + self.assertFalse(c.verify(b"audio")) + + def test_verify_fail_open(self): + """verify() ignores a raising verifier (fail-open).""" + c = MiniHotwordContainer({}, verifiers=[_raising(), _accepting()]) + self.assertTrue(c.verify(b"audio")) + + def test_verify_no_verifiers(self): + """verify() returns True when no verifiers are configured.""" + c = MiniHotwordContainer({}) + self.assertTrue(c.verify(b"audio")) + + +# --------------------------------------------------------------------------- +# MiniVoiceLoop (requires ovos-dinkum-listener) +# --------------------------------------------------------------------------- + +@unittest.skipUnless(HAS_DINKUM, "ovos-dinkum-listener not installed") +class TestMiniVoiceLoop(unittest.TestCase): + """MiniVoiceLoop integration tests against a real DinkumVoiceLoop.""" + + def _loop(self, trigger_after=3, verifiers=None): + ww = MockHotWordEngine("hey_mycroft", trigger_after=trigger_after) + return MiniVoiceLoop( + ww_instances={"hey_mycroft": ww}, verifiers=verifiers + ) + + def test_accept_emits_record_begin(self): + """WW detected + accepting verifier emits the record-begin sequence.""" + with self._loop(verifiers=[_accepting()]) as vl: + msgs = vl.feed_chunks([_SILENCE] * 5) + types = {m.msg_type for m in msgs} + self.assertIn("recognizer_loop:wakeword", types) + self.assertIn("recognizer_loop:record_begin", types) + + def test_no_wakeword_no_events(self): + """No detection emits no recognizer_loop events.""" + with self._loop(trigger_after=100, verifiers=[_accepting()]) as vl: + msgs = vl.feed_chunks([_SILENCE] * 5) + self.assertFalse( + any(m.msg_type.startswith("recognizer_loop:") for m in msgs) + ) + + def test_raising_verifier_fails_open(self): + """A raising verifier does not suppress detection (fail-open).""" + with self._loop(verifiers=[_raising()]) as vl: + vl.assert_record_begin_emitted(vl.feed_chunks([_SILENCE] * 5)) + + def test_default_ww_instances(self): + """Omitting ww_instances uses a default hey_mycroft engine.""" + with MiniVoiceLoop() as vl: + vl.assert_record_begin_emitted(vl.feed_chunks([_SILENCE] * 3)) + + def test_assert_wakeword_detected_helper(self): + """assert_wakeword_detected passes on a real detection.""" + with self._loop() as vl: + vl.feed_chunks([_SILENCE] * 5) + vl.assert_wakeword_detected() # operates on last feed result + + def test_assert_wakeword_detected_fails_without_detection(self): + """assert_wakeword_detected raises when nothing fired.""" + with self._loop(trigger_after=100) as vl: + vl.feed_chunks([_SILENCE] * 5) + with self.assertRaises(AssertionError): + vl.assert_wakeword_detected() + + def test_factory(self): + """get_mini_voice_loop wires a usable harness.""" + vl = get_mini_voice_loop( + ww_instances={"hey_mycroft": MockHotWordEngine(trigger_after=2)}, + ) + try: + vl.assert_record_begin_emitted(vl.feed_chunks([_SILENCE] * 3)) + finally: + vl.shutdown() + + def test_feed_file_full_sequence(self): + """feed_file drives the whole loop to a transcribed utterance.""" + ww = MockHotWordEngine("hey_mycroft", trigger_after=2) + stt = MockStreamingSTT(transcript="what time is it") + with MiniVoiceLoop(ww_instances={"hey_mycroft": ww}, stt_instance=stt) as vl: + msgs = vl.feed_file(_wav(), silence_tail_chunks=30) + types = [m.msg_type for m in msgs] + self.assertIn("recognizer_loop:record_begin", types) + self.assertIn("recognizer_loop:record_end", types) + vl.assert_utterance_emitted("what time is it", msgs) + + def test_feed_file_empty_transcript_unknown(self): + """feed_file with no transcript emits the recognition-unknown event.""" + ww = MockHotWordEngine("hey_mycroft", trigger_after=2) + with MiniVoiceLoop(ww_instances={"hey_mycroft": ww}, + stt_instance=MockStreamingSTT(transcript="")) as vl: + msgs = vl.feed_file(_wav(), silence_tail_chunks=30) + types = [m.msg_type for m in msgs] + self.assertIn("recognizer_loop:speech.recognition.unknown", types) + self.assertNotIn("recognizer_loop:utterance", types) + + +# --------------------------------------------------------------------------- +# Verifier suppression gate (requires the hotword-verifier feature) +# --------------------------------------------------------------------------- + +@unittest.skipUnless( + HAS_VERIFY_GATE, "ovos-dinkum-listener build lacks the hotword-verifier gate" +) +class TestVerifierGate(unittest.TestCase): + """Tests that depend on DinkumVoiceLoop gating on hotwords.verify().""" + + def test_reject_suppresses(self): + """A rejecting verifier suppresses the whole record sequence.""" + ww = MockHotWordEngine("hey_mycroft", trigger_after=3) + with MiniVoiceLoop( + ww_instances={"hey_mycroft": ww}, verifiers=[_rejecting()] + ) as vl: + msgs = vl.feed_chunks([_SILENCE] * 5) + vl.assert_wakeword_suppressed(msgs) + + def test_voice_loop_test_expect_suppressed(self): + """VoiceLoopTest(expect_record_begin=False) passes on rejection.""" + VoiceLoopTest( + ww_instances={"hey_mycroft": MockHotWordEngine(trigger_after=3)}, + verifiers=[_rejecting()], + audio_chunks=[_SILENCE] * 5, + expect_record_begin=False, + ).execute() + + +# --------------------------------------------------------------------------- +# VoiceLoopTest declarative helper (requires ovos-dinkum-listener) +# --------------------------------------------------------------------------- + +@unittest.skipUnless(HAS_DINKUM, "ovos-dinkum-listener not installed") +class TestVoiceLoopTest(unittest.TestCase): + """VoiceLoopTest declarative helper tests.""" + + def test_expect_record_begin_passes(self): + """Passes when record_begin is expected and emitted.""" + VoiceLoopTest( + ww_instances={"hey_mycroft": MockHotWordEngine(trigger_after=3)}, + verifiers=[_accepting()], + audio_chunks=[_SILENCE] * 5, + expect_record_begin=True, + ).execute() + + def test_expect_record_begin_fails_without_detection(self): + """Raises AssertionError when record_begin expected but absent.""" + with self.assertRaises(AssertionError): + VoiceLoopTest( + ww_instances={"hey_mycroft": MockHotWordEngine(trigger_after=100)}, + audio_chunks=[_SILENCE] * 5, + expect_record_begin=True, + ).execute() + + def test_audio_file_with_expected_utterance(self): + """audio_file path drives the full loop and asserts the utterance.""" + VoiceLoopTest( + ww_instances={"hey_mycroft": MockHotWordEngine(trigger_after=2)}, + stt_instance=MockStreamingSTT(transcript="hello world"), + audio_file=_wav(), + expect_utterance="hello world", + ).execute() + + +# --------------------------------------------------------------------------- +# TestMiniVoiceLoopNamespaceBridging +# --------------------------------------------------------------------------- + +@unittest.skipUnless(HAS_DINKUM, "ovos-dinkum-listener not installed") +class TestMiniVoiceLoopNamespaceBridging(unittest.TestCase): + """The MiniVoiceLoop callbacks emit the LEGACY ``recognizer_loop:*`` topics; + ``record_begin``/``record_end``/``utterance`` are migrated to the ovos.* + spec namespace. These tests pin that the FakeBus namespace bridging + connects the two namespaces, and that turning it off isolates one. + """ + + def test_full_loop_legacy_reaches_spec_via_bridging(self): + """Default loop (bridging on): the legacy record-begin/record-end/ + utterance emitted while driving the full loop are also delivered to + subscribers on the spec topics.""" + with MiniVoiceLoop( + ww_instances={"hey_mycroft": MockHotWordEngine(trigger_after=2)}, + stt_instance=MockStreamingSTT(transcript="hello world"), + ) as vl: + spec = {"begin": [], "end": [], "utt": []} + vl.bus.on(str(SpecMessage.LISTENER_RECORD_STARTED), + lambda m: spec["begin"].append(m)) + vl.bus.on(str(SpecMessage.LISTENER_RECORD_ENDED), + lambda m: spec["end"].append(m)) + vl.bus.on(str(SpecMessage.UTTERANCE), + lambda m: spec["utt"].append(m)) + + msgs = vl.feed_file(_wav()) + time.sleep(0.05) + legacy_types = [m.msg_type for m in msgs] + self.assertIn("recognizer_loop:record_begin", legacy_types) + self.assertTrue(spec["begin"], + "ovos.listener.record.started not seen via bridge") + self.assertTrue(spec["end"], + "ovos.listener.record.ended not seen via bridge") + self.assertTrue(spec["utt"], + "ovos.utterance.handle not seen via bridge") + + def test_record_begin_spec_native(self): + """A SPEC producer reaches a spec subscriber natively.""" + with MiniVoiceLoop( + ww_instances={"hey_mycroft": MockHotWordEngine(trigger_after=2)}, + ) as vl: + spec_hits = [] + vl.bus.on(str(SpecMessage.LISTENER_RECORD_STARTED), + lambda m: spec_hits.append(m)) + vl.bus.emit(Message(str(SpecMessage.LISTENER_RECORD_STARTED))) + time.sleep(0.05) + self.assertTrue(spec_hits, + "ovos.listener.record.started not delivered natively") + + def test_no_bridging_isolates_legacy_from_spec(self): + """With bridging OFF, a legacy record-begin does NOT reach a spec-only + subscriber.""" + with MiniVoiceLoop( + ww_instances={"hey_mycroft": MockHotWordEngine(trigger_after=2)}, + modernize=False, emit_legacy=False, + ) as vl: + legacy_hits, spec_hits = [], [] + vl.bus.on("recognizer_loop:record_begin", + lambda m: legacy_hits.append(m)) + vl.bus.on(str(SpecMessage.LISTENER_RECORD_STARTED), + lambda m: spec_hits.append(m)) + vl.bus.emit(Message("recognizer_loop:record_begin")) + time.sleep(0.1) + self.assertTrue(legacy_hits, "legacy record_begin should still fire") + self.assertEqual(spec_hits, [], + "spec topic must not fire with bridging off") + + +if __name__ == "__main__": + unittest.main()