diff --git a/.github/workflows/build-tests.yml b/.github/workflows/build-tests.yml new file mode 100644 index 0000000..e1fc761 --- /dev/null +++ b/.github/workflows/build-tests.yml @@ -0,0 +1,15 @@ +name: Build Tests + +on: + pull_request: + branches: [dev, master, main] + workflow_dispatch: + +jobs: + build: + uses: OpenVoiceOS/gh-automations/.github/workflows/build-tests.yml@dev + secrets: inherit + with: + python_versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]' + install_extras: 'audio,pydantic' + test_path: 'test/unittests/' diff --git a/.github/workflows/build_tests.yml b/.github/workflows/build_tests.yml deleted file mode 100644 index 02fa235..0000000 --- a/.github/workflows/build_tests.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Run Build Tests -on: - push: - branches: - - master - pull_request: - branches: - - dev - workflow_dispatch: - -jobs: - build_tests: - uses: OpenVoiceOS/gh-automations/.github/workflows/build-tests.yml@dev - secrets: inherit - with: - python_versions: '["3.10", "3.11", "3.12", "3.13"]' - install_extras: "audio,pydantic" - test_path: "test/unittests/" diff --git a/.github/workflows/conventional-label.yaml b/.github/workflows/conventional-label.yml similarity index 77% rename from .github/workflows/conventional-label.yaml rename to .github/workflows/conventional-label.yml index 0a449cb..9894c1b 100644 --- a/.github/workflows/conventional-label.yaml +++ b/.github/workflows/conventional-label.yml @@ -7,4 +7,4 @@ jobs: label: runs-on: ubuntu-latest steps: - - uses: bcoe/conventional-release-labels@v1 \ No newline at end of file + - uses: bcoe/conventional-release-labels@v1 diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/coverage-pages.yml similarity index 67% rename from .github/workflows/unit_tests.yml rename to .github/workflows/coverage-pages.yml index a0b8dfc..f76ae94 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/coverage-pages.yml @@ -1,20 +1,20 @@ -name: Run Tests +name: Coverage Pages on: - pull_request: + push: branches: - dev workflow_dispatch: permissions: - pull-requests: write - contents: read + contents: write jobs: - unit_tests: + coverage_pages: uses: OpenVoiceOS/gh-automations/.github/workflows/coverage.yml@dev secrets: inherit with: python_version: "3.12" - install_extras: "audio" test_path: "test/unittests/" coverage_source: "ovoscope" + install_extras: "audio,pydantic" + deploy_pages: true diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 0000000..22cc219 --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,16 @@ +name: Code Coverage + +on: + pull_request: + branches: [dev] + workflow_dispatch: + +jobs: + coverage: + uses: OpenVoiceOS/gh-automations/.github/workflows/coverage.yml@dev + secrets: inherit + with: + python_version: '3.11' + coverage_source: 'ovoscope' + test_path: 'test/unittests/' + min_coverage: 0 diff --git a/.github/workflows/coverage_pages.yml b/.github/workflows/coverage_pages.yml deleted file mode 100644 index f4c43c0..0000000 --- a/.github/workflows/coverage_pages.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Coverage Pages -on: - push: - branches: - - dev - workflow_dispatch: - -permissions: - contents: write - -jobs: - coverage_pages: - uses: OpenVoiceOS/gh-automations/.github/workflows/coverage-pages.yml@dev - secrets: inherit - with: - python_version: "3.14" - test_path: "test/unittests/" - coverage_source: "ovoscope" diff --git a/.github/workflows/downstream_check.yml b/.github/workflows/downstream-check.yml similarity index 100% rename from .github/workflows/downstream_check.yml rename to .github/workflows/downstream-check.yml diff --git a/.github/workflows/license_check.yml b/.github/workflows/license-check.yml similarity index 100% rename from .github/workflows/license_check.yml rename to .github/workflows/license-check.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..9a6b7a5 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,14 @@ +name: Lint + +on: + pull_request: + branches: [dev, master, main] + workflow_dispatch: + +jobs: + lint: + uses: OpenVoiceOS/gh-automations/.github/workflows/lint.yml@dev + secrets: inherit + with: + ruff: true + pre_commit: false # set true if .pre-commit-config.yaml exists diff --git a/.github/workflows/pip_audit.yml b/.github/workflows/pip-audit.yml similarity index 100% rename from .github/workflows/pip_audit.yml rename to .github/workflows/pip-audit.yml diff --git a/.github/workflows/publish_stable.yml b/.github/workflows/publish-stable.yml similarity index 100% rename from .github/workflows/publish_stable.yml rename to .github/workflows/publish-stable.yml diff --git a/.github/workflows/release_preview.yml b/.github/workflows/release-preview.yml similarity index 74% rename from .github/workflows/release_preview.yml rename to .github/workflows/release-preview.yml index ea5542b..78797cb 100644 --- a/.github/workflows/release_preview.yml +++ b/.github/workflows/release-preview.yml @@ -1,4 +1,5 @@ name: Release Preview + on: pull_request: branches: [dev] @@ -9,5 +10,5 @@ jobs: uses: OpenVoiceOS/gh-automations/.github/workflows/release-preview.yml@dev secrets: inherit with: - package_name: "ovoscope" - version_file: "ovoscope/version.py" + package_name: 'ovoscope' + version_file: 'ovoscope/version.py' diff --git a/.github/workflows/release_workflow.yml b/.github/workflows/release-workflow.yml similarity index 100% rename from .github/workflows/release_workflow.yml rename to .github/workflows/release-workflow.yml diff --git a/.github/workflows/repo_health.yml b/.github/workflows/repo-health.yml similarity index 67% rename from .github/workflows/repo_health.yml rename to .github/workflows/repo-health.yml index ebfb0ac..bf2d648 100644 --- a/.github/workflows/repo_health.yml +++ b/.github/workflows/repo-health.yml @@ -1,10 +1,13 @@ name: Repo Health + on: pull_request: - branches: [dev] + branches: [dev, master, main] workflow_dispatch: jobs: repo_health: uses: OpenVoiceOS/gh-automations/.github/workflows/repo-health.yml@dev secrets: inherit + with: + version_file: 'ovoscope/version.py' diff --git a/AUDIT.md b/AUDIT.md index f456185..d4dd424 100644 --- a/AUDIT.md +++ b/AUDIT.md @@ -56,3 +56,129 @@ builds non-reproducible if the upstream action changes. ## Next Steps (Priority Order) 1. Address CaptureSession.capture() return value — usability 2. Pin CI action refs — reproducibility + +--- + +## Audit [2026-03-12] — Correctness Bugs, Coverage Gaps, Docs + +### ~~[CRITICAL] `pipeline.py` race condition in `match()`~~ ✅ FIXED + +**Evidence**: `ovoscope/pipeline.py:159–181` — a single `threading.Event` was +shared for both success (`intent.service.skills.activated`) and failure +(`intent_failure`, `mycroft.skill.handler.start`) handlers. If `intent_failure` +fired first, `captured[0]` would be a failure message returned as success. +Additionally, `done.wait()` return value was not checked; a timeout silently +returned `captured[0]` (or raised `IndexError` on empty list). + +**Fix**: Separate `_matched` and `_failed` events; only the success handler +populates `captured`. Timeout and failure both return `None`. +Source: `ovoscope/pipeline.py:149`. + +### ~~[MAJOR] `diff.py` — subset comparison silently ignores extra keys~~ ✅ FIXED + +**Evidence**: `ovoscope/diff.py:121–138` — `_dict_diff()` only iterated keys +in `expected`, so unexpected keys in `actual` were never flagged. + +**Fix**: Added `strict: bool = False` parameter to `_dict_diff()` and +`diff_fixtures()`. When `strict=True`, extra keys in `actual` are included in +the diff detail. Default `False` preserves existing behaviour. +Source: `ovoscope/diff.py:121`. + +### ~~[MINOR] `bus_coverage.py` — dead method `_skill_id_for_handler()`~~ ✅ FIXED + +**Evidence**: `ovoscope/bus_coverage.py:745–763` — `_skill_id_for_handler()` +was never called anywhere in the codebase (verified by grep). Its logic is +a subset of `_skill_id_for_closure()` which is the method actually used. + +**Fix**: Method deleted. +Source: formerly `ovoscope/bus_coverage.py:745`. + +### ~~[MAJOR] No unit tests for `media.py` (`MockOCPBackend`, `OCPCaptureSession`, `OCPPlayerHarness`)~~ ✅ FIXED + +**Evidence**: `ovoscope/media.py` had no corresponding test file. + +**Fix**: Created `test/unittests/test_media.py` — 20 tests covering +`MockOCPBackend` state transitions, `OCPCaptureSession` message accumulation, +and assertion helpers. + +### ~~[MAJOR] No unit tests for `remote_recorder.py`~~ ✅ FIXED + +**Evidence**: `ovoscope/remote_recorder.py` had no corresponding test file. + +**Fix**: Created `test/unittests/test_remote_recorder.py` — 15 tests covering +constructor defaults, `_parse_url`, connect/disconnect lifecycle, `record()` with +mocked bus client, timeout handling, and fixture serialization. + +### ~~[MINOR] Deprecated `ovos_utils.messagebus.Message` import in `test_phal.py`~~ ✅ FIXED + +**Evidence**: `test/unittests/test_phal.py:23` — imported `Message` from +deprecated `ovos_utils.messagebus` instead of canonical `ovos_bus_client.message`. + +**Fix**: Import changed to `from ovos_bus_client.message import Message`. +Source: `test/unittests/test_phal.py:23`. + +### ~~[MINOR] `SUGGESTIONS.md` missing~~ ✅ FIXED + +**Evidence**: `SUGGESTIONS.md` was absent despite being required by `AGENTS.md`. + +**Fix**: `SUGGESTIONS.md` created with 10 structured proposals. + +--- +## Bus Coverage Module — Full Audit [2026-03-12] + +### ~~[CRITICAL] async_responses excluded from emitter coverage~~ ✅ FIXED +`execute()` now passes `messages + list(capture.async_responses)` to `record_session()`. Source: `ovoscope/__init__.py:666`. + +### ~~[CRITICAL] Unattributed expected_messages silently disappear~~ ✅ FIXED +Both observed and expected messages with no `skill_id` in context now fall back to the `"__core__"` sentinel bucket. Source: `BusCoverageTracker.record_session` — `ovoscope/bus_coverage.py:510`. + +### [CRITICAL] Registration-time handlers always show NOT TESTED — misleading +**Evidence**: `ovoscope/bus_coverage.py:368` / `ovoscope/__init__.py:647` — `snapshot_listeners()` is called after `MiniCroft` reaches READY. Handlers invoked *during* skill loading were called *before* the snapshot and will always show 0 invocations. +**Impact**: Users see `register_intent: NOT TESTED` and conclude their intent registration failed. +**Status**: Documented in `docs/bus-coverage.md` Limitations as a known structural constraint. A `LOAD_TIME` tag or pre-READY tracking remains a future enhancement. + +### ~~[CRITICAL] Non-skill messages silently excluded from emitter coverage~~ ✅ FIXED +Messages with no `skill_id` in context are now attributed to the `"__core__"` bucket in `record_session()`. Source: `ovoscope/bus_coverage.py:510`. + +### ~~[MAJOR] Unused `skill_map` variable in `record_session()`~~ ✅ FIXED +Dead `skill_map = self._skill_instance_map()` call removed from `record_session()`. + +### ~~[MAJOR] `_get_bus_events()` called three times in `snapshot_listeners()`~~ ✅ FIXED +`bus_events = self._get_bus_events()` is now called once and reused across all three passes. Source: `ovoscope/bus_coverage.py:432`. + +### ~~[MAJOR] Double-stop risk in `cmd_bus_coverage()`~~ ✅ FIXED +`test.managed = False` is now set explicitly before `test.execute()`. The `finally` block is the sole owner of `mc.stop()`. The redundant `mc.stop()` in the `except Exception` branch was also removed. Source: `ovoscope/cli.py:349`. + +### ~~[MAJOR] `pytest_terminal_summary` hook uses private pytest internals~~ ✅ FIXED +`bus_coverage_session` fixture now stores merged reports on `request.config._bus_coverage_reports` in its teardown. `pytest_terminal_summary` reads that list — no private attrs. Source: `ovoscope/pytest_plugin.py:191`. + +### [MAJOR] `once()` handlers invisible after firing +**Evidence**: `bus_coverage.py:368` — one-shot handlers fired during skill loading are de-registered before the snapshot. +**Status**: Documented in `docs/bus-coverage.md` Limitations. Pre-READY tracking is a future enhancement. + +### [MAJOR] `ignore_messages` list silently excludes messages from emitter coverage +**Evidence**: `ovoscope/__init__.py:504-505` — messages in `ignore_messages` never reach `responses`. +**Status**: Documented in `docs/bus-coverage.md` Limitations. Passing ignored messages as a separate bucket is a future enhancement. + +### ~~[MAJOR] No JSON schema version field~~ ✅ FIXED +`to_json()` now includes `"schema_version": "1"` as the first key. Source: `ovoscope/bus_coverage.py:297`. + +### ~~[MINOR] Column widths hardcoded in `print_report()`~~ ✅ FIXED +`col_w = max(len(s.skill_id) for s in self.skills) + 2` now drives column width dynamically. Source: `ovoscope/bus_coverage.py:237`. + +### ~~[MINOR] Coverage summary printed before test assertions~~ ✅ FIXED +`print_bus_coverage` block moved to after all assertions, just before `managed` teardown. Source: `ovoscope/__init__.py:795`. + +### ~~[MINOR] `bus_coverage_report` type hint uses `Optional[Any]`~~ ✅ FIXED +Type hint changed to `Optional["BusCoverageReport"]`. Source: `ovoscope/__init__.py:589`. + +### ~~[NITPICK] `_SKIP` set has fragile `"type"` entry~~ ✅ FIXED +Pass 2 now uses `isinstance(owner, type)` check to skip class objects, and the `"type"` string entry removed from `_SKIP`. Source: `ovoscope/bus_coverage.py:450`. + +### ~~Docs gaps (bus-coverage.md)~~ ✅ FIXED +All five missing Limitations items added to `docs/bus-coverage.md`: +- Registration-time handlers always show NOT TESTED +- Pipeline matching is not bus-driven +- `async_responses` now included (fix note) +- `ignore_messages` types excluded +- Core services use `__core__` bucket (not 0/0 — fix note) diff --git a/FAQ.md b/FAQ.md index dad5647..1304056 100644 --- a/FAQ.md +++ b/FAQ.md @@ -1,4 +1,46 @@ # FAQ — `ovoscope` +## How do I measure which bus message handlers my tests actually exercise? + +Use bus coverage: set `track_bus_coverage=True` on `End2EndTest`. After +`execute()`, `test.bus_coverage_report` contains a `BusCoverageReport` with +per-skill listener coverage (which `bus.on()` registrations were triggered) +and emitter coverage (which message types were observed / asserted). + +```python +test = End2EndTest( + skill_ids=["my-skill.author"], + source_message=message, + expected_messages=[...], + track_bus_coverage=True, + print_bus_coverage=True, # print inline summary +) +test.execute() +print(test.bus_coverage_report.to_json()) +``` + +See [docs/bus-coverage.md](docs/bus-coverage.md) for the full reference. +`BusCoverageTracker` — `ovoscope/bus_coverage.py:242`. + +## How do I get an aggregate bus coverage report across an entire test suite? + +Use the `bus_coverage_session` pytest fixture. Each test calls +`bus_coverage_session.add(test.bus_coverage_report)` after `execute()`. A +merged table is printed automatically at session end. See +[docs/bus-coverage.md](docs/bus-coverage.md). + +## How do I run bus coverage from the command line without writing pytest tests? + +Use the `ovoscope bus-coverage` subcommand: + +```bash +ovoscope bus-coverage path/to/fixtures/ # table report +ovoscope bus-coverage path/to/fixtures/ --format json +ovoscope bus-coverage path/to/fixtures/ --verbose # per-msg detail +``` + +`cmd_bus_coverage` — `ovoscope/cli.py`. + + ## How do I test AudioService or PlaybackService without real audio hardware? Use `AudioServiceHarness` or `PlaybackServiceHarness` from `ovoscope.audio`. Both run on a `FakeBus` with `MockAudioBackend`/`MockTTS` respectively — no real audio device, TTS engine, @@ -356,6 +398,15 @@ with GUICaptureSession(mc.bus) as gui: gui.assert_page_shown("my_skill", "main.qml") ``` +### How do I assert that a skill set a specific session data key in a namespace? +Use `assert_namespace_has_key()`: +```python +with GUICaptureSession(mc.bus) as gui: + # ... trigger interaction ... + gui.assert_namespace_has_key("my_skill", "temperature") +``` +This checks that a `mycroft.session.set` message was captured containing the given key in the specified namespace. See [docs/gui-testing.md](docs/gui-testing.md). + --- ## Coverage Scanner diff --git a/MAINTENANCE_REPORT.md b/MAINTENANCE_REPORT.md index 198c5d2..f17aadd 100644 --- a/MAINTENANCE_REPORT.md +++ b/MAINTENANCE_REPORT.md @@ -1,4 +1,70 @@ # Maintenance Report — `ovoscope` + +## [2026-03-13] — GUICaptureSession: assert_namespace_has_key + unit tests + +- **AI Model**: Claude Opus 4.6 +- **Actions Taken**: + - Added `assert_namespace_has_key()` method to `GUICaptureSession` for asserting that a skill set a specific session data key in a GUI namespace + - Updated `docs/gui-testing.md` with documentation for the new method + - Created `test/unittests/test_gui_capture.py` with 10 unit tests covering the new assertion method + - Updated `FAQ.md` with Q&A for `assert_namespace_has_key()` +- **Oversight**: Human-reviewed plan execution + +--- + +## [2026-03-12] — Full Audit Improvements (Correctness, Coverage, Docs, Packaging) + +- **AI Model**: claude-sonnet-4-6 +- **Actions Taken**: + - **P1.1** Fixed `pipeline.py` race condition: split success/failure into + separate `threading.Event` objects; `match()` now returns `None` on timeout + or failure instead of returning a failure message as success. + Source: `ovoscope/pipeline.py:149–200`. + - **P1.2** Added `strict: bool = False` to `diff.py` `_dict_diff()` and + `diff_fixtures()`. When `strict=True`, extra keys in `actual` not in + `expected` are flagged. Default `False` preserves existing behaviour. + Source: `ovoscope/diff.py:121`. + - **P1.3** Deleted dead `_skill_id_for_handler()` from `bus_coverage.py` + (lines 745–763, never called anywhere in the codebase). + - **P2.1** Created `test/unittests/test_media.py` — 20 unit tests for + `MockOCPBackend` state transitions and `OCPCaptureSession` accumulation. + - **P2.2** Created `test/unittests/test_remote_recorder.py` — 15 unit tests + for `RemoteRecorder` using mocked `MessageBusClient`. + - **P2.3** Fixed deprecated `ovos_utils.messagebus.Message` import in + `test/unittests/test_phal.py` → `ovos_bus_client.message.Message`. + - **P3.1** Created `SUGGESTIONS.md` with 10 structured proposals. + - **P3.2** Updated `QUICK_FACTS.md` test count: 243 → 306. + - **P3.3** Expanded `docs/pipeline.md` to full API reference with `pipeline.py:LINE` + citations, `_SinkSkill` explanation, Adapt/Padatious examples, and + pipeline success/failure signal documentation. + - **P3.4** Fixed `docs/ocp.md` to correctly reference `OCPTest` (the class + in `ocp.py`) and add cross-reference to `OCPPlayerHarness` in `media.py`. + - **P3.5** Updated `AUDIT.md` with 7 new findings (5 fixed, 2 pre-existing). + - **P4.1** Updated `pyproject.toml`: added Documentation and Issue Tracker + URLs, `[tool.setuptools.package-data]`, `timeout = 60` in pytest options, + and comment explaining the `ovos-core>=2.0.4a2` alpha pin. + - **P5.1** Made `_count_fixtures()` in `coverage.py` use `Path.rglob("*.json")` + for recursive fixture counting instead of `os.listdir()`. + - **P5.2** Added `TYPE_CHECKING` guard and proper `List["BusCoverageReport"]` + type annotation to `BusCoverageCollector._reports` in `pytest_plugin.py`. +- **Oversight**: Human review pending. 348 unit tests pass locally (was 301; +35 new, +12 from new files). + +--- + +## [2026-03-12] — Bus Coverage Report Feature + +- **AI Model**: Claude Sonnet 4.6 +- **Actions Taken**: + - Created `ovoscope/bus_coverage.py` — `BusCoverageTracker`, `BusCoverageReport`, `SkillBusCoverage`, `HandlerEntry`, `EmitterEntry` dataclasses. Tracks listener invocations (via `bus.emit` monkey-patch) and emitter observed/asserted counts per skill_id. Handler attribution via `handler.__self__` → `minicroft.plugin_skills`. Handles pyee v9 `OrderedDict` storage format. + - Modified `ovoscope/__init__.py`: added `track_bus_coverage`, `print_bus_coverage`, `bus_coverage_report` fields to `End2EndTest`; hooked `BusCoverageTracker` into `execute()` around the capture block. + - Modified `ovoscope/pytest_plugin.py`: added `BusCoverageCollector`, `bus_coverage_session` session fixture, `pytest_terminal_summary` hook for merged end-of-session report. + - Modified `ovoscope/cli.py`: added `cmd_bus_coverage` subcommand and `bus-coverage` parser entry. + - Created `docs/bus-coverage.md` — full API reference with source citations. + - Updated `FAQ.md` with three new Q&A entries. + - Created `test/unittests/test_bus_coverage.py` — 32 unit tests, all passing. +- **Oversight**: 301 unit tests pass locally. `bus_coverage.py` at 97% coverage. + + ## [2026-03-11] — Add ovoscope-setup entrypoint for AI assistant skill installation - **AI Model**: Claude Sonnet 4.6 diff --git a/QUICK_FACTS.md b/QUICK_FACTS.md index 335cb12..7a9a2c6 100644 --- a/QUICK_FACTS.md +++ b/QUICK_FACTS.md @@ -18,7 +18,7 @@ End-to-end test framework for OpenVoiceOS skills ## Testing & CI | Feature | Details | |---------|---------| -| Unit Tests | 243 tests across `test/unittests/` (all passing) | +| Unit Tests | 348 tests across `test/unittests/` (all passing) | | Coverage | 53% overall (transformer/remote code excluded — requires optional deps) | | Test Framework | pytest with custom fixtures | | Coverage Reporter | py-cov-action/python-coverage-comment-action@v3 | diff --git a/SKILL.md b/SKILL.md index 06c79c1..e783f9c 100644 --- a/SKILL.md +++ b/SKILL.md @@ -56,6 +56,40 @@ test.execute(minicroft) ovoscope record "" --output fixtures/hello.json ``` +### Bus Coverage Tracking + +Enable bus-level message coverage to ensure your tests trigger all expected event handlers and emissions. + +#### CLI (via pytest) +```bash +# Enable bus coverage for the session +pytest test/end2end/ --ovoscope-bus-cov + +# Enable verbose mode to see exact message types +pytest test/end2end/ --ovoscope-bus-cov --ovoscope-bus-cov-verbose + +# Filter by skill_id or component name (regex) +pytest test/end2end/ --ovoscope-bus-cov --ovoscope-bus-cov-include="my-skill" +pytest test/end2end/ --ovoscope-bus-cov --ovoscope-bus-cov-exclude="^Thread-|^__core__$" + +# Save the merged report to a JSON file (useful for CI) +pytest test/end2end/ --ovoscope-bus-cov --ovoscope-bus-cov-file=coverage/bus-coverage.json +``` + +#### Manual Opt-in (per test) +```python +def test_something(self, minicroft, bus_coverage_session): + test = End2EndTest(..., track_bus_coverage=True) + test.execute() + # Add to the session-level collector + bus_coverage_session.add(test.bus_coverage_report) +``` + +**What is tracked:** +- **Listeners:** Which message types the skill is listening for and if they were invoked. +- **Emitters:** Which message types the skill emitted and if they were asserted in the test. +- **Coverage %:** Per-skill and session-wide coverage statistics. + ### Validate Expectations ```python diff --git a/SUGGESTIONS.md b/SUGGESTIONS.md new file mode 100644 index 0000000..19ff107 --- /dev/null +++ b/SUGGESTIONS.md @@ -0,0 +1,153 @@ +# Suggestions — `ovoscope` + +Agent-generated proposals for refactors and enhancements. +Each item includes a rationale, affected file, and implementation sketch. + +--- + +## 1. Share MiniCroft Across Fixtures in `cmd_bus_coverage()` [PERFORMANCE] + +**File**: `ovoscope/cli.py` — `cmd_bus_coverage()` + +**Problem**: The current implementation creates a new `MiniCroft` for every +fixture file it runs. When a workspace has many fixtures for the same skill, +this means repeated skill loading, plugin initialisation, and READY-wait +overhead for each fixture — typically 5–20 seconds per fixture. + +**Suggestion**: Group fixture files by their `skill_ids` list, create a single +`MiniCroft` per unique skill set, then replay all fixtures against that shared +instance. Expected speedup: 10–50× for typical skill test suites. + +**Sketch**: +```python +from itertools import groupby +fixtures_by_skills = groupby(sorted(fixtures, key=lambda f: f.skill_ids_key), ...) +for skill_key, group in fixtures_by_skills: + mc = get_minicroft(skill_key) + for fixture in group: + fixture.execute(minicroft=mc) + mc.stop() +``` + +--- + +## 2. Add `PipelineHarness.assert_no_match()` Convenience Method [DONE] + +**File**: `ovoscope/pipeline.py` + +**Status**: Already implemented — `assert_no_match(utterance, timeout=2.0)` is +present at `pipeline.py:213`. No further action needed. + +--- + +## 3. Add `LOAD_TIME` Tag for Registration-Time Handlers [BUS COVERAGE] + +**File**: `ovoscope/bus_coverage.py` + +**Problem**: Handlers invoked during skill loading (before the snapshot) always +show `0 invocations` and `NOT TESTED` in bus coverage reports. This is +misleading because intent registration handlers *are* exercised — just before +the snapshot window. + +**Suggestion**: Capture a pre-READY handler snapshot in `MiniCroft.__init__` +(before `super().__init__()`), then tag any handler present in both the +pre-READY and post-READY snapshots with a `LOAD_TIME` label in the report. +These handlers should be excluded from the `NOT TESTED` count. + +--- + +## 4. `diff.py` — Strict Mode for Extra Keys [DONE] + +**File**: `ovoscope/diff.py` + +**Status**: Implemented in this audit cycle. `_dict_diff()` now accepts +`strict: bool = False`; when `True`, keys present in `actual` but not in +`expected` are flagged as unexpected extras. `diff_fixtures()` exposes the +same `strict` parameter. Default `False` preserves existing behaviour. + +--- + +## 5. Make Coverage Fixture Search Recursive [DONE] + +**File**: `ovoscope/coverage.py` — `_count_fixtures()` + +**Status**: Implemented in this audit cycle. The search now uses +`Path.rglob("*.json")` instead of `os.listdir()`, so fixtures in +sub-directories are counted correctly. + +--- + +## 6. Add Noise-Floor Tolerance to `MockVADEngine.is_silence()` [LISTENER] + +**File**: `ovoscope/listener.py` + +**Problem**: `MockVADEngine.is_silence()` currently returns a fixed value +configured at construction time. Real VAD engines apply a noise-floor +threshold; tests that simulate borderline audio may need to replicate this +behaviour. + +**Suggestion**: Add `noise_floor: float = 0.0` parameter to +`MockVADEngine.__init__()`. When `noise_floor > 0`, `is_silence()` returns +`True` only if the sample RMS is below `noise_floor`. Default `0.0` +preserves existing behaviour (fixed return value). + +--- + +## 7. Make OCP HTTP Patch Targets Configurable [OCP] + +**File**: `ovoscope/ocp.py` + +**Problem**: The default patch targets (`requests.Session.get` and +`requests.get`) are hardcoded in `_apply_patches`. Skills that use +`httpx`, `aiohttp`, or a custom HTTP wrapper cannot be mocked without +specifying `patch_targets`. + +**Suggestion**: Expose `default_patch_targets: List[str]` as a class-level +constant on `OCPTest` so subclasses can override it without rewriting each +test instance: + +```python +class OCPTest: + default_patch_targets: List[str] = ["requests.Session.get", "requests.get"] +``` + +--- + +## 8. Support Env Var for GitHub URL in `setup_skill.py` [SETUP] + +**File**: `ovoscope/setup_skill.py` + +**Problem**: The GitHub URL for skill assets is hardcoded. CI environments or +forks may need to point to a different repository. + +**Suggestion**: Read `OVOSCOPE_SKILL_URL` environment variable as an override: + +```python +import os +SKILL_URL = os.environ.get("OVOSCOPE_SKILL_URL", DEFAULT_SKILL_URL) +``` + +--- + +## 9. Add `RemoteRecorder` Usage to Docs [DOCUMENTATION] + +**File**: `docs/index.md`, `docs/usage-guide.md` + +**Problem**: `RemoteRecorder` — `ovoscope/remote_recorder.py:46` — is not +documented in any public-facing doc file. Users who want to capture fixtures +from a live OVOS instance have no guide. + +**Suggestion**: Add a "Pattern 13: Recording from a Live OVOS Instance" section +to `docs/usage-guide.md` showing the `connect()`/`record()`/`disconnect()` +workflow and the `--live` CLI flag. + +--- + +## 10. Expand `docs/pipeline.md` to Full API Reference [DONE] + +**File**: `docs/pipeline.md` + +**Status**: Expanded in this audit cycle. The file now includes the full +`PipelineHarness` API table with `pipeline.py:LINE` citations, examples for +Adapt and Padatious pipelines, an explanation of `_SinkSkill`, and notes on +pipeline stage ordering and success/failure signals. diff --git a/docs/bus-coverage.md b/docs/bus-coverage.md new file mode 100644 index 0000000..cfc1d49 --- /dev/null +++ b/docs/bus-coverage.md @@ -0,0 +1,151 @@ +# Bus Coverage + +Bus coverage measures how thoroughly an end-to-end test exercises a skill's +MessageBus interface. Unlike line coverage (which measures code paths), bus +coverage answers: + +> *Which message handlers did my tests actually trigger? Which messages did +> the skill emit, and which of those did I explicitly assert?* + +## Summary + +| Dimension | What it measures | +|-----------|-----------------| +| **Listener coverage** | Which message types the skill is listening for and if they were invoked. | +| **Emitter coverage** | Which message types the skill emitted and if they were asserted in the test. | + +--- + +## Enabling Coverage Tracking + +There are three ways to enable bus coverage: + +### 1. Global (Recommended for Pytest) +Pass the `--ovoscope-bus-cov` flag to pytest. This automatically enables tracking for all `End2EndTests` in the session and captures the full boot sequence. + +```bash +# Basic report +pytest test/end2end/ --ovoscope-bus-cov + +# Verbose report (shows exact message types) +pytest test/end2end/ --ovoscope-bus-cov --ovoscope-bus-cov-verbose + +# Filter by skill_id regex +pytest test/end2end/ --ovoscope-bus-cov --ovoscope-bus-cov-include="my-skill" +pytest test/end2end/ --ovoscope-bus-cov --ovoscope-bus-cov-exclude="^Thread-|^__core__$" + +# Save to JSON for CI +pytest test/end2end/ --ovoscope-bus-cov --ovoscope-bus-cov-file=bus-cov.json +``` + +### 2. Manual (Per Test) +Add `track_bus_coverage=True` to an `End2EndTest` instance. + +```python +test = End2EndTest(..., track_bus_coverage=True) +test.execute() +report = test.bus_coverage_report +``` + +### 3. CLI Subcommand +Run coverage against a directory of JSON fixtures. + +```bash +ovoscope bus-coverage Skills/ovos-skill-hello-world/test/end2end/ --verbose +``` + +--- + +## How it Works + +Ovoscope uses a multi-layered approach to capture 100% of bus activity, including events that happen before the tests officially start. + +### Implementation Details + +1. **Global Monkey-Patching**: When enabled, `ovoscope` monkey-patches `ovos_utils.fakebus.FakeBus.on`, `.once`, and `.emit`. This ensures that even "boot sequence" activity (like vocab registration or internal service setup) is captured from the moment the process starts. +2. **Skill Attribution**: To accurately link message handlers to specific skills, `ovoscope` patches `ovos_workshop.skills.ovos.OVOSSkill.add_event` and `.bind`. + * This allows capturing registrations that happen during skill `__init__`. + * It handles skill renames (where a skill starts with a generic name and is later assigned a unique `skill_id` by the loader). +3. **Instance Introspection**: After `MiniCroft` is READY, `ovoscope` performs a final sweep by introspecting `skill.events.events` and walking the bus's internal handler map. + +### Data Attribution Logic + +Messages and handlers are attributed in this order of precedence: +1. **Direct Skill ID**: If the handler was registered via a patched `OVOSSkill` method. +2. **Closure Introspection**: If the handler closure contains a reference to a skill instance. +3. **Component Name**: If the handler belongs to a core component (e.g., `IntentService`, `AdaptPipeline`). +4. **`__core__` Bucket**: A fallback for any message type registered or emitted that cannot be linked to a specific skill or component. + +--- + +## Reading the Report + +The report table displays three main columns: + +``` +Skill Listeners Observed Asserted +────────────────────────────────────────────────────────────────────── +my-skill.author 8/12 66.7% 10/15 6/15 +IntentService 2/4 50.0% 0/0 0/0 +────────────────────────────────────────────────────────────────────── +TOTAL 10/16 62.5% 10/15 6/15 +``` + +### 1. Listeners (The "What can it hear?" metric) +* **Formula**: `(Invoked Message Types) / (Registered Message Types)` +* **Registered**: Total unique message types the skill called `.on()` or `.add_event()` for. +* **Invoked**: How many of those message types were actually emitted on the bus during the session. +* **Meaning**: High percentage means your tests are triggering most of the skill's logic paths. + +### 2. Observed Emitters (The "What did it say?" metric) +* **Formula**: `(Emitted Message Types) / (Known Message Types)` +* **Known**: The set of message types that were *either* emitted by this skill during this session *or* were listed in the test's `expected_messages` for this skill. +* **Observed**: How many of those message types actually appeared on the bus. +* **Meaning**: Usually 100% unless you have conditional emissions that didn't fire. + +### 3. Asserted Emitters (The "Did I check it?" metric) +* **Formula**: `(Asserted Message Types) / (Known Message Types)` +* **Asserted**: How many of the observed message types were explicitly listed in `End2EndTest.expected_messages`. +* **Meaning**: High percentage means your test suite is strictly validating the skill's output, not just letting it happen. + +--- + +## Verbose Breakdown + +In verbose mode (`--ovoscope-bus-cov-verbose`), `ovoscope` lists every message type: + +``` +LISTENERS — my-skill.author + ✓ my-intent.intent 2 invocation(s) + ✗ some-unused-event NOT TESTED + +EMITTERS — my-skill.author + ✓ speak observed 1x ✓ asserted + ✓ my-skill.done observed 1x ✗ not asserted +``` + +* **✓ (Checked)**: The listener was triggered or the emitter was asserted. +* **✗ (Cross)**: The listener was never triggered or the emitter was seen but not checked in the test. + +--- + +## Filtering and Tuning + +By default, the bus report can be noisy because core services register many internal handlers. Use filtering to focus on your code: + +* **Include**: Only show skills matching a regex. + * `--ovoscope-bus-cov-include="my-skill"` +* **Exclude**: Hide matches. The standard CI/CD workflow excludes threads and internal metadata helpers by default. + * `--ovoscope-bus-cov-exclude="^Thread-|^intents$|^skills$"` + +--- + +## Calculations API + +If you are building custom tooling, you can access these values via `SkillBusCoverage` properties: + +* `listener_coverage_pct`: `(covered_listeners / total_listeners) * 100` +* `observed_emitter_pct`: `(observed_emitters / total_emitters) * 100` +* `asserted_emitter_pct`: `(asserted_emitters / total_emitters) * 100` + +Source: `SkillBusCoverage` — `ovoscope/bus_coverage.py:118` diff --git a/docs/gui-testing.md b/docs/gui-testing.md index 78284cc..7a04675 100644 --- a/docs/gui-testing.md +++ b/docs/gui-testing.md @@ -152,6 +152,26 @@ gui.assert_namespace_value("helloworldskill", "greeting", "Hello!") Raises `AssertionError` if no matching message is found. +#### `assert_namespace_has_key(namespace, key)` + +`GUICaptureSession.assert_namespace_has_key` — `ovoscope/__init__.py:1093` + +Assert that a `gui.value.set` or `gui.namespace.update` message set a +specific key in the given namespace, regardless of value. Useful for +dynamic data (weather API responses, timestamps) where the exact value +is unpredictable. + +```python +gui.assert_namespace_has_key("weatherskill", "current_temp") +``` + +| Argument | Type | Description | +|----------|------|-------------| +| `namespace` | `str` | GUI namespace to check. | +| `key` | `str` | Data key that should exist. | + +Raises `AssertionError` if no matching message is found. + #### `assert_namespace_cleared(namespace)` `GUICaptureSession.assert_namespace_cleared` — `ovoscope/__init__.py:1069` diff --git a/docs/index.md b/docs/index.md index e9bcf4b..0c113f5 100644 --- a/docs/index.md +++ b/docs/index.md @@ -12,6 +12,7 @@ | [audio-testing.md](audio-testing.md) | `AudioServiceHarness`, `PlaybackServiceHarness` — testing audio services | | [listener.md](listener.md) | `MiniListener`, `get_mini_listener`, `ListenerTest`, `MockVADEngine`, `MockHotWordEngine`, `VADTest`, `WakeWordTest` — testing audio transformer plugins, STT pipeline, VAD, and wake-word | | [gui-testing.md](gui-testing.md) | `GUICaptureSession` — asserting GUI page navigation and namespace values | +| [bus-coverage.md](bus-coverage.md) | `BusCoverageTracker`, `BusCoverageReport` — measuring handler and emitter coverage per skill | ## Conceptual Model ``` Test FakeBus diff --git a/docs/media-testing.md b/docs/media-testing.md new file mode 100644 index 0000000..3516676 --- /dev/null +++ b/docs/media-testing.md @@ -0,0 +1,320 @@ +# Media / OCP Testing with ovoscope + +This document describes how to test `ovos-media` services — specifically the +`OCPMediaPlayer` state machine — using the harness classes provided in +`ovoscope.media`. + +> **Prerequisite:** Media testing harnesses require `ovos-media` to be installed. +> Install it with: `pip install ovoscope ovos-media` + +## When to Use Which Class + +| Scenario | Class | +|---|---| +| Testing `OCPMediaPlayer` play/pause/stop/next/prev state machine | `OCPPlayerHarness` | +| Testing duck/unduck (volume lowering) and cork/uncork (pause on listen) | `OCPPlayerHarness` | +| Capturing and asserting OCP bus message sequences | `OCPCaptureSession` | +| Simulating a broken or unplayable stream | `MockOCPBackend.simulate_invalid_stream()` | +| Simulating end-of-track to trigger auto-advance | `MockOCPBackend.simulate_end()` | + +## OCPPlayerHarness + +`OCPPlayerHarness` — `ovoscope/media.py` + +Wraps a real `OCPMediaPlayer` (`ovos_media.player`) with a `MockOCPBackend` on a +`FakeBus`. All heavy dependencies are patched out: + +- `ovos_media.player.AudioService` — mocked; `MockOCPBackend` injected as the + sole audio backend +- `ovos_media.player.VideoService` — mocked +- `ovos_media.player.WebService` — mocked +- `ovos_media.player.OcpMprisExporter` — mocked (no D-Bus session required) +- `ovos_media.player.GUIInterface` — mocked (exposed as `harness.gui`) +- `ovos_media.player.OCPMediaCatalog` — mocked +- `ovos_media.player.Configuration` — returns `{"media": {}}` + +### Basic Usage + +```python +from ovoscope.media import OCPPlayerHarness +from ovos_utils.ocp import MediaEntry, PlaybackType, PlayerState + +with OCPPlayerHarness() as h: + entry = MediaEntry( + uri="http://example.com/song.mp3", + playback=PlaybackType.AUDIO, + title="Test Track", + ) + h.play(entry) + h.assert_player_state(PlayerState.PLAYING) + assert h.backend.is_playing + + h.pause() + h.assert_player_state(PlayerState.PAUSED) + + h.resume() + h.assert_player_state(PlayerState.PLAYING) + + h.stop() + h.assert_player_state(PlayerState.STOPPED) +``` + +### Queue Navigation + +```python +from ovoscope.media import OCPPlayerHarness +from ovos_utils.ocp import MediaEntry, PlaybackType, PlayerState +from ovos_bus_client.message import Message + +with OCPPlayerHarness() as h: + track1 = MediaEntry(uri="http://example.com/1.mp3", + playback=PlaybackType.AUDIO) + track2 = MediaEntry(uri="http://example.com/2.mp3", + playback=PlaybackType.AUDIO) + + # Emit play with a playlist so the queue has two tracks + h.bus.emit(Message("ovos.common_play.play", { + "media": track1.as_dict, + "playlist": [track1.as_dict, track2.as_dict], + })) + import time; time.sleep(0.05) + + h.assert_now_playing_uri("http://example.com/1.mp3") + h.next_track() + h.assert_now_playing_uri("http://example.com/2.mp3") +``` + +### Duck / Unduck vs Cork / Uncork + +`OCPMediaPlayer` distinguishes two separate mechanisms for voice-assistant +interruptions. Understanding the difference is essential for writing correct +tests. + +#### Ducking — lower volume, keep playing + +Ducking happens when the assistant **speaks** (TTS output). The player stays +in ``PLAYING`` state; only the audio backend volume is reduced. + +| Bus message | Handler | Effect | +|---|---|---| +| `recognizer_loop:audio_output_start` / `ovos.common_play.duck` | `handle_duck_request` | Calls `audio_service.lower_volume()`, sets `_paused_on_duck=True` | +| `recognizer_loop:audio_output_end` / `ovos.common_play.unduck` | `handle_unduck_request` | Calls `audio_service.restore_volume()` whenever `_paused_on_duck` is True, **regardless of player state** | + +```python +from ovoscope.media import OCPPlayerHarness +from ovos_utils.ocp import MediaEntry, PlaybackType, PlayerState + +with OCPPlayerHarness() as h: + entry = MediaEntry(uri="http://example.com/song.mp3", + playback=PlaybackType.AUDIO) + h.play(entry) + h.duck() # lower_volume called; player stays PLAYING + h.assert_player_state(PlayerState.PLAYING) + assert h.player._paused_on_duck # flag set + h.unduck() # restore_volume called; _paused_on_duck cleared + h.assert_player_state(PlayerState.PLAYING) + assert not h.player._paused_on_duck +``` + +#### Corking — pause the player, resume after listening + +Corking happens when the **microphone opens** (wake-word recognised, user +speaking). The player is fully **paused** and resumes after the interaction. + +| Bus message | Handler | Effect | +|---|---|---| +| `recognizer_loop:record_begin` / `ovos.common_play.cork` | `handle_cork_request` | Pauses player, sets `_paused_on_duck=True` | +| `ovos.common_play.uncork` | `handle_uncork_request` | Resumes player **only if PAUSED and `_paused_on_duck`** | +| `recognizer_loop:record_end` | `handle_record_end` | Waits up to 8 s for `speak`; if none → uncork | + +```python +from ovoscope.media import OCPPlayerHarness +from ovos_utils.ocp import MediaEntry, PlaybackType, PlayerState + +with OCPPlayerHarness() as h: + entry = MediaEntry(uri="http://example.com/song.mp3", + playback=PlaybackType.AUDIO) + h.play(entry) + h.cork() # player → PAUSED + h.assert_player_state(PlayerState.PAUSED) + assert h.player._paused_on_duck + + h.uncork() # player → PLAYING + h.assert_player_state(PlayerState.PLAYING) + assert not h.player._paused_on_duck +``` + +#### Uncork-guard: manual pause is not overridden by uncork + +``handle_uncork_request`` checks ``_paused_on_duck`` before resuming. If the +user paused manually, ``_paused_on_duck`` is ``False`` and ``uncork()`` is a +no-op, preventing a spurious resume. + +```python +with OCPPlayerHarness() as h: + h.play(entry) + h.pause() # manual pause — _paused_on_duck stays False + h.uncork() # no-op — _paused_on_duck is False + h.assert_player_state(PlayerState.PAUSED) +``` + +#### record_end auto-uncork + +When the mic closes without any TTS following (utterance not recognised), +``handle_record_end`` uncorks automatically after an 8-second timeout. Tests +should patch ``bus.wait_for_message`` to avoid the real wait: + +```python +from unittest.mock import patch + +with OCPPlayerHarness() as h: + h.play(entry) + h.cork() + with patch.object(h.bus, "wait_for_message", return_value=None): + h.bus.emit(Message("recognizer_loop:record_end")) + import time; time.sleep(0.05) + h.assert_player_state(PlayerState.PLAYING) +``` + +### Simulating Stream End + +```python +from ovoscope.media import OCPPlayerHarness +from ovos_utils.ocp import MediaEntry, MediaState, PlaybackType + +with OCPPlayerHarness() as h: + entry = MediaEntry(uri="http://example.com/song.mp3", + playback=PlaybackType.AUDIO) + h.play(entry) + h.simulate_track_end() # backend emits END_OF_MEDIA + # Player auto-advances or stops depending on queue + autoplay config +``` + +## OCPCaptureSession + +`OCPCaptureSession` — `ovoscope/media.py` + +Captures all `ovos.common_play.*` and `ovos.audio.*` bus messages during a +block of code and lets you assert that specific message types appeared in order. + +```python +from ovoscope.media import OCPPlayerHarness, OCPCaptureSession +from ovos_utils.ocp import MediaEntry, PlaybackType + +with OCPPlayerHarness() as h: + entry = MediaEntry(uri="http://example.com/song.mp3", + playback=PlaybackType.AUDIO) + with OCPCaptureSession(h.bus) as session: + h.play(entry) + + # Check that player.state was announced after play + session.assert_sequence("ovos.common_play.player.state") +``` + +### Custom Prefixes + +```python +from ovoscope.media import OCPCaptureSession + +with OCPPlayerHarness() as h: + with OCPCaptureSession(h.bus, + track_prefixes=["ovos.common_play.player."]) as s: + h.play(entry) + print(s.message_types) # ['ovos.common_play.player.state'] +``` + +## API Reference + +### MockOCPBackend + +`MockOCPBackend` — `ovoscope/media.py` + +| Attribute / Method | Type | Description | +|---|---|---| +| `played_uris` | `List[str]` | All URIs passed to `add_list()` or `load_track()` | +| `is_playing` | `bool` | True after `play()`, False after `stop()` | +| `is_paused` | `bool` | True after `pause()`, False after `resume()` | +| `current_uri` | `Optional[str]` | Most recently loaded URI | +| `namespace` | `str` | Backend namespace string (default `"audio"`) | +| `stop()` | `bool` | Always returns `True` (required by BaseMediaService) | +| `simulate_end()` | `None` | Emit `END_OF_MEDIA` on the bus | +| `simulate_invalid_stream()` | `None` | Emit `INVALID_MEDIA` on the bus | +| `reset()` | `None` | Clear all recorded state | + +### OCPPlayerHarness + +`OCPPlayerHarness` — `ovoscope/media.py` + +**Control methods** (each emits the corresponding bus message + `time.sleep(0.05)`): + +| Method | Bus message emitted | +|---|---| +| `play(track: MediaEntry)` | `ovos.common_play.play` | +| `pause()` | `ovos.common_play.pause` | +| `resume()` | `ovos.common_play.resume` | +| `stop()` | `ovos.common_play.stop` | +| `next_track()` | `ovos.common_play.next` | +| `prev_track()` | `ovos.common_play.previous` | +| `duck()` | `recognizer_loop:audio_output_start` — lower volume, player stays PLAYING | +| `unduck()` | `recognizer_loop:audio_output_end` — restore volume whenever `_paused_on_duck` is True (duck or cork path) | +| `cork()` | `ovos.common_play.cork` — pause player, set `_paused_on_duck=True` | +| `uncork()` | `ovos.common_play.uncork` — resume player if PAUSED and `_paused_on_duck` | +| `simulate_track_end()` | `ovos.common_play.media.state` END_OF_MEDIA | +| `simulate_invalid_stream()` | `ovos.common_play.media.state` INVALID_MEDIA | + +**Assertion helpers:** + +| Method | Description | +|---|---| +| `assert_player_state(state)` | Raise if `player.state != state` | +| `assert_media_state(state)` | Raise if `player.media_state != state` | +| `assert_backend_playing()` | Raise if `backend.is_playing` is False | +| `assert_backend_paused()` | Raise if `backend.is_paused` is False | +| `assert_backend_stopped()` | Raise if `is_playing` or `is_paused` is True | +| `assert_now_playing_uri(uri)` | Raise if `now_playing.uri != uri` | + +**Exposed attributes:** + +| Attribute | Type | Description | +|---|---|---| +| `player` | `OCPMediaPlayer` | Real player instance | +| `bus` | `FakeBus` | Shared in-process bus | +| `backend` | `MockOCPBackend` | Injected mock audio backend | +| `gui` | `MagicMock` | Mocked GUIInterface | + +### OCPCaptureSession + +`OCPCaptureSession` — `ovoscope/media.py` + +| Method / Property | Description | +|---|---| +| `start()` / `stop()` | Subscribe/unsubscribe from FakeBus | +| `__enter__` / `__exit__` | Context manager interface | +| `messages` | List of captured `Message` objects | +| `message_types` | List of captured `msg_type` strings | +| `assert_sequence(*types)` | Assert types appear in order as a subsequence | + +Default `track_prefixes` captures: `"ovos.common_play."`, `"ovos.audio."`. + +## Limitations + +- **No real audio**: `MockOCPBackend` never plays audio. Use `simulate_end()` to + trigger end-of-track logic. +- **No MPRIS**: `OcpMprisExporter` is mocked out — MPRIS D-Bus integration is + not exercised. +- **No GUI rendering**: `GUIInterface` is a `MagicMock`. Test GUI calls via + `harness.gui.show_media_player.assert_called_with(...)`. +- **No VideoService / WebService**: Only audio playback (`PlaybackType.AUDIO`) + is wired with a real mock backend. +- **FakeBus is synchronous**: Handlers run in the same thread that calls + `bus.emit()`. The `time.sleep(0.05)` in control methods is sufficient for + synchronous delivery; async or threaded handlers may need explicit waits. + +## Cross-References + +- `OCPMediaPlayer` — `ovos-media/ovos_media/player.py` +- `BaseMediaService` — `ovos-media/ovos_media/media_backends/base.py` +- `AudioBackend` (base class) — `ovos_plugin_manager.templates.audio.AudioBackend` +- `MediaEntry`, `PlayerState`, `MediaState` — `ovos_utils.ocp` +- `MockAudioBackend` / `AudioServiceHarness` (audio pattern) — `ovoscope/audio.py` +- End-to-end tests — `ovos-media/test/end2end/test_ocp_player.py` diff --git a/docs/ocp.md b/docs/ocp.md index 2a435c2..34a7788 100644 --- a/docs/ocp.md +++ b/docs/ocp.md @@ -1,7 +1,8 @@ # OCP / Common Play Testing -`ovoscope.ocp` provides `OCPTest` and `assert_ocp_query_response` for -testing OCP (OpenVoiceOS Common Play) skills that handle media queries. +`ovoscope.ocp` provides `OCPTest` and `assert_ocp_query_response` for testing +OCP (OpenVoiceOS Common Play) skills that handle media queries. For testing +the OCP player state machine, see `OCPPlayerHarness` in `ovoscope.media`. ## OCP Message Flow diff --git a/docs/pipeline.md b/docs/pipeline.md index cac7b00..943a957 100644 --- a/docs/pipeline.md +++ b/docs/pipeline.md @@ -9,9 +9,21 @@ Pipeline plugins (Adapt, Padatious, Padacioso, OCP, etc.) match utterances to intents. `PipelineHarness` loads the specified stages on a `MiniCroft` that has no skills, so only the pipeline matching logic is exercised. +## `_SinkSkill` — Internal Catch-all + +`_SinkSkill` — `ovoscope/pipeline.py:37` + +When `PipelineHarness` creates a `MiniCroft`, it injects an internal +`__ovoscope_sink__` skill as a routing target for matched intents. This is +necessary because OVOS routes intent matches to a skill handler; without a +skill present the match is discarded. `_SinkSkill` simply records the matched +intent message and signals the waiting `match()` call. + +Users never interact with `_SinkSkill` directly. + ## `PipelineHarness` — Context Manager -`PipelineHarness` — `pipeline.py:PipelineHarness` +`PipelineHarness` — `ovoscope/pipeline.py:71` ```python from ovoscope.pipeline import PipelineHarness @@ -28,21 +40,78 @@ with PipelineHarness( | Argument | Type | Default | Description | |----------|------|---------|-------------| -| `pipeline` | `List[str]` | `[]` | Pipeline stage IDs to load. | -| `pipeline_config` | `Dict[str, Dict]` | `{}` | Per-stage config overrides. | +| `pipeline` | `List[str]` | `[]` | OPM pipeline stage IDs to load. | +| `pipeline_config` | `Dict[str, Dict]` | `{}` | Per-stage config overrides keyed by stage ID. | | `lang` | `str` | `"en-US"` | Language tag. | ### Methods -| Method | Returns | Description | -|--------|---------|-------------| -| `match(utterance, timeout=5.0)` — `ovoscope/pipeline.py:135` | `Optional[Message]` | Send utterance; return matched `Message` or `None` if no pipeline stage matched within `timeout` seconds. | -| `assert_matches(utterance, intent_type=None, timeout=5.0)` — `ovoscope/pipeline.py:183` | `Message` | Assert at least one pipeline stage matches. Raises `AssertionError` if no match. If `intent_type` is provided, the matched message's `msg_type` must **contain** `intent_type` as a substring (case-sensitive). | -| `assert_no_match(utterance, timeout=2.0)` — `ovoscope/pipeline.py:213` | `None` | Assert the utterance is NOT matched by any loaded stage within `timeout` seconds. Raises `AssertionError` if a match is found. | +| Method | Source | Returns | Description | +|--------|--------|---------|-------------| +| `match(utterance, timeout=5.0)` | `ovoscope/pipeline.py:135` | `Optional[Message]` | Send utterance; return matched `Message` or `None` on timeout/failure. | +| `assert_matches(utterance, intent_type=None, timeout=5.0)` | `ovoscope/pipeline.py:183` | `Message` | Assert at least one stage matches. Raises `AssertionError` if no match. `intent_type` is a substring check on `msg_type`. | +| `assert_no_match(utterance, timeout=2.0)` | `ovoscope/pipeline.py:213` | `None` | Assert no stage matches. Raises `AssertionError` if a match is found. | + +### Pipeline Stage Ordering and Success vs Failure + +OVOS evaluates pipeline stages in the order listed in `pipeline`. The first +stage that returns a non-empty match list wins; remaining stages are skipped. + +**Success signal**: `intent.service.skills.activated` bus message — emitted +when a stage commits to handling the utterance. + +**Failure signal**: `intent_failure` or `mycroft.skill.handler.start` bus +messages — emitted when no stage matched after all stages have been consulted. + +`match()` — `ovoscope/pipeline.py:135` — uses separate `threading.Event` +objects for success and failure so that an `intent_failure` arriving first +does not mask a subsequent late success match. On timeout or failure the +method returns `None`; on success it returns the captured `Message`. + +## Examples + +### Testing Adapt Pipeline Matching + +```python +from ovoscope.pipeline import PipelineHarness + +with PipelineHarness( + pipeline=["ovos-adapt-pipeline-plugin.openvoiceos"], + lang="en-US", +) as harness: + # Adapt must have registered an intent containing "LightsOnIntent" + msg = harness.assert_matches( + "turn on the kitchen lights", + intent_type="LightsOnIntent", + ) + print(msg.data) # {"LightsOnKeyword": "lights", ...} + + # Unrecognised utterance must not match + harness.assert_no_match("garbled xyz 123") +``` -#### `assert_matches(intent_type=...)` semantics +### Testing Padatious Entity Extraction -`intent_type` is a **substring** check on the matched message's `msg_type`: +```python +from ovoscope.pipeline import PipelineHarness + +with PipelineHarness( + pipeline=["ovos-padatious-pipeline-plugin.openvoiceos"], + lang="en-US", +) as harness: + # Padatious must have a trained intent that matches this utterance + msg = harness.assert_matches( + "set a timer for 5 minutes", + intent_type="timer.intent", + ) + # Entity extraction is in msg.data + assert msg.data.get("duration") == "5 minutes" +``` + +### `assert_matches(intent_type=...)` semantics + +`intent_type` is a **substring** check on the matched message's `msg_type` +— `ovoscope/pipeline.py:208`: ```python # Pass: msg_type "padatious:0.95:LightsOnIntent" contains "LightsOnIntent" @@ -56,9 +125,13 @@ msg = harness.assert_matches("turn on the lights", intent_type="LightsOffIntent" # → AssertionError: Expected intent type to contain 'LightsOffIntent', got '...' ``` -## Implementation Note +## Implementation Notes + +`PipelineHarness.__enter__` — `ovoscope/pipeline.py:104` — creates a +`MiniCroft` with `skill_ids=[]` and the specified pipeline. -`PipelineHarness.__enter__` — `pipeline.py:PipelineHarness.__enter__` creates -a `MiniCroft` with `skill_ids=[]` and the specified pipeline. Intent-matched -messages are captured via a `threading.Event` subscription on -`intent.service.skills.activated`. +`PipelineHarness.match()` — `ovoscope/pipeline.py:135` — subscribes to +`intent.service.skills.activated` (success) and `intent_failure` / +`mycroft.skill.handler.start` (failure) before emitting the utterance, +then waits on a `threading.Event` with the given timeout. Bus handlers are +removed after the wait completes to avoid cross-test leakage. diff --git a/ovoscope/__init__.py b/ovoscope/__init__.py index 27da261..3d88df6 100644 --- a/ovoscope/__init__.py +++ b/ovoscope/__init__.py @@ -98,6 +98,134 @@ "ovos-stop-pipeline-plugin-medium", ] +# --------------------------------------------------------------------------- +# Global bus-coverage state (managed by pytest plugin or CLI) +# --------------------------------------------------------------------------- +GLOBAL_BUS_COVERAGE: bool = False +GLOBAL_BUS_COVERAGE_FILE: Optional[str] = None + + +class GlobalBusCoverageCollector: + """Accumulates bus events globally across all FakeBus instances.""" + def __init__(self): + # msg_type -> count + self.invocations: Dict[str, int] = {} + # msg_type -> count (total times .on was called for this type) + self.registrations: Dict[str, int] = {} + # skill_id -> {msg_type -> count} + self.skill_registrations: Dict[str, Dict[str, int]] = {} + + def record_invocation(self, msg_type: str): + self.invocations[msg_type] = self.invocations.get(msg_type, 0) + 1 + + def record_registration(self, msg_type: str): + self.registrations[msg_type] = self.registrations.get(msg_type, 0) + 1 + + def record_skill_registration(self, skill_id: str, msg_type: str): + if not skill_id: + return + if skill_id not in self.skill_registrations: + self.skill_registrations[skill_id] = {} + self.skill_registrations[skill_id][msg_type] = ( + self.skill_registrations[skill_id].get(msg_type, 0) + 1 + ) + + def rename_skill(self, old_id: str, new_id: str): + """Merge registrations from old_id into new_id.""" + if not old_id or not new_id or old_id == new_id: + return + if old_id in self.skill_registrations: + old_data = self.skill_registrations.pop(old_id) + if new_id not in self.skill_registrations: + self.skill_registrations[new_id] = {} + for mt, count in old_data.items(): + self.skill_registrations[new_id][mt] = ( + self.skill_registrations[new_id].get(mt, 0) + count + ) + + +GLOBAL_BUS_COVERAGE_COLLECTOR: Optional[GlobalBusCoverageCollector] = None + + +def _patch_fakebus(): + """Monkey-patch FakeBus and ovos-workshop classes to track global coverage.""" + from ovos_utils.fakebus import FakeBus + + original_on = FakeBus.on + original_once = getattr(FakeBus, "once", None) + original_emit = FakeBus.emit + + def patched_on(self, event, handler): + if GLOBAL_BUS_COVERAGE and GLOBAL_BUS_COVERAGE_COLLECTOR: + GLOBAL_BUS_COVERAGE_COLLECTOR.record_registration(event) + return original_on(self, event, handler) + + def patched_once(self, event, handler): + if GLOBAL_BUS_COVERAGE and GLOBAL_BUS_COVERAGE_COLLECTOR: + GLOBAL_BUS_COVERAGE_COLLECTOR.record_registration(event) + if original_once: + return original_once(self, event, handler) + return original_on(self, event, handler) + + def patched_emit(self, message): + if GLOBAL_BUS_COVERAGE and GLOBAL_BUS_COVERAGE_COLLECTOR: + msg_type = getattr(message, "msg_type", None) or getattr(message, "type", None) + if msg_type: + GLOBAL_BUS_COVERAGE_COLLECTOR.record_invocation(msg_type) + return original_emit(self, message) + + FakeBus.on = patched_on + FakeBus.once = patched_once + FakeBus.emit = patched_emit + + # --- Patch ovos-workshop for better attribution --- + try: + from ovos_workshop.skills.ovos import OVOSSkill + original_add_event = OVOSSkill.add_event + original_bind = OVOSSkill.bind + + def patched_add_event(self, name, handler, *args, **kwargs): + if GLOBAL_BUS_COVERAGE and GLOBAL_BUS_COVERAGE_COLLECTOR: + # Fallback to .name if skill_id is not yet set + sid = getattr(self, "skill_id", None) or getattr(self, "name", None) + if sid: + GLOBAL_BUS_COVERAGE_COLLECTOR.record_skill_registration(sid, name) + return original_add_event(self, name, handler, *args, **kwargs) + + def patched_bind(self, bus): + if GLOBAL_BUS_COVERAGE and GLOBAL_BUS_COVERAGE_COLLECTOR: + old_id = getattr(self, "skill_id", None) or getattr(self, "name", None) + res = original_bind(self, bus) + new_id = getattr(self, "skill_id", None) + if old_id and new_id and old_id != new_id: + GLOBAL_BUS_COVERAGE_COLLECTOR.rename_skill(old_id, new_id) + return res + return original_bind(self, bus) + + OVOSSkill.add_event = patched_add_event + OVOSSkill.bind = patched_bind + except ImportError: + pass + + try: + from ovos_utils.events import EventContainer + original_container_add = EventContainer.add + + def patched_container_add(self, name, handler, once=False): + if GLOBAL_BUS_COVERAGE and GLOBAL_BUS_COVERAGE_COLLECTOR: + # EventContainer usually belongs to a skill, but we don't have easy + # access to skill_id here without more complex patching. + # However, many skills call self.add_event which we already patched. + pass + return original_container_add(self, name, handler, once) + # EventContainer.add = patched_container_add + except ImportError: + pass + + +# Apply the patch immediately when ovoscope is imported +_patch_fakebus() + # Lightweight test pipeline — no C extensions (swig) required. # Uses only pure-Python stages that are dependencies of ovos-core/workshop. # Use this when you want fast CI without building Padatious or Adapt. @@ -581,6 +709,13 @@ class End2EndTest: test_routing: bool = True test_final_session: bool = True + ########################### + # bus coverage + ########################### + track_bus_coverage: bool = False # enable BusCoverageTracker for this test + print_bus_coverage: bool = False # print inline summary after execute() + bus_coverage_report: Optional["BusCoverageReport"] = dataclasses.field(default=None, init=False, repr=False) + ########################### # test runner internals ########################### @@ -589,6 +724,10 @@ class End2EndTest: managed: bool = False def __post_init__(self): + # global coverage opt-in + if GLOBAL_BUS_COVERAGE: + self.track_bus_coverage = True + # standardize to be a list if isinstance(self.source_message, Message): self.source_message = [self.source_message] @@ -632,6 +771,14 @@ def execute(self, timeout: int = 30) -> List[Message]: print(f"💡 original message.context source: '{o_src}'") print(f"💡 original message.context destination: '{o_dst}'") + # bus coverage tracking (optional) + _bus_tracker = None + if self.track_bus_coverage: + from ovoscope.bus_coverage import BusCoverageTracker + _bus_tracker = BusCoverageTracker(self.minicroft.bus, self.minicroft) + _bus_tracker.snapshot_listeners() + _bus_tracker.start_tracking() + # the capture session will store all messages until capture.finish() # even if multiple messages are emitted capture = CaptureSession(self.minicroft, eof_msgs=self.eof_msgs, @@ -646,6 +793,12 @@ def execute(self, timeout: int = 30) -> List[Message]: # final message list messages = capture.finish() + if _bus_tracker is not None: + _bus_tracker.stop_tracking() + all_responses = messages + list(getattr(capture, "async_responses", [])) + _bus_tracker.record_session(all_responses, self.expected_messages) + self.bus_coverage_report = _bus_tracker.build_report() + if self.test_message_number: n1 = len(self.expected_messages) n2 = len(messages) @@ -771,6 +924,9 @@ def execute(self, timeout: int = 30) -> List[Message]: if self.verbose: print(f"✅ final session matches: {expected_sess.serialize()}") + if self.print_bus_coverage and self.bus_coverage_report is not None: + print(self.bus_coverage_report.summary_line()) + if self.managed: self.minicroft.stop() del self.minicroft @@ -1032,8 +1188,12 @@ def assert_page_shown(self, namespace: str, page: str, timeout: float = 2.0) -> while time.monotonic() < deadline: for msg in self.messages: if "page.show" in msg.msg_type: - data_ns = msg.data.get("namespace", "") or msg.context.get("skill_id", "") - pages = msg.data.get("pages", []) or [msg.data.get("page", "")] + data_ns = (msg.data.get("namespace", "") + or msg.data.get("__from", "") + or msg.context.get("skill_id", "")) + pages = (msg.data.get("pages", []) + or msg.data.get("page_names", []) + or [msg.data.get("page", "")]) if namespace in data_ns and any(page in str(p) for p in pages): return time.sleep(0.05) @@ -1056,7 +1216,9 @@ def assert_namespace_value(self, namespace: str, key: str, value: Any) -> None: """ for msg in self.messages: if "value.set" in msg.msg_type or "namespace.update" in msg.msg_type: - data_ns = msg.data.get("namespace", "") or msg.context.get("skill_id", "") + data_ns = (msg.data.get("namespace", "") + or msg.data.get("__from", "") + or msg.context.get("skill_id", "")) if namespace in data_ns: data = msg.data.get("data", msg.data) if data.get(key) == value: @@ -1066,6 +1228,34 @@ def assert_namespace_value(self, namespace: str, key: str, value: Any) -> None: f"Captured GUI messages: {[m.msg_type for m in self.messages]}" ) + def assert_namespace_has_key(self, namespace: str, key: str) -> None: + """Assert that a key was set in a namespace, regardless of value. + + Useful for dynamic data (e.g. weather API responses, timestamps) + where the exact value is unpredictable but the key must exist. + + Args: + namespace: GUI namespace to check. + key: Data key that should exist within the namespace. + + Raises: + AssertionError: If no matching message with the key is found. + """ + for msg in self.messages: + if "value.set" in msg.msg_type or "namespace.update" in msg.msg_type: + data_ns = (msg.data.get("namespace", "") + or msg.data.get("__from", "") + or msg.context.get("skill_id", "")) + if namespace in data_ns: + data = msg.data.get("data", msg.data) + if key in data: + return + raise AssertionError( + f"Expected namespace {namespace!r} to contain key {key!r}, " + f"but it was never set.\n" + f"Captured GUI messages: {[m.msg_type for m in self.messages]}" + ) + def assert_namespace_cleared(self, namespace: str) -> None: """Assert that a namespace was cleared/removed. @@ -1077,7 +1267,9 @@ def assert_namespace_cleared(self, namespace: str) -> None: """ for msg in self.messages: if "namespace.remove" in msg.msg_type or "namespace.clear" in msg.msg_type: - data_ns = msg.data.get("namespace", "") or msg.context.get("skill_id", "") + data_ns = (msg.data.get("namespace", "") + or msg.data.get("__from", "") + or msg.context.get("skill_id", "")) if namespace in data_ns: return raise AssertionError( diff --git a/ovoscope/bus_coverage.py b/ovoscope/bus_coverage.py new file mode 100644 index 0000000..1499539 --- /dev/null +++ b/ovoscope/bus_coverage.py @@ -0,0 +1,821 @@ +# Copyright 2024 Jarbas AI +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Bus-level coverage tracking for ovoscope end-to-end tests. + +Measures two independent dimensions of coverage: + +1. **Listener coverage** — which ``bus.on(msg_type, handler)`` registrations + were actually invoked (i.e. ``bus.emit`` was called for that msg_type) + during the test, grouped by owning skill. + +2. **Emitter coverage** — which message types were: + + * *observed*: appeared in ``CaptureSession.responses`` + * *asserted*: appeared in ``End2EndTest.expected_messages`` + + Both sub-metrics are tracked per-skill via ``msg.context["skill_id"]``. + +Usage example:: + + from ovoscope import End2EndTest + + test = End2EndTest( + skill_ids=["my-skill.author"], + source_message=message, + expected_messages=[...], + track_bus_coverage=True, + print_bus_coverage=True, + ) + test.execute() + report = test.bus_coverage_report + print(report.to_json()) + +See ``docs/bus-coverage.md`` for the full reference. +""" +from __future__ import annotations + +import dataclasses +import json +from copy import deepcopy +from typing import Any, Dict, List, Optional + +import ovoscope +from ovos_bus_client.message import Message + + +# --------------------------------------------------------------------------- +# Data classes +# --------------------------------------------------------------------------- + + +@dataclasses.dataclass +class HandlerEntry: + """Coverage record for a single message-type listener registration. + + Attributes: + msg_type: Bus message type this handler was registered for. + handler_count: Number of distinct handlers registered for this type + within the owning skill. + invocation_count: Number of times ``bus.emit`` was called for this + msg_type during the tracked test session. + covered: ``True`` when ``invocation_count > 0``. + """ + + msg_type: str + handler_count: int + invocation_count: int + covered: bool + + def to_dict(self) -> Dict[str, Any]: + """Serialize to a JSON-compatible dict. + + Returns: + Dict with keys ``msg_type``, ``handler_count``, + ``invocation_count``, ``covered``. + """ + return { + "msg_type": self.msg_type, + "handler_count": self.handler_count, + "invocation_count": self.invocation_count, + "covered": self.covered, + } + + +@dataclasses.dataclass +class EmitterEntry: + """Coverage record for a single message type emitted by a skill. + + Attributes: + msg_type: Bus message type that was emitted. + observed_count: Times this type appeared in + ``CaptureSession.responses``. + asserted_count: Times this type appeared in + ``End2EndTest.expected_messages``. + observed: ``True`` when ``observed_count > 0``. + asserted: ``True`` when ``asserted_count > 0``. + """ + + msg_type: str + observed_count: int + asserted_count: int + observed: bool + asserted: bool + + def to_dict(self) -> Dict[str, Any]: + """Serialize to a JSON-compatible dict. + + Returns: + Dict with keys ``msg_type``, ``observed_count``, + ``asserted_count``, ``observed``, ``asserted``. + """ + return { + "msg_type": self.msg_type, + "observed_count": self.observed_count, + "asserted_count": self.asserted_count, + "observed": self.observed, + "asserted": self.asserted, + } + + +@dataclasses.dataclass +class SkillBusCoverage: + """Bus coverage data for a single skill. + + Attributes: + skill_id: OPM skill identifier. + listeners: Per-msg-type listener coverage entries. + emitters: Per-msg-type emitter coverage entries. + """ + + skill_id: str + listeners: List[HandlerEntry] = dataclasses.field(default_factory=list) + emitters: List[EmitterEntry] = dataclasses.field(default_factory=list) + + @property + def listener_coverage_pct(self) -> float: + """Return the percentage of registered listener msg_types that were invoked. + + Returns: + Float in [0.0, 100.0]. Returns 0.0 when no listeners are registered. + """ + if not self.listeners: + return 0.0 + covered = sum(1 for h in self.listeners if h.covered) + return 100.0 * covered / len(self.listeners) + + @property + def observed_emitter_pct(self) -> float: + """Return the percentage of emitter entries that were observed. + + Returns: + Float in [0.0, 100.0]. Returns 0.0 when no emitters are tracked. + """ + if not self.emitters: + return 0.0 + observed = sum(1 for e in self.emitters if e.observed) + return 100.0 * observed / len(self.emitters) + + @property + def asserted_emitter_pct(self) -> float: + """Return the percentage of emitter entries that appear in expected_messages. + + Returns: + Float in [0.0, 100.0]. Returns 0.0 when no emitters are tracked. + """ + if not self.emitters: + return 0.0 + asserted = sum(1 for e in self.emitters if e.asserted) + return 100.0 * asserted / len(self.emitters) + + def to_dict(self) -> Dict[str, Any]: + """Serialize to a JSON-compatible dict. + + Returns: + Dict with summary percentages and full ``listeners`` / + ``emitters`` lists. + """ + return { + "skill_id": self.skill_id, + "listener_coverage_pct": round(self.listener_coverage_pct, 1), + "observed_emitter_pct": round(self.observed_emitter_pct, 1), + "asserted_emitter_pct": round(self.asserted_emitter_pct, 1), + "listeners": [h.to_dict() for h in self.listeners], + "emitters": [e.to_dict() for e in self.emitters], + } + + +@dataclasses.dataclass +class BusCoverageReport: + """Aggregated bus coverage report across all skills in a test run. + + Attributes: + skills: Per-skill coverage data, one entry per skill that had any + listener or emitter activity. + """ + + skills: List[SkillBusCoverage] = dataclasses.field(default_factory=list) + + def filter(self, include: Optional[str] = None, exclude: Optional[str] = None) -> BusCoverageReport: + """Return a new report containing only skills matching the filters. + + Args: + include: Regex pattern. If provided, only skills matching this + pattern are kept. + exclude: Regex pattern. If provided, skills matching this pattern + are removed. + + Returns: + A new filtered :class:`BusCoverageReport`. + """ + import re + + filtered_skills = [] + for skill in self.skills: + if include and not re.search(include, skill.skill_id): + continue + if exclude and re.search(exclude, skill.skill_id): + continue + filtered_skills.append(skill) + return BusCoverageReport(skills=filtered_skills) + + def summary_line(self) -> str: + """Return per-skill single-line summaries joined by newlines. + + Suitable for inline test output (``print_bus_coverage=True``). + + Returns: + Multi-line string, one line per skill. + """ + lines: List[str] = [] + for skill in self.skills: + n_l = len(skill.listeners) + c_l = sum(1 for h in skill.listeners if h.covered) + n_e = len(skill.emitters) + c_obs = sum(1 for e in skill.emitters if e.observed) + c_ass = sum(1 for e in skill.emitters if e.asserted) + pct = f"{skill.listener_coverage_pct:.1f}%" + lines.append( + f"[bus-coverage] {skill.skill_id} — " + f"listeners: {c_l}/{n_l} ({pct}) | " + f"observed: {c_obs}/{n_e} | asserted: {c_ass}/{n_e}" + ) + return "\n".join(lines) + + def print_report(self, verbose: bool = False) -> None: + """Print a formatted coverage table to stdout. + + Args: + verbose: When ``True``, print per-msg-type detail rows for every + skill after the summary table. + """ + col_w = max((len(s.skill_id) for s in self.skills), default=5) + 2 + col_w = max(col_w, 7) # at least wide enough for "Skill" header + total_w = col_w + 36 + print() + print("━" * total_w) + print("Bus Coverage Report") + print("━" * total_w) + header = f"{'Skill':<{col_w}} {'Listeners':>14} {'Observed':>8} {'Asserted':>8}" + print(header) + print("─" * total_w) + + total_l = total_cl = total_e = total_obs = total_ass = 0 + for skill in self.skills: + n_l = len(skill.listeners) + c_l = sum(1 for h in skill.listeners if h.covered) + n_e = len(skill.emitters) + c_obs = sum(1 for e in skill.emitters if e.observed) + c_ass = sum(1 for e in skill.emitters if e.asserted) + total_l += n_l + total_cl += c_l + total_e += n_e + total_obs += c_obs + total_ass += c_ass + + pct = f"{skill.listener_coverage_pct:.1f}%" + listener_col = f"{c_l}/{n_l} {pct}" + print( + f"{skill.skill_id:<{col_w}} {listener_col:>14} " + f"{c_obs}/{n_e:>6} {c_ass}/{n_e:>6}" + ) + + if self.skills: + print("─" * total_w) + total_pct = (100.0 * total_cl / total_l) if total_l else 0.0 + total_listener_col = f"{total_cl}/{total_l} {total_pct:.1f}%" + print( + f"{'TOTAL':<{col_w}} {total_listener_col:>14} " + f"{total_obs}/{total_e:>6} {total_ass}/{total_e:>6}" + ) + + if verbose: + for skill in self.skills: + print() + print(f"LISTENERS — {skill.skill_id}") + for h in sorted(skill.listeners, key=lambda x: (not x.covered, x.msg_type)): + mark = "✓" if h.covered else "✗" + detail = f"{h.invocation_count} invocation(s)" if h.covered else "NOT TESTED" + print(f" {mark} {h.msg_type:<50} {detail}") + + print() + print(f"EMITTERS — {skill.skill_id}") + for e in sorted(skill.emitters, key=lambda x: (not x.observed, x.msg_type)): + obs_mark = "✓" if e.observed else "✗" + ass_tag = "✓ asserted" if e.asserted else "✗ not asserted" + obs_detail = f"observed {e.observed_count}x" if e.observed else "not observed" + print(f" {obs_mark} {e.msg_type:<50} {obs_detail} {ass_tag}") + + def to_json(self) -> str: + """Serialize the full report to a JSON string. + + Returns: + Pretty-printed JSON with ``skills`` and ``totals`` keys. + """ + data = { + "schema_version": "1", + "skills": [s.to_dict() for s in self.skills], + "totals": self._totals_dict(), + } + return json.dumps(data, indent=2) + + def _totals_dict(self) -> Dict[str, Any]: + """Compute aggregate totals across all skills. + + Returns: + Dict with total listener/emitter counts and percentages. + """ + total_l = sum(len(s.listeners) for s in self.skills) + total_cl = sum(sum(1 for h in s.listeners if h.covered) for s in self.skills) + total_e = sum(len(s.emitters) for s in self.skills) + total_obs = sum(sum(1 for e in s.emitters if e.observed) for s in self.skills) + total_ass = sum(sum(1 for e in s.emitters if e.asserted) for s in self.skills) + return { + "listener_covered": total_cl, + "listener_total": total_l, + "listener_coverage_pct": round(100.0 * total_cl / total_l, 1) if total_l else 0.0, + "observed_count": total_obs, + "asserted_count": total_ass, + "emitter_total": total_e, + } + + +# --------------------------------------------------------------------------- +# Tracker +# --------------------------------------------------------------------------- + + +class BusCoverageTracker: + """Tracks bus listener and emitter coverage for one or more test sessions. + + Call order:: + + tracker = BusCoverageTracker(bus, minicroft) + tracker.snapshot_listeners() # after MiniCroft READY + tracker.start_tracking() + # ... run test / CaptureSession.capture() ... + tracker.stop_tracking() + tracker.record_session(session.responses, test.expected_messages) + report = tracker.build_report() + + Multiple ``record_session`` calls accumulate data across test sessions + before a single ``build_report`` call. + + Args: + bus: The :class:`~ovos_utils.fakebus.FakeBus` instance in use. + minicroft: The running :class:`~ovoscope.MiniCroft` instance. + """ + + def __init__(self, bus: Any, minicroft: Any) -> None: + self._bus = bus + self._minicroft = minicroft + # skill_id -> {msg_type -> handler_count} + self._registered: Dict[str, Dict[str, int]] = {} + # msg_type -> invocation_count (total across all emits during tracking) + self._invocations: Dict[str, int] = {} + # Global counts from the ovoscope collector (captures boot sequence) + self._global_invocations: Dict[str, int] = {} + self._global_registrations: Dict[str, int] = {} + self._global_skill_registrations: Dict[str, Dict[str, int]] = {} + + collector = ovoscope.GLOBAL_BUS_COVERAGE_COLLECTOR + if collector: + # Snapshot the global state at initialization + self._global_invocations = dict(collector.invocations) + self._global_registrations = dict(collector.registrations) + self._global_skill_registrations = deepcopy(collector.skill_registrations) + + # skill_id -> {msg_type -> observed_count} + self._observed: Dict[str, Dict[str, int]] = {} + # skill_id -> {msg_type -> asserted_count} + self._asserted: Dict[str, Dict[str, int]] = {} + self._original_emit: Optional[Any] = None + self._tracking: bool = False + + # ------------------------------------------------------------------ + # Public API + # ------------------------------------------------------------------ + + def snapshot_listeners(self) -> None: + """Record every bus handler grouped by owning component. + + Must be called **after** ``MiniCroft`` reaches READY state. + + Attribution strategy (in priority order): + + 1. **Skills via EventContainer** — for each entry in + ``minicroft.plugin_skills``, unwrap ``PluginSkillLoader.instance`` + and read ``skill.events.events``. This is the authoritative list + because ovos-workshop wraps handlers in ``create_wrapper`` closures + before calling ``bus.on()``, making ``handler.__self__`` unreliable + for skill handlers. + + 2. **Core components via direct ``__self__``** — for every remaining + bus handler whose ``__self__`` is *not* already attributed to a + skill, use ``type(owner).__name__`` as the component name + (e.g. ``IntentService``, ``AdaptPipeline``, ``FallbackService``). + + 3. **Closure scan** — handlers whose ``__self__`` is ``None`` are + scanned for bound-method cell variables (catches ovos-workshop + closures that weren't in ``EventContainer``). + + The resulting ``_registered`` dict maps + ``component_name → {msg_type → handler_count}``. + """ + listener_map: Dict[str, Dict[str, int]] = {} + + # ── Pass 1: skills via EventContainer ────────────────────────────── + plugin_skills: Dict[str, Any] = ( + getattr(self._minicroft, "plugin_skills", {}) or {} + ) + # Build a set of instance ids that belong to skills so Pass 2 can skip them + skill_instance_ids: set = set() + + for skill_id, loader in plugin_skills.items(): + instance = getattr(loader, "instance", loader) + if instance is None: + instance = loader + skill_instance_ids.add(id(instance)) + + ec = getattr(instance, "events", None) + if ec is not None: + event_list = getattr(ec, "events", None) + if event_list is not None: + for msg_type, _handler in event_list: + listener_map.setdefault(skill_id, {}) + listener_map[skill_id][msg_type] = ( + listener_map[skill_id].get(msg_type, 0) + 1 + ) + continue # authoritative — no need for bus scan + + # Skill has no EventContainer: fall through to closure scan below + # (handled in Pass 3 with skill_id as the component label) + skill_instance_ids.discard(id(instance)) # re-enable for pass 3 + + # ── Build id→component name map for all known objects ─────────────── + # Seed with skill instances (skill_id takes priority over type name) + combined_map: Dict[int, str] = {} + for skill_id, loader in plugin_skills.items(): + instance = getattr(loader, "instance", loader) or loader + combined_map[id(instance)] = skill_id + + # Cache the bus events dict once — used across all three passes. + bus_events = self._get_bus_events() + + # Discover remaining owners by walking all bus handlers + for _msg_type, handlers in bus_events.items(): + for handler in self._iter_handlers(handlers): + owner = getattr(handler, "__self__", None) + if owner is None: + continue + if id(owner) not in combined_map: + component = ( + getattr(owner, "skill_id", None) + or getattr(owner, "name", None) + or type(owner).__name__ + ) + combined_map[id(owner)] = component + + # ── Pass 2: direct __self__ handlers ──────────────────────────────── + for msg_type, handlers in bus_events.items(): + for handler in self._iter_handlers(handlers): + owner = getattr(handler, "__self__", None) + if owner is None: + continue # handled in Pass 3 + if id(owner) in skill_instance_ids: + continue # already covered by EventContainer in Pass 1 + # Skip FakeBus itself and bare class objects (type instances) + if isinstance(owner, type): + continue + component = combined_map.get(id(owner), type(owner).__name__) + if component == "FakeBus": + continue + listener_map.setdefault(component, {}) + listener_map[component][msg_type] = ( + listener_map[component].get(msg_type, 0) + 1 + ) + + # ── Pass 3: closure scan for handlers with no direct __self__ ──────── + for msg_type, handlers in bus_events.items(): + for handler in self._iter_handlers(handlers): + if getattr(handler, "__self__", None) is not None: + continue # already handled above + component = self._skill_id_from_closure(handler, combined_map) + if component is None or component == "FakeBus": + continue + # Only add if not already covered by EventContainer + if component in listener_map and msg_type in listener_map[component]: + continue + listener_map.setdefault(component, {}) + listener_map[component][msg_type] = ( + listener_map[component].get(msg_type, 0) + 1 + ) + + # ── Pass 4: Global skill registrations (from patched OVOSSkill) ───── + # High-confidence registrations captured via monkey-patching. + for skill_id, handlers in self._global_skill_registrations.items(): + for msg_type, count in handlers.items(): + listener_map.setdefault(skill_id, {}) + # Use max to avoid double-counting if already found via introspection + listener_map[skill_id][msg_type] = max( + listener_map[skill_id].get(msg_type, 0), count + ) + + # ── Pass 5: Unclaimed global registrations (boot sequence fallback) ── + # Handlers registered and then unregistered during boot are captured + # by the global collector. We attribute them to __core__ if not already + # claimed by a skill/component during the snapshot. + for msg_type, count in self._global_registrations.items(): + # Check if any skill/component already has this msg_type + already_claimed = any(msg_type in handlers for handlers in listener_map.values()) + if not already_claimed: + listener_map.setdefault("__core__", {}) + listener_map["__core__"][msg_type] = ( + listener_map["__core__"].get(msg_type, 0) + count + ) + + self._registered = listener_map + + def start_tracking(self) -> None: + """Monkey-patch ``bus.emit`` to count per-msg-type invocations. + + Each call to ``bus.emit(msg)`` increments the invocation counter for + ``msg.msg_type``. Call :meth:`stop_tracking` to restore the original. + """ + if self._tracking: + return + original_emit = self._bus.emit + invocations = self._invocations + + def _patched_emit(message: Any) -> None: + msg_type = getattr(message, "msg_type", None) or getattr(message, "type", None) + if msg_type: + invocations[msg_type] = invocations.get(msg_type, 0) + 1 + original_emit(message) + + self._original_emit = original_emit + self._bus.emit = _patched_emit + self._tracking = True + + def stop_tracking(self) -> None: + """Restore the original ``bus.emit`` and stop counting invocations.""" + if not self._tracking: + return + self._bus.emit = self._original_emit + self._original_emit = None + self._tracking = False + + def record_session( + self, + responses: List[Message], + expected_messages: List[Message], + ) -> None: + """Accumulate observed and asserted emitter data from one test session. + + Can be called multiple times (once per ``End2EndTest.execute()`` call) + before :meth:`build_report`. + + Messages with no ``skill_id`` in context (core services, pipeline + components) are attributed to the ``"__core__"`` bucket so they are + never silently dropped. + + Args: + responses: Messages from ``CaptureSession.responses`` (observed). + expected_messages: Messages from ``End2EndTest.expected_messages`` + (asserted). + """ + # Observed: messages that actually appeared in CaptureSession + for msg in responses: + skill_id = self._skill_id_for_message(msg) or "__core__" + if skill_id not in self._observed: + self._observed[skill_id] = {} + self._observed[skill_id][msg.msg_type] = ( + self._observed[skill_id].get(msg.msg_type, 0) + 1 + ) + + # Asserted: messages listed in expected_messages + for msg in expected_messages: + skill_id = self._skill_id_for_message(msg) + if skill_id is None: + # Fall back: attribute to the first skill that already observed + # this msg_type so the assertion still shows up in the report. + for sid, obs in self._observed.items(): + if msg.msg_type in obs: + skill_id = sid + break + if skill_id is None: + skill_id = "__core__" + if skill_id not in self._asserted: + self._asserted[skill_id] = {} + self._asserted[skill_id][msg.msg_type] = ( + self._asserted[skill_id].get(msg.msg_type, 0) + 1 + ) + + def build_report(self) -> BusCoverageReport: + """Compile a :class:`BusCoverageReport` from all accumulated data. + + Returns: + Fully populated :class:`BusCoverageReport` instance. + """ + all_skill_ids = ( + set(self._registered) + | set(self._observed) + | set(self._asserted) + ) + skills: List[SkillBusCoverage] = [] + + for skill_id in sorted(all_skill_ids): + # --- listener entries --- + listener_entries: List[HandlerEntry] = [] + for msg_type, handler_count in sorted( + (self._registered.get(skill_id) or {}).items() + ): + # Total invocations = global (boot) + local (test execution) + invocations = ( + self._invocations.get(msg_type, 0) + + self._global_invocations.get(msg_type, 0) + ) + listener_entries.append( + HandlerEntry( + msg_type=msg_type, + handler_count=handler_count, + invocation_count=invocations, + covered=invocations > 0, + ) + ) + + # --- emitter entries --- + all_emitted = set( + (self._observed.get(skill_id) or {}).keys() + ) | set( + (self._asserted.get(skill_id) or {}).keys() + ) + emitter_entries: List[EmitterEntry] = [] + for msg_type in sorted(all_emitted): + obs_count = (self._observed.get(skill_id) or {}).get(msg_type, 0) + ass_count = (self._asserted.get(skill_id) or {}).get(msg_type, 0) + emitter_entries.append( + EmitterEntry( + msg_type=msg_type, + observed_count=obs_count, + asserted_count=ass_count, + observed=obs_count > 0, + asserted=ass_count > 0, + ) + ) + + skills.append( + SkillBusCoverage( + skill_id=skill_id, + listeners=listener_entries, + emitters=emitter_entries, + ) + ) + + return BusCoverageReport(skills=skills) + + # ------------------------------------------------------------------ + # Private helpers + # ------------------------------------------------------------------ + + def _get_bus_events(self) -> Dict[str, Any]: + """Return the raw event-name → handlers mapping from the bus. + + Tries ``bus.ee._events`` (pyee v8+) first, then ``bus._events`` as a + fallback for older versions or custom subclasses. + + Returns: + Dict mapping event name strings to handler containers. + """ + ee = getattr(self._bus, "ee", None) + if ee is not None: + events = getattr(ee, "_events", None) + if events is not None: + return dict(events) + # Fallback: FakeBus exposes handlers directly + events = getattr(self._bus, "_events", None) + if events is not None: + return dict(events) + return {} + + @staticmethod + def _iter_handlers(handlers: Any): + """Yield raw handler callables from a pyee handler container. + + pyee stores handlers in different container types depending on version: + + * v8 / v9: ``OrderedDict`` mapping ``{handler: handler}`` — iterate + keys. + * v8 with wrappers: list of ``_ListenerWrapper`` objects with a ``.fn`` + attribute. + * Legacy: a single callable. + + Args: + handlers: The handler container from ``bus.ee._events[msg_type]``. + + Yields: + Unwrapped callable objects (the registered handler functions). + """ + import collections + + # pyee v9: OrderedDict {handler: handler} — keys are the raw callables + if isinstance(handlers, dict): + for key in handlers.keys(): + # unwrap pyee listener wrappers if present + yield getattr(key, "fn", key) + return + + # Single callable (not a container) + if callable(handlers) and not isinstance(handlers, (list, tuple)): + yield getattr(handlers, "fn", handlers) + return + + # List / tuple of handlers or wrappers + try: + it = iter(handlers) + except TypeError: + return + for item in it: + yield getattr(item, "fn", item) + + def _skill_instance_map(self) -> Dict[int, str]: + """Build a mapping from ``id(skill_instance)`` to ``skill_id``. + + Unwraps ``PluginSkillLoader`` objects (which store the real skill + instance at ``.instance``) before building the map. + + Returns: + Dict of ``{id(skill_instance): skill_id}`` for all loaded plugin skills. + """ + mapping: Dict[int, str] = {} + plugin_skills: Dict[str, Any] = ( + getattr(self._minicroft, "plugin_skills", {}) or {} + ) + for skill_id, loader in plugin_skills.items(): + instance = getattr(loader, "instance", loader) + if instance is None: + instance = loader + mapping[id(instance)] = skill_id + return mapping + + @staticmethod + def _skill_id_from_closure( + handler: Any, + skill_instance_map: Dict[int, str], + ) -> Optional[str]: + """Attempt to attribute a closure-wrapped handler to a skill. + + OVOS wraps all skill handlers in ``create_wrapper`` closures. The + original bound method is captured as a cell variable whose + ``__self__`` is the skill instance. + + Args: + handler: A callable registered via ``bus.on``. + skill_instance_map: Mapping from ``id(skill_instance)`` to skill_id. + + Returns: + The ``skill_id`` string, or ``None`` if no skill found in closure. + """ + closure = getattr(handler, "__closure__", None) + if not closure: + return None + for cell in closure: + try: + val = cell.cell_contents + except ValueError: + continue + # Direct skill instance in closure + sid = skill_instance_map.get(id(val)) + if sid is not None: + return sid + # Bound method whose __self__ is the skill instance + owner = getattr(val, "__self__", None) + if owner is not None: + sid = skill_instance_map.get(id(owner)) + if sid is not None: + return sid + return None + + @staticmethod + def _skill_id_for_message(msg: Message) -> Optional[str]: + """Extract ``skill_id`` from a message's context field. + + Args: + msg: A bus :class:`~ovos_bus_client.message.Message`. + + Returns: + The ``skill_id`` value from ``msg.context``, or ``None``. + """ + if not msg.context: + return None + return msg.context.get("skill_id") diff --git a/ovoscope/cli.py b/ovoscope/cli.py index 87a4087..722d91d 100644 --- a/ovoscope/cli.py +++ b/ovoscope/cli.py @@ -283,6 +283,149 @@ def cmd_coverage(args: argparse.Namespace) -> int: return 0 +def cmd_bus_coverage(args: argparse.Namespace) -> int: + """Run fixture files and report bus-level handler and emitter coverage. + + Loads each ``.json`` fixture found under *test_dir*, executes it with + ``track_bus_coverage=True``, aggregates the results, and prints a table + (or JSON) report. + + Args: + args: Parsed CLI arguments with test_dir, skill_id, format, verbose. + + Returns: + Exit code (0 = success, 1 = failure). + """ + import glob as _glob + import os + + try: + from ovoscope import End2EndTest, get_minicroft + from ovoscope.bus_coverage import BusCoverageReport, SkillBusCoverage, HandlerEntry, EmitterEntry + except ImportError as exc: + _die(f"ovoscope import failed: {exc}") + + # Collect fixture files + test_dir: str = args.test_dir + if os.path.isfile(test_dir) and test_dir.endswith(".json"): + fixture_paths = [test_dir] + else: + fixture_paths = sorted( + _glob.glob(os.path.join(test_dir, "**", "*.json"), recursive=True) + ) + + if not fixture_paths: + _die(f"No fixture JSON files found under: {test_dir}") + + filter_skill_id: Optional[str] = getattr(args, "skill_id", None) + + # Merge buckets: skill_id -> {msg_type -> (handler_count, invocation_count)} + merged_listeners: dict = {} + merged_observed: dict = {} + merged_asserted: dict = {} + errors: List[str] = [] + + for fixture_path in fixture_paths: + print(f"[bus-coverage] Running fixture: {fixture_path}") + try: + test = End2EndTest.from_path(fixture_path) + except Exception as exc: + print(f"[bus-coverage] SKIP (load error): {exc}") + errors.append(fixture_path) + continue + + skill_ids = test.skill_ids or [] + if filter_skill_id and filter_skill_id not in skill_ids: + continue + + try: + mc = get_minicroft(skill_ids, max_wait=60) + except TimeoutError: + print(f"[bus-coverage] SKIP (MiniCroft timeout): {fixture_path}") + errors.append(fixture_path) + continue + + try: + test.minicroft = mc + test.managed = False # prevent execute() from stopping mc; finally block owns it + test.track_bus_coverage = True + test.execute() + except AssertionError as exc: + print(f"[bus-coverage] WARN (test failure, coverage still collected): {exc}") + except Exception as exc: + print(f"[bus-coverage] SKIP (execution error): {exc}") + errors.append(fixture_path) + continue + finally: + mc.stop() + + report = test.bus_coverage_report + if report is None: + continue + + # Merge into global buckets + for skill in report.skills: + sid = skill.skill_id + if sid not in merged_listeners: + merged_listeners[sid] = {} + for h in skill.listeners: + existing = merged_listeners[sid].get(h.msg_type, (h.handler_count, 0)) + merged_listeners[sid][h.msg_type] = ( + existing[0], + existing[1] + h.invocation_count, + ) + if sid not in merged_observed: + merged_observed[sid] = {} + if sid not in merged_asserted: + merged_asserted[sid] = {} + for e in skill.emitters: + merged_observed[sid][e.msg_type] = ( + merged_observed[sid].get(e.msg_type, 0) + e.observed_count + ) + merged_asserted[sid][e.msg_type] = ( + merged_asserted[sid].get(e.msg_type, 0) + e.asserted_count + ) + + # Build final merged report + skills = [] + for skill_id in sorted(set(merged_listeners) | set(merged_observed)): + listeners = [ + HandlerEntry( + msg_type=mt, + handler_count=hc, + invocation_count=ic, + covered=ic > 0, + ) + for mt, (hc, ic) in sorted(merged_listeners.get(skill_id, {}).items()) + ] + all_emitted = set(merged_observed.get(skill_id, {}).keys()) | set( + merged_asserted.get(skill_id, {}).keys() + ) + emitters = [ + EmitterEntry( + msg_type=mt, + observed_count=merged_observed.get(skill_id, {}).get(mt, 0), + asserted_count=merged_asserted.get(skill_id, {}).get(mt, 0), + observed=merged_observed.get(skill_id, {}).get(mt, 0) > 0, + asserted=merged_asserted.get(skill_id, {}).get(mt, 0) > 0, + ) + for mt in sorted(all_emitted) + ] + skills.append(SkillBusCoverage(skill_id=skill_id, listeners=listeners, emitters=emitters)) + + final_report = BusCoverageReport(skills=skills) + + if args.format == "json": + print(final_report.to_json()) + else: + final_report.print_report(verbose=args.verbose) + + if errors: + print(f"\n[bus-coverage] {len(errors)} fixture(s) skipped due to errors.") + + return 0 + + # --------------------------------------------------------------------------- # Argument parser # --------------------------------------------------------------------------- @@ -339,6 +482,35 @@ def _build_parser() -> argparse.ArgumentParser: p_coverage.add_argument("--format", choices=["table", "json"], default="table", help="Output format (default: table).") + # --- bus-coverage --- + p_bus = sub.add_parser( + "bus-coverage", + help="Run fixture files and report bus handler/emitter coverage.", + ) + p_bus.add_argument( + "test_dir", + metavar="TEST_DIR", + help="Path to a directory of fixture JSON files (or a single fixture file).", + ) + p_bus.add_argument( + "--skill-id", + default=None, + metavar="ID", + help="Only report on fixtures that include this skill_id.", + ) + p_bus.add_argument( + "--format", + choices=["table", "json"], + default="table", + help="Output format (default: table).", + ) + p_bus.add_argument( + "--verbose", + "-v", + action="store_true", + help="Print per-msg-type detail rows.", + ) + return parser @@ -358,6 +530,7 @@ def main() -> None: "diff": cmd_diff, "validate": cmd_validate, "coverage": cmd_coverage, + "bus-coverage": cmd_bus_coverage, } handler = dispatch.get(args.command) diff --git a/ovoscope/coverage.py b/ovoscope/coverage.py index a0a9047..b4577eb 100644 --- a/ovoscope/coverage.py +++ b/ovoscope/coverage.py @@ -358,7 +358,11 @@ def _has_e2e_tests(repo_root: str) -> bool: def _count_fixtures(repo_root: str) -> int: - """Count ``.json`` fixture files in common fixture directories. + """Count ``.json`` fixture files in common fixture directories (recursive). + + Searches recursively under each candidate directory so that fixtures + organised in sub-directories (e.g. ``test/end2end/skill_name/*.json``) + are counted correctly. Args: repo_root: Repository root directory. @@ -366,16 +370,18 @@ def _count_fixtures(repo_root: str) -> int: Returns: Total count of JSON fixture files found. """ + from pathlib import Path count = 0 - candidates = [ - os.path.join(repo_root, "test", "end2end"), - os.path.join(repo_root, "tests", "end2end"), - os.path.join(repo_root, "test", "fixtures"), - os.path.join(repo_root, "tests", "fixtures"), + candidate_names = [ + ("test", "end2end"), + ("tests", "end2end"), + ("test", "fixtures"), + ("tests", "fixtures"), ] - for candidate in candidates: - if os.path.isdir(candidate): - count += sum(1 for f in os.listdir(candidate) if f.endswith(".json")) + for parts in candidate_names: + candidate = Path(repo_root).joinpath(*parts) + if candidate.is_dir(): + count += sum(1 for _ in candidate.rglob("*.json")) return count diff --git a/ovoscope/diff.py b/ovoscope/diff.py index c10513d..0a267ee 100644 --- a/ovoscope/diff.py +++ b/ovoscope/diff.py @@ -118,23 +118,36 @@ def to_json(self) -> Dict[str, Any]: } -def _dict_diff(expected: Dict[str, Any], actual: Dict[str, Any]) -> Dict[str, Tuple[Any, Any]]: +def _dict_diff( + expected: Dict[str, Any], + actual: Dict[str, Any], + strict: bool = False, +) -> Dict[str, Tuple[Any, Any]]: """Return keys whose values differ between *expected* and *actual*. - Only keys present in *expected* are checked (subset comparison). + By default only keys present in *expected* are checked (subset comparison). + When *strict* is ``True``, keys present in *actual* but absent from + *expected* are also flagged as unexpected extras. Args: expected: Reference dict. actual: Dict to compare against. + strict: When ``True``, flag extra keys in *actual* not in *expected*. + Default ``False`` preserves the original subset-comparison behaviour. Returns: Mapping of key → (expected_value, actual_value) for differing keys. + For extra keys (strict mode only) the expected_value is ``None``. """ diffs: Dict[str, Tuple[Any, Any]] = {} for k, exp_v in expected.items(): act_v = actual.get(k) if act_v != exp_v: diffs[k] = (exp_v, act_v) + if strict: + for k, act_v in actual.items(): + if k not in expected: + diffs[k] = (None, act_v) return diffs @@ -160,6 +173,7 @@ def diff_fixtures( actual_path: str, *, ignore_context: bool = True, + strict: bool = False, ) -> FixtureDiffResult: """Compare two fixture JSON files and return a structured diff. @@ -173,6 +187,8 @@ def diff_fixtures( expected_path: Path to the reference fixture file. actual_path: Path to the fixture file being validated. ignore_context: Skip context comparison (default True). + strict: When ``True``, flag extra keys in *actual* data/context dicts + not present in *expected*. Default ``False`` (subset comparison). Returns: A :class:`FixtureDiffResult` describing all differences found. @@ -228,10 +244,10 @@ def diff_fixtures( continue # Same type — compare data and context - data_diffs = _dict_diff(exp_msg.get("data", {}), act_msg.get("data", {})) + data_diffs = _dict_diff(exp_msg.get("data", {}), act_msg.get("data", {}), strict=strict) ctx_diffs: Dict[str, Tuple[Any, Any]] = {} if not ignore_context: - ctx_diffs = _dict_diff(exp_msg.get("context", {}), act_msg.get("context", {})) + ctx_diffs = _dict_diff(exp_msg.get("context", {}), act_msg.get("context", {}), strict=strict) if data_diffs or ctx_diffs: diff = MessageDiff( diff --git a/ovoscope/media.py b/ovoscope/media.py new file mode 100644 index 0000000..0c07ee1 --- /dev/null +++ b/ovoscope/media.py @@ -0,0 +1,648 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""OCP / Media testing harnesses for ovoscope. + +Provides a mock OCP audio backend, a context-manager harness that wires a real +``OCPMediaPlayer`` onto a ``FakeBus`` with all heavy dependencies mocked out, +and a lightweight bus-message capture helper — enabling fast, dependency-free +end-to-end player state-machine tests. + +Classes: + MockOCPBackend -- no-op AudioBackend that tracks state and can simulate + media events + OCPPlayerHarness -- context manager wrapping OCPMediaPlayer + MockOCPBackend + OCPCaptureSession -- records OCP bus messages matching given prefixes +""" + +import dataclasses +import time +from typing import List, Optional +from unittest.mock import MagicMock, patch + +from ovos_bus_client.message import Message +from ovos_plugin_manager.templates.audio import AudioBackend +from ovos_utils.fakebus import FakeBus +from ovos_utils.ocp import MediaEntry, MediaState, PlayerState + + +# --------------------------------------------------------------------------- +# MockOCPBackend +# --------------------------------------------------------------------------- + +class MockOCPBackend(AudioBackend): + """A no-op AudioBackend that records state transitions and can emit media + lifecycle events to simulate a real backend during testing. + + No actual audio is played. Every mutating method updates simple Python + attributes so tests can inspect them without mocks. + + Args: + config: Backend configuration dict (may be empty). + bus: FakeBus instance shared with the service under test. + namespace: Backend namespace string, e.g. ``"audio"``. Used when + emitting ``ovos.{namespace}.service.media.state`` events. + """ + + def __init__(self, config: dict, bus: FakeBus, + namespace: str = "audio") -> None: + """Initialise the backend with clean state. + + Args: + config: Configuration dict forwarded to AudioBackend.__init__. + bus: FakeBus used by the enclosing service. + namespace: Namespace prefix for bus events. + """ + super().__init__(config=config, bus=bus) + self.namespace: str = namespace + self.played_uris: List[str] = [] + self.is_playing: bool = False + self.is_paused: bool = False + self.current_uri: Optional[str] = None + + # ------------------------------------------------------------------ + # AudioBackend abstract interface + # ------------------------------------------------------------------ + + def supported_uris(self) -> List[str]: + """Return URI schemes supported by this backend. + + Returns: + List of scheme strings: ``["file", "http", "https"]``. + """ + return ["file", "http", "https"] + + def add_list(self, playlist: List[str]) -> None: + """Record tracks and set ``current_uri``. + + Args: + playlist: List of track URIs to add. + """ + self.played_uris.extend(playlist) + if playlist: + self.current_uri = playlist[0] + + def clear_list(self) -> None: + """Clear the recorded playlist and current URI.""" + self.played_uris.clear() + self.current_uri = None + + def load_track(self, uri: str) -> None: + """Record *uri* and emit a ``LOADED_MEDIA`` state event. + + This triggers ``BaseMediaService.handle_media_state_change`` which + calls ``self.current.play()`` to start real playback in production. + In tests the ``play()`` call on this backend simply sets ``is_playing``. + + Args: + uri: URI of the track to load. + """ + self.current_uri = uri + if uri not in self.played_uris: + self.played_uris.append(uri) + self.bus.emit(Message( + f"ovos.{self.namespace}.service.media.state", + {"state": MediaState.LOADED_MEDIA}, + )) + + def play(self, repeat: bool = False) -> None: + """Mark the backend as playing. + + Args: + repeat: Whether to loop (not implemented in mock). + """ + self.is_playing = True + self.is_paused = False + + def stop(self) -> bool: + """Stop playback and clear state. + + Returns: + True — ``BaseMediaService._perform_stop()`` gates on the return value. + """ + self.is_playing = False + self.is_paused = False + return True + + def pause(self) -> None: + """Pause playback.""" + self.is_paused = True + + def resume(self) -> None: + """Resume paused playback.""" + self.is_paused = False + + def next(self) -> None: + """Skip to next track (no-op).""" + + def previous(self) -> None: + """Skip to previous track (no-op).""" + + def lower_volume(self) -> None: + """Duck volume (no-op in mock).""" + + def restore_volume(self) -> None: + """Restore ducked volume (no-op in mock).""" + + def track_info(self) -> dict: + """Return minimal track info. + + Returns: + Dict with ``"track"`` key containing ``current_uri``. + """ + return {"track": self.current_uri} + + def shutdown(self) -> None: + """Shut down the backend (no-op).""" + + def get_track_length(self) -> int: + """Return track duration in ms. + + Returns: + Always 0 — mock backend has no real audio. + """ + return 0 + + def get_track_position(self) -> int: + """Return current playback position in ms. + + Returns: + Always 0 — mock backend has no real audio. + """ + return 0 + + def set_track_position(self, milliseconds: int) -> None: + """Seek to position (no-op). + + Args: + milliseconds: Target position in ms. + """ + + def seek_forward(self, seconds: int = 1) -> None: + """Seek forward (no-op). + + Args: + seconds: Seconds to seek forward. + """ + + def seek_backward(self, seconds: int = 1) -> None: + """Seek backward (no-op). + + Args: + seconds: Seconds to seek backward. + """ + + # ------------------------------------------------------------------ + # Test helpers + # ------------------------------------------------------------------ + + def simulate_end(self) -> None: + """Emit an ``END_OF_MEDIA`` state event on the shared bus. + + Call this in tests to simulate the backend finishing a track without + real audio hardware. + """ + self.is_playing = False + self.bus.emit(Message( + "ovos.common_play.media.state", + {"state": MediaState.END_OF_MEDIA}, + )) + + def simulate_invalid_stream(self) -> None: + """Emit an ``INVALID_MEDIA`` state event on the shared bus. + + Call this in tests to simulate a broken or unplayable stream. + """ + self.is_playing = False + self.bus.emit(Message( + "ovos.common_play.media.state", + {"state": MediaState.INVALID_MEDIA}, + )) + + def reset(self) -> None: + """Reset all recorded state back to initial values.""" + self.played_uris.clear() + self.is_playing = False + self.is_paused = False + self.current_uri = None + + +# --------------------------------------------------------------------------- +# OCPPlayerHarness +# --------------------------------------------------------------------------- + +class OCPPlayerHarness: + """Context manager that runs ``OCPMediaPlayer`` on a ``FakeBus`` with all + heavy dependencies mocked out and a ``MockOCPBackend`` injected as the + sole audio backend. + + Usage:: + + with OCPPlayerHarness() as h: + entry = MediaEntry(uri="http://example.com/song.mp3", + playback=PlaybackType.AUDIO) + h.play(entry) + h.assert_player_state(PlayerState.PLAYING) + assert h.backend.is_playing + + The harness patches: + + - ``ovos_media.player.AudioService`` + - ``ovos_media.player.VideoService`` + - ``ovos_media.player.WebService`` + - ``ovos_media.player.OcpMprisExporter`` + - ``ovos_media.player.GUIInterface`` (exposed as ``harness.gui``) + - ``ovos_media.player.OCPMediaCatalog`` + - ``ovos_media.player.Configuration`` (returns ``{"media": {}}``) + + Args: + backend_namespace: Namespace for ``MockOCPBackend``; default ``"audio"``. + """ + + def __init__(self, backend_namespace: str = "audio") -> None: + """Initialise harness parameters. + + Args: + backend_namespace: Namespace prefix passed to ``MockOCPBackend``. + """ + self.backend_namespace: str = backend_namespace + self.bus: Optional[FakeBus] = None + self.player = None # OCPMediaPlayer instance + self.backend: Optional[MockOCPBackend] = None + self.gui: Optional[MagicMock] = None + self._patches: list = [] + + def __enter__(self) -> "OCPPlayerHarness": + """Start ``OCPMediaPlayer`` with mocked deps and inject ``MockOCPBackend``. + + Returns: + self + """ + from ovos_media.player import OCPMediaPlayer + + self.bus = FakeBus() + self.backend = MockOCPBackend( + config={}, bus=self.bus, namespace=self.backend_namespace + ) + + # Build patch targets + gui_mock = MagicMock() + self.gui = gui_mock + + # Patch Playlist so that Playlist("Search Results") doesn't try to + # add the string as a media entry. The installed ovos_utils.ocp.Playlist + # treats all positional args as entries; we need a subclass that silently + # drops bare string args (which are titles, not entries) and still + # satisfies isinstance(x, Playlist) checks inside player.py. + from ovos_utils.ocp import Playlist as _RealPlaylist + + class _TolerantPlaylist(_RealPlaylist): + """Playlist subclass that ignores bare string constructor args.""" + + def __init__(self, *args, **kwargs): + valid = [a for a in args if not isinstance(a, str)] + super().__init__(*valid, **kwargs) + + p_playlist = patch("ovos_media.player.Playlist", _TolerantPlaylist) + p_playlist.start() + self._patches.append(p_playlist) + + simple_targets = [ + "ovos_media.player.AudioService", + "ovos_media.player.VideoService", + "ovos_media.player.WebService", + "ovos_media.player.OcpMprisExporter", + "ovos_media.player.OCPMediaCatalog", + ] + for target in simple_targets: + p = patch(target) + p.start() + self._patches.append(p) + + p_cfg = patch("ovos_media.player.Configuration", + return_value={"media": {}}) + p_cfg.start() + self._patches.append(p_cfg) + + p_gui = patch("ovos_media.player.GUIInterface", + return_value=gui_mock) + p_gui.start() + self._patches.append(p_gui) + + # Instantiate the real player (all heavy deps are now mocked) + self.player = OCPMediaPlayer(self.bus, config={}) + + # Inject MockOCPBackend as the sole audio backend + audio_svc = self.player.audio_service + audio_svc.services = [self.backend] + audio_svc.default = self.backend + self.backend.set_track_start_callback(audio_svc.track_start) + + # Register the audio service bus handlers manually + # (normally done inside BaseMediaService.load_services) + ns = self.backend_namespace + self.bus.on(f"ovos.{ns}.service.play", audio_svc.handle_play) + self.bus.on(f"ovos.{ns}.service.pause", audio_svc.pause) + self.bus.on(f"ovos.{ns}.service.resume", audio_svc.resume) + self.bus.on(f"ovos.{ns}.service.stop", audio_svc.stop) + self.bus.on("ovos.common_play.media.state", + audio_svc.handle_media_state_change) + audio_svc._loaded.set() + + return self + + def __exit__(self, *args) -> None: + """Shut down the player, close the bus, and stop all patches.""" + if self.player: + try: + self.player.shutdown() + except Exception: + pass + if self.bus: + try: + self.bus.close() + except Exception: + pass + for p in reversed(self._patches): + try: + p.stop() + except RuntimeError: + pass + + # ------------------------------------------------------------------ + # Control methods — emit the correct bus message and yield briefly + # ------------------------------------------------------------------ + + def play(self, track: MediaEntry) -> None: + """Emit ``ovos.common_play.play`` and wait for synchronous delivery. + + Args: + track: ``MediaEntry`` to play. + """ + self.bus.emit(Message("ovos.common_play.play", { + "media": track.as_dict, + "playlist": [track.as_dict], + })) + time.sleep(0.05) + + def pause(self) -> None: + """Emit ``ovos.common_play.pause``.""" + self.bus.emit(Message("ovos.common_play.pause")) + time.sleep(0.05) + + def resume(self) -> None: + """Emit ``ovos.common_play.resume``.""" + self.bus.emit(Message("ovos.common_play.resume")) + time.sleep(0.05) + + def stop(self) -> None: + """Emit ``ovos.common_play.stop``.""" + self.bus.emit(Message("ovos.common_play.stop")) + time.sleep(0.05) + + def next_track(self) -> None: + """Emit ``ovos.common_play.next``.""" + self.bus.emit(Message("ovos.common_play.next")) + time.sleep(0.05) + + def prev_track(self) -> None: + """Emit ``ovos.common_play.previous``.""" + self.bus.emit(Message("ovos.common_play.previous")) + time.sleep(0.05) + + def duck(self) -> None: + """Lower the audio backend volume via ``recognizer_loop:audio_output_start``. + + Ducking lowers volume while the voice assistant speaks. The player + **stays PLAYING** — only the backend volume is reduced. + + Equivalent OCP message: ``ovos.common_play.duck``. + Handler: ``OCPMediaPlayer.handle_duck_request`` — + ``ovos_media/player.py:1216``. + """ + self.bus.emit(Message("recognizer_loop:audio_output_start")) + time.sleep(0.05) + + def unduck(self) -> None: + """Restore the audio backend volume via ``recognizer_loop:audio_output_end``. + + Note: ``handle_unduck_request`` only restores volume when the player is + PAUSED (``state == PlayerState.PAUSED``). After a pure duck cycle the + player remains PLAYING, so this call is a no-op in that case. + + Equivalent OCP message: ``ovos.common_play.unduck``. + Handler: ``OCPMediaPlayer.handle_unduck_request`` — + ``ovos_media/player.py:1228``. + """ + self.bus.emit(Message("recognizer_loop:audio_output_end")) + time.sleep(0.05) + + def cork(self) -> None: + """Pause the player via ``ovos.common_play.cork`` (microphone opens). + + Corking fully **pauses** the player and sets ``_paused_on_duck = True`` + so ``uncork()`` / ``record_end`` can resume it automatically. + + Equivalent legacy message: ``recognizer_loop:record_begin``. + Handler: ``OCPMediaPlayer.handle_cork_request`` — + ``ovos_media/player.py:1198``. + """ + self.bus.emit(Message("ovos.common_play.cork")) + time.sleep(0.05) + + def uncork(self) -> None: + """Resume the player via ``ovos.common_play.uncork`` (microphone closes). + + Only resumes if the player is PAUSED **and** ``_paused_on_duck`` is True + (i.e. the pause was caused by a cork, not a manual pause). + + Equivalent legacy message: ``recognizer_loop:record_end`` followed by + 8-second no-speak timeout. + Handler: ``OCPMediaPlayer.handle_uncork_request`` — + ``ovos_media/player.py:1207``. + """ + self.bus.emit(Message("ovos.common_play.uncork")) + time.sleep(0.05) + + def simulate_track_end(self) -> None: + """Emit ``ovos.common_play.media.state`` ``END_OF_MEDIA`` via the backend. + + Triggers ``OCPMediaPlayer.handle_player_media_update`` → + ``handle_playback_ended``, which auto-advances the queue when + ``autoplay`` is enabled. + """ + self.backend.simulate_end() + time.sleep(0.05) + + def simulate_invalid_stream(self) -> None: + """Emit ``ovos.common_play.media.state`` ``INVALID_MEDIA`` via the backend. + + Triggers ``OCPMediaPlayer.handle_player_media_update`` → + ``handle_invalid_media``, then ``play_next()`` when ``autoplay`` is + enabled. + """ + self.backend.simulate_invalid_stream() + time.sleep(0.05) + + # ------------------------------------------------------------------ + # Assertion helpers + # ------------------------------------------------------------------ + + def assert_player_state(self, state: PlayerState) -> None: + """Assert the player is in the given ``PlayerState``. + + Args: + state: Expected ``PlayerState``. + """ + assert self.player.state == state, ( + f"Expected PlayerState.{state.name}, " + f"got PlayerState.{self.player.state.name}" + ) + + def assert_media_state(self, state: MediaState) -> None: + """Assert the player's media state matches *state*. + + Args: + state: Expected ``MediaState``. + """ + assert self.player.media_state == state, ( + f"Expected MediaState.{state.name}, " + f"got MediaState.{self.player.media_state.name}" + ) + + def assert_backend_playing(self) -> None: + """Assert the mock backend is currently playing.""" + assert self.backend.is_playing, "Expected backend to be playing" + + def assert_backend_paused(self) -> None: + """Assert the mock backend is currently paused.""" + assert self.backend.is_paused, "Expected backend to be paused" + + def assert_backend_stopped(self) -> None: + """Assert the mock backend is neither playing nor paused.""" + assert not self.backend.is_playing, \ + "Expected backend to be stopped (is_playing=True)" + assert not self.backend.is_paused, \ + "Expected backend to be stopped (is_paused=True)" + + def assert_now_playing_uri(self, uri: str) -> None: + """Assert the currently playing URI matches *uri*. + + Args: + uri: Expected URI string. + """ + actual = self.player.now_playing.uri if self.player.now_playing else None + assert actual == uri, f"Expected now_playing.uri={uri!r}, got {actual!r}" + + +# --------------------------------------------------------------------------- +# OCPCaptureSession +# --------------------------------------------------------------------------- + +@dataclasses.dataclass +class OCPCaptureSession: + """Records bus messages whose types match given prefixes. + + Designed as a lightweight companion to ``OCPPlayerHarness`` for asserting + that specific OCP message sequences were emitted during a media interaction. + + Args: + bus: The ``FakeBus`` to subscribe to. + track_prefixes: Message-type prefix strings to capture. + + Attributes: + messages: All captured ``Message`` objects in emission order. + + Example:: + + with OCPPlayerHarness() as h: + with OCPCaptureSession(h.bus) as session: + h.play(entry) + session.assert_sequence( + "ovos.common_play.play", + "ovos.common_play.player.state", + ) + """ + + bus: FakeBus + track_prefixes: List[str] = dataclasses.field(default_factory=lambda: [ + "ovos.common_play.", + "ovos.audio.", + ]) + messages: List[Message] = dataclasses.field(default_factory=list) + + def _handle(self, msg: str) -> None: + """Internal handler subscribed to the raw ``message`` event. + + Args: + msg: Serialised message string from ``FakeBus``. + """ + m = Message.deserialize(msg) + for prefix in self.track_prefixes: + if m.msg_type.startswith(prefix): + self.messages.append(m) + break + + def start(self) -> None: + """Begin capturing messages.""" + self.messages.clear() + self.bus.on("message", self._handle) + + def stop(self) -> None: + """Stop capturing messages.""" + self.bus.remove("message", self._handle) + + def __enter__(self) -> "OCPCaptureSession": + """Start capturing on context entry. + + Returns: + self + """ + self.start() + return self + + def __exit__(self, *args) -> None: + """Stop capturing on context exit.""" + self.stop() + + @property + def message_types(self) -> List[str]: + """Return the list of captured message type strings. + + Returns: + List of ``msg_type`` strings in the order they were received. + """ + return [m.msg_type for m in self.messages] + + def assert_sequence(self, *types: str) -> None: + """Assert that captured messages contain all given types in order. + + Args: + *types: Expected message type strings (subsequence check). + + Raises: + AssertionError: If any type is missing or order is violated. + """ + received = self.message_types + pos = 0 + for t in types: + found = False + while pos < len(received): + if received[pos] == t: + pos += 1 + found = True + break + pos += 1 + assert found, ( + f"Expected message '{t}' not found in sequence after position. " + f"Full captured sequence: {received}" + ) diff --git a/ovoscope/pipeline.py b/ovoscope/pipeline.py index 0bb31c9..e8ccbd2 100644 --- a/ovoscope/pipeline.py +++ b/ovoscope/pipeline.py @@ -146,38 +146,60 @@ def match(self, utterance: str, timeout: float = 5.0) -> Optional[Message]: if self._mc is None: raise RuntimeError("PipelineHarness must be used as a context manager.") + import threading + captured: List[Message] = [] - event_types = [ - "intent.service.skills.activated", - "intent_failure", - "mycroft.skill.handler.start", - ] + _matched = threading.Event() + _failed = threading.Event() - import threading - done = threading.Event() + success_type = "intent.service.skills.activated" + failure_types = ["intent_failure", "mycroft.skill.handler.start"] - def _capture(msg: Any) -> None: + def _on_success(msg: Any) -> None: if isinstance(msg, str): try: msg = Message.deserialize(msg) except Exception: return captured.append(msg) - done.set() + _matched.set() + + def _on_failure(msg: Any) -> None: + _failed.set() - for et in event_types: - self._mc.bus.on(et, _capture) + self._mc.bus.on(success_type, _on_success) + for et in failure_types: + self._mc.bus.on(et, _on_failure) src = Message( "recognizer_loop:utterance", data={"utterances": [utterance], "lang": self.lang}, ) self._mc.bus.emit(src) - done.wait(timeout=timeout) - for et in event_types: - self._mc.bus.remove(et, _capture) + # Wait for either a match or a failure signal + import threading as _threading + done = _threading.Event() + + def _wait_either() -> None: + while not _matched.is_set() and not _failed.is_set(): + _matched.wait(timeout=0.05) + if _matched.is_set() or _failed.is_set(): + break + done.set() + + watcher = _threading.Thread(target=_wait_either, daemon=True) + watcher.start() + timed_out = not done.wait(timeout=timeout) + + self._mc.bus.remove(success_type, _on_success) + for et in failure_types: + self._mc.bus.remove(et, _on_failure) + if timed_out: + return None + if _failed.is_set(): + return None return captured[0] if captured else None def assert_matches( diff --git a/ovoscope/pytest_plugin.py b/ovoscope/pytest_plugin.py index c4eb4ec..5745d87 100644 --- a/ovoscope/pytest_plugin.py +++ b/ovoscope/pytest_plugin.py @@ -1,8 +1,7 @@ -"""ovoscope pytest plugin — provides the ``minicroft`` fixture. +"""ovoscope pytest plugin — provides the ``minicroft`` fixture and bus-coverage hooks. -Registered automatically via the ``pytest11`` entry point in ``setup.py`` / -``pyproject.toml``. Import directly in a ``conftest.py`` if you need to -customise the fixture scope: +Registered automatically via the ``pytest11`` entry point in ``pyproject.toml``. +Import directly in a ``conftest.py`` if you need to customise the fixture scope:: # conftest.py from ovoscope.pytest_plugin import minicroft # noqa: F401 @@ -30,13 +29,73 @@ def test_something(self, minicroft): expected_messages=[...], ) test.execute(timeout=10) + +Bus coverage opt-in:: + + class TestMySkill: + skill_ids = ["my-skill.author"] + + def test_something(self, minicroft, bus_coverage_session): + test = End2EndTest( + minicroft=minicroft, + skill_ids=self.skill_ids, + source_message=message, + expected_messages=[...], + track_bus_coverage=True, + ) + test.execute() + bus_coverage_session.add(test.bus_coverage_report) """ -from typing import Iterator, List, Union +from typing import TYPE_CHECKING, Iterator, List, Optional, Union + +if TYPE_CHECKING: + from ovoscope.bus_coverage import BusCoverageReport import pytest -from ovoscope import MiniCroft, get_minicroft +from ovoscope import MiniCroft, get_minicroft, End2EndTest + +# Global collector for autouse fixture and monkey-patched End2EndTest +_SESSION_COLLECTOR: Optional["BusCoverageCollector"] = None + + +def pytest_addoption(parser): + """Add CLI options for ovoscope bus coverage.""" + group = parser.getgroup("ovoscope") + group.addoption( + "--ovoscope-bus-cov", + action="store_true", + default=False, + help="Enable bus-level coverage tracking for all End2EndTests.", + ) + group.addoption( + "--ovoscope-bus-cov-file", + action="store", + default=None, + metavar="PATH", + help="Save the merged bus coverage report to a JSON file.", + ) + group.addoption( + "--ovoscope-bus-cov-verbose", + action="store_true", + default=False, + help="Show detailed list of covered/uncovered message types in the terminal.", + ) + group.addoption( + "--ovoscope-bus-cov-include", + action="store", + default=None, + metavar="PATTERN", + help="Only include skills/components matching this regex in the coverage report.", + ) + group.addoption( + "--ovoscope-bus-cov-exclude", + action="store", + default=None, + metavar="PATTERN", + help="Exclude skills/components matching this regex from the coverage report.", + ) @pytest.fixture(scope="class") @@ -64,3 +123,201 @@ def test_intent(self, minicroft): finally: if mc is not None: mc.stop() + + +class BusCoverageCollector: + """Accumulates :class:`~ovoscope.bus_coverage.BusCoverageReport` objects + across a pytest session and merges them for the terminal summary. + + Usage:: + + # In a test + def test_something(self, minicroft, bus_coverage_session): + test = End2EndTest(..., track_bus_coverage=True) + test.execute() + bus_coverage_session.add(test.bus_coverage_report) + """ + + def __init__(self) -> None: + self._reports: List["BusCoverageReport"] = [] + + def add(self, report: Optional["BusCoverageReport"]) -> None: + """Add a :class:`~ovoscope.bus_coverage.BusCoverageReport` to the collector. + + Silently ignores ``None`` so callers do not need to guard against tests + where ``track_bus_coverage=False``. + + Args: + report: A :class:`~ovoscope.bus_coverage.BusCoverageReport` or ``None``. + """ + if report is not None: + self._reports.append(report) + + def merged_report(self) -> Optional[object]: + """Return a merged :class:`~ovoscope.bus_coverage.BusCoverageReport`. + + Merges all accumulated reports by summing per-skill listener invocations, + observed counts, and asserted counts. + + Returns: + A merged report, or ``None`` if no reports have been added. + """ + if not self._reports: + return None + try: + from ovoscope.bus_coverage import ( + BusCoverageReport, + SkillBusCoverage, + HandlerEntry, + EmitterEntry, + ) + except ImportError: + return None + + # Merge by skill_id + listener_data: dict = {} # skill_id -> {msg_type -> (handler_count, invocation_count)} + observed_data: dict = {} # skill_id -> {msg_type -> count} + asserted_data: dict = {} # skill_id -> {msg_type -> count} + + for report in self._reports: + for skill in report.skills: + sid = skill.skill_id + if sid not in listener_data: + listener_data[sid] = {} + for h in skill.listeners: + existing = listener_data[sid].get(h.msg_type, (h.handler_count, 0)) + listener_data[sid][h.msg_type] = ( + existing[0], + existing[1] + h.invocation_count, + ) + if sid not in observed_data: + observed_data[sid] = {} + for e in skill.emitters: + observed_data[sid][e.msg_type] = ( + observed_data[sid].get(e.msg_type, 0) + e.observed_count + ) + if sid not in asserted_data: + asserted_data[sid] = {} + for e in skill.emitters: + asserted_data[sid][e.msg_type] = ( + asserted_data[sid].get(e.msg_type, 0) + e.asserted_count + ) + + skills = [] + for skill_id in sorted(set(listener_data) | set(observed_data)): + listeners = [ + HandlerEntry( + msg_type=mt, + handler_count=hc, + invocation_count=ic, + covered=ic > 0, + ) + for mt, (hc, ic) in sorted(listener_data.get(skill_id, {}).items()) + ] + all_emitted = set(observed_data.get(skill_id, {}).keys()) | set( + asserted_data.get(skill_id, {}).keys() + ) + emitters = [ + EmitterEntry( + msg_type=mt, + observed_count=observed_data.get(skill_id, {}).get(mt, 0), + asserted_count=asserted_data.get(skill_id, {}).get(mt, 0), + observed=observed_data.get(skill_id, {}).get(mt, 0) > 0, + asserted=asserted_data.get(skill_id, {}).get(mt, 0) > 0, + ) + for mt in sorted(all_emitted) + ] + skills.append(SkillBusCoverage(skill_id=skill_id, listeners=listeners, emitters=emitters)) + + return BusCoverageReport(skills=skills) + + +@pytest.fixture(scope="session", autouse=True) +def bus_coverage_session(request) -> Iterator[BusCoverageCollector]: + """Session-scoped fixture that collects bus coverage reports from all tests. + + Automatically enabled if ``--ovoscope-bus-cov`` is passed to pytest. + When enabled, it monkey-patches ``End2EndTest`` to + automatically add reports to the session collector after ``execute()``. + + Tests can also opt in manually without the CLI flag by requesting this + fixture and calling ``bus_coverage_session.add(test.bus_coverage_report)``. + """ + import ovoscope + global _SESSION_COLLECTOR + enabled = request.config.getoption("--ovoscope-bus-cov") + cov_file = request.config.getoption("--ovoscope-bus-cov-file") + + collector = BusCoverageCollector() + _SESSION_COLLECTOR = collector + + original_execute = End2EndTest.execute + + if enabled: + ovoscope.GLOBAL_BUS_COVERAGE = True + ovoscope.GLOBAL_BUS_COVERAGE_FILE = cov_file + # Initialize the global collector to catch boot-time events + ovoscope.GLOBAL_BUS_COVERAGE_COLLECTOR = ovoscope.GlobalBusCoverageCollector() + + # Auto-collect report after execution + def patched_execute(self, *args, **kwargs): + res = original_execute(self, *args, **kwargs) + if self.bus_coverage_report: + collector.add(self.bus_coverage_report) + return res + + End2EndTest.execute = patched_execute + + try: + yield collector + finally: + _SESSION_COLLECTOR = None + if enabled: + ovoscope.GLOBAL_BUS_COVERAGE = False + ovoscope.GLOBAL_BUS_COVERAGE_COLLECTOR = None + # Restore original behavior + End2EndTest.execute = original_execute + # Note: we don't restore track_bus_coverage default because it's a + # class attribute and we might have stepped on a manual True/False + # in some tests, but in pytest context it usually doesn't matter + # after the session ends. + + # Store the merged report on the config object so the terminal hook can + # retrieve it without touching private pytest internals. + report = collector.merged_report() + if report is not None: + # Apply filters + include = request.config.getoption("--ovoscope-bus-cov-include") + exclude = request.config.getoption("--ovoscope-bus-cov-exclude") + report = report.filter(include=include, exclude=exclude) + + if not hasattr(request.config, "_bus_coverage_reports"): + request.config._bus_coverage_reports = [] + request.config._bus_coverage_reports.append(report) + + # Save to file if requested + cov_file = request.config.getoption("--ovoscope-bus-cov-file") + if cov_file: + import os + try: + os.makedirs(os.path.dirname(os.path.abspath(cov_file)), exist_ok=True) + with open(cov_file, "w", encoding="utf-8") as f: + f.write(report.to_json()) + except Exception as exc: + print(f"\nERROR: Failed to save bus coverage report to {cov_file}: {exc}") + + +def pytest_terminal_summary(terminalreporter, exitstatus, config): # noqa: ARG001 + """Print the merged bus coverage report at the end of the pytest session. + + Only runs if at least one test used the ``bus_coverage_session`` fixture + (or ``--ovoscope-bus-cov`` was used) and reports were collected. + """ + reports = getattr(config, "_bus_coverage_reports", None) + if not reports: + return + verbose = config.getoption("--ovoscope-bus-cov-verbose") + for report in reports: + terminalreporter.write_sep("=", "Bus Coverage Report") + report.print_report(verbose=verbose) + terminalreporter.write_line("") diff --git a/pyproject.toml b/pyproject.toml index 7403b8c..e061f24 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,8 @@ authors = [ ] requires-python = ">=3.10" dependencies = [ + # Alpha pin: 2.0.4 stable not yet released; 2.0.4a2 is the minimum that + # includes the FakeBus-compatible SkillManager changes ovoscope depends on. "ovos-core>=2.0.4a2", ] classifiers = [ @@ -27,7 +29,8 @@ classifiers = [ pydantic = ["ovos-pydantic-models>=0.1.0"] audio = ["ovos-audio>=1.2.0"] dev = [ - "ovoscope[audio,pydantic]", + "ovos-audio>=1.2.0", + "ovos-pydantic-models>=0.1.0", "pytest", "pytest-cov", ] @@ -38,6 +41,8 @@ ovoscope-setup = "ovoscope.setup_skill:main" [project.urls] Homepage = "https://github.com/TigreGotico/ovoscope" +Documentation = "https://github.com/TigreGotico/ovoscope/tree/master/docs" +"Issue Tracker" = "https://github.com/TigreGotico/ovoscope/issues" [project.entry-points."pytest11"] ovoscope = "ovoscope.pytest_plugin" @@ -45,9 +50,12 @@ ovoscope = "ovoscope.pytest_plugin" [tool.setuptools.dynamic] version = {attr = "ovoscope.version.__version__"} +[tool.setuptools.package-data] +ovoscope = ["skill_data/*", "*.md"] [tool.pytest.ini_options] testpaths = ["test"] +timeout = 60 [tool.coverage.run] relative_files = true diff --git a/test/unittests/test_bus_coverage.py b/test/unittests/test_bus_coverage.py new file mode 100644 index 0000000..23b9934 --- /dev/null +++ b/test/unittests/test_bus_coverage.py @@ -0,0 +1,571 @@ +# Copyright 2024 Jarbas AI +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for ovoscope.bus_coverage.""" + +import json +from unittest.mock import MagicMock, patch + +import pytest +from ovos_bus_client.message import Message +from ovos_utils.fakebus import FakeBus + +from ovoscope.bus_coverage import ( + BusCoverageReport, + BusCoverageTracker, + EmitterEntry, + HandlerEntry, + SkillBusCoverage, +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_message(msg_type: str, skill_id: str = None) -> Message: + """Create a minimal Message with optional skill_id in context.""" + context = {} + if skill_id: + context["skill_id"] = skill_id + return Message(msg_type, {}, context) + + +def _make_tracker_with_handlers(): + """Build a BusCoverageTracker whose bus has two fake skill handlers.""" + bus = FakeBus() + + skill_a = MagicMock() + skill_b = MagicMock() + skill_a.__class__.__name__ = "SkillA" + skill_b.__class__.__name__ = "SkillB" + + # Register bound-like handlers using lambdas bound to skill instances + handler_a1 = MagicMock() + handler_a1.__self__ = skill_a + handler_a2 = MagicMock() + handler_a2.__self__ = skill_a + handler_b1 = MagicMock() + handler_b1.__self__ = skill_b + + bus.on("speak", handler_a1) + bus.on("intent.service.skills.activate", handler_a2) + bus.on("speak", handler_b1) + + minicroft = MagicMock() + minicroft.plugin_skills = { + "skill-a.author": skill_a, + "skill-b.author": skill_b, + } + + tracker = BusCoverageTracker(bus, minicroft) + return tracker, bus, skill_a, skill_b + + +# --------------------------------------------------------------------------- +# HandlerEntry +# --------------------------------------------------------------------------- + + +class TestHandlerEntry: + def test_covered_true_when_invoked(self): + h = HandlerEntry(msg_type="speak", handler_count=1, invocation_count=3, covered=True) + assert h.covered is True + + def test_covered_false_when_not_invoked(self): + h = HandlerEntry(msg_type="speak", handler_count=1, invocation_count=0, covered=False) + assert h.covered is False + + def test_to_dict_keys(self): + h = HandlerEntry(msg_type="speak", handler_count=2, invocation_count=5, covered=True) + d = h.to_dict() + assert set(d.keys()) == {"msg_type", "handler_count", "invocation_count", "covered"} + assert d["msg_type"] == "speak" + assert d["handler_count"] == 2 + assert d["invocation_count"] == 5 + assert d["covered"] is True + + +# --------------------------------------------------------------------------- +# EmitterEntry +# --------------------------------------------------------------------------- + + +class TestEmitterEntry: + def test_observed_true_when_count_positive(self): + e = EmitterEntry(msg_type="speak", observed_count=1, asserted_count=0, observed=True, asserted=False) + assert e.observed is True + assert e.asserted is False + + def test_to_dict_keys(self): + e = EmitterEntry(msg_type="speak", observed_count=2, asserted_count=1, observed=True, asserted=True) + d = e.to_dict() + assert set(d.keys()) == {"msg_type", "observed_count", "asserted_count", "observed", "asserted"} + + +# --------------------------------------------------------------------------- +# SkillBusCoverage properties +# --------------------------------------------------------------------------- + + +class TestSkillBusCoverage: + def _make_skill(self) -> SkillBusCoverage: + skill = SkillBusCoverage(skill_id="test.skill") + skill.listeners = [ + HandlerEntry("speak", 1, 3, True), + HandlerEntry("intent.activate", 1, 0, False), + HandlerEntry("intent.deactivate", 1, 1, True), + ] + skill.emitters = [ + EmitterEntry("speak", 3, 2, True, True), + EmitterEntry("gui.page.show", 1, 0, True, False), + EmitterEntry("ovos.utterance.handled", 1, 0, True, False), + ] + return skill + + def test_listener_coverage_pct(self): + skill = self._make_skill() + # 2 out of 3 listeners covered + assert abs(skill.listener_coverage_pct - 66.7) < 0.5 + + def test_observed_emitter_pct(self): + skill = self._make_skill() + # all 3 emitters observed + assert skill.observed_emitter_pct == pytest.approx(100.0) + + def test_asserted_emitter_pct(self): + skill = self._make_skill() + # only 1 of 3 asserted + assert abs(skill.asserted_emitter_pct - 33.3) < 0.5 + + def test_empty_listeners_pct(self): + skill = SkillBusCoverage(skill_id="empty.skill") + assert skill.listener_coverage_pct == 0.0 + assert skill.observed_emitter_pct == 0.0 + assert skill.asserted_emitter_pct == 0.0 + + def test_to_dict_structure(self): + skill = self._make_skill() + d = skill.to_dict() + assert "skill_id" in d + assert "listener_coverage_pct" in d + assert "observed_emitter_pct" in d + assert "asserted_emitter_pct" in d + assert isinstance(d["listeners"], list) + assert isinstance(d["emitters"], list) + + +# --------------------------------------------------------------------------- +# BusCoverageReport +# --------------------------------------------------------------------------- + + +class TestBusCoverageReport: + def _make_report(self) -> BusCoverageReport: + skill = SkillBusCoverage(skill_id="my.skill") + skill.listeners = [ + HandlerEntry("speak", 1, 2, True), + HandlerEntry("intent.activate", 1, 0, False), + ] + skill.emitters = [ + EmitterEntry("speak", 2, 1, True, True), + ] + return BusCoverageReport(skills=[skill]) + + def test_summary_line_format(self): + report = self._make_report() + line = report.summary_line() + assert "my.skill" in line + assert "listeners:" in line + assert "observed:" in line + assert "asserted:" in line + + def test_to_json_is_valid(self): + report = self._make_report() + raw = report.to_json() + data = json.loads(raw) + assert "skills" in data + assert "totals" in data + assert isinstance(data["skills"], list) + + def test_totals_dict_counts(self): + report = self._make_report() + totals = report._totals_dict() + assert totals["listener_total"] == 2 + assert totals["listener_covered"] == 1 + assert totals["emitter_total"] == 1 + assert totals["observed_count"] == 1 + assert totals["asserted_count"] == 1 + + def test_print_report_no_error(self, capsys): + report = self._make_report() + report.print_report() + captured = capsys.readouterr() + assert "Bus Coverage Report" in captured.out + assert "my.skill" in captured.out + + def test_print_report_verbose(self, capsys): + report = self._make_report() + report.print_report(verbose=True) + captured = capsys.readouterr() + assert "LISTENERS" in captured.out + assert "EMITTERS" in captured.out + + +# --------------------------------------------------------------------------- +# BusCoverageTracker +# --------------------------------------------------------------------------- + + +class _FakeSkill: + """Minimal stand-in for a skill instance with a bound handler.""" + + def __init__(self): + pass + + def on_speak(self, message): + pass + + +class _FakeEventContainer: + """Minimal EventContainer stub matching ovos_workshop's EventContainer.""" + + def __init__(self, events): + self.events = events # list of (msg_type, handler) + + +class TestBusCoverageTrackerSnapshotListeners: + def test_snapshot_uses_event_container_when_available(self): + """snapshot_listeners should read skill.events.events (primary path).""" + bus = FakeBus() + skill_a = _FakeSkill() + skill_a.events = _FakeEventContainer([ + ("speak", skill_a.on_speak), + ("intent.activate", skill_a.on_speak), + ]) + + loader = MagicMock() + loader.instance = skill_a + + minicroft = MagicMock() + minicroft.plugin_skills = {"skill-a.author": loader} + + tracker = BusCoverageTracker(bus, minicroft) + tracker.snapshot_listeners() + + assert "skill-a.author" in tracker._registered + assert "speak" in tracker._registered["skill-a.author"] + assert "intent.activate" in tracker._registered["skill-a.author"] + assert len(tracker._registered["skill-a.author"]) == 2 + + def test_snapshot_fallback_bus_introspection(self): + """snapshot_listeners falls back to bus introspection when no EventContainer.""" + bus = FakeBus() + skill_a = _FakeSkill() + # No .events attribute on skill_a + + loader = MagicMock() + loader.instance = skill_a + + # Register a bound method directly on the bus + bus.on("speak", skill_a.on_speak) + + minicroft = MagicMock() + minicroft.plugin_skills = {"skill-a.author": loader} + + tracker = BusCoverageTracker(bus, minicroft) + tracker.snapshot_listeners() + + # Should have found the handler via fallback bus introspection + assert "skill-a.author" in tracker._registered + assert "speak" in tracker._registered["skill-a.author"] + + def test_snapshot_ignores_unattributed_handlers(self): + """Skills with no handlers produce no entries.""" + bus = FakeBus() + skill_a = _FakeSkill() + skill_a.events = _FakeEventContainer([]) # no handlers + + loader = MagicMock() + loader.instance = skill_a + + minicroft = MagicMock() + minicroft.plugin_skills = {"skill-a.author": loader} + + tracker = BusCoverageTracker(bus, minicroft) + tracker.snapshot_listeners() + + # skill registered but no msg_types + assert tracker._registered.get("skill-a.author", {}) == {} + + +class TestBusCoverageTrackerEmitPatch: + def test_start_tracking_counts_emits(self): + """start_tracking should increment _invocations per emit call.""" + bus = FakeBus() + minicroft = MagicMock() + minicroft.plugin_skills = {} + + tracker = BusCoverageTracker(bus, minicroft) + tracker.start_tracking() + + bus.emit(Message("speak", {}, {})) + bus.emit(Message("speak", {}, {})) + bus.emit(Message("intent.activate", {}, {})) + + assert tracker._invocations.get("speak") == 2 + assert tracker._invocations.get("intent.activate") == 1 + + def test_stop_tracking_restores_emit(self): + """stop_tracking should restore the original emit callable.""" + bus = FakeBus() + minicroft = MagicMock() + minicroft.plugin_skills = {} + + tracker = BusCoverageTracker(bus, minicroft) + tracker.start_tracking() + patched_emit = bus.emit + + tracker.stop_tracking() + # After stop_tracking the tracker should no longer be active + assert not tracker._tracking + assert tracker._original_emit is None + # The patched emit should be gone + assert bus.emit is not patched_emit + + def test_double_start_is_idempotent(self): + """Calling start_tracking twice should not double-wrap.""" + bus = FakeBus() + minicroft = MagicMock() + minicroft.plugin_skills = {} + + tracker = BusCoverageTracker(bus, minicroft) + tracker.start_tracking() + patched = bus.emit + tracker.start_tracking() + assert bus.emit is patched # still the same patched function + + +class TestBusCoverageTrackerRecordSession: + def test_record_session_accumulates_observed(self): + """Responses with skill_id in context should be recorded as observed.""" + bus = FakeBus() + minicroft = MagicMock() + minicroft.plugin_skills = {} + tracker = BusCoverageTracker(bus, minicroft) + + responses = [ + _make_message("speak", "my.skill"), + _make_message("speak", "my.skill"), + _make_message("gui.page.show", "my.skill"), + ] + tracker.record_session(responses, []) + + assert tracker._observed["my.skill"]["speak"] == 2 + assert tracker._observed["my.skill"]["gui.page.show"] == 1 + + def test_record_session_accumulates_asserted(self): + """expected_messages with skill_id should be recorded as asserted.""" + bus = FakeBus() + minicroft = MagicMock() + minicroft.plugin_skills = {} + tracker = BusCoverageTracker(bus, minicroft) + + expected = [_make_message("speak", "my.skill")] + tracker.record_session([], expected) + + assert tracker._asserted["my.skill"]["speak"] == 1 + + def test_record_session_fallback_attribution(self): + """expected_messages without skill_id should fall back to observed skill.""" + bus = FakeBus() + minicroft = MagicMock() + minicroft.plugin_skills = {} + tracker = BusCoverageTracker(bus, minicroft) + + # observed has the type under skill-a + responses = [_make_message("speak", "skill-a.author")] + # expected has no skill_id + expected = [Message("speak", {}, {})] + + tracker.record_session(responses, expected) + + assert "skill-a.author" in tracker._asserted + assert tracker._asserted["skill-a.author"]["speak"] == 1 + + +class TestBusCoverageTrackerBuildReport: + def test_build_report_populates_skills(self): + """build_report should return SkillBusCoverage entries for each skill.""" + bus = FakeBus() + minicroft = MagicMock() + minicroft.plugin_skills = {} + + tracker = BusCoverageTracker(bus, minicroft) + tracker._registered = {"my.skill": {"speak": 1}} + tracker._invocations = {"speak": 3} + tracker._observed = {"my.skill": {"speak": 3}} + tracker._asserted = {"my.skill": {"speak": 2}} + + report = tracker.build_report() + + assert len(report.skills) == 1 + skill = report.skills[0] + assert skill.skill_id == "my.skill" + assert len(skill.listeners) == 1 + assert skill.listeners[0].covered is True + assert skill.listeners[0].invocation_count == 3 + assert len(skill.emitters) == 1 + assert skill.emitters[0].observed is True + assert skill.emitters[0].asserted is True + + def test_build_report_uncovered_listener(self): + """Listeners whose msg_type was never emitted should have covered=False.""" + bus = FakeBus() + minicroft = MagicMock() + minicroft.plugin_skills = {} + + tracker = BusCoverageTracker(bus, minicroft) + tracker._registered = {"my.skill": {"never.emitted": 1}} + tracker._invocations = {} + + report = tracker.build_report() + assert report.skills[0].listeners[0].covered is False + + def test_build_report_empty(self): + """build_report on a fresh tracker should return an empty report.""" + bus = FakeBus() + minicroft = MagicMock() + minicroft.plugin_skills = {} + tracker = BusCoverageTracker(bus, minicroft) + report = tracker.build_report() + assert report.skills == [] + + +class TestBusCoverageTrackerGetBusEvents: + def test_returns_ee_events(self): + """_get_bus_events should read bus.ee._events for pyee EventEmitter.""" + bus = FakeBus() + bus.on("test.event", lambda m: None) + minicroft = MagicMock() + minicroft.plugin_skills = {} + tracker = BusCoverageTracker(bus, minicroft) + events = tracker._get_bus_events() + assert "test.event" in events + + def test_returns_empty_for_unknown_bus(self): + """_get_bus_events should return {} if no known event attributes exist.""" + bus = MagicMock() + del bus.ee + del bus._events + minicroft = MagicMock() + minicroft.plugin_skills = {} + tracker = BusCoverageTracker(bus, minicroft) + # Should not raise + events = tracker._get_bus_events() + assert isinstance(events, dict) + + +class TestBusCoverageTrackerIterHandlers: + def test_iter_handlers_ordered_dict(self): + """_iter_handlers should yield keys from an OrderedDict (pyee v9 format).""" + import collections + + def my_handler(): + pass + + od = collections.OrderedDict({my_handler: my_handler}) + result = list(BusCoverageTracker._iter_handlers(od)) + assert result == [my_handler] + + def test_iter_handlers_list(self): + """_iter_handlers should yield each item from a list (no .fn attr).""" + def my_handler(): + pass + + result = list(BusCoverageTracker._iter_handlers([my_handler])) + assert result == [my_handler] + + def test_iter_handlers_single_callable(self): + """_iter_handlers should yield a single callable directly.""" + def my_handler(): + pass + + result = list(BusCoverageTracker._iter_handlers(my_handler)) + assert result == [my_handler] + + def test_iter_handlers_pyee_wrapper_in_list(self): + """_iter_handlers should unwrap .fn from pyee listener wrappers in a list.""" + def fn(): + pass + + class _Wrapper: + def __init__(self, fn): + self.fn = fn + + wrapper = _Wrapper(fn) + result = list(BusCoverageTracker._iter_handlers([wrapper])) + assert result == [fn] + + +class TestCoreAttributionBucket: + """Tests for __core__ bucket fallback (C1/C4 fixes).""" + + def test_message_without_skill_id_goes_to_core_bucket(self): + """Observed messages with no skill_id in context must land in __core__.""" + bus = FakeBus() + minicroft = MagicMock() + minicroft.plugin_skills = {} + tracker = BusCoverageTracker(bus, minicroft) + + responses = [Message("intent.service.result", {}, {})] + tracker.record_session(responses, []) + + assert "__core__" in tracker._observed + assert tracker._observed["__core__"]["intent.service.result"] == 1 + + def test_expected_message_without_skill_id_goes_to_core_when_unobserved(self): + """expected_messages with no skill_id and no prior observation go to __core__.""" + bus = FakeBus() + minicroft = MagicMock() + minicroft.plugin_skills = {} + tracker = BusCoverageTracker(bus, minicroft) + + expected = [Message("ovos.utterance.handled", {}, {})] + tracker.record_session([], expected) + + assert "__core__" in tracker._asserted + assert tracker._asserted["__core__"]["ovos.utterance.handled"] == 1 + + def test_core_bucket_appears_in_report(self): + """build_report must include __core__ as a SkillBusCoverage row.""" + bus = FakeBus() + minicroft = MagicMock() + minicroft.plugin_skills = {} + tracker = BusCoverageTracker(bus, minicroft) + + responses = [Message("speak", {}, {})] + tracker.record_session(responses, []) + + report = tracker.build_report() + skill_ids = [s.skill_id for s in report.skills] + assert "__core__" in skill_ids + + +class TestJsonSchemaVersion: + def test_to_json_includes_schema_version(self): + """to_json() output must contain 'schema_version': '1'.""" + report = BusCoverageReport(skills=[]) + data = json.loads(report.to_json()) + assert data.get("schema_version") == "1" diff --git a/test/unittests/test_global_bus_coverage.py b/test/unittests/test_global_bus_coverage.py new file mode 100644 index 0000000..f486f13 --- /dev/null +++ b/test/unittests/test_global_bus_coverage.py @@ -0,0 +1,118 @@ +# Copyright 2024 Jarbas AI +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for global bus coverage tracking.""" + +from unittest.mock import MagicMock +import pytest +from ovos_bus_client.message import Message +from ovos_utils.fakebus import FakeBus + +import ovoscope +from ovoscope.bus_coverage import BusCoverageTracker + + +class TestGlobalBusCoverage: + @pytest.fixture(autouse=True) + def setup_globals(self): + """Reset global state before and after each test.""" + orig_enabled = ovoscope.GLOBAL_BUS_COVERAGE + orig_collector = ovoscope.GLOBAL_BUS_COVERAGE_COLLECTOR + + ovoscope.GLOBAL_BUS_COVERAGE = False + ovoscope.GLOBAL_BUS_COVERAGE_COLLECTOR = None + + yield + + ovoscope.GLOBAL_BUS_COVERAGE = orig_enabled + ovoscope.GLOBAL_BUS_COVERAGE_COLLECTOR = orig_collector + + def test_collector_records_activity(self): + """GlobalBusCoverageCollector should store registration and invocation counts.""" + collector = ovoscope.GlobalBusCoverageCollector() + collector.record_registration("test.on") + collector.record_registration("test.on") + collector.record_invocation("test.emit") + + assert collector.registrations["test.on"] == 2 + assert collector.invocations["test.emit"] == 1 + + def test_fakebus_patches_work(self): + """FakeBus.on and .emit should update the global collector when enabled.""" + ovoscope.GLOBAL_BUS_COVERAGE = True + ovoscope.GLOBAL_BUS_COVERAGE_COLLECTOR = ovoscope.GlobalBusCoverageCollector() + + bus = FakeBus() + bus.on("global.on", lambda m: None) + bus.emit(Message("global.emit")) + + assert ovoscope.GLOBAL_BUS_COVERAGE_COLLECTOR.registrations["global.on"] == 1 + assert ovoscope.GLOBAL_BUS_COVERAGE_COLLECTOR.invocations["global.emit"] == 1 + + def test_tracker_snapshots_global_state(self): + """BusCoverageTracker should snapshot global state at __init__.""" + ovoscope.GLOBAL_BUS_COVERAGE = True + ovoscope.GLOBAL_BUS_COVERAGE_COLLECTOR = ovoscope.GlobalBusCoverageCollector() + ovoscope.GLOBAL_BUS_COVERAGE_COLLECTOR.record_invocation("boot.event") + ovoscope.GLOBAL_BUS_COVERAGE_COLLECTOR.record_registration("boot.handler") + + bus = FakeBus() + minicroft = MagicMock() + minicroft.plugin_skills = {} + + tracker = BusCoverageTracker(bus, minicroft) + + assert tracker._global_invocations["boot.event"] == 1 + assert tracker._global_registrations["boot.handler"] == 1 + + def test_tracker_merges_global_registrations(self): + """snapshot_listeners should merge global registrations into __core__ if unclaimed.""" + ovoscope.GLOBAL_BUS_COVERAGE = True + ovoscope.GLOBAL_BUS_COVERAGE_COLLECTOR = ovoscope.GlobalBusCoverageCollector() + ovoscope.GLOBAL_BUS_COVERAGE_COLLECTOR.record_registration("boot.unclaimed") + + bus = FakeBus() + minicroft = MagicMock() + minicroft.plugin_skills = {} + + tracker = BusCoverageTracker(bus, minicroft) + tracker.snapshot_listeners() + + assert "__core__" in tracker._registered + assert "boot.unclaimed" in tracker._registered["__core__"] + + def test_tracker_merges_global_invocations(self): + """build_report should sum global and local invocations.""" + ovoscope.GLOBAL_BUS_COVERAGE = True + ovoscope.GLOBAL_BUS_COVERAGE_COLLECTOR = ovoscope.GlobalBusCoverageCollector() + ovoscope.GLOBAL_BUS_COVERAGE_COLLECTOR.record_invocation("shared.event") # 1x during boot + + bus = FakeBus() + minicroft = MagicMock() + minicroft.plugin_skills = {} + + tracker = BusCoverageTracker(bus, minicroft) + tracker.start_tracking() + bus.emit(Message("shared.event")) # 1x during test + tracker.stop_tracking() + + # Manually register the listener so it shows up in report + tracker._registered = {"__core__": {"shared.event": 1}} + + report = tracker.build_report() + skill = next(s for s in report.skills if s.skill_id == "__core__") + handler = next(h for h in skill.listeners if h.msg_type == "shared.event") + + # 1 (boot) + 1 (test) = 2 + assert handler.invocation_count == 2 + assert handler.covered is True diff --git a/test/unittests/test_gui_capture.py b/test/unittests/test_gui_capture.py new file mode 100644 index 0000000..11cc11d --- /dev/null +++ b/test/unittests/test_gui_capture.py @@ -0,0 +1,126 @@ +"""Unit tests for GUICaptureSession assertion methods.""" +import unittest +from unittest.mock import MagicMock + +from ovos_bus_client.message import Message +from ovoscope import GUICaptureSession + + +class TestGUICaptureSessionAssertions(unittest.TestCase): + """Test GUICaptureSession assertion methods with synthetic messages.""" + + def _make_session(self) -> GUICaptureSession: + """Create a GUICaptureSession with a mock bus (not started).""" + session = GUICaptureSession(bus=MagicMock()) + return session + + def _value_set_msg(self, namespace: str, data: dict) -> Message: + """Build a gui.value.set message.""" + return Message( + "gui.value.set", + {"namespace": namespace, "data": data}, + ) + + def _page_show_msg(self, namespace: str, page: str) -> Message: + """Build a gui.page.show message.""" + return Message( + "gui.page.show", + {"namespace": namespace, "pages": [page]}, + ) + + # -- assert_namespace_has_key -- + + def test_assert_namespace_has_key_found(self) -> None: + """Key present in namespace data should pass.""" + session = self._make_session() + session.messages = [self._value_set_msg("weatherskill", {"current_temp": 22})] + session.assert_namespace_has_key("weatherskill", "current_temp") + + def test_assert_namespace_has_key_missing(self) -> None: + """Missing key should raise AssertionError.""" + session = self._make_session() + session.messages = [self._value_set_msg("weatherskill", {"current_temp": 22})] + with self.assertRaises(AssertionError): + session.assert_namespace_has_key("weatherskill", "location") + + def test_assert_namespace_has_key_wrong_namespace(self) -> None: + """Key in different namespace should not match.""" + session = self._make_session() + session.messages = [self._value_set_msg("otherskill", {"current_temp": 22})] + with self.assertRaises(AssertionError): + session.assert_namespace_has_key("weatherskill", "current_temp") + + def test_assert_namespace_has_key_none_value(self) -> None: + """Key with None value should still pass (key exists).""" + session = self._make_session() + session.messages = [self._value_set_msg("skill", {"key": None})] + session.assert_namespace_has_key("skill", "key") + + # -- assert_namespace_value -- + + def test_assert_namespace_value_match(self) -> None: + """Exact value match should pass.""" + session = self._make_session() + session.messages = [self._value_set_msg("skill", {"greeting": "Hello!"})] + session.assert_namespace_value("skill", "greeting", "Hello!") + + def test_assert_namespace_value_mismatch(self) -> None: + """Wrong value should raise AssertionError.""" + session = self._make_session() + session.messages = [self._value_set_msg("skill", {"greeting": "Hello!"})] + with self.assertRaises(AssertionError): + session.assert_namespace_value("skill", "greeting", "Goodbye!") + + # -- assert_page_shown -- + + def test_assert_page_shown_match(self) -> None: + """Page in namespace should pass.""" + session = self._make_session() + session.messages = [self._page_show_msg("helloworldskill", "hello.qml")] + session.assert_page_shown("helloworldskill", "hello.qml") + + def test_assert_page_shown_missing(self) -> None: + """Missing page should raise AssertionError.""" + session = self._make_session() + session.messages = [self._page_show_msg("helloworldskill", "hello.qml")] + with self.assertRaises(AssertionError): + session.assert_page_shown("helloworldskill", "goodbye.qml") + + # -- __from and page_names wire format -- + + def test_assert_page_shown_from_field(self) -> None: + """Page show using __from and page_names (real wire format).""" + session = self._make_session() + session.messages = [Message( + "gui.page.show", + {"page_names": ["SYSTEM_clock"], "__from": "ovos-skill-date-time.openvoiceos"}, + )] + session.assert_page_shown("date-time", "SYSTEM_clock") + + def test_assert_namespace_has_key_from_field(self) -> None: + """Value set using __from (real wire format).""" + session = self._make_session() + session.messages = [Message( + "gui.value.set", + {"__from": "ovos-skill-weather.openvoiceos", "current_temp": 22}, + )] + session.assert_namespace_has_key("weather", "current_temp") + + # -- assert_namespace_cleared -- + + def test_assert_namespace_cleared_match(self) -> None: + """Namespace clear message should pass.""" + session = self._make_session() + session.messages = [Message("gui.namespace.remove", {"namespace": "skill"})] + session.assert_namespace_cleared("skill") + + def test_assert_namespace_cleared_missing(self) -> None: + """No clear message should raise AssertionError.""" + session = self._make_session() + session.messages = [] + with self.assertRaises(AssertionError): + session.assert_namespace_cleared("skill") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/unittests/test_media.py b/test/unittests/test_media.py new file mode 100644 index 0000000..5c0e405 --- /dev/null +++ b/test/unittests/test_media.py @@ -0,0 +1,204 @@ +# Copyright 2024 Jarbas AI +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for ovoscope.media (MockOCPBackend, OCPCaptureSession, OCPPlayerHarness).""" + +import pytest +from unittest.mock import MagicMock + +from ovos_bus_client.message import Message +from ovos_utils.fakebus import FakeBus +from ovos_utils.ocp import MediaState + +from ovoscope.media import MockOCPBackend, OCPCaptureSession + + +# --------------------------------------------------------------------------- +# MockOCPBackend tests +# --------------------------------------------------------------------------- + +class TestMockOCPBackendInit: + """Constructor and initial state.""" + + def test_initial_state(self) -> None: + """Backend starts with clean state.""" + bus = FakeBus() + backend = MockOCPBackend(config={}, bus=bus) + assert backend.is_playing is False + assert backend.is_paused is False + assert backend.current_uri is None + assert backend.played_uris == [] + + def test_namespace_default(self) -> None: + """Default namespace is 'audio'.""" + bus = FakeBus() + backend = MockOCPBackend(config={}, bus=bus) + assert backend.namespace == "audio" + + def test_namespace_custom(self) -> None: + """Custom namespace is stored.""" + bus = FakeBus() + backend = MockOCPBackend(config={}, bus=bus, namespace="video") + assert backend.namespace == "video" + + +class TestMockOCPBackendStateTransitions: + """State mutation methods.""" + + def setup_method(self) -> None: + self.bus = FakeBus() + self.backend = MockOCPBackend(config={}, bus=self.bus) + + def test_play_sets_playing(self) -> None: + self.backend.play() + assert self.backend.is_playing is True + assert self.backend.is_paused is False + + def test_pause_sets_paused(self) -> None: + self.backend.play() + self.backend.pause() + assert self.backend.is_paused is True + + def test_resume_clears_paused(self) -> None: + self.backend.play() + self.backend.pause() + self.backend.resume() + assert self.backend.is_paused is False + + def test_stop_clears_state(self) -> None: + self.backend.play() + result = self.backend.stop() + assert self.backend.is_playing is False + assert self.backend.is_paused is False + assert result is True + + def test_load_track_sets_uri(self) -> None: + self.backend.load_track("http://example.com/song.mp3") + assert self.backend.current_uri == "http://example.com/song.mp3" + assert "http://example.com/song.mp3" in self.backend.played_uris + + def test_load_track_emits_state_event(self) -> None: + received: list = [] + self.bus.on(f"ovos.audio.service.media.state", lambda m: received.append(m)) + self.backend.load_track("http://example.com/song.mp3") + assert len(received) == 1 + + def test_add_list_records_uris(self) -> None: + self.backend.add_list(["track1.mp3", "track2.mp3"]) + assert "track1.mp3" in self.backend.played_uris + assert self.backend.current_uri == "track1.mp3" + + def test_clear_list(self) -> None: + self.backend.add_list(["track1.mp3"]) + self.backend.clear_list() + assert self.backend.played_uris == [] + assert self.backend.current_uri is None + + def test_reset(self) -> None: + self.backend.play() + self.backend.add_list(["track1.mp3"]) + self.backend.reset() + assert self.backend.is_playing is False + assert self.backend.is_paused is False + assert self.backend.current_uri is None + assert self.backend.played_uris == [] + + def test_supported_uris(self) -> None: + uris = self.backend.supported_uris() + assert "file" in uris + assert "http" in uris + assert "https" in uris + + def test_track_info(self) -> None: + self.backend.current_uri = "http://example.com/song.mp3" + info = self.backend.track_info() + assert info["track"] == "http://example.com/song.mp3" + + def test_get_track_length_returns_zero(self) -> None: + assert self.backend.get_track_length() == 0 + + def test_get_track_position_returns_zero(self) -> None: + assert self.backend.get_track_position() == 0 + + def test_simulate_end_emits_event(self) -> None: + received: list = [] + self.bus.on("ovos.common_play.media.state", lambda m: received.append(m)) + self.backend.simulate_end() + assert len(received) == 1 + assert self.backend.is_playing is False + + def test_simulate_invalid_stream(self) -> None: + received: list = [] + self.bus.on("ovos.common_play.media.state", lambda m: received.append(m)) + self.backend.simulate_invalid_stream() + assert len(received) == 1 + assert self.backend.is_playing is False + + +# --------------------------------------------------------------------------- +# OCPCaptureSession tests +# --------------------------------------------------------------------------- + +class TestOCPCaptureSessionMessageAccumulation: + """Message capture and filtering.""" + + def setup_method(self) -> None: + self.bus = FakeBus() + + def test_captures_matching_prefix(self) -> None: + session = OCPCaptureSession(bus=self.bus) + session.start() + self.bus.emit(Message("ovos.common_play.play")) + session.stop() + assert "ovos.common_play.play" in session.message_types + + def test_does_not_capture_non_matching(self) -> None: + session = OCPCaptureSession(bus=self.bus) + session.start() + self.bus.emit(Message("some.other.message")) + session.stop() + assert "some.other.message" not in session.message_types + + def test_start_clears_previous(self) -> None: + session = OCPCaptureSession(bus=self.bus) + session.start() + self.bus.emit(Message("ovos.common_play.play")) + session.stop() + session.start() + session.stop() + assert session.messages == [] + + def test_context_manager(self) -> None: + with OCPCaptureSession(bus=self.bus) as session: + self.bus.emit(Message("ovos.common_play.pause")) + assert "ovos.common_play.pause" in session.message_types + + def test_assert_sequence_passes(self) -> None: + with OCPCaptureSession(bus=self.bus) as session: + self.bus.emit(Message("ovos.common_play.play")) + self.bus.emit(Message("ovos.common_play.pause")) + session.assert_sequence("ovos.common_play.play", "ovos.common_play.pause") + + def test_assert_sequence_fails_on_missing(self) -> None: + with OCPCaptureSession(bus=self.bus) as session: + self.bus.emit(Message("ovos.common_play.play")) + with pytest.raises(AssertionError): + session.assert_sequence("ovos.common_play.stop") + + def test_custom_prefixes(self) -> None: + session = OCPCaptureSession(bus=self.bus, track_prefixes=["custom.prefix."]) + session.start() + self.bus.emit(Message("custom.prefix.event")) + self.bus.emit(Message("ovos.common_play.play")) # should not be captured + session.stop() + assert session.message_types == ["custom.prefix.event"] diff --git a/test/unittests/test_phal.py b/test/unittests/test_phal.py index 639ff8f..bccc368 100644 --- a/test/unittests/test_phal.py +++ b/test/unittests/test_phal.py @@ -20,7 +20,7 @@ import pytest from ovos_utils.fakebus import FakeBus -from ovos_utils.messagebus import Message +from ovos_bus_client.message import Message from ovoscope.phal import MiniPHAL, PHALTest diff --git a/test/unittests/test_remote_recorder.py b/test/unittests/test_remote_recorder.py new file mode 100644 index 0000000..2c450d7 --- /dev/null +++ b/test/unittests/test_remote_recorder.py @@ -0,0 +1,249 @@ +# Copyright 2024 Jarbas AI +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for ovoscope.remote_recorder.RemoteRecorder.""" + +import threading +from typing import Any, List +from unittest.mock import MagicMock, patch + +import pytest + +from ovos_bus_client.message import Message + +from ovoscope.remote_recorder import RemoteRecorder + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_mock_client(messages_to_emit: List[Message]) -> MagicMock: + """Build a mock MessageBusClient that emits *messages_to_emit* when subscribed.""" + client = MagicMock() + client.connected_event = threading.Event() + client.connected_event.set() + + handlers: dict = {} + + def on_side_effect(event_type: str, handler: Any) -> None: + handlers.setdefault(event_type, []).append(handler) + + def remove_side_effect(event_type: str, handler: Any) -> None: + if event_type in handlers: + try: + handlers[event_type].remove(handler) + except ValueError: + pass + + def emit_side_effect(msg: Message) -> None: + # Deliver mocked response messages to subscribed handlers + for m in messages_to_emit: + for h in list(handlers.get("message", [])): + h(m.serialize()) + + client.on.side_effect = on_side_effect + client.remove.side_effect = remove_side_effect + client.emit.side_effect = emit_side_effect + + return client + + +# --------------------------------------------------------------------------- +# Constructor / config +# --------------------------------------------------------------------------- + + +class TestRemoteRecorderConstructor: + """Constructor and default field values.""" + + def test_default_url(self) -> None: + r = RemoteRecorder() + assert r.bus_url == "ws://localhost:8181/core" + + def test_custom_url(self) -> None: + r = RemoteRecorder(bus_url="ws://192.168.1.5:8181/core") + assert r.bus_url == "ws://192.168.1.5:8181/core" + + def test_initial_state(self) -> None: + r = RemoteRecorder() + assert r._client is None + assert r._captured == [] + + +# --------------------------------------------------------------------------- +# _parse_url +# --------------------------------------------------------------------------- + + +class TestParseUrl: + """URL parsing helper.""" + + def test_full_ws_url(self) -> None: + host, port, path = RemoteRecorder._parse_url("ws://localhost:8181/core") + assert host == "localhost" + assert port == 8181 + assert path == "/core" + + def test_no_port(self) -> None: + host, port, path = RemoteRecorder._parse_url("ws://myhost/core") + assert host == "myhost" + assert port == 8181 + assert path == "/core" + + def test_no_path(self) -> None: + host, port, path = RemoteRecorder._parse_url("ws://localhost:8181") + assert host == "localhost" + assert port == 8181 + assert path == "/core" + + def test_wss_scheme(self) -> None: + host, port, path = RemoteRecorder._parse_url("wss://secure.host:443/bus") + assert host == "secure.host" + assert port == 443 + assert path == "/bus" + + +# --------------------------------------------------------------------------- +# connect / disconnect +# --------------------------------------------------------------------------- + + +class TestConnectDisconnect: + """Connection lifecycle.""" + + def test_connect_sets_client(self) -> None: + r = RemoteRecorder() + mock_client = MagicMock() + mock_client.connected_event = threading.Event() + mock_client.connected_event.set() + with patch("ovoscope.remote_recorder.RemoteRecorder._parse_url", return_value=("localhost", 8181, "/core")): + with patch("ovos_bus_client.client.MessageBusClient", return_value=mock_client): + r.connect() + assert r._client is not None + + def test_disconnect_clears_client(self) -> None: + r = RemoteRecorder() + mock_client = MagicMock() + mock_client.connected_event = threading.Event() + mock_client.connected_event.set() + with patch("ovoscope.remote_recorder.RemoteRecorder._parse_url", return_value=("localhost", 8181, "/core")): + with patch("ovos_bus_client.client.MessageBusClient", return_value=mock_client): + r.connect() + r.disconnect() + assert r._client is None + + def test_disconnect_without_connect_is_safe(self) -> None: + r = RemoteRecorder() + r.disconnect() # should not raise + + +# --------------------------------------------------------------------------- +# record +# --------------------------------------------------------------------------- + + +class TestRecord: + """record() method with mocked bus client.""" + + def test_record_requires_connect(self) -> None: + r = RemoteRecorder() + with pytest.raises(RuntimeError, match="connect"): + r.record("hello") + + def test_record_returns_end2endtest(self) -> None: + """record() returns an End2EndTest when an EOF message is emitted.""" + eof_msg = Message("ovos.utterance.handled") + speak_msg = Message("speak", {"utterance": "hello world"}) + mock_client = _make_mock_client([speak_msg, eof_msg]) + + r = RemoteRecorder() + r._client = mock_client + + result = r.record("hello", timeout=5.0) + + from ovoscope import End2EndTest + assert isinstance(result, End2EndTest) + + def test_record_captures_messages(self) -> None: + """record() captures all messages before the EOF signal.""" + eof_msg = Message("ovos.utterance.handled") + speak_msg = Message("speak", {"utterance": "greetings"}) + mock_client = _make_mock_client([speak_msg, eof_msg]) + + r = RemoteRecorder() + r._client = mock_client + + result = r.record("greetings", timeout=5.0) + types = [m.msg_type for m in result.expected_messages] + assert "speak" in types + + def test_record_timeout_raises(self) -> None: + """record() raises TimeoutError when no EOF arrives.""" + mock_client = _make_mock_client([]) # no messages → never completes + + r = RemoteRecorder() + r._client = mock_client + + with pytest.raises(TimeoutError): + r.record("silent utterance", timeout=0.1) + + def test_record_passes_skill_id_in_context(self) -> None: + """record() attaches skill_id to the emitted source message context.""" + eof_msg = Message("ovos.utterance.handled") + mock_client = _make_mock_client([eof_msg]) + + r = RemoteRecorder() + r._client = mock_client + result = r.record("hello", skill_id="ovos-skill-hello-world.openvoiceos", timeout=5.0) + assert result.skill_ids == ["ovos-skill-hello-world.openvoiceos"] + + +# --------------------------------------------------------------------------- +# Fixture serialization +# --------------------------------------------------------------------------- + + +class TestFixtureSerialization: + """Output format of record().""" + + def test_source_message_is_utterance(self) -> None: + eof_msg = Message("ovos.utterance.handled") + mock_client = _make_mock_client([eof_msg]) + + r = RemoteRecorder() + r._client = mock_client + result = r.record("what time is it", timeout=5.0) + + # source_message may be stored as a list or a single Message + src = result.source_message + if isinstance(src, list): + src = src[0] + assert src.msg_type == "recognizer_loop:utterance" + assert "what time is it" in src.data["utterances"] + + def test_save_produces_json(self, tmp_path) -> None: + """Fixture can be saved to JSON without errors.""" + import json + eof_msg = Message("ovos.utterance.handled") + mock_client = _make_mock_client([eof_msg]) + + r = RemoteRecorder() + r._client = mock_client + result = r.record("hello", timeout=5.0) + + out = tmp_path / "fixture.json" + result.save(str(out)) + payload = json.loads(out.read_text()) + assert "expected_messages" in payload