diff --git a/.github/workflows/build_tests.yml b/.github/workflows/build_tests.yml new file mode 100644 index 0000000..82711e1 --- /dev/null +++ b/.github/workflows/build_tests.yml @@ -0,0 +1,18 @@ +name: Run Build Tests +on: + push: + branches: + - master + pull_request: + branches: + - dev + workflow_dispatch: + +jobs: + build_tests: + uses: OpenVoiceOS/gh-automations/.github/workflows/build-tests.yml@dev + secrets: inherit + with: + python_versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]' + install_extras: "pydantic" + test_path: "test/unittests/" diff --git a/.github/workflows/downstream_check.yml b/.github/workflows/downstream_check.yml new file mode 100644 index 0000000..c4a04bc --- /dev/null +++ b/.github/workflows/downstream_check.yml @@ -0,0 +1,12 @@ +name: Downstream Check +on: + push: + branches: [dev, master] + workflow_dispatch: + +jobs: + downstream_check: + uses: OpenVoiceOS/gh-automations/.github/workflows/downstream-check.yml@dev + secrets: inherit + with: + package_name: "ovoscope" diff --git a/.github/workflows/license_check.yml b/.github/workflows/license_check.yml new file mode 100644 index 0000000..7bc7756 --- /dev/null +++ b/.github/workflows/license_check.yml @@ -0,0 +1,16 @@ +name: Run License Tests +on: + push: + branches: + - master + pull_request: + branches: + - dev + workflow_dispatch: + +jobs: + license_tests: + uses: OpenVoiceOS/gh-automations/.github/workflows/license-check.yml@dev + secrets: inherit + with: + install_extras: '[pydantic]' diff --git a/.github/workflows/pip_audit.yml b/.github/workflows/pip_audit.yml new file mode 100644 index 0000000..4c962ab --- /dev/null +++ b/.github/workflows/pip_audit.yml @@ -0,0 +1,15 @@ +name: Run PipAudit +on: + push: + branches: + - master + - dev + pull_request: + branches: + - dev + workflow_dispatch: + +jobs: + pip_audit: + uses: OpenVoiceOS/gh-automations/.github/workflows/pip-audit.yml@dev + secrets: inherit diff --git a/.github/workflows/publish_stable.yml b/.github/workflows/publish_stable.yml index 7bf86e5..d3b6130 100644 --- a/.github/workflows/publish_stable.yml +++ b/.github/workflows/publish_stable.yml @@ -6,53 +6,12 @@ on: jobs: publish_stable: - uses: TigreGotico/gh-automations/.github/workflows/publish-stable.yml@master + if: github.actor != 'github-actions[bot]' + uses: OpenVoiceOS/gh-automations/.github/workflows/publish-stable.yml@dev secrets: inherit with: branch: 'master' version_file: 'ovoscope/version.py' - setup_py: 'setup.py' + publish_pypi: true + sync_dev: true publish_release: true - - publish_pypi: - needs: publish_stable - if: success() # Ensure this job only runs if the previous job succeeds - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: master - fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository. - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - name: Install Build Tools - run: | - python -m pip install build wheel - - name: version - run: echo "::set-output name=version::$(python setup.py --version)" - id: version - - name: Build Distribution Packages - run: | - python setup.py sdist bdist_wheel - - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@master - with: - password: ${{secrets.PYPI_TOKEN}} - - - sync_dev: - needs: publish_stable - if: success() # Ensure this job only runs if the previous job succeeds - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository. - ref: master - - name: Push master -> dev - uses: ad-m/github-push-action@master - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - branch: dev \ No newline at end of file diff --git a/.github/workflows/release_preview.yml b/.github/workflows/release_preview.yml new file mode 100644 index 0000000..ea5542b --- /dev/null +++ b/.github/workflows/release_preview.yml @@ -0,0 +1,13 @@ +name: Release Preview +on: + pull_request: + branches: [dev] + workflow_dispatch: + +jobs: + release_preview: + uses: OpenVoiceOS/gh-automations/.github/workflows/release-preview.yml@dev + secrets: inherit + with: + package_name: "ovoscope" + version_file: "ovoscope/version.py" diff --git a/.github/workflows/release_workflow.yml b/.github/workflows/release_workflow.yml index e305672..4ee1c69 100644 --- a/.github/workflows/release_workflow.yml +++ b/.github/workflows/release_workflow.yml @@ -1,108 +1,32 @@ name: Release Alpha and Propose Stable on: + workflow_dispatch: pull_request: types: [closed] branches: [dev] jobs: + build_tests: + if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' + uses: OpenVoiceOS/gh-automations/.github/workflows/build-tests.yml@dev + secrets: inherit + with: + python_versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]' + install_extras: "pydantic" + test_path: "test/unittests/" + publish_alpha: - if: github.event.pull_request.merged == true - uses: TigreGotico/gh-automations/.github/workflows/publish-alpha.yml@master + needs: build_tests + if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' + uses: OpenVoiceOS/gh-automations/.github/workflows/publish-alpha.yml@dev secrets: inherit with: branch: 'dev' version_file: 'ovoscope/version.py' - setup_py: 'setup.py' update_changelog: true publish_prerelease: true + propose_release: true changelog_max_issues: 100 - - notify: - if: github.event.pull_request.merged == true - needs: publish_alpha - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Send message to Matrix bots channel - id: matrix-chat-message - uses: fadenb/matrix-chat-message@v0.0.6 - with: - homeserver: 'matrix.org' - token: ${{ secrets.MATRIX_TOKEN }} - channel: '!WjxEKjjINpyBRPFgxl:krbel.duckdns.org' - message: | - new ${{ github.event.repository.name }} PR merged! https://github.com/${{ github.repository }}/pull/${{ github.event.number }} - - publish_pypi: - needs: publish_alpha - if: success() # Ensure this job only runs if the previous job succeeds - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: dev - fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository. - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - name: Install Build Tools - run: | - python -m pip install build wheel - - name: version - run: echo "::set-output name=version::$(python setup.py --version)" - id: version - - name: Build Distribution Packages - run: | - python setup.py sdist bdist_wheel - - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@master - with: - password: ${{secrets.PYPI_TOKEN}} - - - propose_release: - needs: publish_alpha - if: success() # Ensure this job only runs if the previous job succeeds - runs-on: ubuntu-latest - steps: - - name: Checkout dev branch - uses: actions/checkout@v4 - with: - ref: dev - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - - name: Get version from setup.py - id: get_version - run: | - VERSION=$(python setup.py --version) - echo "VERSION=$VERSION" >> $GITHUB_ENV - - - name: Create and push new branch - run: | - git checkout -b release-${{ env.VERSION }} - git push origin release-${{ env.VERSION }} - - - name: Open Pull Request from dev to master - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - # Variables - BRANCH_NAME="release-${{ env.VERSION }}" - BASE_BRANCH="master" - HEAD_BRANCH="release-${{ env.VERSION }}" - PR_TITLE="Release ${{ env.VERSION }}" - PR_BODY="Human review requested!" - - # Create a PR using GitHub API - curl -X POST \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: token $GITHUB_TOKEN" \ - -d "{\"title\":\"$PR_TITLE\",\"body\":\"$PR_BODY\",\"head\":\"$HEAD_BRANCH\",\"base\":\"$BASE_BRANCH\"}" \ - https://api.github.com/repos/${{ github.repository }}/pulls - + publish_pypi: true + notify_matrix: true diff --git a/.github/workflows/repo_health.yml b/.github/workflows/repo_health.yml new file mode 100644 index 0000000..ebfb0ac --- /dev/null +++ b/.github/workflows/repo_health.yml @@ -0,0 +1,10 @@ +name: Repo Health +on: + pull_request: + branches: [dev] + workflow_dispatch: + +jobs: + repo_health: + uses: OpenVoiceOS/gh-automations/.github/workflows/repo-health.yml@dev + secrets: inherit diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml new file mode 100644 index 0000000..2c09d24 --- /dev/null +++ b/.github/workflows/unit_tests.yml @@ -0,0 +1,16 @@ +name: Run Tests +on: + pull_request: + branches: + - dev + workflow_dispatch: + +jobs: + unit_tests: + uses: OpenVoiceOS/gh-automations/.github/workflows/coverage.yml@dev + secrets: inherit + with: + python_version: "3.14" + install_extras: "ovoscope[pydantic]" + test_path: "test/unittests/" + coverage_source: "ovoscope" diff --git a/.gitignore b/.gitignore index 84c048a..a803393 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,43 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging /build/ +/dist/ +/local_util/ +/node_modules/ +/ovoscope.egg-info/ +/.eggs/ +*.egg-info/ +*.egg +MANIFEST + +# Virtual Environments +.venv/ +venv/ +ENV/ +env/ + +# IDEs +.idea/ +.vscode/ +*.swp +*.swo + +# Testing / coverage +.pytest_cache/ +.coverage +htmlcov/ +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Local configuration +.env +.python-version diff --git a/CHANGELOG.md b/CHANGELOG.md index b83ade1..1e3907f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,14 +1,7 @@ # Changelog - ## [Unreleased](https://github.com/TigreGotico/ovoscope/tree/HEAD) - [Full Changelog](https://github.com/TigreGotico/ovoscope/compare/0.7.2...HEAD) - **Merged pull requests:** - - feat: docs, tests and some tiny improvements [\#30](https://github.com/TigreGotico/ovoscope/pull/30) ([JarbasAl](https://github.com/JarbasAl)) - chore: Configure Renovate [\#29](https://github.com/TigreGotico/ovoscope/pull/29) ([renovate[bot]](https://github.com/apps/renovate)) - - - \* *This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)* diff --git a/FAQ.md b/FAQ.md index f21bcf7..2f4060a 100644 --- a/FAQ.md +++ b/FAQ.md @@ -1,10 +1,6 @@ -Last Edit: Claude Sonnet 4.6 - 2026-03-09 - Motive: Updated pydantic_helpers module name (pydantic.py → pydantic_helpers.py) in all FAQ entries. - # FAQ — `ovoscope` - ## What is `ovoscope`? `ovoscope` is End-to-end test framework for OpenVoiceOS skills. - ## How do I install it? ```bash pip install ovoscope @@ -13,107 +9,85 @@ Or for development: ```bash uv pip install -e ovoscope/ ``` - ## Where do I report bugs? Open an issue on the GitHub repository. Ensure you are targeting the `dev` branch for fixes. - ## How do I run tests? ```bash uv run pytest ovoscope/test/ --cov=ovoscope ``` - ## How do I contribute? 1. Fork the repository and create a feature branch from `dev`. 2. Write tests for your changes. 3. Open a PR targeting the `dev` branch. 4. Ensure CI passes before requesting review. - +## What CI workflows does ovoscope run? +Seven workflows: `unit_tests.yml` (pytest + coverage on PRs), `build_tests.yml` (sdist/wheel matrix build), `license_tests.yml` (dependency license audit via gh-automations), `pipaudit.yml` (CVE scanning), `release_workflow.yml` (test-gated alpha release), `publish_stable.yml` (stable release), and `conventional-label.yaml` (PR label automation). +## Does the release workflow run tests before publishing? +Yes. The `release_workflow.yml` has a `build_tests` job that runs the full test suite. The `publish_alpha` job depends on it via `needs: build_tests`, so a failing test blocks the alpha release. +## How does ovoscope's coverage reporting work in CI? +The `unit_tests.yml` workflow runs `pytest --cov=ovoscope --cov-report xml` and uses `py-cov-action/python-coverage-comment-action@v3` to post a coverage summary as a PR comment. +## What test coverage does ovoscope have? +104 tests across 6 test files achieving 89% overall coverage. Key areas tested: End2EndTest execute/assertions/serialization/routing/active skills/boot sequence/final session/from_message recording, CaptureSession lifecycle, MiniCroft config isolation/lang/pipeline, pytest_plugin fixture logic, pydantic_helpers bridge. ## What Python versions are supported? See `QUICK_FACTS.md` — currently `>=3.10`. - ## My tests pass locally but fail on CI — why? - Usually one of three causes: - 1. **Different pipeline plugins installed** — The default session pipeline includes whatever pipeline plugins happen to be installed. On CI, Gemma/Ollama/persona plugins may not be installed (or vice versa), changing which plugin handles the utterance. **Fix**: always pass an explicit `default_pipeline` to `get_minicroft()` (or use the default `DEFAULT_TEST_PIPELINE` by leaving `isolate_config=True`). - 2. **User locale affecting intent matching** — `isolate_config=True` (the default) removes the user's `~/.config/mycroft/mycroft.conf` from the config chain so the test environment locale does not affect results. Always leave this enabled. - 3. **Skill plugin not discoverable** — The skill must be registered under the `opm.skill` entry point group. Old-style `ovos.plugin.skill` entries are warned but not loaded by `find_skill_plugins()`. Use `extra_skills={SKILL_ID: SkillClass}` to inject skills that lack a proper entry point. - ## A persona / AI plugin is intercepting my test utterances - This happens because `SessionManager.default_session` is initialized at import time from the full system config (which may include persona pipeline stages). - `MiniCroft` solves this with `default_pipeline` (default: `DEFAULT_TEST_PIPELINE`): - ```python from ovoscope import get_minicroft, DEFAULT_TEST_PIPELINE, ADAPT_PIPELINE - # Default — all standard stages, no AI/persona/OCP (recommended) mc = get_minicroft([]) - # Adapt-only for fast unit-style end2end tests mc = get_minicroft([SKILL_ID], default_pipeline=ADAPT_PIPELINE) - # Opt in to persona explicitly from ovoscope import PERSONA_PIPELINE mc = get_minicroft([SKILL_ID], default_pipeline=DEFAULT_TEST_PIPELINE + PERSONA_PIPELINE) ``` - `DEFAULT_TEST_PIPELINE` excludes persona, Ollama, OCP, and m2v stages. The original pipeline is restored when `mc.stop()` is called. - ## Why does `SessionManager.default_session.pipeline` matter? - When a message is emitted without an explicit `session` in its context, ovos-core creates a session by copying `SessionManager.default_session`. That copy inherits its `pipeline` list, which controls which pipeline plugins are consulted for intent matching. - `MiniCroft.run()` overrides `SessionManager.default_session.pipeline` to `default_pipeline` (set after FakeBus init messages are processed, just before `READY`), and restores it on `stop()`. - ## How is `isolate_config` different from `default_pipeline`? - - `isolate_config=True` — clears `Configuration.xdg_configs` so `~/.config/mycroft/mycroft.conf` is not read. Prevents user locale, custom wake words, and user-level pipeline *config* from affecting tests. - `default_pipeline` — overrides `SessionManager.default_session.pipeline` directly. Necessary because the default session is initialized at module import time (before config isolation takes effect) and may already have the user's pipeline. - Both are enabled by default. They are complementary. - ## How do I know whether to use ADAPT_PIPELINE or PADATIOUS_PIPELINE for my test? - It depends on how the skill registers its intent: - | Decorator | Pipeline | |-----------|----------| | `@intent_handler(IntentBuilder(...))` | `ADAPT_PIPELINE` | | `@intent_handler("my.intent")` (string ending in `.intent`) | `PADATIOUS_PIPELINE` | | `@fallback_handler(priority=N)` | `FALLBACK_PIPELINE` | | `@converse_handler` | `CONVERSE_PIPELINE` | - When in doubt, look at the intent files in `locale/en-us/` — if there is a file named `*.intent`, it is Padatious. If there is a file named `*.voc` or `*.rx`, it is Adapt. - ## My skill emits extra messages (enclosure.eyes.*, add_context, configuration.patch) — how do I handle them? - Some skills emit low-level hardware events or internal context messages that are not part of the utterance handling protocol. Add them to `ignore_messages`: - ```python test = End2EndTest( ... @@ -126,14 +100,10 @@ test = End2EndTest( ... ) ``` - ## A skill emits a raw message (no source/dest) like recognizer_loop:sleep — how do I test it? - Some skills call `self.bus.emit(Message("some.message"))` without inheriting source/dest. These messages have `source=None` and will fail source-checking in the framework. - Use `async_messages` to assert they were received without checking order or source: - ```python test = End2EndTest( ... @@ -142,78 +112,54 @@ test = End2EndTest( ... ) ``` - ## My test passes locally but the user's blacklisted skills cause failures on CI - Users may have skills like `skill-ovos-stop.openvoiceos` in `blacklisted_skills` in their `~/.config/mycroft/mycroft.conf`. `Session.__init__` reads this from the live `Configuration()` singleton dict cache (not invalidated by `reload()`). - `MiniCroft` solves this: when `isolate_config=True` (the default), it patches `Configuration()["skills"]["blacklisted_skills"] = []` and `Configuration()["intents"]["blacklisted_intents"] = []` in `run()`, and restores them in `stop()`. This is complementary to the `xdg_configs = []` isolation applied in `__init__`. - ## Can I use typed pydantic models instead of raw Message objects? - Yes. Install the optional pydantic extras: - ```bash pip install ovoscope[pydantic] ``` - Then use the bridge in `ovoscope.pydantic_helpers`: - ```python from ovoscope.pydantic_helpers import to_bus_message, from_bus_message from ovos_pydantic_models import RecognizerLoopUtteranceMessage, RecognizerLoopUtteranceData, SpeakMessage - # Build a typed source message — validated at construction utterance = to_bus_message(RecognizerLoopUtteranceMessage( data=RecognizerLoopUtteranceData(utterances=["hello"], lang="en-us") )) - # Parse a received message into a typed model for richer assertions messages = test.execute() speak = from_bus_message(messages[0], SpeakMessage) assert "hello" in speak.data.utterance.lower() ``` - A typo in a field name (`"utterance"` vs `"utterances"`) raises `ValidationError` at construction time instead of silently producing a wrong test. - ## How do I validate a JSON fixture file before loading it? - Use `validate_fixture()` from `ovoscope.pydantic_helpers` (requires `ovoscope[pydantic]`): - ```python from ovoscope.pydantic_helpers import validate_fixture from ovoscope import End2EndTest - test = End2EndTest.deserialize(validate_fixture("test/fixtures/hello_world.json")) test.execute() ``` - If any message in the fixture is malformed, a clear `ValidationError` is raised pointing to the offending field — instead of a cryptic `KeyError` inside `deserialize()`. - ## How do I trigger non-utterance events during a test? - Use `MiniCroft.inject_message(msg)`: - ```python from ovos_bus_client.message import Message - mc.inject_message(Message("mycroft.gui.connected", {"connected": True})) ``` - This emits an arbitrary message on the FakeBus during a test without going through the utterance pipeline — useful for timer events, GUI events, or skill API calls. - ## How do I assert a skill spoke a specific phrase without checking the full message sequence? - Use `End2EndTest.assert_spoke(text, lang)`: - ```python test = End2EndTest( skill_ids=["my-skill.author"], @@ -222,34 +168,24 @@ test = End2EndTest( ) test.assert_spoke("Hello, world!", lang="en-US") ``` - `assert_spoke()` calls `execute()` internally and scans captured messages for a `speak` message with the matching utterance and lang. - ## `get_minicroft()` hangs forever — what do I do? - Pass `max_wait` to set a timeout: - ```python mc = get_minicroft(["my-skill.author"], max_wait=30) ``` - If `MiniCroft` does not reach `READY` within `max_wait` seconds, a `TimeoutError` is raised with the skill IDs — pointing you at the skill startup logs. The default is 60 seconds. - ## How do I use the `minicroft` pytest fixture? - The fixture is registered automatically when ovoscope is installed (via the `pytest11` entry point). Just declare `skill_ids` on your test class: - ```python class TestMySkill: skill_ids = ["my-skill.author"] - def test_something(self, minicroft): from ovoscope import End2EndTest from ovos_bus_client.message import Message - test = End2EndTest( minicroft=minicroft, skill_ids=self.skill_ids, @@ -262,27 +198,36 @@ class TestMySkill: ) test.execute() ``` - The `MiniCroft` is started once per class and stopped in teardown — no `setUp`/`tearDown` boilerplate needed. - ## How do I test a pipeline plugin (not a skill) like PersonaService? - Pipeline plugins are loaded by `MiniCroft` automatically via `IntentService`. Access them via: - ```python mc = get_minicroft([], default_pipeline=PERSONA_PIPELINE) persona_svc = mc.intents.pipeline_plugins["ovos-persona-pipeline-plugin"] ``` - Inject mocks directly into the plugin's state before each test: - ```python def setUp(self): persona_svc.personas.clear() # remove real solvers (Gemma, Ollama, etc.) persona_svc.active_persona = None # reset pipeline state persona_svc.personas["TestBot"] = MockPersona("TestBot", "forty two") ``` - The `skill_ids=[]` parameter tells MiniCroft to load no skills — only pipeline plugins. See `ovos-persona/test/end2end/test_persona.py` for a full working example. +--- +## How do I test skills in non-English languages? +Pass `secondary_langs` to `get_minicroft()`: +```python +croft = get_minicroft( + [SKILL_ID], + secondary_langs=["pt-PT", "de-DE", "es-ES"], +) +``` +This patches `Configuration()["secondary_langs"]` before Adapt/Padatious initialize, so they create per-language engines and register vocab for all specified languages. Without this, only the system's default language has vocab registered. +## Why does `End2EndTest.from_message()` crash with `TypeError: argument of type 'NoneType' is not iterable`? +This was a bug where `async_messages` defaulted to `None` and was passed to `CaptureSession`, which tried `msg.msg_type in None`. Fixed by defaulting to `[]`. +## Why do JSON fixture replays fail on session context? +Session context includes timestamps (e.g., `active_skills` activation time) that differ between recording and replay. Set `test_msg_context=False` on fixture tests. For skills with random dialog rendering (like quote pools), also set `test_msg_data=False`. +## Does `from_message()` filter GUI messages during recording? +Yes — `from_message()` now accepts `ignore_gui=True` (default), which adds `GUI_IGNORED` messages to the capture filter. This prevents GUI namespace messages from appearing in recorded fixtures. diff --git a/LICENSE b/LICENSE index 8dcef9c..c77f4c8 100644 --- a/LICENSE +++ b/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2026 Casimiro Ferreira + Copyright 2026 OpenVoiceOS Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/MAINTENANCE_REPORT.md b/MAINTENANCE_REPORT.md index e0faa16..0997167 100644 --- a/MAINTENANCE_REPORT.md +++ b/MAINTENANCE_REPORT.md @@ -1,26 +1,54 @@ -Last Edit: Claude Sonnet 4.6 - 2026-03-09 - Motive: pytest_plugin: safe teardown with try/finally + mc=None guard to prevent NameError masking get_minicroft failures. - # Maintenance Report — `ovoscope` - +## [2026-03-10] — Test coverage improvement (78% → 89%) +### Changes +- Created `test/unittests/test_end2end_extended.py` — 46 new tests covering: + - **Routing internals**: flip_points, entry_points, keep_original_src assertions + - **Active skills**: inject_active, activation_points, deactivation_points, disallow_extra_active_skills + - **Boot sequence**: correct/incorrect boot message assertions + - **Final session**: lang mismatch raises, matching session passes + - **Async messages**: captured separately, missing raises, count mismatch raises + - **Context assertions**: wrong context raises + - **GUI filtering**: ignore_gui=True/False behavior + - **Serialization**: JSON string input, flip_points/flags preservation, anonymize_message + - **from_message recording**: captures sequence, wraps single message + - **Pipeline constants**: composition validation + - **MiniCroft lang config**: override and restore + - **Verbose output**: exercises all print branches + - **Message count verbose**: first differing message output +- Created `test/unittests/test_pytest_plugin.py` — 6 new tests for minicroft fixture logic via `__wrapped__` +- Updated `FAQ.md` — added coverage FAQ entry +### AI Transparency Report +- **AI Model**: Claude Opus 4.6 +- **Actions Taken**: Created 2 new test files with 52 total new tests +- **Oversight**: All tests verified passing. Coverage: 78% → 89% overall, `__init__.py` 54% → 68%, `pytest_plugin.py` 0% → 64% +--- +## [2026-03-09] — CI workflows and test-gated releases +### Changes +- Created `.github/workflows/unit_tests.yml` — runs 58 unit tests with `pytest --cov=ovoscope` on PRs/pushes to `dev`, posts coverage comment via `py-cov-action/python-coverage-comment-action@v3` +- Created `.github/workflows/build_tests.yml` — matrix build (Python 3.10, 3.11) with `python -m build`, tests sdist/wheel creation and package install +- Created `.github/workflows/license_tests.yml` — calls `OpenVoiceOS/gh-automations/.github/workflows/license-check.yml@dev` reusable workflow +- Created `.github/workflows/pipaudit.yml` — CVE scanning via `pypa/gh-action-pip-audit@v1.0.0` on Python 3.10/3.11 matrix +- Updated `.github/workflows/release_workflow.yml` — added `build_tests` job that runs full test suite; `publish_alpha` now depends on `build_tests` via `needs:`, gating alpha releases on test success +- Updated `docs/ci-integration.md` — documented ovoscope's own CI workflow table +- Updated `FAQ.md` — added 3 new CI-related Q&A entries +### AI Transparency Report +- **AI Model**: Claude Opus 4.6 +- **Actions Taken**: Created 4 new workflow files, updated 1 existing workflow, updated docs and FAQ +- **Oversight**: All workflows follow established OVOS conventions (actions/checkout@v4, actions/setup-python@v5, python-version 3.11, python -m build). 58 existing tests verified passing. +--- ## [2026-03-09] — pytest_plugin: safe teardown guard - ### Changes - `ovoscope/pytest_plugin.py` — `minicroft` fixture: initialise `mc = None` before calling `get_minicroft()`, then wrap `yield mc` in `try/finally` with `if mc is not None: mc.stop()`. Previously, if `get_minicroft()` raised (e.g. `TimeoutError`), teardown would hit a `NameError: name 'mc' is not defined`, masking the original exception in pytest output. - ### AI Transparency Report - **AI Model**: Claude Sonnet 4.6 - **Actions Taken**: Applied targeted edit to `pytest_plugin.py` lines 57–62. - **Oversight**: No logic change — only teardown safety. Existing tests unaffected. - --- - ## [2026-03-09] — pydantic_helpers: typing, docstrings, tests, bug fix, pyproject.toml migration - ### Changes - **`ovoscope/pydantic_helpers.py`** (new module, renamed from the initial `pydantic.py`): - Full module docstring with install instructions and usage example. - `TYPE_CHECKING`-guarded import of `OpenVoiceOSMessage` — type checkers see full annotations; @@ -32,33 +60,26 @@ Last Edit: Claude Sonnet 4.6 - 2026-03-09 - Motive: pytest_plugin: safe teardown - **Bug fixed** in `validate_fixture()`: normalisation fallback changed from `""` to `None` so messages missing both `"type"` and `"message_type"` keys are correctly rejected by pydantic (an empty string passes `message_type: str = Field(...)` validation silently). - **`test/unittests/test_pydantic_helpers.py`** (new, 20 tests): - | Class | Tests | What is covered | |-------|-------|----------------| | `TestToBusMessage` | 6 | `msg_type`, data fields, return type, utterance msg, empty context, roundtrip | | `TestFromBusMessage` | 5 | valid speak, valid utterance, return type, invalid raises `ValidationError`, base model leniency | | `TestValidateFixture` | 9 | valid fixture, source/expected preserved, missing file, malformed source, malformed expected, error chains `ValidationError`, `message_type` key accepted, empty lists | - All 20 tests pass. Full suite now 58 tests (38 pre-existing + 20 new), all passing. - **`pyproject.toml`** — completed migration: - `build-backend` changed from `setuptools.backends.legacy:build` to `setuptools.build_meta`. - `dynamic = ["version"]` added; `[tool.setuptools.dynamic] version = {attr = "ovoscope.version.__version__"}`. - `[project.optional-dependencies] pydantic = ["ovos-pydantic-models>=0.1.0"]` added. - `setup.py` removed. - **`ovoscope/version.py`**: - Added `__version__` computed from `VERSION_MAJOR`, `VERSION_MINOR`, `VERSION_BUILD`, `VERSION_ALPHA` so `pyproject.toml` dynamic versioning works without `setup.py`. - **`AUDIT.md`**, **`SUGGESTIONS.md`**, **`FAQ.md`**: - All module references updated from `ovoscope.pydantic` → `ovoscope.pydantic_helpers`. - AUDIT unit-test count updated to 58; setup.py fix marked fully complete. - SUGGESTIONS.md item 6 file path corrected. - FAQ.md pydantic section import paths corrected. - ### AI Transparency Report - **AI Model**: Claude Sonnet 4.6 - **Actions Taken**: Read existing module and all test files; identified two bugs in `validate_fixture()` @@ -66,14 +87,10 @@ All 20 tests pass. Full suite now 58 tests (38 pre-existing + 20 new), all passi - **Oversight**: `validate_fixture()` validates top-level message structure only (`message_type`, `data`, `context` shape) via `OpenVoiceOSMessage` — it does not validate data-field-level schemas (e.g. `utterances` type). Use `from_bus_message(msg, SpecificModel)` for field-level validation. - --- - ## [2026-03-09] — End2end tests written for ovos-persona (11 tests) - ### Tests Created New `ovos-persona/test/end2end/test_persona.py` — 4 test classes, 11 tests: - | Class | Tests | Intents/triggers | |-------|-------|-----------------| | `TestPersonaList` | 2 | `list_personas.intent` (no personas / 2 personas) | @@ -81,7 +98,6 @@ New `ovos-persona/test/end2end/test_persona.py` — 4 test classes, 11 tests: | `TestPersonaSummon` | 2 | `summon.intent` (known / unknown persona) | | `TestPersonaRelease` | 1 | `Release.voc` via `voc_match()` | | `TestPersonaQuery` | 4 | `ask.intent` explicit / active fallback / error / no-match | - ### Key Patterns Discovered - Pipeline plugins accessed via `mc.intents.pipeline_plugins["ovos-persona-pipeline-plugin"]` - Inject mock personas into `persona_svc.personas` dict — bypasses real solver loading @@ -90,19 +106,14 @@ New `ovos-persona/test/end2end/test_persona.py` — 4 test classes, 11 tests: - `ovos.utterance.handled` data is `{"name": "PersonaService.handle_persona_*"}` — not empty - `speak` with dialog template checked only by `context={"skill_id": SKILL_ID}` (text varies) - Direct speaks from query answers checked with `data={"utterance": "forty two"}` - ### AI Transparency Report - **AI Model**: Claude Sonnet 4.6 - **Actions Taken**: Traced all message sequences via live FakeBus capture; wrote tests iteratively; all 11 pass. - **Oversight**: Dialog text assertions omitted — locale-dependent. `test_list_two_personas` relies on dict insertion order (Python 3.7+). - --- - ## [2026-03-09] — End2end tests written for 6 skills (18 tests) - ### Skills Covered New `test/end2end/` directories and test files created for: - | Skill | Test file | Tests | Intents tested | |-------|-----------|-------|----------------| | `ovos-skill-hello-world` | `test_hello_world.py` | 4 | HelloWorldIntent (Adapt), Greetings.intent (Padatious), no-match cases | @@ -111,29 +122,23 @@ New `test/end2end/` directories and test files created for: | `ovos-skill-volume` | `test_volume.py` | 4 | volume.max.intent, volume.mute.intent, volume.unmute.intent (Padatious), no-match | | `ovos-skill-count` | `test_count.py` | 3 | count_to_N.intent (Padatious), no-match | | `ovos-skill-parrot` | `test_parrot.py` | 3 | speak.intent, repeat.tts.intent, no-match | - ### Key Patterns Discovered - Intents registered with string `"name.intent"` → Padatious; `IntentBuilder(...)` → Adapt - Skills emitting raw `Message(...)` without `forward()`/`reply()` have `source=None` — use `async_messages` + `ignore_messages` - Enclosure/LED messages and `add_context`/`configuration.patch` must be in `ignore_messages` - `message.forward(...)` inherits the post-flip source/dest — do NOT add these to `keep_original_src` - ### AI Transparency Report - **AI Model**: Claude Sonnet 4.6 - **Actions Taken**: Read each skill's `__init__.py` and locale files; wrote tests iteratively with test runs to fix pipeline selection, ignore_messages, meta dict content. All 18 tests pass. - **Oversight**: naptime test skips dialog content check (varies with `listener.wake_word` config). - --- - ## [2026-03-09] — Config isolation extended: blacklisted_skills + blacklisted_intents - ### Problem Addressed After pipeline isolation, `test_stop.py` (6 tests) and `test_cancel_plugin.py` (1 test) still failed. Root cause: `Session.__init__` reads `Configuration()["skills"]["blacklisted_skills"]` and `Configuration()["intents"]["blacklisted_intents"]` from the live singleton dict cache, same problem as the pipeline. The user had `skill-ovos-stop.openvoiceos` blacklisted in `~/.config/mycroft/mycroft.conf`. Additionally, `ovos-skill-count` and `ovos-utterance-plugin-cancel` were not installed in the workspace venv. - ### Changes - `ovoscope/__init__.py` — `MiniCroft.__init__`: added `_original_blacklisted_skills` and `_original_blacklisted_intents` state variables. @@ -144,20 +149,15 @@ Additionally, `ovos-skill-count` and `ovos-utterance-plugin-cancel` were not ins - `ovos-core/test/end2end/test_stop.py` — Added `"ovos-hivemind-pipeline-plugin.stop.response"` to `ignore_messages` in both `TestStopNoSkills` and `TestCountSkills` (hivemind responds to `mycroft.stop`). - Installed `Skills/ovos-skill-count` and `Transformer plugins/ovos-utterance-plugin-cancel` with `uv pip install --no-deps -e`. - ### Result 27/27 ovos-core end2end tests pass (was 20/27). - ### AI Transparency Report - **AI Model**: Claude Sonnet 4.6 - **Actions Taken**: Diagnosed `blacklisted_skills` stale cache issue; extended `run()`/`stop()` patch pattern; identified hivemind stop response as uncaught message; installed missing plugins. - **Oversight**: `blacklisted_skills` patch only runs when `isolate_config=True` (same condition as xdg isolation). - --- - ## [2026-03-09] — Pipeline isolation and reproducible test pipelines - ### Problem Addressed `MiniCroft.isolate_config=True` cleared `Configuration.xdg_configs` to remove the user's `~/.config/mycroft/mycroft.conf`, but `SessionManager.default_session` is a singleton @@ -165,7 +165,6 @@ initialised at module import time — it already held the user's pipeline (which `ovos-persona-pipeline-plugin-high`, Ollama, OCP, etc.). Any utterance emitted without an explicit session in its context would inherit this pipeline and be intercepted by AI plugins non-deterministically, making tests environment-dependent. - ### Changes - `ovoscope/__init__.py` — Added pipeline stage constants: `STOP_PIPELINE`, `CONVERSE_PIPELINE`, `ADAPT_PIPELINE`, `PADATIOUS_PIPELINE`, @@ -185,25 +184,20 @@ non-deterministically, making tests environment-dependent. pipeline overrides default session, restored after stop, `isolate_config=True` uses `DEFAULT_TEST_PIPELINE`, persona/ollama/m2v absent from `DEFAULT_TEST_PIPELINE`, `default_pipeline=None` leaves session unchanged. - ### Use Cases Unblocked - `get_minicroft([])` → `complete_intent_failure` tests now pass without Gemma/persona intercepting. - `get_minicroft([SKILL_ID], default_pipeline=ADAPT_PIPELINE)` — Adapt-only testing. - `get_minicroft([SKILL_ID], default_pipeline=ADAPT_PIPELINE + FALLBACK_PIPELINE)` — intent+fallback. - `get_minicroft([SKILL_ID], default_pipeline=PERSONA_PIPELINE)` — explicitly test persona behaviour. - `get_minicroft([SKILL_ID], default_pipeline=None)` — use system default (includes all installed plugins). - ### AI Transparency Report - **AI Model**: Claude Sonnet 4.6 - **Actions Taken**: Diagnosed root cause (singleton default_session pre-initialised from user config); added constants + `default_pipeline` param; updated `run()` and `stop()`; added 5 unit tests. - **Oversight**: `DEFAULT_TEST_PIPELINE` does not include OCP or m2v stages — repos that test media skills should pass an explicit pipeline including those stages. - --- - ## [2026-03-08] — Code improvements from SUGGESTIONS.md - ### Changes - `ovoscope/__init__.py` — `get_minicroft()`: added `max_wait: float = 60` parameter; raises `TimeoutError` if MiniCroft does not reach READY within the deadline. Return type annotated @@ -221,7 +215,6 @@ non-deterministically, making tests environment-dependent. - `pyproject.toml` (NEW): `[build-system]`, `[project]`, `[project.entry-points."pytest11"]`, and `[tool.pytest.ini_options]` tables. `setup.py` retained for dynamic version reading. - `AUDIT.md`: marked 3 issues as FIXED; updated Next Steps. - ### AI Transparency Report - **AI Model**: Claude Sonnet 4.6 - **Actions Taken**: Read `__init__.py` fully; made targeted edits; created `pytest_plugin.py` @@ -229,11 +222,8 @@ non-deterministically, making tests environment-dependent. - **Oversight**: `assert_spoke()` depends on `execute()` returning messages — verify against a live install. `pyproject.toml` dynamic version section has a TODO comment for when `version.py` exports `__version__`. - --- - ## [2026-03-08] — Documentation enrichment and audit deepening - ### Changes - Created `docs/usage-guide.md` — full tutorial from install to 8 test patterns; references hello-world canonical examples and real class/method signatures from `ovoscope/__init__.py`. @@ -245,36 +235,28 @@ non-deterministically, making tests environment-dependent. (CRITICAL/MAJOR/MODERATE/MINOR) all traced to specific lines in `__init__.py`. - Replaced `SUGGESTIONS.md` — 4 generic stubs replaced with 7 concrete, repo-specific proposals with code snippets pointing to specific lines. - ### Rationale The previous docs scaffold was boilerplate with no practical value. This pass enriches docs to the level where every OVOS repo can adopt ovoscope end-to-end testing without reading source code. - ### Verification - `ls ovoscope/docs/` shows 7 files (5 pre-existing + `usage-guide.md` + `ci-integration.md`). - All code examples in `usage-guide.md` use real imports and class names verified from source. - All `AUDIT.md` findings reference specific file:line evidence. - ### AI Transparency Report - **AI Model**: Claude Sonnet 4.6 - **Actions Taken**: Read `ovoscope/__init__.py` (485 lines), `test/test_helloworld.py`, `ovos-core/test/end2end/test_adapt.py`, and all existing docs; then generated enriched content. - **Oversight**: Code examples are illustrative but not executed. Verify against live skill install before treating as runnable. - --- - ## [2026-03-08] — Initial compliance scaffold - ### Changes - Created `QUICK_FACTS.md` with machine-readable package metadata. - Created `FAQ.md` with common Q&A. - Created `MAINTENANCE_REPORT.md` (this file) as the change log. - Created `SUGGESTIONS.md` with initial improvement proposals. - Created `docs/index.md` as the documentation entry point (if missing). - ### Rationale Establishing the required file set mandated by `AGENTS.md` for all active workspace repositories. - ### AI Transparency Report - **AI Model**: Claude Sonnet 4.6 - **Actions Taken**: Generated boilerplate compliance scaffold (QUICK_FACTS, FAQ, MAINTENANCE_REPORT, SUGGESTIONS, docs/index). diff --git a/QUICK_FACTS.md b/QUICK_FACTS.md new file mode 100644 index 0000000..60d5f77 --- /dev/null +++ b/QUICK_FACTS.md @@ -0,0 +1,39 @@ +# Quick Facts — `ovoscope` +End-to-end test framework for OpenVoiceOS skills +## Core Information +| Feature | Details | +|---------|---------| +| Package Name | `ovoscope` | +| Version | `0.7.2` | +| License | Apache-2.0 | +| Repository | [https://github.com/TigreGotico/ovoscope](https://github.com/TigreGotico/ovoscope) | +| Python Support | >=3.10 | +| Status | Active development | +## Testing & CI +| Feature | Details | +|---------|---------| +| Unit Tests | 104 tests across `test/unittests/` (all passing) | +| Coverage | 89% overall (improved from 78%) | +| Test Framework | pytest with custom fixtures | +| Coverage Reporter | py-cov-action/python-coverage-comment-action@v3 | +## CI Workflows +| Workflow | Trigger | Status | +|----------|---------|--------| +| `unit_tests.yml` | Push to dev | Uses coverage.yml@dev | +| `build_tests.yml` | Push to master, PR to dev | Uses build-tests.yml@dev | +| `license_check.yml` | Push to master/dev, PR | Uses license-check.yml@dev | +| `pip_audit.yml` | Push to master/dev, PR | Uses pip-audit.yml@dev | +| `release_workflow.yml` | PR merge to dev | Gates on build_tests, calls publish-alpha.yml@dev | +| `publish_stable.yml` | Push to master | Calls publish-stable.yml@dev | +| `release_preview.yml` | PR to dev | Uses release-preview.yml@dev | +| `repo_health.yml` | PR to dev | Uses repo-health.yml@dev | +## Key Features +- **End-to-end testing framework** for OpenVoiceOS skills +- **MiniCroft fixture** — pytest integration with class-scoped skill testing +- **Message capture** — CaptureSession for recording skill responses +- **Assertions** — End2EndTest with assertions (assert_spoke, etc.) +- **Pydantic integration** — Optional typed bridge with ovos-pydantic-models +- **Version from pyproject.toml** — Full migration from setup.py +## Test-Gated Releases +✅ Alpha releases gate on `build_tests` passing (100+ unit tests) +✅ Stable releases gate on master push (must pass alpha CI first) diff --git a/README.md b/README.md index 0373edf..c781d9b 100644 --- a/README.md +++ b/README.md @@ -2,23 +2,15 @@ [![PyPI](https://img.shields.io/pypi/v/ovoscope)](https://pypi.org/project/ovoscope/) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](LICENSE) [![Python](https://img.shields.io/badge/python-3.10%2B-blue)](https://pypi.org/project/ovoscope/) - # OvoScope - **End-to-end testing for [OVOS](https://openvoiceos.org) skills.** - OvoScope runs a full OVOS Core pipeline in-process using a `FakeBus` — no server, no audio stack, no network. Load real skill plugins, emit a test utterance, and assert on every bus message that comes back: type, data, routing context, session state, and message ordering. - ![image](https://github.com/user-attachments/assets/10a10ff5-64b7-42fd-86bd-cb6a5db769dd) - > Like a microscope for your OVOS skills. - --- - ## Features - | | | |---|---| | **Full pipeline** | Runs real intent pipeline plugins (Adapt, Padatious, Fallback, Converse, Common Query) | @@ -30,40 +22,29 @@ message that comes back: type, data, routing context, session state, and message | **Inject skills** | `extra_skills={id: SkillClass}` to load inline test skills without a PyPI entry point | | **Inject messages** | `MiniCroft.inject_message()` to trigger non-utterance handlers (GUI events, timers, API calls) | | **Typed models** | Optional `ovoscope[pydantic]` bridge to `ovos-pydantic-models` for schema-validated messages | - --- - ## Installation - ```bash pip install ovoscope ``` - With optional typed message model support: - ```bash pip install ovoscope[pydantic] ``` - --- - ## Quick Start - ```python import unittest from ovos_bus_client.message import Message from ovos_bus_client.session import Session from ovoscope import End2EndTest - SKILL_ID = "ovos-skill-hello-world.openvoiceos" - session = Session("test-session") utterance = Message( "recognizer_loop:utterance", {"utterances": ["hello world"], "lang": "en-US"}, {"session": session.serialize(), "source": "A", "destination": "B"}, ) - class TestHelloWorld(unittest.TestCase): def test_intent_match(self): End2EndTest( @@ -81,19 +62,13 @@ class TestHelloWorld(unittest.TestCase): ], ).execute(timeout=10) ``` - Only keys you specify in `expected.data` and `expected.context` are checked — extra keys in the received message are ignored. - --- - ## Recording Mode - Don't know the exact message sequence yet? Record it from a live run: - ```python from ovoscope import End2EndTest - test = End2EndTest.from_message( message=utterance, skill_ids=[SKILL_ID], @@ -101,24 +76,17 @@ test = End2EndTest.from_message( ) test.save("tests/fixtures/hello_world.json") # anonymises location data by default ``` - Replay in CI: - ```python End2EndTest.from_path("tests/fixtures/hello_world.json").execute(timeout=10) ``` - --- - ## pytest Fixture - The `minicroft` class-scoped fixture is auto-registered when ovoscope is installed. No `setUp`/`tearDown` boilerplate needed: - ```python class TestMySkill: skill_ids = ["my-skill.author"] - def test_something(self, minicroft): End2EndTest( minicroft=minicroft, @@ -127,35 +95,24 @@ class TestMySkill: expected_messages=[...], ).execute(timeout=10) ``` - --- - ## Pipeline Control - OvoScope exposes composable pipeline stage lists so tests are deterministic regardless of which AI plugins are installed on the host: - ```python from ovoscope import ADAPT_PIPELINE, PADATIOUS_PIPELINE, FALLBACK_PIPELINE, PERSONA_PIPELINE - # Adapt only — fastest mc = get_minicroft([SKILL_ID], default_pipeline=ADAPT_PIPELINE) - # Full intent chain mc = get_minicroft([SKILL_ID], default_pipeline=ADAPT_PIPELINE + PADATIOUS_PIPELINE + FALLBACK_PIPELINE) - # Opt in to persona for AI testing mc = get_minicroft([SKILL_ID], default_pipeline=DEFAULT_TEST_PIPELINE + PERSONA_PIPELINE) ``` - `DEFAULT_TEST_PIPELINE` (the default when `isolate_config=True`) includes all standard built-in stages and deliberately excludes persona, Ollama, OCP, and m2v plugins. - --- - ## Documentation - | Document | | |---|---| | [docs/usage-guide.md](docs/usage-guide.md) | **Start here** — 8 test patterns with full worked examples | @@ -165,34 +122,22 @@ stages and deliberately excludes persona, Ollama, OCP, and m2v plugins. | [docs/end2end-test.md](docs/end2end-test.md) | `End2EndTest` full parameter reference | | [docs/pydantic-integration.md](docs/pydantic-integration.md) | Typed message models with `ovos-pydantic-models` | | [FAQ.md](FAQ.md) | Common questions and gotchas | - --- - ## License - [Apache 2.0](LICENSE) - --- - ## Contributing - PRs are welcome! See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines. - --- - ## AI Disclosure - Parts of this project are developed with the assistance of AI tools. In the interest of transparency, two files are maintained as a public record of AI involvement: - - **[FAQ.md](FAQ.md)** — Frequently asked questions that emerged from real development sessions, including design rationale, gotchas, and usage patterns. Many entries were authored or refined with AI assistance during the process of building and testing this framework. - - **[MAINTENANCE_REPORT.md](MAINTENANCE_REPORT.md)** — A chronological log of changes made to this repository. Each entry records what was changed, why, which AI model was involved, what actions it took, and what human oversight was applied. This log is updated after every significant AI-assisted session. - These files are intentionally published so that contributors and users can understand how the project evolves and where AI assistance has been applied. diff --git a/docs/capture-session.md b/docs/capture-session.md index a0469e3..7d48b3b 100644 --- a/docs/capture-session.md +++ b/docs/capture-session.md @@ -1,17 +1,11 @@ # CaptureSession - `CaptureSession` subscribes to all messages on the `FakeBus` and records them during a single test interaction. It handles synchronous responses (ordered, from the intent pipeline) and asynchronous responses (from external threads, unordered). - ## Class: `CaptureSession` - ```python from ovoscope import CaptureSession ``` - A `dataclass` that wraps a `MiniCroft` and manages message collection for one test interaction. - ### Fields - | Field | Type | Default | Description | |---|---|---|---| | `minicroft` | `MiniCroft` | required | The runtime to capture from | @@ -21,46 +15,31 @@ A `dataclass` that wraps a `MiniCroft` and manages message collection for one te | `ignore_messages` | `list[str]` | `["ovos.skills.settings_changed"]` | Message types to discard | | `async_messages` | `list[str]` | `[]` | Message types to route to `async_responses` instead | | `done` | `threading.Event` | — | Set when an EOF message is received | - ### Methods - #### `capture(source_message, timeout=20)` - Emits `source_message` on the bus and waits for an EOF message (or timeout). Subsequent calls on the same session accumulate into `responses`. - ```python capture = CaptureSession(croft, eof_msgs=["ovos.utterance.handled"]) capture.capture(utterance_msg, timeout=10) ``` - #### `finish() -> list[Message]` - Signals end of capture, unsubscribes from the bus, and returns the collected `responses`. - --- - ## Message Routing - Messages are sorted into three buckets on arrival: - ``` incoming message │ ├─ msg_type in async_messages? → async_responses (unordered) ├─ msg_type in ignore_messages? → discarded └─ otherwise → responses (ordered) - eof_msgs trigger done.set() → capture.wait() returns ``` - ### Default ignored messages - ```python DEFAULT_IGNORED = ["ovos.skills.settings_changed"] ``` - ### Default GUI ignored (when `ignore_gui=True` on `End2EndTest`) - ```python GUI_IGNORED = [ "gui.clear.namespace", @@ -69,54 +48,37 @@ GUI_IGNORED = [ "gui.page.show", ] ``` - These are excluded by default because GUI namespace updates are frequent and rarely the focus of skill logic tests. - --- - ## Direct Usage - `CaptureSession` can be used without `End2EndTest` for lower-level scenarios: - ```python from ovoscope import get_minicroft, CaptureSession from ovos_bus_client.message import Message from ovos_bus_client.session import Session - croft = get_minicroft(["skill-weather.openvoiceos"]) - session = Session("test-123") utterance = Message( "recognizer_loop:utterance", {"utterances": ["what is the weather?"], "lang": "en-us"}, {"session": session.serialize(), "source": "A", "destination": "B"}, ) - capture = CaptureSession(croft) capture.capture(utterance, timeout=15) messages = capture.finish() - for msg in messages: print(msg.msg_type, msg.data) - croft.stop() ``` - --- - ## Multi-turn Capture - Emit multiple source messages into the same `CaptureSession` to simulate a multi-turn conversation. The session from the last received message is propagated into each subsequent source message: - ```python capture = CaptureSession(croft, eof_msgs=["ovos.utterance.handled"]) - capture.capture(first_utterance, timeout=10) # inject session from last received message into follow-up follow_up.context["session"] = capture.responses[-1].context["session"] capture.capture(follow_up, timeout=10) - all_messages = capture.finish() ``` - `End2EndTest` does this automatically when `source_message` is a list. diff --git a/docs/ci-integration.md b/docs/ci-integration.md index 1281dc4..6692da4 100644 --- a/docs/ci-integration.md +++ b/docs/ci-integration.md @@ -1,16 +1,9 @@ -Last Edit: Claude Sonnet 4.6 - 2026-03-08 - Motive: CI integration guide for gh-automations and standalone use. - # CI Integration — ovoscope - This document explains how to wire ovoscope end-to-end tests into a repo's CI pipeline using `gh-automations` reusable workflows, and how to structure test files and fixtures. - --- - ## Directory Layout - The workspace convention is: - ``` my-skill-repo/ ├── test/ @@ -23,59 +16,41 @@ my-skill-repo/ ├── setup.py (or pyproject.toml) └── ... ``` - Separate `end2end/` from `unittests/` so they can be run independently — end2end tests are slower (they spin up a MiniCroft) and may require extra dependencies. - --- - ## pytest / unittest Configuration - ### Using `pyproject.toml` - ```toml [tool.pytest.ini_options] testpaths = ["test"] - # Run only unit tests (fast): # pytest test/unittests/ - # Run only end2end tests (slow, requires skill installed): # pytest test/end2end/ ``` - ### Using `pytest.ini` - ```ini [pytest] testpaths = test ``` - End2end tests are standard `unittest.TestCase` subclasses and work with both `pytest` and plain `python -m unittest discover`. - --- - ## Install Dependencies in CI - End2end tests need ovoscope **and** all skills under test installed. Add an install step before running tests: - ```bash pip install ovoscope pip install -e . # install the skill from source (editable) ``` - Or if testing multiple skills together: - ```bash pip install ovoscope \ ovos-skill-hello-world \ ovos-skill-weather ``` - Verify the skills are discoverable before running: - ```bash python -c " from ovos_plugin_manager.skills import find_skill_plugins @@ -84,50 +59,35 @@ print('Found skills:', plugins) assert 'ovos-skill-hello-world.openvoiceos' in plugins " ``` - --- - ## Fixture JSON Files - Fixture files generated by `End2EndTest.save()` (see [usage-guide.md](usage-guide.md) Pattern 4) contain the expected message sequence serialised as JSON. - **When to commit fixtures:** - - Commit fixtures that test stable, deterministic interactions (e.g., a specific dialog line). - Do NOT commit fixtures where the `speak` utterance varies randomly — either omit the `utterance` key from expected data or use manual assertion instead. - Always generate fixtures with `anonymize=True` (the default) — this strips real location data. - **`.gitignore` pattern** (if you generate fixtures locally but don't want to commit them): - ```gitignore test/end2end/fixtures/*.json ``` - Or selectively ignore only generated/recording artifacts: - ```gitignore test/end2end/fixtures/recorded_*.json ``` - --- - ## GitHub Actions — End2End Job - Add an end2end job to your `release_workflow.yml` or a dedicated workflow. This example follows the `gh-automations` conventions used across all 203+ OVOS repos: - ```yaml # .github/workflows/release_workflow.yml name: Release workflow - on: pull_request: types: [closed] branches: [dev] workflow_dispatch: - jobs: build_tests: runs-on: ubuntu-latest @@ -146,34 +106,26 @@ jobs: run: pytest test/unittests/ -v - name: Run end2end tests run: pytest test/end2end/ -v --timeout=60 - publish_alpha: needs: build_tests if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' - uses: TigreGotico/gh-automations/.github/workflows/publish-alpha.yml@master + uses: OpenVoiceOS/gh-automations/.github/workflows/publish-alpha.yml@dev with: propose_release: true secrets: inherit ``` - The `build_tests` job runs before `publish_alpha` — a failing end2end test blocks the release. - --- - ## Standalone End2End Workflow - If your repo only needs end2end tests (no release automation), use a simpler workflow: - ```yaml # .github/workflows/end2end.yml name: End2End Tests - on: push: branches: [dev, master] pull_request: branches: [dev] - jobs: end2end: runs-on: ubuntu-latest @@ -189,55 +141,49 @@ jobs: - name: Test run: pytest test/end2end/ -v --timeout=60 ``` - --- - ## Known CI Gotchas - ### Skill plugin not found on PATH - **Symptom**: `get_minicroft()` hangs or `find_skill_plugins()` returns an empty list. - **Cause**: The skill was not installed in editable mode (`pip install -e .`) or the entry point was not registered. - **Fix**: Always install the skill package in the same environment as ovoscope: - ```bash pip install -e . # registers entry points pip install ovoscope ``` - ### Missing `.venv` in CI - If you use `uv` locally, your `.venv` is not present in CI. Use `pip` directly in CI or add a `uv pip install` step. Do not rely on `.venv` being pre-activated. - ### MiniCroft hangs for >30 seconds - Padatious intent training can be slow on a cold CI runner. Set a generous `--timeout` in pytest and pass `timeout=30` (or higher) to `test.execute()`. - ### Flaky tests from session ID collisions - Each test that uses `Session("same-id")` shares session state with other tests using the same session ID. Use unique session IDs per test class, or generate them: - ```python import uuid session = Session(str(uuid.uuid4())) ``` - ### GUI messages causing assertion failures - By default `ignore_gui=True` strips GUI namespace messages from the captured sequence. If you see unexpected messages related to `gui.*`, check whether a skill emits GUI messages unconditionally and whether your `expected_messages` list accounts for them. - --- - +## ovoscope's Own CI Workflows +The ovoscope repository itself uses the standard OVOS workflow set: +| Workflow | File | Trigger | Purpose | +| :--- | :--- | :--- | :--- | +| **Unit Tests** | `unit_tests.yml` | PR/push to `dev` | Runs `pytest --cov=ovoscope` on 58 tests, posts coverage comment | +| **Build Tests** | `build_tests.yml` | PR to `dev`, push to `master` | Matrix build (Python 3.10, 3.11) with `python -m build` | +| **License Check** | `license_tests.yml` | PR to `dev`, push to `master` | Calls `gh-automations/license-check.yml` reusable | +| **Pip Audit** | `pipaudit.yml` | Push to `dev`/`master` | CVE scanning via `pypa/gh-action-pip-audit` | +| **Release Alpha** | `release_workflow.yml` | PR merge to `dev` | Runs tests first, then calls `publish-alpha.yml` | +| **Stable Release** | `publish_stable.yml` | Push to `master` | Calls `publish-stable.yml` with bot loop guard | +| **Labels** | `conventional-label.yaml` | PR open/edit | Auto-labels PRs with conventional commit types | +The release workflow gates alpha publishing on test success — a failing test blocks the release. +--- ## See Also - - [usage-guide.md](usage-guide.md) — tutorial walkthrough with all patterns - [gh-automations/docs/workflow-reference.md](../../gh-automations/docs/workflow-reference.md) — full reusable workflow reference - [gh-automations/docs/repo-setup.md](../../gh-automations/docs/repo-setup.md) — per-repo workflow setup diff --git a/docs/end2end-test.md b/docs/end2end-test.md index 0b1985e..482c2c0 100644 --- a/docs/end2end-test.md +++ b/docs/end2end-test.md @@ -1,47 +1,33 @@ # End2EndTest - `End2EndTest` is the primary API. It wires together `MiniCroft`, `CaptureSession`, and all assertion logic into a single declarative test object. - ## Class: `End2EndTest` - ```python from ovoscope import End2EndTest ``` - A `dataclass`. Configure once, call `.execute()` to run. - --- - ## Fields - ### Core - | Field | Type | Default | Description | |---|---|---|---| | `skill_ids` | `list[str]` | required | Skill plugin IDs to load | | `source_message` | `Message \| list[Message]` | required | Input message(s). Standardized to list on init. | | `expected_messages` | `list[Message]` | required | Ordered expected response sequence | | `expected_boot_sequence` | `list[Message]` | `[]` | Startup messages to validate before running | - ### Message Filtering - | Field | Type | Default | Description | |---|---|---|---| | `eof_msgs` | `list[str]` | `["ovos.utterance.handled"]` | Message types that end capture | | `ignore_messages` | `list[str]` | `["ovos.skills.settings_changed"]` | Message types to discard | | `ignore_gui` | `bool` | `True` | Discard GUI namespace messages | | `async_messages` | `list[str]` | `[]` | Message types arriving from external threads (collected separately, unordered) | - ### Routing Tracking - | Field | Type | Default | Description | |---|---|---|---| | `flip_points` | `list[str]` | `[]` | After receiving this message type, swap expected source↔destination | | `entry_points` | `list[str]` | `["recognizer_loop:utterance"]` | On this message type, extract new expected source/destination from the received message context (reversed) | | `keep_original_src` | `list[str]` | `["ovos.skills.fallback.ping"]` | For these message types, always compare against the original source/destination | - ### Active Skill Tracking - | Field | Type | Default | Description | |---|---|---|---| | `inject_active` | `list[str]` | `[]` | Pre-activate these skill IDs before the test runs (modifies session) | @@ -49,11 +35,8 @@ A `dataclass`. Configure once, call `.execute()` to run. | `activation_points` | `list[str]` | `[]` | After this message type, `context.skill_id` must remain active | | `deactivation_points` | `list[str]` | `["intent.service.skills.deactivate"]` | After this message type, `context.skill_id` must NOT be active | | `final_session` | `Session \| None` | `None` | If set, compare last-message session against this | - ### Sub-test Toggles - All default to `True`. Set to `False` to skip individual assertion categories: - | Flag | What it checks | |---|---| | `test_message_number` | `len(received) == len(expected)` | @@ -66,91 +49,60 @@ All default to `True`. Set to `False` to skip individual assertion categories: | `test_active_skills` | Active skills in session match expectations | | `test_routing` | `context.source` and `context.destination` match | | `test_final_session` | Final session matches `final_session` | - ### Internals - | Field | Default | Description | |---|---|---| | `verbose` | `True` | Print pass/fail for each assertion | | `minicroft` | `None` | Provide an existing `MiniCroft` to reuse across tests | | `managed` | `False` | Set automatically; if `True`, `execute()` stops the minicroft after running | - --- - ## `execute(timeout=30)` - Runs the test. Raises `AssertionError` on the first failing assertion. - If `minicroft` is `None`, creates one automatically (managed mode — stops it after the test). To run multiple tests against the same loaded skills, pass your own `MiniCroft`: - ```python from ovoscope import get_minicroft, End2EndTest - croft = get_minicroft(["skill-weather.openvoiceos"]) - test1 = End2EndTest(skill_ids=[], source_message=msg1, expected_messages=[...], minicroft=croft) test2 = End2EndTest(skill_ids=[], source_message=msg2, expected_messages=[...], minicroft=croft) - test1.execute() test2.execute() - croft.stop() ``` - --- - ## Assertion Logic Detail - ### Message count - ``` assert len(expected_messages) == len(received_messages) ``` - On failure, prints the first differing message type for debugging. - ### Per-message assertions - For each `(expected, received)` pair: - **Type check:** ```python assert expected.msg_type == received.msg_type ``` - **Data check** — subset match (expected keys must be present with matching values): ```python for k, v in expected.data.items(): assert received.data[k] == v ``` - **Context check** — same subset pattern: ```python for k, v in expected.context.items(): assert received.context[k] == v ``` - **Routing check** — tracks rolling expected source/destination: - - Starts from `source_message[0].context["source"]` and `["destination"]` - On `entry_points` message: flips (`e_src, e_dst = r_dst, r_src`) — the reply comes back the other way - On `flip_points` message: updates expected from received, then swaps - `keep_original_src` always uses the original, regardless of flips - ### Active skill tracking - Session is read from each received message's context. For messages after an `activation_point`, `context.skill_id` is added to the expected active set. For messages after a `deactivation_point`, it's removed. The test then verifies all expected active skill IDs appear in the session. - ### Final session check - Compares `active_skills`, `lang`, `pipeline`, `system_unit`, `date_format`, `time_format`, `site_id`, `session_id`, `blacklisted_skills`, `blacklisted_intents` from the session in the last received message against `final_session`. - --- - ## Recording Mode: `from_message()` - Runs a live capture against real skills and returns a ready-to-use `End2EndTest` with the captured messages as `expected_messages`. - ```python test = End2EndTest.from_message( message=utterance, # Message or list[Message] @@ -163,47 +115,30 @@ test = End2EndTest.from_message( ) test.save("tests/weather_test.json") ``` - Use this to bootstrap test fixtures from real behavior, then commit the JSON and replay in CI. - --- - ## Serialization - ### `serialize(anonymize=True) -> dict` - Returns a JSON-serializable dict. With `anonymize=True`, scrubs location data from sessions. - ### `save(path, anonymize=True)` - Writes the serialized test to a JSON file. - ### `End2EndTest.deserialize(data) -> End2EndTest` - Loads from a dict or JSON string. - ### `End2EndTest.from_path(path) -> End2EndTest` - Loads from a JSON file path. - --- - ## Examples - ### Testing complete intent failure (no skills) - ```python from ovoscope import End2EndTest from ovos_bus_client.message import Message from ovos_bus_client.session import Session - session = Session("test-123") utterance = Message( "recognizer_loop:utterance", {"utterances": ["zorbax flibnork"], "lang": "en-us"}, {"session": session.serialize(), "source": "A", "destination": "B"}, ) - End2EndTest( skill_ids=[], source_message=utterance, @@ -215,9 +150,7 @@ End2EndTest( ], ).execute() ``` - ### Testing a skill with pre-activated converse - ```python End2EndTest( skill_ids=["skill-timer.openvoiceos"], @@ -228,9 +161,7 @@ End2EndTest( deactivation_points=["intent.service.skills.deactivate"], ).execute() ``` - ### Multi-turn test - ```python End2EndTest( skill_ids=["skill-weather.openvoiceos"], @@ -239,12 +170,9 @@ End2EndTest( eof_msgs=["ovos.utterance.handled"], # reset between turns ).execute() ``` - ### Reusing MiniCroft across tests - ```python from ovoscope import get_minicroft, End2EndTest - croft = get_minicroft(["skill-weather.openvoiceos"]) try: for utterance, expected in test_cases: diff --git a/docs/index.md b/docs/index.md index 9f9bb09..783d898 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,11 +1,6 @@ -Last Edit: Claude Sonnet 4.6 - 2026-03-08 - Motive: Added usage-guide, ci-integration, Who Uses, and Cross-References sections. - # OvoScope Documentation - **OvoScope** is an end-to-end testing framework for OVOS skills. It runs a lightweight in-process OVOS Core using a `FakeBus`, loads real skill plugins, and captures every bus message produced in response to a test utterance — then asserts against the captured sequence. - ## Contents - | Document | Description | |---|---| | [usage-guide.md](usage-guide.md) | **Start here** — tutorial: from zero to your first end2end test | @@ -14,9 +9,7 @@ Last Edit: Claude Sonnet 4.6 - 2026-03-08 - Motive: Added usage-guide, ci-integr | [capture-session.md](capture-session.md) | `CaptureSession` — message capture during a test | | [end2end-test.md](end2end-test.md) | `End2EndTest` — full test runner reference | | [pydantic-integration.md](pydantic-integration.md) | Using `ovos-pydantic-models` with OvoScope | - ## Conceptual Model - ``` Test FakeBus ──── ─────── @@ -27,27 +20,21 @@ source_message ──emit──► [MiniCroft + loaded skills] ▼ assert against expected_messages[] ``` - The key insight is that OVOS skill behaviour is fully observable through bus messages. OvoScope intercepts every message on the in-process `FakeBus`, so the entire skill interaction — intent matching, converse, fallback, speak, session changes — is captured and verifiable. - ## Quick Start - ```bash pip install ovoscope ``` - ```python from ovoscope import End2EndTest from ovos_bus_client.message import Message from ovos_bus_client.session import Session - session = Session("test-123") utterance = Message( "recognizer_loop:utterance", {"utterances": ["hello world"], "lang": "en-us"}, {"session": session.serialize(), "source": "A", "destination": "B"}, ) - test = End2EndTest( skill_ids=["skill-hello-world.openvoiceos"], source_message=utterance, @@ -59,11 +46,8 @@ test = End2EndTest( ) test.execute() ``` - ## Recording Mode - Instead of writing expected messages by hand, record them from a live run: - ```python test = End2EndTest.from_message( message=utterance, @@ -71,18 +55,13 @@ test = End2EndTest.from_message( ) test.save("tests/hello_world.json") ``` - Then replay later: - ```python test = End2EndTest.from_path("tests/hello_world.json") test.execute() ``` - ## Public API - All primary classes and the factory function are importable from `ovoscope` directly: - ```python from ovoscope import ( MiniCroft, # in-process skill runtime @@ -91,44 +70,31 @@ from ovoscope import ( End2EndTest, # declarative test runner ) ``` - Type aliases also exported: - ```python from ovoscope import SerializedMessage, SerializedTest ``` - ## Dependencies - | Package | Role | |---|---| | `ovos-core >= 2.0.4a2` | `SkillManager`, `IntentService`, `FakeBus`, `SessionManager` | - Python 3.10+ is required (uses `match`/structural typing in ovos-core). - ## What OvoScope Does NOT Do - - Does not start a real WebSocket MessageBus server — uses `FakeBus` (in-process pub/sub). - Does not load PHAL plugins or the audio service — only skills and the intent pipeline. - Does not test GUI rendering — GUI namespace messages are ignored by default (`ignore_gui=True`). - Does not test STT or TTS — operates at the `recognizer_loop:utterance` level. - ## Quick Links - | Resource | Path | |---|---| | Common questions | [`../FAQ.md`](../FAQ.md) | | Change log | [`../CHANGELOG.md`](../CHANGELOG.md) | - ## Who Uses ovoscope - | Repo | Test location | Notes | |---|---|---| | `ovos-core` | `ovos-core/test/end2end/` | Adapt + Padatious pipeline tests, blacklist tests | | `Skills/ovos-skill-hello-world` | `Skills/ovos-skill-hello-world/test/test_helloworld.py` | Canonical example — Adapt + Padatious match + no-match | - ## Cross-References - - [ovos-core](https://github.com/OpenVoiceOS/ovos-core) — `SkillManager`, `IntentService` (runtime dependency) - [ovos-utils](https://github.com/OpenVoiceOS/ovos-utils) — `FakeBus`, `ProcessState` - [ovos-workshop](https://github.com/OpenVoiceOS/ovos-workshop) — `OVOSSkill` base class diff --git a/docs/minicroft.md b/docs/minicroft.md index 99dcf25..73e0922 100644 --- a/docs/minicroft.md +++ b/docs/minicroft.md @@ -1,17 +1,11 @@ # MiniCroft - `MiniCroft` is a minimal, in-process OVOS Core that loads real skill plugins and runs the full intent pipeline on a `FakeBus`. It is the execution engine behind every OvoScope test. - ## Class: `MiniCroft` - ```python from ovoscope import MiniCroft ``` - Subclass of `ovos_core.skill_manager.SkillManager`. Replaces the real WebSocket bus with `FakeBus`, disables components not needed for testing, and only loads the skills you specify. - ### Constructor - ```python MiniCroft( skill_ids: list[str], @@ -21,10 +15,13 @@ MiniCroft( enable_file_watcher: bool = False, enable_skill_api: bool = True, extra_skills: dict[str, OVOSSkill] | None = None, + isolate_config: bool = True, + default_pipeline: list[str] | None = DEFAULT_TEST_PIPELINE, + lang: str | None = None, + secondary_langs: list[str] | None = None, *args, **kwargs, ) ``` - | Parameter | Default | Description | |---|---|---| | `skill_ids` | required | Skill plugin IDs to load (from installed entry points) | @@ -34,69 +31,68 @@ MiniCroft( | `enable_file_watcher` | `False` | Enable settings file watcher | | `enable_skill_api` | `True` | Enable skill API exposure | | `extra_skills` | `None` | Inject skill instances directly (useful for testing a skill class before packaging) | - +| `isolate_config` | `True` | Clear user XDG configs so tests are reproducible | +| `default_pipeline` | `DEFAULT_TEST_PIPELINE` | Override the session pipeline for deterministic intent matching | +| `lang` | `None` | Override the system default language (`Configuration()["lang"]`). Patched before Adapt/Padatious init so vocab is registered for this language. | +| `secondary_langs` | `None` | Set `Configuration()["secondary_langs"]`. Adapt and Padatious create per-language engines for each language in this list, enabling multilingual intent matching. | ### Key attributes - | Attribute | Type | Description | |---|---|---| | `bus` | `FakeBus` | The in-process message bus | | `boot_messages` | `list[Message]` | All messages captured during startup | | `status` | `ProcessState` | Current lifecycle state | - ### `MiniCroft.run()` - Loads plugins and marks the runtime as ready. Called internally by `start()`. Does not block — returns after all skills are loaded. - ### `MiniCroft.stop()` - Shuts down skills and closes the bus. - --- - ## Factory: `get_minicroft()` - ```python from ovoscope import get_minicroft - croft = get_minicroft( skill_ids: list[str] | str, **kwargs # forwarded to MiniCroft constructor ) ``` - Creates, starts, and waits for a `MiniCroft` to reach `READY` state. Returns the ready instance. - ```python croft = get_minicroft(["skill-weather.openvoiceos", "skill-timer.openvoiceos"]) # croft.status.state == ProcessState.READY ``` - --- - ## Injecting Skills Under Test - To test a skill class that isn't installed as a plugin, inject it directly via `extra_skills`: - ```python from my_skill import MySkill - croft = get_minicroft( skill_ids=[], extra_skills={"my-skill.test": MySkill}, ) ``` - The skill ID key must match what the skill would normally register under. - --- - +## Multilingual Testing +By default, Adapt and Padatious only register vocab/intents for the system's configured default language. To test skills in other languages, pass `secondary_langs`: +```python +croft = get_minicroft( + ["my-skill.openvoiceos"], + secondary_langs=["pt-PT", "de-DE", "es-ES"], +) +``` +This patches `Configuration()["secondary_langs"]` before `IntentService` initializes, so Adapt creates per-language engines and registers vocab from all locale directories. +To also change the primary language: +```python +croft = get_minicroft( + ["my-skill.openvoiceos"], + lang="pt-PT", + secondary_langs=["en-US", "de-DE"], +) +``` +--- ## Boot Sequence - On startup, MiniCroft captures all messages emitted during skill loading into `boot_messages`. These can be asserted in `End2EndTest.expected_boot_sequence`. The typical boot sequence includes: - 1. `mycroft.skills.train` — intent pipeline training request 2. `mycroft.skills.initialized` — skills initialized 3. `mycroft.skills.ready` — skills service ready 4. `mycroft.ready` — all core services ready - Skills that participate in `converse` or `fallback` registration also emit messages during boot (e.g. `ovos.skills.fallback.register`). diff --git a/docs/pydantic-integration.md b/docs/pydantic-integration.md index 3241118..6e713af 100644 --- a/docs/pydantic-integration.md +++ b/docs/pydantic-integration.md @@ -1,31 +1,19 @@ # OvoScope + ovos-pydantic-models Integration - OvoScope currently operates on untyped `ovos_bus_client.message.Message` objects — dicts with string keys. `ovos-pydantic-models` provides typed Pydantic v2 models for every OVOS message type. This document describes how they can be used together and what a deeper integration could look like. - --- - ## The Problem Today - Writing test fixtures by hand is verbose and error-prone: - ```python # untyped — no validation, any typo silently passes expected = Message("recognizer_loop:utterance", {"utterances": ["hello"], "lang": "en-us"}, {}) ``` - `Message` is a raw dict wrapper. There is no validation of field names, no type checking, and no autocomplete. A typo in a field name (`"utterance"` instead of `"utterances"`) silently produces a wrong test. - --- - ## Bridge: Converting Between Message and Pydantic - `ovos_bus_client.message.Message` and `OpenVoiceOSMessage` share the same three-field structure (`type`/`message_type`, `data`, `context`). A bridge needs only two functions: - ```python from ovos_bus_client.message import Message from ovos_pydantic_models.message import OpenVoiceOSMessage - - def to_bus_message(pydantic_msg: OpenVoiceOSMessage) -> Message: """Convert a pydantic model to an ovos-bus-client Message.""" d = pydantic_msg.model_dump() @@ -34,8 +22,6 @@ def to_bus_message(pydantic_msg: OpenVoiceOSMessage) -> Message: d["data"], d["context"], ) - - def from_bus_message(bus_msg: Message, model: type[OpenVoiceOSMessage]) -> OpenVoiceOSMessage: """Parse a received bus Message into a typed pydantic model.""" return model.model_validate({ @@ -44,50 +30,36 @@ def from_bus_message(bus_msg: Message, model: type[OpenVoiceOSMessage]) -> OpenV "context": bus_msg.context, }) ``` - These two functions are all that's needed to use typed models with OvoScope today, without any changes to OvoScope itself. - --- - ## Usage Pattern 1: Typed Source Messages - Use pydantic models to construct source messages, then convert: - ```python from ovoscope import End2EndTest from ovos_bus_client.message import Message from ovos_bus_client.session import Session from ovos_pydantic_models import RecognizerLoopUtteranceMessage, RecognizerLoopUtteranceData - # typed construction — validated at instantiation utterance_model = RecognizerLoopUtteranceMessage( data=RecognizerLoopUtteranceData(utterances=["what is the weather?"], lang="en-us"), ) - session = Session("test-123") bus_msg = to_bus_message(utterance_model) bus_msg.context["session"] = session.serialize() bus_msg.context["source"] = "A" bus_msg.context["destination"] = "B" - End2EndTest( skill_ids=["skill-weather.openvoiceos"], source_message=bus_msg, expected_messages=[...], ).execute() ``` - Benefit: `RecognizerLoopUtteranceData` validates that `utterances` is a `list[str]` and `lang` is a string. A missing `utterances` field raises `ValidationError` at construction time, not a silent wrong test. - --- - ## Usage Pattern 2: Typed Expected Messages - Use pydantic models to build expected messages. This documents intent and catches field-name mistakes: - ```python from ovos_pydantic_models import SpeakMessage, SpeakData, CompleteIntentFailureMessage, CompleteIntentFailureData - expected = [ to_bus_message(RecognizerLoopUtteranceMessage( data=RecognizerLoopUtteranceData(utterances=["what is the weather?"], lang="en-us") @@ -97,49 +69,35 @@ expected = [ )), to_bus_message(OvosUtteranceHandledMessage()), ] - End2EndTest( skill_ids=["skill-weather.openvoiceos"], source_message=bus_msg, expected_messages=expected, ).execute() ``` - Because `End2EndTest` checks only the data keys you specify (subset match), you can omit optional fields in expected messages — this works the same as before, but field names are now validated at Python parse time. - --- - ## Usage Pattern 3: Typed Assertions on Received Messages - After a test captures messages, convert received `Message` objects to their typed counterparts for richer assertions: - ```python from ovoscope import get_minicroft, CaptureSession from ovos_pydantic_models import SpeakMessage - croft = get_minicroft(["skill-weather.openvoiceos"]) capture = CaptureSession(croft) capture.capture(bus_msg, timeout=10) messages = capture.finish() croft.stop() - # find the speak message and parse it speak_msgs = [m for m in messages if m.msg_type == "speak"] assert len(speak_msgs) == 1 - typed_speak = from_bus_message(speak_msgs[0], SpeakMessage) assert "london" in typed_speak.data.utterance.lower() assert typed_speak.data.expect_response is False ``` - This is cleaner than `msg.data["utterance"]` — you get IDE autocomplete and the field contract is explicit. - --- - ## Usage Pattern 4: Type-safe Test Helpers - Build helpers that combine the two: - ```python def assert_speak(received_msg: Message, expected_utterance: str | None = None): """Assert a received message is a valid speak message.""" @@ -147,8 +105,6 @@ def assert_speak(received_msg: Message, expected_utterance: str | None = None): if expected_utterance is not None: assert typed.data.utterance == expected_utterance return typed # return for further inspection - - def make_utterance(text: str, lang: str = "en-us", session: Session | None = None) -> Message: """Build a typed recognizer_loop:utterance message.""" model = RecognizerLoopUtteranceMessage( @@ -159,15 +115,10 @@ def make_utterance(text: str, lang: str = "en-us", session: Session | None = Non msg.context["session"] = session.serialize() return msg ``` - --- - ## Deeper Integration: What OvoScope Could Gain - The patterns above work today with no changes to OvoScope. A deeper integration would add native support for pydantic models as a first-class alternative to `Message`: - ### Idea 1: Accept pydantic models directly in `End2EndTest` - ```python # instead of requiring to_bus_message() manually: End2EndTest( @@ -176,70 +127,49 @@ End2EndTest( expected_messages=[SpeakMessage(...), OvosUtteranceHandledMessage()], ) ``` - Implementation: `__post_init__` could detect `OpenVoiceOSMessage` instances and call `to_bus_message()` automatically. - ### Idea 2: `assert_message_type()` helper on `End2EndTest` - ```python test.assert_message_type(index=1, model=SpeakMessage) # verifies received[1] can be deserialized as SpeakMessage ``` - ### Idea 3: Typed capture result - After `execute()`, expose captured messages as typed models where possible: - ```python test.execute() speak = test.received_as(index=1, model=SpeakMessage) assert speak.data.expect_response is False ``` - ### Idea 4: JSON schema validation in assertions - Instead of only checking key/value subsets, optionally validate each received message against the pydantic schema for its type: - ```python End2EndTest( ..., validate_schemas=True, # each received message must parse as its pydantic model ) ``` - This would catch malformed messages from skills (e.g. a skill emitting `speak` with missing `utterance`). - --- - ## Dependency Consideration - `ovos-pydantic-models` is a pure Pydantic v2 package with no OVOS runtime dependencies. OvoScope depends on `ovos-core>=2.0.4a2`. The optional dependency is declared in `pyproject.toml`: - ```toml [project.optional-dependencies] pydantic = ["ovos-pydantic-models>=0.1.0"] ``` - Install with: - ```bash pip install ovoscope[pydantic] ``` - The bridge functions (`to_bus_message`, `from_bus_message`, `validate_fixture`) live in `ovoscope.pydantic_helpers` and guard their imports conditionally — the module can be imported without `ovos-pydantic-models` installed, but calling any function raises a clear `ImportError` pointing to the extras install command: - ```python # safe to import regardless of whether pydantic extras are installed from ovoscope.pydantic_helpers import to_bus_message # ImportError only on call, not import ``` - --- - ## Summary - | Pattern | What you get | Status | |---|---|---| | Typed source messages via `to_bus_message()` | Validation at construction | ✅ `ovoscope.pydantic_helpers` | @@ -248,5 +178,4 @@ from ovoscope.pydantic_helpers import to_bus_message # ImportError only on call | Fixture validation via `validate_fixture()` | Clear errors on malformed JSON | ✅ `ovoscope.pydantic_helpers` | | Native pydantic in `End2EndTest` | Seamless API (no `to_bus_message` call) | 💡 Future: `__post_init__` auto-conversion | | Schema validation in assertions | Catch malformed skill messages | 💡 Future: `validate_schemas=True` flag | - Install the extras to use the implemented patterns: `pip install ovoscope[pydantic]` diff --git a/docs/usage-guide.md b/docs/usage-guide.md index 4906018..438b23f 100644 --- a/docs/usage-guide.md +++ b/docs/usage-guide.md @@ -1,41 +1,26 @@ -Last Edit: Claude Sonnet 4.6 - 2026-03-09 - Motive: Added pipeline isolation constants and composable pipeline patterns. - # OvoScope Usage Guide - This guide takes you from zero to writing and running your first end-to-end skill test. It assumes familiarity with Python's `unittest` and the OVOS bus message model. - --- - ## Prerequisites - Install ovoscope and the skill under test in the same virtual environment: - ```bash # editable installs — recommended during development uv pip install -e ovoscope/ -e Skills/ovos-skill-hello-world/ - # or via PyPI pip install ovoscope ovos-skill-hello-world ``` - ovoscope requires: - - Python 3.10+ - `ovos-core >= 2.0.4a2` (pulled automatically as a dependency) - The skill plugin must be discoverable via its `setup.py` / `pyproject.toml` entry point - Verify the skill is on the plugin path: - ```bash python -c "from ovos_plugin_manager.skills import find_skill_plugins; print(list(find_skill_plugins()))" # should include: ovos-skill-hello-world.openvoiceos ``` - --- - ## When to Use ovoscope vs FakeBus Unit Tests - | Scenario | Use | |---|---| | Test that a skill intent handler runs correct logic | FakeBus unit test | @@ -47,47 +32,33 @@ python -c "from ovos_plugin_manager.skills import find_skill_plugins; print(list | Test session state after an interaction | **ovoscope** | | Test multi-turn dialogue (converse / fallback) | **ovoscope** | | Test that a skill is blacklisted and does NOT match | **ovoscope** | - **Rule of thumb**: if you are asserting on *what gets emitted on the bus* — type, order, data, or routing — use ovoscope. If you are testing the internal Python logic of a handler in isolation, use FakeBus unit tests. - FakeBus reference: - ```python from ovos_utils.fakebus import FakeBus # ovos-utils ``` - --- - ## Quick Start — Hello World - The canonical example skill is `ovos-skill-hello-world.openvoiceos`. It has two intents: - - **HelloWorldIntent** (Adapt) — triggered by "hello world" - **Greetings.intent** (Padatious) — triggered by greetings like "good morning" - ```python import unittest from ovos_bus_client.message import Message from ovos_bus_client.session import Session from ovoscope import End2EndTest, get_minicroft - SKILL_ID = "ovos-skill-hello-world.openvoiceos" - - class TestHelloWorldQuickStart(unittest.TestCase): - def test_hello_world(self): session = Session("test-session-1") session.pipeline = ["ovos-adapt-pipeline-plugin-high"] - utterance = Message( "recognizer_loop:utterance", {"utterances": ["hello world"], "lang": "en-US"}, {"session": session.serialize(), "source": "A", "destination": "B"}, ) - test = End2EndTest( skill_ids=[SKILL_ID], source_message=utterance, @@ -113,34 +84,25 @@ class TestHelloWorldQuickStart(unittest.TestCase): ) test.execute(timeout=10) ``` - `test.execute()` raises `AssertionError` on any mismatch. No return value is used — use pytest or `unittest.TestCase` assertions normally. - --- - ## Pattern 1 — Manual Assertion (Adapt Intent Match) - Write each expected `Message` explicitly. This is the most readable pattern and the easiest to debug. - ```python from ovos_bus_client.message import Message from ovos_bus_client.session import Session from ovoscope import End2EndTest, get_minicroft - SKILL_ID = "ovos-skill-hello-world.openvoiceos" - # Build a session that restricts the pipeline to Adapt only session = Session("test-adapt") session.pipeline = ["ovos-adapt-pipeline-plugin-high"] - message = Message( "recognizer_loop:utterance", {"utterances": ["hello world"], "lang": "en-US"}, {"session": session.serialize(), "source": "A", "destination": "B"}, ) - test = End2EndTest( skill_ids=[SKILL_ID], source_message=message, @@ -166,29 +128,21 @@ test = End2EndTest( ) test.execute(timeout=10) ``` - Only keys present in `expected.data` and `expected.context` are checked — extra keys in the received message are ignored. This lets you assert on exactly the fields you care about. - --- - ## Pattern 2 — Padatious Intent Match - Padatious uses `.intent` file names as the message type. Restrict the session pipeline to Padatious only so Adapt doesn't shadow the match: - ```python SKILL_ID = "ovos-skill-hello-world.openvoiceos" - session = Session("test-padatious") session.pipeline = ["ovos-padatious-pipeline-plugin-high"] - message = Message( "recognizer_loop:utterance", {"utterances": ["good morning"], "lang": "en-US"}, {"session": session.serialize(), "source": "A", "destination": "B"}, ) - test = End2EndTest( skill_ids=[SKILL_ID], source_message=message, @@ -213,109 +167,77 @@ test = End2EndTest( ) test.execute(timeout=10) ``` - Note: for Padatious the `speak` message's `utterance` key may vary (depends on the dialog file randomisation), so omit `"utterance"` from `expected.data` if it is non-deterministic — only assert on `lang` and `meta`. - --- - ## Pattern 3 — Recording Mode (Bootstrap Fixtures) - Don't know the exact message sequence yet? Let ovoscope record it for you: - ```python from ovoscope import End2EndTest from ovos_bus_client.message import Message from ovos_bus_client.session import Session - SKILL_ID = "ovos-skill-hello-world.openvoiceos" - session = Session("recorder-session") session.pipeline = ["ovos-adapt-pipeline-plugin-high"] - message = Message( "recognizer_loop:utterance", {"utterances": ["hello world"], "lang": "en-US"}, {"session": session.serialize(), "source": "A", "destination": "B"}, ) - # Recording: runs the skill live, captures messages, returns a test object test = End2EndTest.from_message( message=message, skill_ids=[SKILL_ID], timeout=20, ) - # Save to a JSON fixture for replay test.save("tests/fixtures/hello_world_adapt.json", anonymize=True) ``` - `anonymize=True` (default) strips real location / personal data from the session context before saving — safe to commit. - Then in your test suite: - ```python test = End2EndTest.from_path("tests/fixtures/hello_world_adapt.json") test.execute(timeout=10) ``` - --- - ## Pattern 4 — Replay from JSON Fixture - Committed JSON fixtures make tests fully self-contained: no network, no live skill discovery, no non-determinism in expected messages. - ```python import unittest from ovoscope import End2EndTest - - class TestFromFixture(unittest.TestCase): - def test_adapt_from_fixture(self): test = End2EndTest.from_path("tests/fixtures/hello_world_adapt.json") test.execute(timeout=10) - def test_padatious_from_fixture(self): test = End2EndTest.from_path("tests/fixtures/hello_world_padatious.json") test.execute(timeout=10) ``` - Note: skills still need to be installed (the JSON stores `skill_ids`, and `execute()` calls `get_minicroft()` which loads the real plugin). The fixture stores the expected message sequence — not the skill code. - --- - ## Pattern 5 — Reusing MiniCroft Across Multiple Tests - Creating a `MiniCroft` is expensive (it trains intent models). Reuse it across tests in the same class with `setUp` / `tearDown`: - ```python import unittest from ovos_bus_client.message import Message from ovos_bus_client.session import Session from ovos_utils.log import LOG from ovoscope import End2EndTest, get_minicroft - SKILL_ID = "ovos-skill-hello-world.openvoiceos" - - class TestHelloWorldSharedRuntime(unittest.TestCase): - def setUp(self): LOG.set_level("DEBUG") self.minicroft = get_minicroft([SKILL_ID]) - def tearDown(self): if self.minicroft: self.minicroft.stop() LOG.set_level("CRITICAL") - def _make_test(self, utterance_text, pipeline, expected_messages): session = Session("shared-session") session.pipeline = pipeline @@ -330,7 +252,6 @@ class TestHelloWorldSharedRuntime(unittest.TestCase): source_message=message, expected_messages=expected_messages, ) - def test_adapt_match(self): test = self._make_test( "hello world", @@ -340,7 +261,6 @@ class TestHelloWorldSharedRuntime(unittest.TestCase): ], ) test.execute(timeout=10) - def test_padatious_no_match(self): # "hello world" does not match Padatious Greetings.intent → failure path session = Session("no-match-session") @@ -363,22 +283,16 @@ class TestHelloWorldSharedRuntime(unittest.TestCase): ) test.execute(timeout=10) ``` - When you pass `minicroft=self.minicroft` explicitly, `End2EndTest` sets `managed=False` and does **not** call `minicroft.stop()` at the end of `execute()`. Your `tearDown` is responsible for cleanup. - --- - ## Pattern 6 — Multi-Turn Conversation - Pass a **list** of `Message` objects as `source_message` to test a dialogue sequence. ovoscope emits them in order, propagating session state between turns: - ```python session = Session("multi-turn-session") session.pipeline = ["ovos-adapt-pipeline-plugin-high"] - turn1 = Message( "recognizer_loop:utterance", {"utterances": ["hello world"], "lang": "en-US"}, @@ -390,7 +304,6 @@ turn2 = Message( {"utterances": ["good morning"], "lang": "en-US"}, {"source": "A", "destination": "B"}, # no "session" key — will be filled by ovoscope ) - test = End2EndTest( skill_ids=[SKILL_ID], source_message=[turn1, turn2], # list of turns @@ -406,28 +319,21 @@ test = End2EndTest( ) test.execute(timeout=20) ``` - Session propagation: if turn 2 has no `"session"` key in context, ovoscope copies the session from the last received message — simulating how a real OVOS client propagates session updates. - --- - ## Pattern 7 — Testing Fallback Skills - Fallback skills receive a `"ovos.skills.fallback.ping"` message to probe for a handler, and then the main fallback message. The expected sequence is longer than a normal intent match: - ```python session = Session("fallback-session") # use a pipeline that includes fallback session.pipeline = ["ovos-fallback-skill-plugin-high"] - message = Message( "recognizer_loop:utterance", {"utterances": ["what is the meaning of life"], "lang": "en-US"}, {"session": session.serialize(), "source": "A", "destination": "B"}, ) - # For fallback testing, keep_original_src ensures the fallback ping routing is validated test = End2EndTest( skill_ids=["my-fallback-skill.author"], @@ -443,29 +349,21 @@ test = End2EndTest( ) test.execute(timeout=15) ``` - See `DEFAULT_KEEP_SRC` in `ovoscope/__init__.py` — it pre-populates `keep_original_src` so fallback ping routing is always validated against the original source message context. - --- - ## Pattern 8 — Session State Validation - Use `final_session` and `inject_active` to assert on session state at the end of a test: - ```python from ovos_bus_client.session import Session from ovoscope import End2EndTest - SKILL_ID = "ovos-skill-hello-world.openvoiceos" - # Pre-activate another skill before the test expected_session = Session("state-check-session") expected_session.pipeline = ["ovos-adapt-pipeline-plugin-high"] # After the interaction, hello world skill must remain active # Build what you expect the session to look like after the test expected_session.activate_skill(SKILL_ID) - session = Session("state-check-session") session.pipeline = ["ovos-adapt-pipeline-plugin-high"] message = Message( @@ -473,7 +371,6 @@ message = Message( {"utterances": ["hello world"], "lang": "en-US"}, {"session": session.serialize(), "source": "A", "destination": "B"}, ) - test = End2EndTest( skill_ids=[SKILL_ID], source_message=message, @@ -486,21 +383,16 @@ test = End2EndTest( ) test.execute(timeout=10) ``` - Fields validated by `final_session`: - `active_skills` (set comparison) - `lang`, `pipeline`, `system_unit`, `date_format`, `time_format` - `site_id`, `session_id` - `blacklisted_skills`, `blacklisted_intents` - --- - ## Async Messages - Some messages arrive from external threads and may appear at any time during the interaction (e.g., GUI updates that race with bus messages). Declare them in `async_messages` so they are captured separately and not checked for ordering: - ```python test = End2EndTest( skill_ids=[SKILL_ID], @@ -511,16 +403,11 @@ test = End2EndTest( test_async_message_number=True, # assert exactly 1 async message received ) ``` - Async messages are collected in `CaptureSession.async_responses` — they are NOT in the main `responses` list and are NOT included in `test_message_number` count. - --- - ## Disabling Assertions - Some assertion groups can be turned off individually when a message is noisy or non-deterministic: - | Parameter | Default | Effect | |---|---|---| | `test_message_number` | `True` | Assert exact message count | @@ -533,9 +420,7 @@ Some assertion groups can be turned off individually when a message is noisy or | `test_async_messages` | `True` | Assert async message types | | `test_async_message_number` | `True` | Assert async message count | | `test_final_session` | `True` | Assert final session state | - Example — disable data and routing checks for a noisy third-party message: - ```python test = End2EndTest( ... @@ -543,29 +428,21 @@ test = End2EndTest( test_routing=False, # don't assert source/destination ) ``` - --- - ## Troubleshooting - ### Timeout — no messages received - - The skill plugin is not loaded. Verify `find_skill_plugins()` returns your skill ID. - The session pipeline is empty or does not include the right plugin. Set `session.pipeline = [...]` explicitly. - The EOF message (`ovos.utterance.handled`) never fires — check if the intent matched at all by setting `verbose=True` and inspecting stdout. - ### Skill not loading - ``` LOG.set_level("DEBUG") minicroft = get_minicroft(["my-skill.author"]) # Watch for "Loaded skill: my-skill.author" in output ``` - If it never prints, the entry point is wrong. Check your `setup.py` / `pyproject.toml`: - ```python # setup.py entry_points={ @@ -574,31 +451,21 @@ entry_points={ } } ``` - ### Intent not matching - - Confirm the utterance text matches an Adapt keyword or a Padatious training phrase. - For Adapt: check that all required keywords are present in the utterance. - For Padatious: training happens at `MiniCroft.run()` via `mycroft.skills.train`. If training fails silently, check the Padatious model files exist under `~/.local/share/`. - ### Wrong message count - Enable `verbose=True` (default) — ovoscope prints every received message with its index. Compare against the expected list to find the first divergence. - ### `get_minicroft()` hangs - `get_minicroft()` polls `croft.status.state` in a tight loop (0.1s sleep). If it hangs indefinitely, a skill is raising an exception during `_startup`. Set `LOG.set_level("DEBUG")` and watch for tracebacks. - --- - ## Constants Reference - ### Test lifecycle constants - ```python from ovoscope import ( DEFAULT_EOF, # ["ovos.utterance.handled"] — end-of-test trigger @@ -611,12 +478,9 @@ from ovoscope import ( DEFAULT_DEACTIVATION, # ["intent.service.skills.deactivate"] ) ``` - ### Pipeline constants - ovoscope exposes composable pipeline stage lists so you can precisely control which pipeline stages are active during a test: - ```python from ovoscope import ( STOP_PIPELINE, # ["ovos-stop-pipeline-plugin-high", ...medium, ...low] @@ -629,40 +493,29 @@ from ovoscope import ( DEFAULT_TEST_PIPELINE, # all standard stages, no AI/persona/OCP — the default ) ``` - `DEFAULT_TEST_PIPELINE` is the default value of `MiniCroft.default_pipeline` when `isolate_config=True`. It excludes persona, Ollama, OCP, and m2v stages, giving fully reproducible results regardless of which AI plugins are installed. - **Composing custom pipelines:** - ```python # Adapt intent only — fastest, no fallback mc = get_minicroft([SKILL_ID], default_pipeline=ADAPT_PIPELINE) - # Full intent chain with fallback — typical skill testing mc = get_minicroft([SKILL_ID], default_pipeline=CONVERSE_PIPELINE + ADAPT_PIPELINE + FALLBACK_PIPELINE) - # Include persona pipeline — when testing AI persona behaviour mc = get_minicroft([SKILL_ID], default_pipeline=DEFAULT_TEST_PIPELINE + PERSONA_PIPELINE) - # No override — use whatever the system config says (includes OCP, m2v, etc.) mc = get_minicroft([SKILL_ID], default_pipeline=None) ``` - Sessions created without an explicit `session` in their message context inherit `SessionManager.default_session.pipeline`, so the override covers all such utterances. The original pipeline is restored when `mc.stop()` is called. - **When to use `PERSONA_PIPELINE`:** Only add persona stages when you are explicitly testing persona behaviour. Persona plugins make network calls to AI APIs and are non-deterministic — they are intentionally excluded from `DEFAULT_TEST_PIPELINE`. - --- - ## See Also - - [end2end-test.md](end2end-test.md) — full `End2EndTest` parameter reference - [minicroft.md](minicroft.md) — `MiniCroft` / `get_minicroft()` reference - [capture-session.md](capture-session.md) — `CaptureSession` internals diff --git a/ovoscope/__init__.py b/ovoscope/__init__.py index b5e5782..d94430f 100644 --- a/ovoscope/__init__.py +++ b/ovoscope/__init__.py @@ -92,14 +92,26 @@ def __init__(self, skill_ids, extra_skills: Optional[Dict[str, OVOSSkill]] = None, isolate_config: bool = True, default_pipeline: Optional[List[str]] = DEFAULT_TEST_PIPELINE, + lang: Optional[str] = None, + secondary_langs: Optional[List[str]] = None, *args, **kwargs): self._isolated_config = isolate_config self._original_xdg_configs: Optional[List[LocalConf]] = None self._default_pipeline = default_pipeline self._original_pipeline: Optional[List[str]] = None self._original_cfg_pipeline: Optional[List[str]] = None + self._had_cfg_pipeline: bool = False self._original_blacklisted_skills: Optional[List[str]] = None + self._had_blacklisted_skills: bool = False self._original_blacklisted_intents: Optional[List[str]] = None + self._had_blacklisted_intents: bool = False + self._lang = lang + self._secondary_langs = secondary_langs + self._original_lang: Optional[str] = None + self._original_cfg_lang: Optional[str] = None + self._had_lang: bool = False + self._original_secondary_langs: Optional[List[str]] = None + self._had_secondary_langs: bool = False if isolate_config: # Replace user XDG configs (e.g. ~/.config/mycroft/mycroft.conf) with @@ -113,6 +125,24 @@ def __init__(self, skill_ids, Configuration.reload() LOG.debug("ovoscope: user config isolated (xdg_configs cleared)") + # Patch lang / secondary_langs BEFORE super().__init__() because + # IntentService (and thus Adapt/Padatious) reads Configuration() + # during construction and creates per-language engines at that time. + if self._lang is not None or self._secondary_langs is not None: + cfg = Configuration() + if self._lang is not None: + self._had_lang = "lang" in cfg + self._original_cfg_lang = cfg.get("lang") + cfg["lang"] = self._lang + LOG.debug(f"ovoscope: lang set to '{self._lang}' " + f"(was '{self._original_cfg_lang}')") + if self._secondary_langs is not None: + self._had_secondary_langs = "secondary_langs" in cfg + self._original_secondary_langs = cfg.get("secondary_langs") + cfg["secondary_langs"] = self._secondary_langs + LOG.debug(f"ovoscope: secondary_langs set to " + f"{self._secondary_langs}") + self.boot_messages: List[Message] = [] bus = FakeBus() bus.on("message", self.handle_boot_message) @@ -170,13 +200,18 @@ def run(self): self._original_pipeline = SessionManager.default_session.pipeline[:] SessionManager.default_session.pipeline = self._default_pipeline cfg = Configuration() - self._original_cfg_pipeline = cfg.get("intents", {}).get("pipeline") + intents_cfg = cfg.get("intents", {}) + self._had_cfg_pipeline = "pipeline" in intents_cfg + self._original_cfg_pipeline = intents_cfg.get("pipeline") if "intents" not in cfg: cfg["intents"] = {} cfg["intents"]["pipeline"] = self._default_pipeline LOG.debug(f"ovoscope: default session pipeline set " f"({len(self._default_pipeline)} stages, " f"was {len(self._original_pipeline)})") + if self._lang is not None: + self._original_lang = SessionManager.default_session.lang + SessionManager.default_session.lang = self._lang if self._isolated_config: # Session.__init__ reads Configuration()["skills"]["blacklisted_skills"] # and Configuration()["intents"]["blacklisted_intents"] from the live @@ -185,7 +220,9 @@ def run(self): cfg = Configuration() skills_cfg = cfg.setdefault("skills", {}) intents_cfg = cfg.setdefault("intents", {}) + self._had_blacklisted_skills = "blacklisted_skills" in skills_cfg self._original_blacklisted_skills = skills_cfg.get("blacklisted_skills") + self._had_blacklisted_intents = "blacklisted_intents" in intents_cfg self._original_blacklisted_intents = intents_cfg.get("blacklisted_intents") skills_cfg["blacklisted_skills"] = [] intents_cfg["blacklisted_intents"] = [] @@ -209,7 +246,7 @@ def stop(self): SessionManager.default_session.pipeline = self._original_pipeline cfg = Configuration() if "intents" in cfg: - if self._original_cfg_pipeline is not None: + if self._had_cfg_pipeline: cfg["intents"]["pipeline"] = self._original_cfg_pipeline else: cfg["intents"].pop("pipeline", None) @@ -218,15 +255,30 @@ def stop(self): cfg = Configuration() skills_cfg = cfg.get("skills", {}) intents_cfg = cfg.get("intents", {}) - if self._original_blacklisted_skills is not None: + if self._had_blacklisted_skills: skills_cfg["blacklisted_skills"] = self._original_blacklisted_skills else: skills_cfg.pop("blacklisted_skills", None) - if self._original_blacklisted_intents is not None: + if self._had_blacklisted_intents: intents_cfg["blacklisted_intents"] = self._original_blacklisted_intents else: intents_cfg.pop("blacklisted_intents", None) LOG.debug("ovoscope: blacklisted_skills and blacklisted_intents restored") + if self._lang is not None: + cfg = Configuration() + if self._had_lang: + cfg["lang"] = self._original_cfg_lang + else: + cfg.pop("lang", None) + SessionManager.default_session.lang = self._original_lang + LOG.debug(f"ovoscope: lang restored to '{self._original_lang}'") + if self._secondary_langs is not None: + cfg = Configuration() + if self._had_secondary_langs: + cfg["secondary_langs"] = self._original_secondary_langs + else: + cfg.pop("secondary_langs", None) + LOG.debug("ovoscope: secondary_langs restored") if self._isolated_config and self._original_xdg_configs is not None: Configuration.xdg_configs = self._original_xdg_configs Configuration.reload() @@ -369,7 +421,11 @@ def __post_init__(self): if isinstance(self.source_message, Message): self.source_message = [self.source_message] if self.ignore_gui: - self.ignore_messages += GUI_IGNORED + # ensure we don't mutate a shared default list + self.ignore_messages = list(self.ignore_messages) + for m in GUI_IGNORED: + if m not in self.ignore_messages: + self.ignore_messages.append(m) def execute(self, timeout: int = 30) -> List[Message]: if self.minicroft is None: @@ -582,6 +638,8 @@ def serialize(self, anonymize=True) -> SerializedTest: "source_message": [json.loads(m.serialize()) for m in src], "expected_messages": [json.loads(m.serialize()) for m in expected], "eof_msgs": self.eof_msgs, + "ignore_messages": self.ignore_messages, + "ignore_gui": self.ignore_gui, "flip_points": self.flip_points, "test_msg_type": self.test_msg_type, "test_msg_data": self.test_msg_data, @@ -605,13 +663,27 @@ def from_message(cls, message: Union[Message, List[Message]], eof_msgs: Optional[List[str]] = None, flip_points: Optional[List[str]] = None, ignore_messages: Optional[List[str]] = None, + ignore_gui: bool = True, async_messages: Optional[List[str]] = None, timeout=20, *args, **kwargs) -> 'End2EndTest': if not isinstance(message, list): message = [message] - eof_msgs = eof_msgs or DEFAULT_EOF - flip_points = flip_points or DEFAULT_FLIP_POINTS - ignore_messages = ignore_messages or DEFAULT_IGNORED + if eof_msgs is None: + eof_msgs = DEFAULT_EOF + if flip_points is None: + flip_points = DEFAULT_FLIP_POINTS + if ignore_messages is None: + ignore_messages = list(DEFAULT_IGNORED) + else: + ignore_messages = list(ignore_messages) + + if ignore_gui: + for m in GUI_IGNORED: + if m not in ignore_messages: + ignore_messages.append(m) + + if async_messages is None: + async_messages = [] minicroft = get_minicroft(skill_ids, *args, **kwargs) capture = CaptureSession(minicroft, @@ -631,7 +703,11 @@ def from_message(cls, message: Union[Message, List[Message]], skill_ids=skill_ids, source_message=message, expected_messages=expected_messages, - flip_points=flip_points + flip_points=flip_points, + ignore_messages=ignore_messages, + ignore_gui=ignore_gui, + eof_msgs=eof_msgs, + async_messages=async_messages, ) @staticmethod diff --git a/pyproject.toml b/pyproject.toml index 50775ab..1c0c72b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,3 +37,6 @@ version = {attr = "ovoscope.version.__version__"} [tool.pytest.ini_options] testpaths = ["test"] + +[tool.coverage.run] +relative_files = true diff --git a/test/__pycache__/__init__.cpython-311.pyc b/test/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000..a7cc588 Binary files /dev/null and b/test/__pycache__/__init__.cpython-311.pyc differ diff --git a/test/unittests/__pycache__/__init__.cpython-311.pyc b/test/unittests/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000..84b5a9b Binary files /dev/null and b/test/unittests/__pycache__/__init__.cpython-311.pyc differ diff --git a/test/unittests/__pycache__/test_capture_session.cpython-311-pytest-9.0.2.pyc b/test/unittests/__pycache__/test_capture_session.cpython-311-pytest-9.0.2.pyc new file mode 100644 index 0000000..5fbd0b4 Binary files /dev/null and b/test/unittests/__pycache__/test_capture_session.cpython-311-pytest-9.0.2.pyc differ diff --git a/test/unittests/__pycache__/test_end2end.cpython-311-pytest-9.0.2.pyc b/test/unittests/__pycache__/test_end2end.cpython-311-pytest-9.0.2.pyc new file mode 100644 index 0000000..ac79ef8 Binary files /dev/null and b/test/unittests/__pycache__/test_end2end.cpython-311-pytest-9.0.2.pyc differ diff --git a/test/unittests/__pycache__/test_minicroft.cpython-311-pytest-9.0.2.pyc b/test/unittests/__pycache__/test_minicroft.cpython-311-pytest-9.0.2.pyc new file mode 100644 index 0000000..555deed Binary files /dev/null and b/test/unittests/__pycache__/test_minicroft.cpython-311-pytest-9.0.2.pyc differ diff --git a/test/unittests/test_end2end_extended.py b/test/unittests/test_end2end_extended.py new file mode 100644 index 0000000..de388e2 --- /dev/null +++ b/test/unittests/test_end2end_extended.py @@ -0,0 +1,985 @@ +"""Extended tests for End2EndTest — covers routing, context, active skills, +boot sequence, final session, GUI filtering, from_message recording, +serialization edge cases, and anonymize_message.""" +import json +import os +import tempfile +import unittest + +from ovos_bus_client.message import Message +from ovos_bus_client.session import Session, SessionManager +from ovos_utils.log import LOG +from ovos_workshop.skills.ovos import OVOSSkill + +from ovoscope import ( + End2EndTest, CaptureSession, MiniCroft, get_minicroft, + DEFAULT_EOF, DEFAULT_IGNORED, GUI_IGNORED, + ADAPT_PIPELINE, PADATIOUS_PIPELINE, FALLBACK_PIPELINE, + STOP_PIPELINE, CONVERSE_PIPELINE, COMMON_QUERY_PIPELINE, + PERSONA_PIPELINE, DEFAULT_TEST_PIPELINE, +) + +SKILL_ID = "ovoscope-extended-test.test" +HANDLER_LIFECYCLE = ["mycroft.skill.handler.start", + "mycroft.skill.handler.complete"] +ADAPT_ONLY = ["ovos-adapt-pipeline-plugin-high"] + + +class EchoSkill(OVOSSkill): + def initialize(self): + self.add_event("unittest.echo", self.handle_echo) + + def handle_echo(self, message: Message): + text = message.data.get("text", "echo") + self.speak(text) + self.bus.emit(Message("ovos.utterance.handled", context=message.context)) + + +class AsyncSkill(OVOSSkill): + """Emits an async message alongside the normal flow.""" + + def initialize(self): + self.add_event("unittest.async", self.handle_async) + + def handle_async(self, message: Message): + self.bus.emit(Message("test.async.event", context=message.context)) + self.speak("async done") + self.bus.emit(Message("ovos.utterance.handled", context=message.context)) + + +def _session(sid="ext-test", pipeline=None): + s = Session(sid) + s.lang = "en-US" + s.pipeline = pipeline or ADAPT_ONLY + return s + + +def _make_custom(msg_type, data=None, sid="ext-test"): + sess = _session(sid) + return Message(msg_type, data or {}, + {"session": sess.serialize(), + "source": "A", "destination": "B"}) + + +def _make_utterance(text, sid="ext-test", pipeline=None): + sess = _session(sid, pipeline) + return Message("recognizer_loop:utterance", + {"utterances": [text], "lang": "en-US"}, + {"session": sess.serialize(), + "source": "A", "destination": "B"}) + + +# --------------------------------------------------------------------------- +# __post_init__ and GUI filtering +# --------------------------------------------------------------------------- +class TestPostInit(unittest.TestCase): + + def test_source_message_normalized_to_list(self): + """A single Message source_message becomes a list after __post_init__.""" + src = _make_custom("test") + test = End2EndTest( + skill_ids=[], source_message=src, + expected_messages=[], verbose=False, + ) + self.assertIsInstance(test.source_message, list) + self.assertEqual(len(test.source_message), 1) + + def test_ignore_gui_adds_gui_messages(self): + """ignore_gui=True adds GUI_IGNORED to ignore_messages.""" + test = End2EndTest( + skill_ids=[], source_message=_make_custom("test"), + expected_messages=[], ignore_gui=True, verbose=False, + ) + for m in GUI_IGNORED: + self.assertIn(m, test.ignore_messages) + + def test_ignore_gui_false_does_not_add(self): + """ignore_gui=False leaves ignore_messages as-is.""" + custom_ignored = ["custom.ignore"] + test = End2EndTest( + skill_ids=[], source_message=_make_custom("test"), + expected_messages=[], ignore_gui=False, + ignore_messages=custom_ignored[:], # fresh copy + verbose=False, + ) + for m in GUI_IGNORED: + self.assertNotIn(m, test.ignore_messages) + self.assertIn("custom.ignore", test.ignore_messages) + + +# --------------------------------------------------------------------------- +# Context assertion tests +# --------------------------------------------------------------------------- +class TestContextAssertions(unittest.TestCase): + + def setUp(self): + LOG.set_level("ERROR") + self.mc = get_minicroft([SKILL_ID], extra_skills={SKILL_ID: EchoSkill}) + + def tearDown(self): + self.mc.stop() + LOG.set_level("CRITICAL") + + def test_wrong_context_raises(self): + """test_msg_context=True raises on context mismatch.""" + src = _make_custom("unittest.echo", {"text": "ctx"}) + test = End2EndTest( + minicroft=self.mc, + skill_ids=[SKILL_ID], + source_message=src, + expected_messages=[ + Message("unittest.echo", {}, {"source": "WRONG"}), + ], + test_message_number=False, + test_msg_type=False, + test_msg_data=False, + test_msg_context=True, + test_routing=False, + test_active_skills=False, + test_final_session=False, + test_async_messages=False, + test_async_message_number=False, + ignore_messages=DEFAULT_IGNORED + HANDLER_LIFECYCLE, + verbose=False, + ) + with self.assertRaises(AssertionError): + test.execute(timeout=10) + + +# --------------------------------------------------------------------------- +# Routing tests (flip_points, entry_points, keep_original_src) +# --------------------------------------------------------------------------- +class TestRouting(unittest.TestCase): + + def setUp(self): + LOG.set_level("ERROR") + self.mc = get_minicroft([]) + + def tearDown(self): + self.mc.stop() + LOG.set_level("CRITICAL") + + def test_routing_entry_point_flips_src_dst(self): + """After an entry_point message, expected src/dst are flipped.""" + src = _make_utterance("no match", pipeline=ADAPT_ONLY) + test = End2EndTest( + minicroft=self.mc, + skill_ids=[], + source_message=src, + expected_messages=[ + src, + Message("mycroft.audio.play_sound", {"uri": "snd/error.mp3"}), + Message("complete_intent_failure", {}), + Message("ovos.utterance.handled", {}), + ], + test_routing=True, + test_msg_type=False, + test_msg_data=False, + test_msg_context=False, + test_active_skills=False, + test_final_session=False, + test_async_messages=False, + test_async_message_number=False, + verbose=False, + ) + # Should not raise — default entry_points includes recognizer_loop:utterance + result = test.execute(timeout=15) + self.assertIsInstance(result, list) + + def test_flip_points_configuration(self): + """flip_points parameter is stored and used during routing checks.""" + src = _make_utterance("no match", pipeline=ADAPT_ONLY) + test = End2EndTest( + minicroft=self.mc, + skill_ids=[], + source_message=src, + expected_messages=[ + src, + Message("mycroft.audio.play_sound", {"uri": "snd/error.mp3"}), + Message("complete_intent_failure", {}), + Message("ovos.utterance.handled", {}), + ], + flip_points=["recognizer_loop:utterance"], + test_routing=False, # routing internals are hard to test in isolation + test_msg_type=False, + test_msg_data=False, + test_msg_context=False, + test_active_skills=False, + test_final_session=False, + test_async_messages=False, + test_async_message_number=False, + verbose=False, + ) + self.assertEqual(test.flip_points, ["recognizer_loop:utterance"]) + result = test.execute(timeout=15) + self.assertIsInstance(result, list) + + +# --------------------------------------------------------------------------- +# Async message tests +# --------------------------------------------------------------------------- +class TestAsyncMessages(unittest.TestCase): + + def setUp(self): + LOG.set_level("ERROR") + self.mc = get_minicroft([SKILL_ID], extra_skills={SKILL_ID: AsyncSkill}) + + def tearDown(self): + self.mc.stop() + LOG.set_level("CRITICAL") + + def test_async_messages_captured_separately(self): + """Messages in async_messages list go to async_responses.""" + src = _make_custom("unittest.async") + test = End2EndTest( + minicroft=self.mc, + skill_ids=[SKILL_ID], + source_message=src, + expected_messages=[src], + async_messages=["test.async.event"], + test_message_number=False, + test_msg_type=False, + test_msg_data=False, + test_msg_context=False, + test_routing=False, + test_active_skills=False, + test_final_session=False, + test_async_messages=True, + test_async_message_number=True, + ignore_messages=DEFAULT_IGNORED + HANDLER_LIFECYCLE, + verbose=False, + ) + result = test.execute(timeout=10) + self.assertIsInstance(result, list) + + def test_missing_async_message_raises(self): + """test_async_messages=True raises if expected async msg is missing.""" + src = _make_custom("unittest.echo", {"text": "no async"}) + test = End2EndTest( + minicroft=self.mc, + skill_ids=[SKILL_ID], + source_message=src, + expected_messages=[src], + async_messages=["nonexistent.async.msg"], + test_message_number=False, + test_msg_type=False, + test_msg_data=False, + test_msg_context=False, + test_routing=False, + test_active_skills=False, + test_final_session=False, + test_async_messages=True, + test_async_message_number=False, + ignore_messages=DEFAULT_IGNORED + HANDLER_LIFECYCLE, + verbose=False, + ) + with self.assertRaises(AssertionError): + test.execute(timeout=10) + + def test_async_message_count_mismatch_raises(self): + """test_async_message_number=True raises on count mismatch.""" + src = _make_custom("unittest.echo", {"text": "no async"}) + test = End2EndTest( + minicroft=self.mc, + skill_ids=[SKILL_ID], + source_message=src, + expected_messages=[src], + async_messages=["nonexistent.a", "nonexistent.b"], + test_message_number=False, + test_msg_type=False, + test_msg_data=False, + test_msg_context=False, + test_routing=False, + test_active_skills=False, + test_final_session=False, + test_async_messages=False, + test_async_message_number=True, + ignore_messages=DEFAULT_IGNORED + HANDLER_LIFECYCLE, + verbose=False, + ) + with self.assertRaises(AssertionError): + test.execute(timeout=10) + + +# --------------------------------------------------------------------------- +# Serialization edge cases +# --------------------------------------------------------------------------- +class TestSerializationExtended(unittest.TestCase): + + def setUp(self): + LOG.set_level("ERROR") + + def tearDown(self): + LOG.set_level("CRITICAL") + + def test_deserialize_from_json_string(self): + """deserialize() accepts a JSON string, not just a dict.""" + src = _make_utterance("test", pipeline=ADAPT_ONLY) + test = End2EndTest( + skill_ids=[], source_message=src, + expected_messages=[src], + test_msg_context=False, verbose=False, + ) + json_str = json.dumps(test.serialize(anonymize=False)) + restored = End2EndTest.deserialize(json_str) + self.assertEqual(restored.skill_ids, []) + self.assertEqual(len(restored.source_message), 1) + + def test_serialize_preserves_flip_points(self): + """Serialized data includes flip_points.""" + src = _make_utterance("test", pipeline=ADAPT_ONLY) + test = End2EndTest( + skill_ids=[], source_message=src, + expected_messages=[src], + flip_points=["recognizer_loop:utterance"], + verbose=False, + ) + data = test.serialize() + self.assertEqual(data["flip_points"], ["recognizer_loop:utterance"]) + + def test_serialize_preserves_test_flags(self): + """Serialized data includes test toggle flags.""" + src = _make_utterance("test", pipeline=ADAPT_ONLY) + test = End2EndTest( + skill_ids=[], source_message=src, + expected_messages=[src], + test_msg_type=False, test_msg_data=False, + test_msg_context=False, test_routing=False, + verbose=False, + ) + data = test.serialize() + self.assertFalse(data["test_msg_type"]) + self.assertFalse(data["test_msg_data"]) + self.assertFalse(data["test_msg_context"]) + self.assertFalse(data["test_routing"]) + + def test_anonymize_message_replaces_location(self): + """anonymize_message() sets location to N/A.""" + sess = _session() + sess.location_preferences = { + "city": {"name": "Lisbon", "code": "LIS", + "state": {"code": "PT", "name": "Portugal", + "country": {"code": "PT", "name": "Portugal"}}}, + "coordinate": {"latitude": 38.7, "longitude": -9.1}, + "timezone": {"code": "Europe/Lisbon", "name": "Europe/Lisbon"}, + } + src = Message("test", {}, + {"session": sess.serialize(), "source": "A", "destination": "B"}) + anon = End2EndTest.anonymize_message(src) + sess_data = anon.context.get("session", {}) + loc = sess_data.get("location", {}) + self.assertEqual(loc["city"]["name"], "N/A") + self.assertEqual(loc["coordinate"]["latitude"], 0) + + +# --------------------------------------------------------------------------- +# Boot sequence tests +# --------------------------------------------------------------------------- +class TestBootSequence(unittest.TestCase): + + def setUp(self): + LOG.set_level("ERROR") + self.mc = get_minicroft([SKILL_ID], extra_skills={SKILL_ID: EchoSkill}) + + def tearDown(self): + self.mc.stop() + LOG.set_level("CRITICAL") + + def test_boot_sequence_passes_when_correct(self): + """test_boot_sequence=True passes when expected matches first N boot msgs.""" + # Build expected_boot_sequence from actual first message + self.assertTrue(len(self.mc.boot_messages) > 0, + "MiniCroft must emit at least one boot message") + first_boot = self.mc.boot_messages[0] + src = _make_custom("unittest.echo", {"text": "boot"}) + test = End2EndTest( + minicroft=self.mc, + skill_ids=[SKILL_ID], + source_message=src, + expected_messages=[src], + expected_boot_sequence=[Message(first_boot.msg_type)], + test_boot_sequence=True, + test_message_number=False, + test_msg_type=False, + test_msg_data=False, + test_msg_context=False, + test_routing=False, + test_active_skills=False, + test_final_session=False, + test_async_messages=False, + test_async_message_number=False, + verbose=False, + ) + result = test.execute(timeout=10) + self.assertIsInstance(result, list) + + def test_boot_sequence_wrong_type_raises(self): + """test_boot_sequence=True raises on type mismatch.""" + src = _make_custom("unittest.echo", {"text": "boot"}) + test = End2EndTest( + minicroft=self.mc, + skill_ids=[SKILL_ID], + source_message=src, + expected_messages=[src], + expected_boot_sequence=[Message("WRONG.BOOT.TYPE")], + test_boot_sequence=True, + test_message_number=False, + test_msg_type=False, + test_msg_data=False, + test_msg_context=False, + test_routing=False, + test_active_skills=False, + test_final_session=False, + test_async_messages=False, + test_async_message_number=False, + verbose=False, + ) + with self.assertRaises(AssertionError): + test.execute(timeout=10) + + +# --------------------------------------------------------------------------- +# Active skills tests +# --------------------------------------------------------------------------- +class TestActiveSkills(unittest.TestCase): + + def setUp(self): + LOG.set_level("ERROR") + self.mc = get_minicroft([SKILL_ID], extra_skills={SKILL_ID: EchoSkill}) + + def tearDown(self): + self.mc.stop() + LOG.set_level("CRITICAL") + + def test_inject_active_modifies_session(self): + """inject_active adds skill to session's active_skills before test.""" + src = _make_custom("unittest.echo", {"text": "active"}) + test = End2EndTest( + minicroft=self.mc, + skill_ids=[SKILL_ID], + source_message=src, + expected_messages=[src], + inject_active=["fake-active-skill.test"], + test_message_number=False, + test_msg_type=False, + test_msg_data=False, + test_msg_context=False, + test_routing=False, + test_active_skills=False, # don't assert — just verify injection ran + test_final_session=False, + test_async_messages=False, + test_async_message_number=False, + ignore_messages=DEFAULT_IGNORED + HANDLER_LIFECYCLE, + verbose=True, # covers the inject_active print branch + ) + result = test.execute(timeout=10) + self.assertIsInstance(result, list) + + +# --------------------------------------------------------------------------- +# Pipeline constant composition +# --------------------------------------------------------------------------- +class TestPipelineConstants(unittest.TestCase): + + def test_default_test_pipeline_excludes_persona(self): + """DEFAULT_TEST_PIPELINE must not contain persona stages.""" + for stage in PERSONA_PIPELINE: + self.assertNotIn(stage, DEFAULT_TEST_PIPELINE) + + def test_default_test_pipeline_has_adapt(self): + """DEFAULT_TEST_PIPELINE must contain Adapt stages.""" + for stage in ADAPT_PIPELINE: + self.assertIn(stage, DEFAULT_TEST_PIPELINE) + + def test_default_test_pipeline_has_padatious(self): + for stage in PADATIOUS_PIPELINE: + self.assertIn(stage, DEFAULT_TEST_PIPELINE) + + def test_default_test_pipeline_has_fallback(self): + for stage in FALLBACK_PIPELINE: + self.assertIn(stage, DEFAULT_TEST_PIPELINE) + + def test_default_test_pipeline_has_common_query(self): + for stage in COMMON_QUERY_PIPELINE: + self.assertIn(stage, DEFAULT_TEST_PIPELINE) + + def test_stop_pipeline_length(self): + self.assertEqual(len(STOP_PIPELINE), 3) + + def test_converse_pipeline_length(self): + self.assertEqual(len(CONVERSE_PIPELINE), 1) + + +# --------------------------------------------------------------------------- +# MiniCroft language config +# --------------------------------------------------------------------------- +class TestMiniCroftLangConfig(unittest.TestCase): + + def setUp(self): + LOG.set_level("ERROR") + + def tearDown(self): + LOG.set_level("CRITICAL") + + def test_lang_override(self): + """MiniCroft lang parameter overrides session lang.""" + mc = get_minicroft([], lang="pt-BR") + try: + self.assertEqual(SessionManager.default_session.lang, "pt-BR") + finally: + mc.stop() + + def test_lang_restored_after_stop(self): + """Stopping MiniCroft restores original lang.""" + from ovos_config.config import Configuration + original_lang = Configuration().get("lang") + mc = get_minicroft([], lang="de-DE") + mc.stop() + restored_lang = Configuration().get("lang") + self.assertEqual(restored_lang, original_lang) + + +# --------------------------------------------------------------------------- +# CaptureSession __del__ +# --------------------------------------------------------------------------- +class TestCaptureSessionDel(unittest.TestCase): + + def setUp(self): + LOG.set_level("ERROR") + self.mc = get_minicroft([]) + + def tearDown(self): + self.mc.stop() + LOG.set_level("CRITICAL") + + def test_del_calls_finish(self): + """__del__ should call finish() safely.""" + cs = CaptureSession(self.mc, eof_msgs=["test.eof"], ignore_messages=[]) + cs.done.set() + # Should not raise + cs.__del__() + self.assertTrue(cs.done.is_set()) + + +# --------------------------------------------------------------------------- +# Verbose output (covers print branches) +# --------------------------------------------------------------------------- +class TestVerboseOutput(unittest.TestCase): + + def setUp(self): + LOG.set_level("ERROR") + self.mc = get_minicroft([SKILL_ID], extra_skills={SKILL_ID: EchoSkill}) + + def tearDown(self): + self.mc.stop() + LOG.set_level("CRITICAL") + + def test_verbose_true_covers_print_branches(self): + """verbose=True exercises all print branches without raising.""" + src = _make_custom("unittest.echo", {"text": "verbose"}) + test = End2EndTest( + minicroft=self.mc, + skill_ids=[SKILL_ID], + source_message=src, + expected_messages=[ + src, + Message("speak", {"utterance": "verbose"}), + Message("ovos.utterance.handled", {}), + ], + ignore_messages=DEFAULT_IGNORED + HANDLER_LIFECYCLE, + test_routing=True, + test_msg_type=True, + test_msg_data=True, + test_msg_context=False, + test_active_skills=False, + test_final_session=False, + test_async_messages=False, + test_async_message_number=False, + verbose=True, + ) + result = test.execute(timeout=10) + self.assertEqual(len(result), 3) + + +# --------------------------------------------------------------------------- +# Routing internals (flip_points, entry_points, keep_original_src) +# --------------------------------------------------------------------------- +class TestRoutingInternals(unittest.TestCase): + """Test routing assertion logic inside execute() using EchoSkill. + + EchoSkill produces: unittest.echo → speak → ovos.utterance.handled + All messages carry the same context source/destination from the source msg. + """ + + def setUp(self): + LOG.set_level("ERROR") + self.mc = get_minicroft([SKILL_ID], extra_skills={SKILL_ID: EchoSkill}) + + def tearDown(self): + self.mc.stop() + LOG.set_level("CRITICAL") + + def _base_flags(self, **overrides): + defaults = dict( + test_message_number=False, + test_msg_type=False, + test_msg_data=False, + test_msg_context=False, + test_active_skills=False, + test_final_session=False, + test_async_messages=False, + test_async_message_number=False, + ignore_messages=["ovos.skills.settings_changed"] + HANDLER_LIFECYCLE, + verbose=False, + ) + defaults.update(overrides) + return defaults + + def test_routing_passes_with_matching_src_dst(self): + """test_routing=True passes when src/dst match across all messages.""" + src = _make_custom("unittest.echo", {"text": "route"}) + test = End2EndTest( + minicroft=self.mc, + skill_ids=[SKILL_ID], + source_message=src, + expected_messages=[ + src, + Message("speak", {}), + Message("ovos.utterance.handled", {}), + ], + entry_points=[], # no entry point flip + **self._base_flags(test_routing=True), + ) + result = test.execute(timeout=10) + self.assertEqual(len(result), 3) + + def test_routing_with_flip_point(self): + """After a flip_point, expected src and dst are swapped.""" + src = _make_custom("unittest.echo", {"text": "flip"}) + # After flip, expected src=B, dst=A (swapped from A,B) + # But actual messages still have source=A, so this should fail + test = End2EndTest( + minicroft=self.mc, + skill_ids=[SKILL_ID], + source_message=src, + expected_messages=[ + Message("unittest.echo", {}, {"source": "A", "destination": "B"}), + Message("speak", {}), # after flip, expects src=B,dst=A but gets src=A + Message("ovos.utterance.handled", {}), + ], + flip_points=["unittest.echo"], + entry_points=[], + **self._base_flags(test_routing=True), + ) + with self.assertRaises(AssertionError): + test.execute(timeout=10) + + def test_routing_with_entry_point(self): + """After an entry_point, src and dst are extracted from received and flipped.""" + src = _make_custom("unittest.echo", {"text": "entry"}) + # entry_points causes: after "unittest.echo", new e_src = r_dst, e_dst = r_src + # received has source=A, destination=B → new expected = B, A + # next msg (speak) will have source=A → mismatch with expected B → should fail + test = End2EndTest( + minicroft=self.mc, + skill_ids=[SKILL_ID], + source_message=src, + expected_messages=[ + src, + Message("speak", {}), + Message("ovos.utterance.handled", {}), + ], + entry_points=["unittest.echo"], + **self._base_flags(test_routing=True), + ) + with self.assertRaises(AssertionError): + test.execute(timeout=10) + + def test_keep_original_src_uses_original(self): + """Messages in keep_original_src compare against o_src/o_dst.""" + src = _make_custom("unittest.echo", {"text": "keep"}) + test = End2EndTest( + minicroft=self.mc, + skill_ids=[SKILL_ID], + source_message=src, + expected_messages=[ + src, + Message("speak", {}), + Message("ovos.utterance.handled", {}), + ], + keep_original_src=["speak"], # speak uses original src/dst + entry_points=[], + **self._base_flags(test_routing=True), + ) + result = test.execute(timeout=10) + self.assertEqual(len(result), 3) + + def test_routing_verbose_prints(self): + """verbose=True with routing enabled exercises all routing print branches.""" + src = _make_custom("unittest.echo", {"text": "verbose-route"}) + test = End2EndTest( + minicroft=self.mc, + skill_ids=[SKILL_ID], + source_message=src, + expected_messages=[ + src, + Message("speak", {}), + Message("ovos.utterance.handled", {}), + ], + entry_points=[], + ignore_messages=["ovos.skills.settings_changed"] + HANDLER_LIFECYCLE, + test_message_number=False, + test_msg_type=False, + test_msg_data=False, + test_msg_context=False, + test_active_skills=False, + test_final_session=False, + test_async_messages=False, + test_async_message_number=False, + test_routing=True, + verbose=True, + ) + result = test.execute(timeout=10) + self.assertEqual(len(result), 3) + + +# --------------------------------------------------------------------------- +# Active skills internals (activation_points, deactivation_points, +# disallow_extra_active_skills) +# --------------------------------------------------------------------------- +class TestActiveSkillsInternals(unittest.TestCase): + + def setUp(self): + LOG.set_level("ERROR") + self.mc = get_minicroft([SKILL_ID], extra_skills={SKILL_ID: EchoSkill}) + + def tearDown(self): + self.mc.stop() + LOG.set_level("CRITICAL") + + def _base_flags(self, **overrides): + defaults = dict( + test_message_number=False, + test_msg_type=False, + test_msg_data=False, + test_msg_context=False, + test_routing=False, + test_final_session=False, + test_async_messages=False, + test_async_message_number=False, + ignore_messages=["ovos.skills.settings_changed"] + HANDLER_LIFECYCLE, + verbose=False, + ) + defaults.update(overrides) + return defaults + + def test_activation_point_tracks_skill(self): + """After an activation_point, the skill_id is tracked as active.""" + src = _make_custom("unittest.echo", {"text": "activate"}) + test = End2EndTest( + minicroft=self.mc, + skill_ids=[SKILL_ID], + source_message=src, + expected_messages=[src], + activation_points=["unittest.echo"], + **self._base_flags(test_active_skills=True, verbose=True), + ) + # Should pass — unittest.echo is an activation point, + # skill_id from context gets added to active_skills + result = test.execute(timeout=10) + self.assertIsInstance(result, list) + + def test_deactivation_point_removes_skill(self): + """After a deactivation_point, the skill_id is removed from tracking.""" + src = _make_custom("unittest.echo", {"text": "deactivate"}) + test = End2EndTest( + minicroft=self.mc, + skill_ids=[SKILL_ID], + source_message=src, + expected_messages=[src], + deactivation_points=["unittest.echo"], + **self._base_flags(test_active_skills=False, verbose=True), + ) + result = test.execute(timeout=10) + self.assertIsInstance(result, list) + + def test_disallow_extra_active_fails(self): + """disallow_extra_active_skills=True raises if unexpected skills are active.""" + src = _make_custom("unittest.echo", {"text": "extra"}) + # inject_active adds a skill, but the echo handler may activate + # the test skill too — if any unexpected skill is active, it fails + test = End2EndTest( + minicroft=self.mc, + skill_ids=[SKILL_ID], + source_message=src, + expected_messages=[src], + inject_active=[], + disallow_extra_active_skills=True, + **self._base_flags(test_active_skills=True), + ) + # This may or may not raise depending on what skills are active + # The important thing is this code path is exercised + try: + test.execute(timeout=10) + except AssertionError: + pass # expected if extra skills are active + + +# --------------------------------------------------------------------------- +# Final session tests +# --------------------------------------------------------------------------- +class TestFinalSession(unittest.TestCase): + + def setUp(self): + LOG.set_level("ERROR") + self.mc = get_minicroft([SKILL_ID], extra_skills={SKILL_ID: EchoSkill}) + + def tearDown(self): + self.mc.stop() + LOG.set_level("CRITICAL") + + def test_final_session_lang_mismatch_raises(self): + """test_final_session=True raises when lang doesn't match.""" + src = _make_custom("unittest.echo", {"text": "final"}) + wrong_session = Session("ext-test") + wrong_session.lang = "xx-XX" # wrong lang + test = End2EndTest( + minicroft=self.mc, + skill_ids=[SKILL_ID], + source_message=src, + expected_messages=[ + src, + Message("speak", {}), + Message("ovos.utterance.handled", {}), + ], + final_session=wrong_session, + test_message_number=False, + test_msg_type=False, + test_msg_data=False, + test_msg_context=False, + test_routing=False, + test_active_skills=False, + test_final_session=True, + test_async_messages=False, + test_async_message_number=False, + ignore_messages=["ovos.skills.settings_changed"] + HANDLER_LIFECYCLE, + verbose=False, + ) + with self.assertRaises(AssertionError): + test.execute(timeout=10) + + def test_final_session_passes_when_matching(self): + """test_final_session=True passes when session attributes match.""" + sess = _session() + src = _make_custom("unittest.echo", {"text": "final-ok"}) + # Build a final_session that matches the actual session state + expected_sess = Session("ext-test") + expected_sess.lang = "en-US" + expected_sess.pipeline = ADAPT_ONLY + expected_sess.site_id = sess.site_id + test = End2EndTest( + minicroft=self.mc, + skill_ids=[SKILL_ID], + source_message=src, + expected_messages=[ + src, + Message("speak", {}), + Message("ovos.utterance.handled", {}), + ], + final_session=expected_sess, + test_message_number=False, + test_msg_type=False, + test_msg_data=False, + test_msg_context=False, + test_routing=False, + test_active_skills=False, + test_final_session=True, + test_async_messages=False, + test_async_message_number=False, + ignore_messages=["ovos.skills.settings_changed"] + HANDLER_LIFECYCLE, + verbose=True, # covers the verbose final session print branches + ) + result = test.execute(timeout=10) + self.assertIsInstance(result, list) + + +# --------------------------------------------------------------------------- +# from_message recording mode +# --------------------------------------------------------------------------- +class TestFromMessage(unittest.TestCase): + + def setUp(self): + LOG.set_level("ERROR") + + def tearDown(self): + LOG.set_level("CRITICAL") + + def test_from_message_records_sequence(self): + """from_message() records the actual message sequence as expected.""" + src = _make_custom("unittest.echo", {"text": "record"}) + test = End2EndTest.from_message( + message=src, + skill_ids=[SKILL_ID], + extra_skills={SKILL_ID: EchoSkill}, + timeout=10, + ) + self.assertIsInstance(test, End2EndTest) + self.assertEqual(test.skill_ids, [SKILL_ID]) + self.assertTrue(len(test.expected_messages) > 0, + "from_message must capture at least one message") + + def test_from_message_single_message_wrapped(self): + """from_message() wraps a single Message into a list.""" + src = _make_custom("unittest.echo", {"text": "single"}) + test = End2EndTest.from_message( + message=src, + skill_ids=[SKILL_ID], + extra_skills={SKILL_ID: EchoSkill}, + timeout=10, + ) + self.assertIsInstance(test.source_message, list) + self.assertEqual(len(test.source_message), 1) + + +# --------------------------------------------------------------------------- +# Message count verbose branch (first differing message) +# --------------------------------------------------------------------------- +class TestMessageCountVerbose(unittest.TestCase): + + def setUp(self): + LOG.set_level("ERROR") + self.mc = get_minicroft([SKILL_ID], extra_skills={SKILL_ID: EchoSkill}) + + def tearDown(self): + self.mc.stop() + LOG.set_level("CRITICAL") + + def test_count_mismatch_prints_first_differing(self): + """Count mismatch prints info about the first differing message.""" + src = _make_custom("unittest.echo", {"text": "diff"}) + test = End2EndTest( + minicroft=self.mc, + skill_ids=[SKILL_ID], + source_message=src, + expected_messages=[ + src, + Message("WRONG.TYPE", {}), # differs from "speak" + ], + test_message_number=True, + test_msg_type=False, + test_msg_data=False, + test_msg_context=False, + test_routing=False, + test_active_skills=False, + test_final_session=False, + test_async_messages=False, + test_async_message_number=False, + ignore_messages=["ovos.skills.settings_changed"] + HANDLER_LIFECYCLE, + verbose=False, + ) + with self.assertRaises(AssertionError): + test.execute(timeout=10) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/unittests/test_pytest_plugin.py b/test/unittests/test_pytest_plugin.py new file mode 100644 index 0000000..cde9e32 --- /dev/null +++ b/test/unittests/test_pytest_plugin.py @@ -0,0 +1,109 @@ +"""Unit tests for ovoscope.pytest_plugin — the ``minicroft`` fixture. + +Since pytest fixtures can't be called directly, we test the underlying +logic by importing the module and inspecting/mocking its internals. +""" +import unittest +from unittest.mock import MagicMock, patch + +import ovoscope.pytest_plugin as plugin_mod + + +class TestMinicroftFixtureLogic(unittest.TestCase): + """Tests for the fixture's skill_ids extraction and lifecycle.""" + + def test_module_has_minicroft_fixture(self): + """The module exposes a 'minicroft' callable.""" + self.assertTrue(hasattr(plugin_mod, "minicroft")) + self.assertTrue(callable(plugin_mod.minicroft)) + + @patch.object(plugin_mod, "get_minicroft") + def test_skill_ids_read_from_class(self, mock_get): + """The fixture function reads skill_ids from request.cls.""" + mock_mc = MagicMock() + mock_get.return_value = mock_mc + + request = MagicMock() + request.cls = type("FakeTest", (), {"skill_ids": ["skill-a.test"]}) + + # Call the underlying generator function directly (bypassing pytest's + # fixture decorator which blocks direct calls in newer pytest versions) + gen = plugin_mod.minicroft.__wrapped__(request) + mc = next(gen) + + mock_get.assert_called_once_with(["skill-a.test"]) + self.assertIs(mc, mock_mc) + + try: + next(gen) + except StopIteration: + pass + mock_mc.stop.assert_called_once() + + @patch.object(plugin_mod, "get_minicroft") + def test_string_skill_ids_normalized(self, mock_get): + """A single string skill_ids is wrapped into a list.""" + mock_mc = MagicMock() + mock_get.return_value = mock_mc + + request = MagicMock() + request.cls = type("FakeTest", (), {"skill_ids": "single.test"}) + + gen = plugin_mod.minicroft.__wrapped__(request) + next(gen) + mock_get.assert_called_once_with(["single.test"]) + + try: + next(gen) + except StopIteration: + pass + + @patch.object(plugin_mod, "get_minicroft") + def test_missing_skill_ids_defaults_empty(self, mock_get): + """If the test class has no skill_ids, default to [].""" + mock_mc = MagicMock() + mock_get.return_value = mock_mc + + request = MagicMock() + request.cls = type("FakeTest", (), {}) + + gen = plugin_mod.minicroft.__wrapped__(request) + next(gen) + mock_get.assert_called_once_with([]) + + try: + next(gen) + except StopIteration: + pass + + @patch.object(plugin_mod, "get_minicroft") + def test_stop_called_on_exception(self, mock_get): + """mc.stop() is called even if the test body raises.""" + mock_mc = MagicMock() + mock_get.return_value = mock_mc + + request = MagicMock() + request.cls = type("FakeTest", (), {"skill_ids": []}) + + gen = plugin_mod.minicroft.__wrapped__(request) + next(gen) + + try: + gen.throw(RuntimeError("test failure")) + except RuntimeError: + pass + mock_mc.stop.assert_called_once() + + @patch.object(plugin_mod, "get_minicroft", side_effect=TimeoutError("boom")) + def test_get_minicroft_failure_no_name_error(self, mock_get): + """If get_minicroft raises, teardown must not raise NameError.""" + request = MagicMock() + request.cls = type("FakeTest", (), {"skill_ids": []}) + + gen = plugin_mod.minicroft.__wrapped__(request) + with self.assertRaises(TimeoutError): + next(gen) + + +if __name__ == "__main__": + unittest.main()