diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7eeaf19..e15a27b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,7 +51,7 @@ jobs: run: pnpm smoke - name: Verify generated dist is committed - run: git diff --exit-code -- dist + run: git diff --exit-code -- packages/mimir/dist packages/mimir-tts/dist - name: Verify npm package metadata run: pnpm package:check diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index a259164..87947b2 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -47,7 +47,8 @@ jobs: - name: Verify version input run: | - test "$(node -p "require('./package.json').version")" = "${{ inputs.version }}" + test "$(node -p "require('./packages/mimir/package.json').version")" = "${{ inputs.version }}" + test "$(node -p "require('./packages/mimir-tts/package.json').version")" = "${{ inputs.version }}" - name: Install dependencies run: pnpm install --frozen-lockfile @@ -68,7 +69,7 @@ jobs: run: pnpm smoke - name: Verify generated dist is committed - run: git diff --exit-code -- dist + run: git diff --exit-code -- packages/mimir/dist packages/mimir-tts/dist - name: Verify npm package metadata run: pnpm package:check @@ -82,7 +83,12 @@ jobs: name: mimir-release-${{ inputs.version }} path: release-artifacts/ - - name: Publish - run: npm publish --access public --provenance + - name: Publish Mimir TTS + run: pnpm --dir packages/mimir-tts publish --access public --provenance --no-git-checks + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + + - name: Publish Mimir + run: pnpm --dir packages/mimir publish --access public --provenance --no-git-checks env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.gitignore b/.gitignore index f040c77..81d3077 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,20 @@ node_modules/ coverage/ .env +.env.* .DS_Store private/** .kb/ .mimir/ *.tgz release-artifacts/ + +# Tracked synthetic examples. Keep generated example runtime state ignored. +!packages/mimir/examples/ +!packages/mimir/examples/**/ +!packages/mimir/examples/**/.kb/ +!packages/mimir/examples/**/.kb/config.json +!packages/mimir/examples/**/.kb/sources.txt +packages/mimir/examples/**/.kb/storage/ +packages/mimir/examples/**/.kb/access.log +packages/mimir/examples/**/.mimir/ diff --git a/AGENTS.md b/AGENTS.md index c497722..6434750 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,28 +14,78 @@ - `kb init` and `kb install-skill` must keep generated local Mimir state ignored in target repositories. By default, add `.kb/`, `.mimir/`, and private raw-document paths to the target repository `.gitignore`. -- Keep confidentiality features low-friction: local-only network policy, redaction before - indexing, metadata-only access logs, bounded MCP retrieval, and `security-audit` should work - from default config. +- Keep confidentiality features low-friction: local-hash retrieval by default, optional + Transformers.js embeddings with remote model loading disabled by default, redaction before + indexing, metadata-only access logs, bounded MCP retrieval, configurable text-extension ingestion, + and `security-audit` should work from default config. +- Keep public positioning focused on sovereign local RAG for confidential datasets and AI agents. + Avoid claiming universal binary-file support; unsupported proprietary formats need extraction or + dedicated parsers. +- Keep optional audio summaries separate from core ingestion/query behavior. The + `mimir-audio-summary` skill must prefer `kb audio` / `@jcode.labs/mimir-tts`, support offline + model loading, and keep generated audio under ignored local Mimir state. +- Keep the repository as a simple pnpm workspace monorepo. Add Turbo only if multiple packages or + apps start needing task caching/orchestration beyond `pnpm --filter`. +- Keep Mimir core free of Ollama. `embeddingProvider: "local-hash"` supports ingestion, search, MCP, + and cited retrieval without a model server, but it must not be described as equivalent to semantic + retrieval. `embeddingProvider: "transformers"` is the optional semantic embedding path. +- Keep `packages/mimir/examples/sovereign-rag-demo` synthetic and safe to commit. It exists for + package/user testing only; never place real confidential documents there. - Use Context7 before changing dependencies or public APIs that rely on external libraries. - Run `pnpm validate` before opening a release pull request or publishing. It covers Biome, TypeScript, Vitest, build output, production CLI/MCP smoke tests, and npm package metadata. - Do not publish from a local machine or direct push to `main`. npm releases must go through - the protected manual `Publish npm` GitHub Actions workflow after `main` has green CI. + the protected manual `Publish npm` GitHub Actions workflow after `main` has green CI. The workflow + publishes `@jcode.labs/mimir-tts` first, then `@jcode.labs/mimir`. + +## Coding Conventions + +General principles (KISS, DRY, YAGNI, SOLID) as applied in this codebase. Match the surrounding style. + +- One responsibility per module. The ingest pipeline is split on purpose: `files` discovers, + `parsing` extracts, `redaction` strips, `chunking` splits, `embeddings` vectorizes, `store` + persists, `query` retrieves. Add logic to the module that owns the concern, or a new small module. +- No duplicated logic. Reuse existing helpers (`loadConfig`, `embedText`/`embedTexts`, + `openRowsTable`, `redactText`, `supportedExtensions`, `recordAccess`); extract instead of copying. + `embedText` delegating to `embedTexts` is the reference pattern. +- No dead or obsolete code. Delete replaced code, unused exports, and commented-out blocks in the + same change; a deletion must cover both source and the regenerated package `dist/`. +- No magic strings or numbers. Name meaningful literals as constants, and put shared paths, provider + defaults, and ignore constants in `packages/mimir/src/defaults.ts` rather than copying them across + modules. +- Validate at the boundary, narrow inside. Use Zod at external edges (config in `config.ts`, MCP + inputs in `mcp.ts`) and CLI parsers (`parsePositiveInt`); trust the types past that point. +- Type-guard instead of casting. Prefer runtime guards over `as`/`!` (`hasToList`, `isNumberArray`, + `isNumberMatrix`); LanceDB row casts at the `store`/`query` driver boundary are the only exception. +- Named exports only; keep the public surface explicit in `index.ts`. Functions stay small and pure; + private helpers sit below the exported function in the same file. +- Comments explain why, not what; the codebase is near comment-free. Only the CLI (`cli.ts`) writes + to stdout/stderr — library, MCP, and pipeline code return data, never log. +- YAGNI: no options, providers, or abstractions ahead of a real need. ## Architecture -- `src/cli.ts` exposes the `kb` CLI. -- `src/config.ts` resolves `.kb/config.json` from the target repository. -- `src/ingest.ts` parses supported files, chunks text, embeds chunks, and rebuilds the +- `packages/mimir` is the core package published as `@jcode.labs/mimir`. +- `packages/mimir/src/cli.ts` exposes the `kb` CLI. +- `packages/mimir/src/config.ts` resolves `.kb/config.json` from the target repository. +- `packages/mimir/src/defaults.ts` owns shared default paths, provider defaults, and generated-state ignore + constants. Keep config/init/security/gitignore aligned through this module instead of copying + literals. +- `packages/mimir/src/ingest.ts` parses supported files, chunks text, embeds chunks, and rebuilds the local LanceDB table. -- `src/query.ts` performs vector search and local Ollama answer synthesis. -- `src/mcp.ts` exposes Mimir as an MCP stdio server for agents. -- `src/gitignore.ts` owns target-repository `.gitignore` entries for local generated Mimir +- `packages/mimir/src/query.ts` performs vector search and returns cited retrieval context; LLM synthesis belongs + outside Mimir core. +- `packages/mimir/src/mcp.ts` exposes Mimir as an MCP stdio server for agents. +- `packages/mimir-tts` is the standalone JS/ONNX TTS package used by `kb audio`. +- `packages/mimir/src/gitignore.ts` owns target-repository `.gitignore` entries for local generated Mimir state. -- `src/security.ts`, `src/network.ts`, `src/redaction.ts`, and `src/access-log.ts` own the +- `packages/mimir/src/security.ts`, `packages/mimir/src/redaction.ts`, and + `packages/mimir/src/access-log.ts` own the privacy and confidentiality hardening layer. -- `skills/mimir/SKILL.md` is the bundled portable agent skill. +- `packages/mimir/skills/mimir/SKILL.md` is the bundled portable agent skill. +- `packages/mimir/skills/mimir-audio-summary/SKILL.md` is the optional bundled audio-summary skill. +- `packages/mimir/examples/sovereign-rag-demo` is the tracked synthetic test workspace for manual + and package validation. - `.kb/`, `.mimir/`, and project `private/` folders are local user data or generated agent state in target repositories and must not be committed. diff --git a/CHANGELOG.md b/CHANGELOG.md index b38d4dd..6d4ed26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,29 @@ # Changelog +## 0.4.0 - 2026-06-28 + +- Reposition Mimir as sovereign local RAG for confidential datasets and AI agents. +- Expand default ingestion to common text, Office/OpenDocument, data, config, log, and source-code + file types. +- Add `includeExtensions` / `KB_INCLUDE_EXTENSIONS` for custom UTF-8 text file extensions. +- Add the optional `mimir-audio-summary` bundled skill for confidential audio summaries. +- Install both the main Mimir skill and optional audio-summary skill with `kb install-skill`. +- Improve agent guidance for deep multi-query retrieval before synthesis. +- Make Mimir core retrieval-only: `kb ask` now returns cited context for external agents or LLMs + instead of generating answers internally. +- Add optional Transformers.js semantic embeddings through `embeddingProvider: "transformers"`. +- Remove Ollama providers and keep `embeddingProvider: "local-hash"` as the no-model default. +- Move the repository to a simple pnpm workspace monorepo without adding Turbo. +- Move the core `@jcode.labs/mimir` package into `packages/mimir`. +- Add `@jcode.labs/mimir-tts` for plug-and-play JS/ONNX WAV rendering without Python or ffmpeg. +- Add `kb audio` and update the audio-summary skill to use Mimir TTS before advanced fallback + engines. + ## 0.3.0 - 2026-06-28 -- Add confidentiality hardening defaults: local-only Ollama network policy, built-in - redaction before indexing, metadata-only access logs, and bounded MCP retrieval. -- Add `kb security-audit` for zero-telemetry, network, redaction, gitignore, storage, and +- Add confidentiality hardening defaults: built-in redaction before indexing, metadata-only access + logs, and bounded MCP retrieval. +- Add `kb security-audit` for zero-telemetry, provider, redaction, gitignore, storage, and MCP posture checks. - Add `kb destroy-index --yes` to remove generated vector indexes. - Add release verification artifacts: npm tarball, SHA256 checksums, SBOM, and manifest. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..29b5e55 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,97 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +`AGENTS.md` is the authoritative source for shared rules — working rules, coding conventions, and +high-level architecture. Read it first. This file adds only the Claude Code operational details and +non-obvious traps that matter when editing here, without duplicating `AGENTS.md`. + +## Commands + +```bash +pnpm build # builds packages/mimir-tts, then packages/mimir; package dist is committed +pnpm check # typecheck only (tsc --noEmit) +pnpm lint # Biome CI (format + lint check, no writes) +pnpm lint:fix # Biome auto-fix +pnpm format # Biome format --write +pnpm test # vitest run for packages/mimir-tts, then packages/mimir +pnpm smoke # build production CLI + MCP smoke test (scripts/smoke.mjs) +pnpm validate # full release gate: lint + check + test + build + smoke + package:check + release:artifacts +``` + +Run a single core test file: `pnpm --filter @jcode.labs/mimir exec vitest run src/config.test.ts` +Run a single core test by name: `pnpm --filter @jcode.labs/mimir exec vitest run -t "applies env overrides"` +Run only the TTS package tests: `pnpm --filter @jcode.labs/mimir-tts test` + +Tests are colocated as `packages/*/src/*.test.ts` and run on the TypeScript sources. + +## Committed `dist/` — critical + +`packages/mimir/dist/` and `packages/mimir-tts/dist/` are checked into Git. CI enforces +`git diff --exit-code -- packages/mimir/dist packages/mimir-tts/dist`. After any change under +`packages/mimir/src/` or `packages/mimir-tts/src/`, run `pnpm build` and commit the regenerated +output in the same commit, or CI fails. This is the single easiest mistake to make in this repo. + +## Naming map (the package has several names on purpose) + +- Product / core package: **Mimir**, published as `@jcode.labs/mimir` from `packages/mimir`. +- TTS package: **Mimir TTS**, published as `@jcode.labs/mimir-tts`. +- CLI binary: **`kb`** (`packages/mimir/bin.kb` -> `packages/mimir/dist/cli.js`). Commands: `init`, + `ingest`, `search`, `ask`, `audit`, `status`, `security-audit`, `destroy-index`, `audio`, + `serve-mcp`, `skill-path`, `install-skill`. +- TTS CLI binary: **`mimir-tts`** (`packages/mimir-tts/dist/cli.js`). Commands: `doctor`, `render`. +- Project config/state in the target repo: **`.kb/`** (`config.json`, `sources.txt`, `access.log`, + `storage/`), raw documents in **`private/`**, agent kit in **`.mimir/`**. +- Environment overrides: **`KB_*`** (e.g. `KB_EMBEDDING_PROVIDER`, `KB_CHUNK_SIZE`). +- MCP tools exposed to agents: **`mimir_*`** (`mimir_status`, `mimir_search`, `mimir_ask`, + `mimir_audit`, `mimir_security_audit`). + +## Architecture and data flow + +This is a pnpm workspace monorepo with the core package in `packages/mimir` and TTS in +`packages/mimir-tts`. Do not add Turbo unless `pnpm --filter` stops being enough. + +The core package is an ESM-only TypeScript library + CLI + MCP server. Same core, three entry +points: `packages/mimir/src/cli.ts` (commander), `packages/mimir/src/index.ts` (public library +exports), `packages/mimir/src/mcp.ts` (stdio MCP server). + +The ingest pipeline (`packages/mimir/src/ingest.ts`) chains single-responsibility modules: +`files.ts` (discover supported files via fast-glob, with sha256 checksums) → +`parsing.ts` (extract text per format: PDF/Office/HTML/etc.) → +`redaction.ts` (strip secrets/PII *before* anything is embedded) → +`chunking.ts` (split into overlapping chunks) → +`embeddings.ts` (vectorize) → `store.ts` (LanceDB). `query.ts` embeds the query and runs vector +search; `ask` returns cited passages only (no LLM synthesis in core). + +`packages/mimir-tts` is a separate ESM package that uses Transformers.js text-to-speech to render +WAV files without Python or ffmpeg. Core `kb audio` imports it dynamically. + +Key behaviors to keep in mind before editing: + +- **Config resolution is caller-relative.** `loadConfig` walks up from `cwd` looking for + `.kb/config.json` (`findProjectRoot`). The package must resolve project data from the caller's + working directory, never from its own install path. Zod validates config; `KB_*` env vars override. +- **Two embedding providers, not interchangeable at runtime.** `local-hash` (default) is a 384-dim + sha256 lexical embedding — fully offline, no model, *not semantic*. `transformers` lazily + `import()`s `@huggingface/transformers` with `allowRemoteModels` off by default. The two produce + different vectors, so **switching providers requires a full re-ingest**. +- **Ingest always full-rebuilds** the LanceDB table (`mode: "overwrite"`). The `--rebuild` flag is a + no-op kept for compatibility. There is no incremental indexing; `audit` only *reports* missing/stale + files against the current index. +- **Privacy is a feature, not a side effect.** Redaction runs before embedding, the access log stores + query hashes/metadata only (`access-log.ts`), MCP top-K is clamped to `mcpMaxTopK`, and + `gitignore.ts` keeps `.kb/`, `.mimir/`, `private/**` ignored in target repos. `security-audit` + reports this posture and `--strict` exits non-zero on warnings. Preserve these guarantees. + +Coding conventions (KISS, DRY, YAGNI, SOLID as applied here) live in `AGENTS.md`. + +## Toolchain constraints + +- Strict TypeScript with `noUncheckedIndexedAccess` and `exactOptionalPropertyTypes`; module mode is + `NodeNext`, so relative imports use `.js` extensions even from `.ts` sources. +- Biome is the formatter and linter (not ESLint/Prettier): 2-space indent, width 100, double quotes, + semicolons as-needed, trailing commas all. +- Conventional Commits are enforced by commitlint in CI. + +Release policy (no local publish, no direct push to `main`, protected `Publish npm` workflow) lives +in `AGENTS.md`. The workflow publishes `@jcode.labs/mimir-tts` before `@jcode.labs/mimir`. diff --git a/README.md b/README.md index 5df9d1a..581d511 100644 --- a/README.md +++ b/README.md @@ -1,293 +1,31 @@ -# Mimir +# Mimir Monorepo -[![CI](https://github.com/jcode-works/jcode-mimir/actions/workflows/ci.yml/badge.svg)](https://github.com/jcode-works/jcode-mimir/actions/workflows/ci.yml) -[![CodeQL](https://github.com/jcode-works/jcode-mimir/actions/workflows/codeql.yml/badge.svg)](https://github.com/jcode-works/jcode-mimir/actions/workflows/codeql.yml) -[![npm](https://img.shields.io/npm/v/@jcode.labs/mimir)](https://www.npmjs.com/package/@jcode.labs/mimir) -[![license: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](./LICENSE) +Open-source packages for Mimir, a sovereign local RAG toolkit for confidential datasets and AI +agents. -Open-source, local-first memory and retrieval for private project knowledge. +## Packages -Mimir provides a TypeScript CLI and library that can be installed in any Node.js -repository. It indexes files from the target repository, stores vectors locally with LanceDB, -and uses Ollama for local embeddings and answers. +- [`@jcode.labs/mimir`](./packages/mimir): core CLI, library, MCP server, bundled agent skills, and + synthetic examples. +- [`@jcode.labs/mimir-tts`](./packages/mimir-tts): plug-and-play JS/ONNX text-to-speech renderer + used by `kb audio`. -Created by Jean-Baptiste Thery and published under the JCode Labs npm scope. - -Built by Jean-Baptiste Thery, freelance full-stack/AI tooling engineer at JCode Labs. - -## Open Source - -Mimir is a public open-source project under the MIT License. It is designed to be -inspectable, forkable, and usable without a JCode Labs account. - -Contributions are welcome through pull requests. Start with [`CONTRIBUTING.md`](./CONTRIBUTING.md). -Security reports should stay private and follow the policy in [`SECURITY.md`](./SECURITY.md). - -## Sponsors - -Mimir stays MIT open source. Sponsorship helps fund maintenance, issue triage, -documentation, and practical agent-workflow improvements. - -Sponsor the project through [GitHub Sponsors](https://github.com/sponsors/jb-thery). - -Suggested GitHub Sponsors tiers: - -- EUR 5/month: support the project. -- EUR 15/month: active sponsor. -- EUR 49/month: priority on issues and questions. -- EUR 199/month: company sponsor and light advisory support. - -## Status - -Early public package. APIs may evolve before `1.0.0`. - -## Requirements - -- Node.js 20+ -- pnpm, npm, yarn or bun -- Ollama running locally -- Embedding model installed once: +## Development ```bash -ollama pull nomic-embed-text +pnpm install +pnpm validate ``` -Optional answer model: +Useful filtered commands: ```bash -ollama pull gemma4 +pnpm --filter @jcode.labs/mimir test +pnpm --filter @jcode.labs/mimir-tts test +pnpm --filter @jcode.labs/mimir build +pnpm --filter @jcode.labs/mimir-tts build ``` -## Install From npm - -The package is public. Users do not need a JCode Labs account or npm token to install it. - -With pnpm: - -```bash -pnpm add -D @jcode.labs/mimir -``` - -With npm: - -```bash -npm install --save-dev @jcode.labs/mimir -``` - -Maintainer tokens are only needed to publish new versions. - -## Install From Git - -```bash -pnpm add -D git+ssh://git@github.com/jcode-works/jcode-mimir.git -``` - -For local development: - -```bash -pnpm add -D file:../jcode-mimir -``` - -Before creating an npm tarball later, run: - -```bash -pnpm build -pnpm pack -``` - -## Use In Any Repository - -Initialize the local project config: - -```bash -pnpm exec kb init -``` - -Add private documents under `private/`, then run: - -```bash -pnpm exec kb ingest -pnpm exec kb search "vendor invoice status" -pnpm exec kb ask "What do the documents prove?" -pnpm exec kb audit -pnpm exec kb security-audit -pnpm exec kb status -``` - -With npm, use `npx` after installing the package: - -```bash -npx kb init -npx kb ingest -npx kb search "vendor invoice status" -npx kb ask "What do the documents prove?" -npx kb audit -npx kb security-audit -npx kb status -``` - -## Agent Skill And MCP - -Mimir ships with a portable agent skill and a standard MCP server. - -Install the agent kit into a repository: - -```bash -pnpm exec kb install-skill -``` - -This creates: - -```plain text -.mimir/skills/mimir/SKILL.md -.mimir/mcp.json -.mimir/README.md -``` - -Agents that support skill folders can load `.mimir/skills/mimir/`. Other agents can read the -generated `.mimir/README.md` and use the MCP config snippet. - -Start the MCP server from the repository root: - -```bash -pnpm exec kb serve-mcp -``` - -MCP tools exposed: - -- `mimir_status` -- `mimir_search` -- `mimir_ask` -- `mimir_audit` -- `mimir_security_audit` - -Print the bundled skill path from the installed package: - -```bash -pnpm exec kb skill-path -``` - -## Data Boundary - -The package code lives in `node_modules` or in this repository. Project data stays in the -repository where you run the CLI: - -```plain text -your-project/ - private/ # raw documents to ingest - .kb/config.json # local config - .kb/sources.txt # optional extra source paths - .kb/storage/ # generated LanceDB index - .kb/access.log # metadata-only access log -``` - -The package never ships project documents. `kb init` adds gitignore entries for `.kb/` -and `private/**`, and `kb install-skill` keeps `.mimir/` ignored as generated local agent -state. - -## Confidentiality Defaults - -Mimir is designed for private repositories and sensitive local evidence. - -- Zero telemetry: no analytics or document content is sent to JCode Labs. -- Local-only network policy: Ollama must be on loopback by default. -- Redaction before indexing: common secrets and identifiers are redacted before chunks are - embedded and stored. -- Metadata-only access logs: query hashes and action metadata are logged, not raw queries. -- MCP is read-focused and bounded by `mcpMaxTopK`. -- Generated local state is ignored by Git. - -Run: - -```bash -pnpm exec kb security-audit --strict -``` - -Remove the generated vector index: - -```bash -pnpm exec kb destroy-index --yes -``` - -For air-gapped operation, release verification, secure deletion limits, and threat model details, -read [`SECURITY-HARDENING.md`](./SECURITY-HARDENING.md). - -## Supported Files - -- Markdown: `.md`, `.mdx` -- Text: `.txt`, `.text` -- JSON: `.json` -- YAML: `.yaml`, `.yml` -- CSV/TSV: `.csv`, `.tsv` -- HTML: `.html`, `.htm` -- PDF: `.pdf` - -## Config - -`.kb/config.json`: - -```json -{ - "rawDir": "private", - "storageDir": ".kb/storage", - "sourcesFile": ".kb/sources.txt", - "accessLogPath": ".kb/access.log", - "tableName": "chunks", - "ollamaHost": "http://localhost:11434", - "networkPolicy": "local-only", - "embedModel": "nomic-embed-text", - "llmModel": "gemma4:latest", - "redaction": { - "enabled": true, - "builtIn": true, - "patterns": [] - }, - "accessLog": true, - "mcpMaxTopK": 10, - "topK": 5, - "chunkSize": 1200, - "chunkOverlap": 150 -} -``` - -Environment overrides: - -- `KB_RAW_DIR` -- `KB_STORAGE_DIR` -- `KB_SOURCES_FILE` -- `KB_ACCESS_LOG_PATH` -- `KB_OLLAMA_HOST` -- `KB_NETWORK_POLICY` -- `KB_EMBED_MODEL` -- `KB_LLM_MODEL` -- `KB_REDACTION_ENABLED` -- `KB_REDACTION_BUILT_IN` -- `KB_ACCESS_LOG` -- `KB_MCP_MAX_TOP_K` -- `KB_TOP_K` -- `KB_CHUNK_SIZE` -- `KB_CHUNK_OVERLAP` - -## Library API - -```ts -import { ingest, search, ask } from "@jcode.labs/mimir" - -await ingest({ rebuild: true }) -const results = await search("vendor invoice status") -const answer = await ask("What documents support the project timeline?") -``` - -## Privacy - -- Embeddings and answers use local Ollama by default. -- Remote Ollama hosts are blocked unless `networkPolicy` explicitly allows them. -- Built-in redaction runs before indexing by default. -- Access logs store query hashes, not raw queries. -- The vector index is stored locally. -- Raw private documents should stay in the target repository's ignored `private/` folder. -- Do not put secrets or scans inside this package repository. - -## License - -MIT © Jean-Baptiste Thery. +The root package is private and only orchestrates workspace tasks. npm publishing is handled by the +protected `Publish npm` GitHub Actions workflow, which publishes `@jcode.labs/mimir-tts` before +`@jcode.labs/mimir`. diff --git a/RELEASING.md b/RELEASING.md index 08ae53d..0616e7b 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -11,7 +11,8 @@ and approved by Jean-Baptiste Thery through the protected `npm-publish` environm 2. Wait for the required CI checks to pass. 3. Merge only after approval and green checks. 4. Trigger the `Publish npm` workflow manually from `main`. -5. Enter the version already committed in `package.json`. +5. Enter the version already committed in `packages/mimir/package.json` and + `packages/mimir-tts/package.json`. 6. Approve the protected `npm-publish` environment when GitHub asks for review. The publish workflow refuses to run from any branch other than `main`, verifies that the diff --git a/SECURITY-HARDENING.md b/SECURITY-HARDENING.md index f534555..f3873b9 100644 --- a/SECURITY-HARDENING.md +++ b/SECURITY-HARDENING.md @@ -1,13 +1,16 @@ # Mimir Security Hardening -Mimir is a local-first knowledge base for private project documents. It is built to minimize -data movement, but it is not a certified high-assurance system. +Mimir is a sovereign local RAG knowledge base for confidential project documents and datasets. It is +built to minimize data movement, but it is not a certified high-assurance system. ## Current Guarantees - Zero telemetry: Mimir does not send usage analytics or document content to JCode Labs. -- Local-only network policy by default: document text can only be sent to loopback Ollama hosts - unless the repository explicitly opts in to broader network access. +- Retrieval-only core: Mimir does not call a chat model or generate LLM answers. +- No-model retrieval mode: `embeddingProvider: "local-hash"` can ingest, search, and return cited + passages without a model server. +- Optional semantic embeddings: `embeddingProvider: "transformers"` uses Transformers.js, with + remote model loading disabled by default through `transformersAllowRemoteModels: false`. - Redaction before indexing: built-in DLP patterns redact common secrets and identifiers before chunks are embedded and stored. - Metadata-only access logs: access logs contain action metadata and query hashes, not raw @@ -16,13 +19,15 @@ data movement, but it is not a certified high-assurance system. default. - MCP is read-focused: destructive tools are not exposed over MCP, and MCP retrieval is capped by `mcpMaxTopK`. +- Optional audio summaries use `kb audio` / `@jcode.labs/mimir-tts` for local WAV rendering with + Transformers.js. They do not require Python, ffmpeg, Piper, XTTS, or a local TTS server. - npm releases are published with provenance from the protected GitHub Actions workflow. - Release artifacts include a package tarball, SHA256 checksums, SBOM, and manifest. ## Threat Model -Mimir protects against accidental repository leaks, accidental remote LLM usage, accidental secret -indexing, and weak release traceability. +Mimir protects against accidental repository leaks, accidental built-in LLM usage, accidental online +TTS usage for generated summaries, accidental secret indexing, and weak release traceability. Mimir does not protect against a compromised local machine, malicious dependencies already present in the runtime, a user with filesystem access to the same checkout, or forensic recovery from an @@ -50,32 +55,45 @@ pnpm build pnpm release:artifacts ``` -Move the generated tarball from `release-artifacts/` into the offline environment and install it: +Move the generated tarballs from `release-artifacts/` into the offline environment and install them: ```bash -pnpm add -D ./jcode.labs-mimir-.tgz +pnpm add -D ./jcode.labs-mimir-tts-.tgz ./jcode.labs-mimir-.tgz pnpm exec kb init pnpm exec kb ingest ``` -Ollama and the required models must also be preloaded inside the offline environment. +For semantic embeddings, preload the Transformers.js-compatible embedding model files inside the +offline environment under the configured `embeddingModelPath`. For audio, preload the TTS model +files under `.mimir/models/tts` and render with `pnpm exec kb audio --offline`. ## Zero Network Posture -Default config: +Default no-model config: ```json { - "ollamaHost": "http://localhost:11434", - "networkPolicy": "local-only" + "embeddingProvider": "local-hash" } ``` -Allowed policies: +Optional semantic config: -- `local-only`: only loopback hosts such as `localhost` and `127.0.0.1`. -- `allow-private`: loopback and private LAN hosts. -- `allow-any`: any host. Use only when the remote endpoint is explicitly trusted. +```json +{ + "embeddingProvider": "transformers", + "embeddingModel": "mixedbread-ai/mxbai-embed-xsmall-v1", + "embeddingModelPath": ".mimir/models", + "transformersAllowRemoteModels": false +} +``` + +The local-hash mode performs lexical/hash retrieval only. It is useful for smoke tests, +dependency-light offline workflows, and handing cited passages to another trusted LLM. It is not +equivalent to model semantic retrieval. + +Keep `transformersAllowRemoteModels` false for confidential or air-gapped work. If it is true, +Transformers.js may download model files from Hugging Face during model loading. Run: @@ -108,6 +126,25 @@ Custom patterns can be added in `.kb/config.json`: Redaction changes the indexed text, not the raw files under `private/`. +## Optional Audio Summaries + +`kb install-skill` installs an optional `mimir-audio-summary` skill. It is designed for listenable +briefings from a local Mimir index. The default renderer is `kb audio`, backed by +`@jcode.labs/mimir-tts` and Transformers.js. + +Confidentiality defaults: + +- narration text is written to a temp file outside the repository; +- generated WAV audio should be written under `.mimir/audio/`; +- `.mimir/` is ignored by Git; +- Python, ffmpeg, Piper, XTTS, and local TTS servers are not required for the default path; +- the first online-enabled render may download public model weights into `.mimir/models/tts`, but + the narration text is processed locally; +- `--offline` disables remote model loading and requires preloaded model files. + +Generated audio can still contain sensitive information. Treat it like a derived confidential +document. + ## MCP Hardening MCP gives an agent access to retrieved private context. Use it only for agents running under the @@ -124,16 +161,17 @@ For team use, prefer one checkout per user or per role. Mimir does not implement ## Release Verification -The protected npm workflow runs validation, generates release artifacts, and publishes with -provenance: +The protected npm workflow runs validation, generates release artifacts, and publishes both +workspace packages with provenance: ```bash -npm publish --access public --provenance +pnpm --dir packages/mimir-tts publish --access public --provenance --no-git-checks +pnpm --dir packages/mimir publish --access public --provenance --no-git-checks ``` Release artifacts include: -- npm tarball; +- npm tarballs for `@jcode.labs/mimir-tts` and `@jcode.labs/mimir`; - `SHA256SUMS`; - CycloneDX SBOM; - `release-manifest.json`. diff --git a/biome.json b/biome.json index 695fb0e..749fded 100644 --- a/biome.json +++ b/biome.json @@ -6,7 +6,7 @@ "useIgnoreFile": true }, "files": { - "includes": ["*.cjs", "*.json", "scripts/**/*.mjs", "src/**/*.ts"] + "includes": ["*.cjs", "*.json", "scripts/**/*.mjs", "packages/*/src/**/*.ts"] }, "formatter": { "enabled": true, diff --git a/dist/chunking.js.map b/dist/chunking.js.map deleted file mode 100644 index 6b91a2c..0000000 --- a/dist/chunking.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"chunking.js","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AAGxC,MAAM,UAAU,aAAa,CAC3B,QAAwB,EACxB,SAAiB,EACjB,YAAoB;IAEpB,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnB,OAAO,EAAE,CAAA;IACX,CAAC;IAED,MAAM,MAAM,GAAgB,EAAE,CAAA;IAC9B,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,UAAU,GAAG,CAAC,CAAA;IAElB,OAAO,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,cAAc,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,CAAC,CAAA;QAC5D,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;QAEpD,IAAI,IAAI,EAAE,CAAC;YACT,MAAM,EAAE,GAAG,UAAU,CAAC,QAAQ,CAAC;iBAC5B,MAAM,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,YAAY,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;iBAC7D,MAAM,CAAC,KAAK,CAAC,CAAA;YAChB,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE;gBACF,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM;gBAC5B,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,YAAY;gBACxC,UAAU;gBACV,IAAI;gBACJ,QAAQ,EAAE,QAAQ,CAAC,IAAI,CAAC,QAAQ;gBAChC,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK;gBAC1B,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,OAAO;aAC/B,CAAC,CAAA;YACF,UAAU,IAAI,CAAC,CAAA;QACjB,CAAC;QAED,IAAI,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAChC,MAAK;QACP,CAAC;QACD,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,YAAY,EAAE,MAAM,GAAG,CAAC,CAAC,CAAA;IACnD,CAAC;IAED,OAAO,MAAM,CAAA;AACf,CAAC;AAED,SAAS,cAAc,CAAC,IAAY,EAAE,MAAc,EAAE,SAAiB;IACrE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;IACzD,IAAI,OAAO,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;QAC5B,OAAO,OAAO,CAAA;IAChB,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAC1C,MAAM,cAAc,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAA;IACjD,IAAI,cAAc,GAAG,SAAS,GAAG,IAAI,EAAE,CAAC;QACtC,OAAO,MAAM,GAAG,cAAc,CAAA;IAChC,CAAC;IAED,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAC5B,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,EACxB,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,EACxB,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CACzB,CAAA;IACD,IAAI,aAAa,GAAG,SAAS,GAAG,IAAI,EAAE,CAAC;QACrC,OAAO,MAAM,GAAG,aAAa,GAAG,CAAC,CAAA;IACnC,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAA;IAC1C,IAAI,UAAU,GAAG,SAAS,GAAG,IAAI,EAAE,CAAC;QAClC,OAAO,MAAM,GAAG,UAAU,CAAA;IAC5B,CAAC;IAED,OAAO,OAAO,CAAA;AAChB,CAAC"} \ No newline at end of file diff --git a/dist/cli.js.map b/dist/cli.js.map deleted file mode 100644 index e6654f3..0000000 --- a/dist/cli.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AACnC,OAAO,EAAE,MAAM,YAAY,CAAA;AAC3B,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AAC3C,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAA;AACvC,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAA;AACnC,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;AACxC,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC7C,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAC3D,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AACtC,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAA;AAEtC,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAA;AAE7B,OAAO;KACJ,IAAI,CAAC,IAAI,CAAC;KACV,WAAW,CAAC,+DAA+D,CAAC;KAC5E,OAAO,CAAC,OAAO,CAAC,CAAA;AAEnB,OAAO;KACJ,OAAO,CAAC,MAAM,CAAC;KACf,WAAW,CAAC,iFAAiF,CAAC;KAC9F,MAAM,CAAC,KAAK,IAAI,EAAE;IACjB,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;IAChD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAA;QAC7C,OAAM;IACR,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAA;IACjC,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QAC3B,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,CAAA;IAC5B,CAAC;AACH,CAAC,CAAC,CAAA;AAEJ,OAAO;KACJ,OAAO,CAAC,QAAQ,CAAC;KACjB,WAAW,CAAC,oFAAoF,CAAC;KACjG,MAAM,CAAC,WAAW,EAAE,qEAAqE,CAAC;KAC1F,MAAM,CAAC,KAAK,IAAI,EAAE;IACjB,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAA;IAClE,OAAO,CAAC,GAAG,CACT,EAAE,CAAC,KAAK,CACN,sBAAsB,MAAM,CAAC,YAAY,WAAW,MAAM,CAAC,MAAM,iBAAiB,MAAM,CAAC,YAAY,eAAe,MAAM,CAAC,UAAU,WAAW,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CACvK,CACF,CAAA;IACD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG,CAAC,OAAO,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;IAC9D,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAA;IACtB,CAAC;AACH,CAAC,CAAC,CAAA;AAEJ,OAAO;KACJ,OAAO,CAAC,QAAQ,CAAC;KACjB,WAAW,CAAC,6DAA6D,CAAC;KAC1E,QAAQ,CAAC,SAAS,EAAE,eAAe,CAAC;KACpC,MAAM,CAAC,sBAAsB,EAAE,+BAA+B,EAAE,gBAAgB,CAAC;KACjF,MAAM,CAAC,KAAK,EAAE,KAAa,EAAE,OAA0B,EAAE,EAAE;IAC1D,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAA;IAC3D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,sDAAsD,CAAC,CAAC,CAAA;QAChF,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAA;QACpB,OAAM;IACR,CAAC;IAED,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;QAChD,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;QAC9E,OAAO,CAAC,GAAG,CACT,KAAK,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,GAAG,CAAC,KAAK,MAAM,CAAC,YAAY,EAAE,CAAC,UAAU,MAAM,CAAC,UAAU,aAAa,QAAQ,EAAE,CACxG,CAAA;QACD,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAA;IACxC,CAAC;AACH,CAAC,CAAC,CAAA;AAEJ,OAAO;KACJ,OAAO,CAAC,KAAK,CAAC;KACd,WAAW,CAAC,sEAAsE,CAAC;KACnF,QAAQ,CAAC,SAAS,EAAE,qBAAqB,CAAC;KAC1C,MAAM,CAAC,sBAAsB,EAAE,4BAA4B,EAAE,gBAAgB,CAAC;KAC9E,MAAM,CAAC,KAAK,EAAE,KAAa,EAAE,OAA0B,EAAE,EAAE;IAC1D,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAA;IACvD,OAAO,CAAC,GAAG,CAAC,KAAK,MAAM,CAAC,MAAM,IAAI,CAAC,CAAA;IACnC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAA;QAC/B,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;YACvD,OAAO,CAAC,GAAG,CAAC,MAAM,KAAK,GAAG,CAAC,KAAK,MAAM,CAAC,YAAY,UAAU,MAAM,CAAC,UAAU,EAAE,CAAC,CAAA;QACnF,CAAC;IACH,CAAC;AACH,CAAC,CAAC,CAAA;AAEJ,OAAO;KACJ,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,gEAAgE,CAAC;KAC7E,MAAM,CAAC,KAAK,IAAI,EAAE;IACjB,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;IACzC,OAAO,CAAC,GAAG,CAAC,kBAAkB,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE,CAAC,CAAA;IAC7D,OAAO,CAAC,GAAG,CAAC,gBAAgB,MAAM,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAA;IACzD,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,WAAW,EAAE,CAAC,CAAA;IAChD,OAAO,CAAC,GAAG,CAAC,oBAAoB,MAAM,CAAC,gBAAgB,CAAC,MAAM,EAAE,CAAC,CAAA;IACjE,OAAO,CAAC,GAAG,CAAC,gBAAgB,MAAM,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAA;IAEzD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,CAAA;IAC5C,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;QACvC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,CAAA;IACvC,CAAC;IAED,IAAI,MAAM,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAA;IACtB,CAAC;AACH,CAAC,CAAC,CAAA;AAEJ,OAAO;KACJ,OAAO,CAAC,QAAQ,CAAC;KACjB,WAAW,CAAC,gDAAgD,CAAC;KAC7D,MAAM,CAAC,KAAK,IAAI,EAAE;IACjB,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;IAC9C,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,CAAA;IACpC,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,WAAW,EAAE,CAAC,CAAA;IAChD,OAAO,CAAC,GAAG,CAAC,UAAU,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;IACtC,OAAO,CAAC,GAAG,CAAC,cAAc,MAAM,CAAC,UAAU,EAAE,CAAC,CAAA;IAC9C,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,WAAW,EAAE,CAAC,CAAA;IAChD,OAAO,CAAC,GAAG,CAAC,iBAAiB,MAAM,CAAC,aAAa,EAAE,CAAC,CAAA;IACpD,OAAO,CAAC,GAAG,CAAC,iBAAiB,MAAM,CAAC,aAAa,EAAE,CAAC,CAAA;IACpD,OAAO,CAAC,GAAG,CAAC,cAAc,MAAM,CAAC,UAAU,EAAE,CAAC,CAAA;IAC9C,OAAO,CAAC,GAAG,CAAC,YAAY,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAA;IAC1C,OAAO,CAAC,GAAG,CAAC,oBAAoB,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,CAAA;IAC3D,OAAO,CAAC,GAAG,CAAC,aAAa,MAAM,CAAC,SAAS,EAAE,CAAC,CAAA;IAC5C,OAAO,CAAC,GAAG,CAAC,cAAc,MAAM,CAAC,UAAU,EAAE,CAAC,CAAA;IAC9C,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAA;AACtC,CAAC,CAAC,CAAA;AAEJ,OAAO;KACJ,OAAO,CAAC,gBAAgB,CAAC;KACzB,WAAW,CAAC,qEAAqE,CAAC;KAClF,MAAM,CAAC,QAAQ,EAAE,8BAA8B,CAAC;KAChD,MAAM,CAAC,UAAU,EAAE,6CAA6C,CAAC;KACjE,MAAM,CAAC,KAAK,EAAE,OAA6C,EAAE,EAAE;IAC9D,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;IACjD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAA;IAC9C,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,iBAAiB,MAAM,CAAC,aAAa,EAAE,CAAC,CAAA;QACpD,OAAO,CAAC,GAAG,CAAC,iBAAiB,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;QACrD,OAAO,CAAC,GAAG,CAAC,cAAc,MAAM,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAA;QACtD,OAAO,CAAC,GAAG,CAAC,4BAA4B,MAAM,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC,CAAA;QACxE,OAAO,CAAC,GAAG,CAAC,oBAAoB,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,CAAA;QAC3D,OAAO,CAAC,GAAG,CAAC,oBAAoB,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,CAAA;QAC3D,OAAO,CAAC,GAAG,CAAC,aAAa,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,CAAA;QACpD,OAAO,CAAC,GAAG,CAAC,6BAA6B,MAAM,CAAC,SAAS,CAAC,gBAAgB,EAAE,CAAC,CAAA;QAC7E,OAAO,CAAC,GAAG,CAAC,qBAAqB,MAAM,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAA;QAC7D,OAAO,CAAC,GAAG,CAAC,cAAc,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAA;QAC/C,OAAO,CAAC,GAAG,CAAC,8BAA8B,MAAM,CAAC,GAAG,CAAC,uBAAuB,EAAE,CAAC,CAAA;QAC/E,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACtC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,YAAY,OAAO,EAAE,CAAC,CAAC,CAAA;QAC/C,CAAC;IACH,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjD,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAA;IACtB,CAAC;AACH,CAAC,CAAC,CAAA;AAEJ,OAAO;KACJ,OAAO,CAAC,eAAe,CAAC;KACxB,WAAW,CAAC,2DAA2D,CAAC;KACxE,MAAM,CAAC,OAAO,EAAE,iDAAiD,CAAC;KAClE,MAAM,CAAC,KAAK,EAAE,OAA0B,EAAE,EAAE;IAC3C,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;QACjB,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG,CAAC,6CAA6C,CAAC,CAAC,CAAA;QACpE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAA;QACpB,OAAM;IACR,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;IAChD,OAAO,CAAC,GAAG,CAAC,cAAc,MAAM,CAAC,UAAU,EAAE,CAAC,CAAA;IAC9C,OAAO,CAAC,GAAG,CAAC,WAAW,MAAM,CAAC,OAAO,EAAE,CAAC,CAAA;IACxC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;AAC1B,CAAC,CAAC,CAAA;AAEJ,OAAO;KACJ,OAAO,CAAC,WAAW,CAAC;KACpB,WAAW,CACV,qFAAqF,CACtF;KACA,MAAM,CAAC,KAAK,IAAI,EAAE;IACjB,MAAM,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;AAC/B,CAAC,CAAC,CAAA;AAEJ,OAAO;KACJ,OAAO,CAAC,YAAY,CAAC;KACrB,WAAW,CAAC,+EAA+E,CAAC;KAC5F,MAAM,CAAC,GAAG,EAAE;IACX,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC,CAAA;AACjC,CAAC,CAAC,CAAA;AAEJ,OAAO;KACJ,OAAO,CAAC,eAAe,CAAC;KACxB,WAAW,CAAC,kFAAkF,CAAC;KAC/F,MAAM,CACL,qBAAqB,EACrB,oDAAoD,EACpD,eAAe,CAChB;KACA,MAAM,CAAC,KAAK,EAAE,OAA8B,EAAE,EAAE;IAC/C,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAA;IACvF,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAA;IACzC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QAClC,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,CAAA;IAC5B,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,SAAS,EAAE,CAAC,CAAA;IAC9C,OAAO,CAAC,GAAG,CAAC,uBAAuB,MAAM,CAAC,aAAa,EAAE,CAAC,CAAA;AAC5D,CAAC,CAAC,CAAA;AAEJ,MAAM,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;AAEtC,SAAS,gBAAgB,CAAC,KAAa;IACrC,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAA;IACzC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAA;IACjD,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC;AAED,SAAS,QAAQ,CAAC,IAAwB;IACxC,OAAO,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,CAAA;AACnF,CAAC"} \ No newline at end of file diff --git a/dist/config.d.ts.map b/dist/config.d.ts.map deleted file mode 100644 index 96ee57f..0000000 --- a/dist/config.d.ts.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;AAuCxC,wBAAgB,eAAe,CAAC,KAAK,SAAgB,GAAG,MAAM,CAc7D;AAED,wBAAsB,UAAU,CAAC,KAAK,SAAgB,GAAG,OAAO,CAAC,MAAM,CAAC,CAgCvE"} \ No newline at end of file diff --git a/dist/config.js.map b/dist/config.js.map deleted file mode 100644 index 99a1dea..0000000 --- a/dist/config.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAGvB,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC;IACrC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,aAAa,CAAC;IAC7C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,iBAAiB,CAAC;IAClD,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,gBAAgB,CAAC;IACnD,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC;IACvC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,wBAAwB,CAAC;IACxD,aAAa,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,eAAe,EAAE,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC;IACzF,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,kBAAkB,CAAC;IAClD,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,CAAC;IAC7C,SAAS,EAAE,CAAC;SACT,MAAM,CAAC;QACN,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;QAClC,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;QAClC,QAAQ,EAAE,CAAC;aACR,KAAK,CACJ,CAAC,CAAC,MAAM,CAAC;YACP,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;YACvB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1B,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YAC5B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;SACnC,CAAC,CACH;aACA,OAAO,CAAC,EAAE,CAAC;KACf,CAAC;SACD,OAAO,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IAC1D,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;IACpC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IACnD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAC5C,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;IACpD,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC;CAC1D,CAAC,CAAA;AAIF,MAAM,WAAW,GAAG,iBAAiB,CAAA;AAErC,MAAM,UAAU,eAAe,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,EAAE;IACnD,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IAEjC,OAAO,IAAI,EAAE,CAAC;QACZ,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,EAAE,CAAC;YAChD,OAAO,OAAO,CAAA;QAChB,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;QACpC,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;QAC5B,CAAC;QACD,OAAO,GAAG,MAAM,CAAA;IAClB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,EAAE;IACpD,MAAM,WAAW,GAAG,eAAe,CAAC,KAAK,CAAC,CAAA;IAC1C,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAA;IACtD,MAAM,GAAG,GAAG,UAAU,CAAC,UAAU,CAAC;QAChC,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,CAAa;QAC7D,CAAC,CAAC,EAAE,CAAA;IAEN,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IACzC,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAA;IAEhC,IAAI,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAA;IAC/D,CAAC;IAED,OAAO;QACL,WAAW;QACX,MAAM,EAAE,eAAe,CAAC,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC;QACpD,UAAU,EAAE,eAAe,CAAC,WAAW,EAAE,OAAO,CAAC,UAAU,CAAC;QAC5D,WAAW,EAAE,eAAe,CAAC,WAAW,EAAE,OAAO,CAAC,WAAW,CAAC;QAC9D,aAAa,EAAE,eAAe,CAAC,WAAW,EAAE,OAAO,CAAC,aAAa,CAAC;QAClE,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,YAAY,EAAE,OAAO,CAAC,YAAY;KACnC,CAAA;AACH,CAAC;AAED,SAAS,eAAe,CAAC,WAAmB,EAAE,KAAa;IACzD,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,KAAK,CAAC,CAAA;AAC1E,CAAC;AAED,SAAS,QAAQ,CAAC,MAAiB;IACjC,OAAO;QACL,GAAG,MAAM;QACT,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,MAAM,CAAC,MAAM;QAC/C,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,MAAM,CAAC,UAAU;QAC3D,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,MAAM,CAAC,WAAW;QAC9D,aAAa,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,MAAM,CAAC,aAAa;QACrE,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,MAAM,CAAC,UAAU;QAC3D,aAAa,EAAE,oBAAoB,CAAC,mBAAmB,EAAE,MAAM,CAAC,aAAa,CAAC;QAC9E,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,MAAM,CAAC,UAAU;QAC3D,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,MAAM,CAAC,QAAQ;QACrD,SAAS,EAAE;YACT,GAAG,MAAM,CAAC,SAAS;YACnB,OAAO,EAAE,cAAc,CAAC,sBAAsB,EAAE,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC;YACzE,OAAO,EAAE,cAAc,CAAC,uBAAuB,EAAE,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC;SAC3E;QACD,SAAS,EAAE,cAAc,CAAC,eAAe,EAAE,MAAM,CAAC,SAAS,CAAC;QAC5D,UAAU,EAAE,kBAAkB,CAAC,kBAAkB,EAAE,MAAM,CAAC,UAAU,CAAC;QACrE,IAAI,EAAE,kBAAkB,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC;QACjD,SAAS,EAAE,kBAAkB,CAAC,eAAe,EAAE,MAAM,CAAC,SAAS,CAAC;QAChE,YAAY,EAAE,qBAAqB,CAAC,kBAAkB,EAAE,MAAM,CAAC,YAAY,CAAC;KAC7E,CAAA;AACH,CAAC;AAED,SAAS,oBAAoB,CAC3B,IAAY,EACZ,QAAoC;IAEpC,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAC7B,IAAI,GAAG,KAAK,YAAY,IAAI,GAAG,KAAK,eAAe,IAAI,GAAG,KAAK,WAAW,EAAE,CAAC;QAC3E,OAAO,GAAG,CAAA;IACZ,CAAC;IACD,OAAO,QAAQ,CAAA;AACjB,CAAC;AAED,SAAS,cAAc,CAAC,IAAY,EAAE,QAAiB;IACrD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,CAAA;IAC5C,IAAI,GAAG,KAAK,GAAG,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;QACnD,OAAO,IAAI,CAAA;IACb,CAAC;IACD,IAAI,GAAG,KAAK,GAAG,IAAI,GAAG,KAAK,OAAO,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;QACnD,OAAO,KAAK,CAAA;IACd,CAAC;IACD,OAAO,QAAQ,CAAA;AACjB,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY,EAAE,QAAgB;IACxD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAC7B,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO,QAAQ,CAAA;IACjB,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAA;IACtC,OAAO,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAA;AAChE,CAAC;AAED,SAAS,qBAAqB,CAAC,IAAY,EAAE,QAAgB;IAC3D,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAC7B,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO,QAAQ,CAAA;IACjB,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAA;IACtC,OAAO,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAA;AACjE,CAAC"} \ No newline at end of file diff --git a/dist/embeddings.d.ts.map b/dist/embeddings.d.ts.map deleted file mode 100644 index 2ed43b8..0000000 --- a/dist/embeddings.d.ts.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"embeddings.d.ts","sourceRoot":"","sources":["../src/embeddings.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;AAExC,wBAAsB,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAmBrF;AAED,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAM/E"} \ No newline at end of file diff --git a/dist/embeddings.js b/dist/embeddings.js deleted file mode 100644 index ad9f116..0000000 --- a/dist/embeddings.js +++ /dev/null @@ -1,25 +0,0 @@ -import { Ollama } from "ollama"; -import { assertNetworkPolicy } from "./network.js"; -export async function embedTexts(texts, config) { - if (texts.length === 0) { - return []; - } - assertNetworkPolicy(config); - const client = new Ollama({ host: config.ollamaHost }); - const response = await client.embed({ - model: config.embedModel, - input: texts, - }); - if (!response.embeddings || response.embeddings.length !== texts.length) { - throw new Error(`Expected ${texts.length} embeddings, received ${response.embeddings?.length ?? 0}.`); - } - return response.embeddings; -} -export async function embedText(text, config) { - const [embedding] = await embedTexts([text], config); - if (!embedding) { - throw new Error("No embedding returned for query."); - } - return embedding; -} -//# sourceMappingURL=embeddings.js.map \ No newline at end of file diff --git a/dist/embeddings.js.map b/dist/embeddings.js.map deleted file mode 100644 index 2e23048..0000000 --- a/dist/embeddings.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"embeddings.js","sourceRoot":"","sources":["../src/embeddings.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAA;AAC/B,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAA;AAGlD,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,KAAe,EAAE,MAAc;IAC9D,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,EAAE,CAAA;IACX,CAAC;IAED,mBAAmB,CAAC,MAAM,CAAC,CAAA;IAC3B,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC,CAAA;IACtD,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC;QAClC,KAAK,EAAE,MAAM,CAAC,UAAU;QACxB,KAAK,EAAE,KAAK;KACb,CAAC,CAAA;IAEF,IAAI,CAAC,QAAQ,CAAC,UAAU,IAAI,QAAQ,CAAC,UAAU,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;QACxE,MAAM,IAAI,KAAK,CACb,YAAY,KAAK,CAAC,MAAM,yBAAyB,QAAQ,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC,GAAG,CACrF,CAAA;IACH,CAAC;IAED,OAAO,QAAQ,CAAC,UAAU,CAAA;AAC5B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAY,EAAE,MAAc;IAC1D,MAAM,CAAC,SAAS,CAAC,GAAG,MAAM,UAAU,CAAC,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,CAAA;IACpD,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAA;IACrD,CAAC;IACD,OAAO,SAAS,CAAA;AAClB,CAAC"} \ No newline at end of file diff --git a/dist/files.d.ts.map b/dist/files.d.ts.map deleted file mode 100644 index 0a31b69..0000000 --- a/dist/files.d.ts.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"files.d.ts","sourceRoot":"","sources":["../src/files.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,YAAY,CAAA;AAEpD,eAAO,MAAM,oBAAoB,aAa/B,CAAA;AAEF,wBAAsB,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAuC3E"} \ No newline at end of file diff --git a/dist/files.js.map b/dist/files.js.map deleted file mode 100644 index 417c890..0000000 --- a/dist/files.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"files.js","sourceRoot":"","sources":["../src/files.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAA;AACjD,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,MAAM,WAAW,CAAA;AAG1B,MAAM,CAAC,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC;IAC1C,MAAM;IACN,MAAM;IACN,OAAO;IACP,OAAO;IACP,KAAK;IACL,MAAM;IACN,MAAM;IACN,OAAO;IACP,MAAM;IACN,MAAM;IACN,OAAO;IACP,MAAM;CACP,CAAC,CAAA;AAEF,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,MAAc;IAClD,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,CAAA;IACvC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAsB,CAAA;IAE3C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACtB,SAAQ;QACV,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,MAAM,EAAE;YAC/B,GAAG,EAAE,IAAI;YACT,QAAQ,EAAE,IAAI;YACd,SAAS,EAAE,IAAI;YACf,GAAG,EAAE,KAAK;YACV,mBAAmB,EAAE,KAAK;YAC1B,MAAM,EAAE,CAAC,YAAY,EAAE,oBAAoB,EAAE,WAAW,EAAE,cAAc,CAAC;SAC1E,CAAC,CAAA;QAEF,KAAK,MAAM,YAAY,IAAI,OAAO,EAAE,CAAC;YACnC,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,WAAW,EAAE,CAAA;YAC1D,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBACzC,SAAQ;YACV,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,CAAA;YACrC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,YAAY,CAAC,CAAA;YAC3C,KAAK,CAAC,GAAG,CAAC,YAAY,EAAE;gBACtB,YAAY;gBACZ,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,YAAY,CAAC;gBAC7D,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC;gBACxE,SAAS;gBACT,KAAK,EAAE,IAAI,CAAC,IAAI;gBAChB,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAC5D,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAA;AACzF,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,MAAc;IACvC,MAAM,KAAK,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;IAC7B,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC;QACpC,OAAO,KAAK,CAAA;IACd,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,CAAA;IAC1D,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;QAC3B,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACxC,SAAQ;QACV,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,CAAA;IAC5F,CAAC;IAED,OAAO,KAAK,CAAA;AACd,CAAC"} \ No newline at end of file diff --git a/dist/gitignore.d.ts b/dist/gitignore.d.ts deleted file mode 100644 index 9871cae..0000000 --- a/dist/gitignore.d.ts +++ /dev/null @@ -1,3 +0,0 @@ -export declare const MIMIR_GITIGNORE_ENTRIES: readonly [".kb/", ".mimir/", "private/**", "!private/", "!private/README.md", "!private/**/", "!private/**/.gitkeep"]; -export declare function ensureMimirGitignore(cwd?: string): Promise; -//# sourceMappingURL=gitignore.d.ts.map \ No newline at end of file diff --git a/dist/gitignore.d.ts.map b/dist/gitignore.d.ts.map deleted file mode 100644 index d3a258b..0000000 --- a/dist/gitignore.d.ts.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"gitignore.d.ts","sourceRoot":"","sources":["../src/gitignore.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,uBAAuB,uHAQ1B,CAAA;AAEV,wBAAsB,oBAAoB,CAAC,GAAG,SAAgB,GAAG,OAAO,CAAC,OAAO,CAAC,CAyBhF"} \ No newline at end of file diff --git a/dist/ingest.js.map b/dist/ingest.js.map deleted file mode 100644 index 36b612a..0000000 --- a/dist/ingest.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"ingest.js","sourceRoot":"","sources":["../src/ingest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAC9C,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;AACxC,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAA;AAC5D,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAUrD,MAAM,gBAAgB,GAAG,EAAE,CAAA;AAE3B,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,UAAyB,EAAE;IACtD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;IACrE,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,CAAA;IAC3C,MAAM,SAAS,GAAgB,EAAE,CAAA;IACjC,MAAM,MAAM,GAA2B,EAAE,CAAA;IACzC,MAAM,eAAe,GAAqB,EAAE,CAAA;IAC5C,IAAI,YAAY,GAAG,CAAC,CAAA;IAEpB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,CAAA;YACpC,MAAM,QAAQ,GAAG,UAAU,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;YAChD,eAAe,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAA;YACxC,MAAM,MAAM,GAAG,aAAa,CAC1B,EAAE,GAAG,MAAM,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,EAClC,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,YAAY,CACpB,CAAA;YACD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACxB,YAAY,IAAI,CAAC,CAAA;YACnB,CAAC;YACD,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAA;QAC3B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,IAAI,CAAC,YAAY;gBACvB,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAChE,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAgB,EAAE,CAAA;IAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,gBAAgB,EAAE,CAAC;QAC5D,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,CAAA;QACtD,MAAM,UAAU,GAAG,MAAM,UAAU,CACjC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,EAChC,MAAM,CACP,CAAA;QACD,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;YAC7C,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,CAAA;YAChC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,+BAA+B,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,UAAU,GAAG,CAAC,CAAA;YAC3F,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,KAAK,EAAE,MAAM,EAAE,CAAC,CAAA;QACjC,CAAC;IACH,CAAC;IAED,MAAM,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IAC7B,MAAM,YAAY,CAAC,MAAM,EAAE;QACzB,MAAM,EAAE,QAAQ;QAChB,WAAW,EAAE,IAAI,CAAC,MAAM;QACxB,UAAU,EAAE,eAAe,CAAC,eAAe,CAAC;KAC7C,CAAC,CAAA;IAEF,OAAO;QACL,YAAY,EAAE,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI;QAC/D,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,YAAY;QACZ,UAAU,EAAE,eAAe,CAAC,eAAe,CAAC;QAC5C,MAAM;KACP,CAAA;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE;IAC7C,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAA;IACpC,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,CAAA;IAC3C,MAAM,cAAc,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;IAC7D,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAA;IAEzC,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO;YACL,YAAY,EAAE,EAAE;YAChB,cAAc;YACd,gBAAgB,EAAE,cAAc;YAChC,YAAY,EAAE,EAAE;YAChB,WAAW,EAAE,CAAC;SACf,CAAA;IACH,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,CAAoC,CAAA;IAC7F,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAA;IACxC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IACvE,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,CAAA;IAC5C,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAA;IAEzC,OAAO;QACL,YAAY,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;aAChC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;aACtC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QAClD,cAAc;QACd,gBAAgB,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACxE,YAAY,EAAE,CAAC,GAAG,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE;QAC9E,WAAW,EAAE,IAAI,CAAC,MAAM;KACzB,CAAA;AACH,CAAC"} \ No newline at end of file diff --git a/dist/init.d.ts.map b/dist/init.d.ts.map deleted file mode 100644 index 2a9f3a4..0000000 --- a/dist/init.d.ts.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":"AA2BA,wBAAsB,WAAW,CAAC,GAAG,SAAgB,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAwCxE"} \ No newline at end of file diff --git a/dist/init.js.map b/dist/init.js.map deleted file mode 100644 index e744441..0000000 --- a/dist/init.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAA;AAErD,MAAM,cAAc,GAAG;IACrB,MAAM,EAAE,SAAS;IACjB,UAAU,EAAE,aAAa;IACzB,WAAW,EAAE,iBAAiB;IAC9B,aAAa,EAAE,gBAAgB;IAC/B,SAAS,EAAE,QAAQ;IACnB,UAAU,EAAE,wBAAwB;IACpC,aAAa,EAAE,YAAY;IAC3B,UAAU,EAAE,kBAAkB;IAC9B,QAAQ,EAAE,eAAe;IACzB,SAAS,EAAE;QACT,OAAO,EAAE,IAAI;QACb,OAAO,EAAE,IAAI;QACb,QAAQ,EAAE,EAAE;KACb;IACD,SAAS,EAAE,IAAI;IACf,UAAU,EAAE,EAAE;IACd,IAAI,EAAE,CAAC;IACP,SAAS,EAAE,IAAI;IACf,YAAY,EAAE,GAAG;CAClB,CAAA;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE;IACnD,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;IAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IACpC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,CAAA;IAC7C,MAAM,OAAO,GAAa,EAAE,CAAA;IAE5B,MAAM,KAAK,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IACvC,MAAM,KAAK,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAE5C,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,aAAa,CAAC,CAAA;IAClD,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC5B,MAAM,SAAS,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;QACnF,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,CAAA;IAC/C,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,aAAa,CAAC,CAAA;IACnD,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC7B,MAAM,SAAS,CACb,WAAW,EACX,8FAA8F,EAC9F,MAAM,CACP,CAAA;QACD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,CAAA;IAChD,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,CAAA;IACrD,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC5B,MAAM,SAAS,CACb,UAAU,EACV,6FAA6F,EAC7F,MAAM,CACP,CAAA;QACD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,CAAA;IAC/C,CAAC;IAED,IAAI,MAAM,oBAAoB,CAAC,IAAI,CAAC,EAAE,CAAC;QACrC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;IAC5B,CAAC;IAED,OAAO,OAAO,CAAA;AAChB,CAAC"} \ No newline at end of file diff --git a/dist/mcp.d.ts.map b/dist/mcp.d.ts.map deleted file mode 100644 index 025724f..0000000 --- a/dist/mcp.d.ts.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"mcp.d.ts","sourceRoot":"","sources":["../src/mcp.ts"],"names":[],"mappings":"AAUA,wBAAsB,QAAQ,CAAC,GAAG,SAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAiFjE"} \ No newline at end of file diff --git a/dist/mcp.js.map b/dist/mcp.js.map deleted file mode 100644 index 49807dd..0000000 --- a/dist/mcp.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"mcp.js","sourceRoot":"","sources":["../src/mcp.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAA;AACnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAA;AAChF,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AACvB,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAA;AACnC,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;AACxC,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC7C,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AACtC,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAA;AAEtC,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE;IAChD,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;QAC3B,IAAI,EAAE,OAAO;QACb,OAAO,EAAE,OAAO;KACjB,CAAC,CAAA;IAEF,MAAM,CAAC,YAAY,CACjB,cAAc,EACd;QACE,KAAK,EAAE,cAAc;QACrB,WAAW,EAAE,0DAA0D;QACvE,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;KAC1B,EACD,KAAK,IAAI,EAAE;QACT,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAA;QACpC,MAAM,aAAa,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,CAAA;QAC7C,MAAM,MAAM,GAAG;YACb,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,aAAa,EAAE,MAAM,CAAC,aAAa;YACnC,gBAAgB,EAAE,MAAM,CAAC,SAAS,CAAC,OAAO;YAC1C,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,aAAa;SACd,CAAA;QAED,OAAO,UAAU,CAAC,MAAM,CAAC,CAAA;IAC3B,CAAC,CACF,CAAA;IAED,MAAM,CAAC,YAAY,CACjB,cAAc,EACd;QACE,KAAK,EAAE,cAAc;QACrB,WAAW,EAAE,iEAAiE;QAC9E,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;YACpB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;YACxB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAC7C,CAAC;KACH,EACD,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,UAAU,CAAC,MAAM,MAAM,CAAC,KAAK,EAAE,MAAM,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CAC3F,CAAA;IAED,MAAM,CAAC,YAAY,CACjB,WAAW,EACX;QACE,KAAK,EAAE,WAAW;QAClB,WAAW,EACT,mFAAmF;QACrF,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;YACpB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;YACxB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAC7C,CAAC;KACH,EACD,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,KAAK,EAAE,MAAM,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CACxF,CAAA;IAED,MAAM,CAAC,YAAY,CACjB,aAAa,EACb;QACE,KAAK,EAAE,aAAa;QACpB,WAAW,EAAE,uEAAuE;QACpF,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;KAC1B,EACD,KAAK,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC,CACzC,CAAA;IAED,MAAM,CAAC,YAAY,CACjB,sBAAsB,EACtB;QACE,KAAK,EAAE,sBAAsB;QAC7B,WAAW,EAAE,qEAAqE;QAClF,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;KAC1B,EACD,KAAK,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,aAAa,CAAC,GAAG,CAAC,CAAC,CACjD,CAAA;IAED,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,oBAAoB,EAAE,CAAC,CAAA;AAClD,CAAC;AAED,SAAS,UAAU,CAAC,KAAc;IAChC,OAAO;QACL,OAAO,EAAE;YACP;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;aACrC;SACF;KACF,CAAA;AACH,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,GAAW,EACX,IAAwB;IAExB,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAA;IACpC,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,UAAU,CAAC,CAAA;IACpE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,CAAA;AACnC,CAAC"} \ No newline at end of file diff --git a/dist/network.d.ts b/dist/network.d.ts deleted file mode 100644 index a3f9ad2..0000000 --- a/dist/network.d.ts +++ /dev/null @@ -1,4 +0,0 @@ -import type { Config, HostClassification } from "./types.js"; -export declare function assertNetworkPolicy(config: Config): void; -export declare function classifyHost(input: string): HostClassification; -//# sourceMappingURL=network.d.ts.map \ No newline at end of file diff --git a/dist/network.d.ts.map b/dist/network.d.ts.map deleted file mode 100644 index abf4861..0000000 --- a/dist/network.d.ts.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"network.d.ts","sourceRoot":"","sources":["../src/network.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,MAAM,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAA;AAE5D,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAsBxD;AAED,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,kBAAkB,CAkB9D"} \ No newline at end of file diff --git a/dist/network.js b/dist/network.js deleted file mode 100644 index d43ffb9..0000000 --- a/dist/network.js +++ /dev/null @@ -1,59 +0,0 @@ -import { isIP } from "node:net"; -export function assertNetworkPolicy(config) { - const classification = classifyHost(config.ollamaHost); - if (config.networkPolicy === "allow-any") { - return; - } - if (config.networkPolicy === "local-only" && classification.kind !== "loopback") { - throw new Error(`Refusing to send document text to non-local Ollama host "${config.ollamaHost}". Set networkPolicy to "allow-private" or "allow-any" only if this is intentional.`); - } - if (config.networkPolicy === "allow-private" && - classification.kind !== "loopback" && - classification.kind !== "private") { - throw new Error(`Refusing to send document text to remote Ollama host "${config.ollamaHost}". Set networkPolicy to "allow-any" only if this is intentional.`); - } -} -export function classifyHost(input) { - let url; - try { - url = new URL(input); - } - catch { - return { kind: "invalid", host: input }; - } - const host = url.hostname.replace(/^\[(.*)\]$/, "$1").toLowerCase(); - if (isLoopbackHost(host)) { - return { kind: "loopback", host }; - } - if (isPrivateHost(host)) { - return { kind: "private", host }; - } - return { kind: "remote", host }; -} -function isLoopbackHost(host) { - if (host === "localhost" || host === "::1") { - return true; - } - if (isIP(host) === 4) { - return host.startsWith("127."); - } - return false; -} -function isPrivateHost(host) { - if (host === "host.docker.internal" || host.endsWith(".local")) { - return true; - } - if (isIP(host) === 4) { - const parts = host.split(".").map((part) => Number.parseInt(part, 10)); - const [first = 0, second = 0] = parts; - return (first === 10 || - (first === 172 && second >= 16 && second <= 31) || - (first === 192 && second === 168) || - (first === 169 && second === 254)); - } - if (isIP(host) === 6) { - return host.startsWith("fc") || host.startsWith("fd") || host.startsWith("fe80"); - } - return false; -} -//# sourceMappingURL=network.js.map \ No newline at end of file diff --git a/dist/network.js.map b/dist/network.js.map deleted file mode 100644 index d0cb015..0000000 --- a/dist/network.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"network.js","sourceRoot":"","sources":["../src/network.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,UAAU,CAAA;AAG/B,MAAM,UAAU,mBAAmB,CAAC,MAAc;IAChD,MAAM,cAAc,GAAG,YAAY,CAAC,MAAM,CAAC,UAAU,CAAC,CAAA;IAEtD,IAAI,MAAM,CAAC,aAAa,KAAK,WAAW,EAAE,CAAC;QACzC,OAAM;IACR,CAAC;IAED,IAAI,MAAM,CAAC,aAAa,KAAK,YAAY,IAAI,cAAc,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;QAChF,MAAM,IAAI,KAAK,CACb,4DAA4D,MAAM,CAAC,UAAU,qFAAqF,CACnK,CAAA;IACH,CAAC;IAED,IACE,MAAM,CAAC,aAAa,KAAK,eAAe;QACxC,cAAc,CAAC,IAAI,KAAK,UAAU;QAClC,cAAc,CAAC,IAAI,KAAK,SAAS,EACjC,CAAC;QACD,MAAM,IAAI,KAAK,CACb,yDAAyD,MAAM,CAAC,UAAU,kEAAkE,CAC7I,CAAA;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAa;IACxC,IAAI,GAAQ,CAAA;IACZ,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAA;IACtB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,CAAA;IACzC,CAAC;IAED,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC,WAAW,EAAE,CAAA;IACnE,IAAI,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;QACzB,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,CAAA;IACnC,CAAC;IAED,IAAI,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAA;IAClC,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAA;AACjC,CAAC;AAED,SAAS,cAAc,CAAC,IAAY;IAClC,IAAI,IAAI,KAAK,WAAW,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;QAC3C,OAAO,IAAI,CAAA;IACb,CAAC;IAED,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACrB,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAA;IAChC,CAAC;IAED,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,aAAa,CAAC,IAAY;IACjC,IAAI,IAAI,KAAK,sBAAsB,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC/D,OAAO,IAAI,CAAA;IACb,CAAC;IAED,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACrB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAA;QACtE,MAAM,CAAC,KAAK,GAAG,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,GAAG,KAAK,CAAA;QACrC,OAAO,CACL,KAAK,KAAK,EAAE;YACZ,CAAC,KAAK,KAAK,GAAG,IAAI,MAAM,IAAI,EAAE,IAAI,MAAM,IAAI,EAAE,CAAC;YAC/C,CAAC,KAAK,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,CAAC;YACjC,CAAC,KAAK,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,CAAC,CAClC,CAAA;IACH,CAAC;IAED,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACrB,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAA;IAClF,CAAC;IAED,OAAO,KAAK,CAAA;AACd,CAAC"} \ No newline at end of file diff --git a/dist/parsing.d.ts.map b/dist/parsing.d.ts.map deleted file mode 100644 index 0369743..0000000 --- a/dist/parsing.d.ts.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"parsing.d.ts","sourceRoot":"","sources":["../src/parsing.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,YAAY,CAAA;AAE5D,wBAAsB,SAAS,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,cAAc,CAAC,CA6BzE"} \ No newline at end of file diff --git a/dist/parsing.js b/dist/parsing.js deleted file mode 100644 index 3fb20d9..0000000 --- a/dist/parsing.js +++ /dev/null @@ -1,46 +0,0 @@ -import { readFile } from "node:fs/promises"; -import { htmlToText } from "html-to-text"; -import { extractText, getDocumentProxy } from "unpdf"; -import YAML from "yaml"; -export async function parseFile(file) { - let text; - switch (file.extension) { - case ".pdf": - text = await parsePdf(file.absolutePath); - break; - case ".html": - case ".htm": - text = htmlToText(await readFile(file.absolutePath, "utf8"), { - wordwrap: false, - selectors: [ - { selector: "a", options: { ignoreHref: true } }, - { selector: "img", format: "skip" }, - ], - }); - break; - case ".json": - text = JSON.stringify(JSON.parse(await readFile(file.absolutePath, "utf8")), null, 2); - break; - case ".yaml": - case ".yml": - text = YAML.stringify(YAML.parse(await readFile(file.absolutePath, "utf8"))); - break; - default: - text = await readFile(file.absolutePath, "utf8"); - } - return { file, text: normalizeText(text) }; -} -async function parsePdf(filePath) { - const buffer = await readFile(filePath); - const pdf = await getDocumentProxy(new Uint8Array(buffer)); - const result = await extractText(pdf, { mergePages: true }); - return result.text; -} -function normalizeText(input) { - return input - .replace(/\r\n/g, "\n") - .replace(/[ \t]+\n/g, "\n") - .replace(/\n{4,}/g, "\n\n\n") - .trim(); -} -//# sourceMappingURL=parsing.js.map \ No newline at end of file diff --git a/dist/parsing.js.map b/dist/parsing.js.map deleted file mode 100644 index 54cd431..0000000 --- a/dist/parsing.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"parsing.js","sourceRoot":"","sources":["../src/parsing.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAA;AACzC,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,OAAO,CAAA;AACrD,OAAO,IAAI,MAAM,MAAM,CAAA;AAGvB,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAgB;IAC9C,IAAI,IAAY,CAAA;IAEhB,QAAQ,IAAI,CAAC,SAAS,EAAE,CAAC;QACvB,KAAK,MAAM;YACT,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;YACxC,MAAK;QACP,KAAK,OAAO,CAAC;QACb,KAAK,MAAM;YACT,IAAI,GAAG,UAAU,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,EAAE;gBAC3D,QAAQ,EAAE,KAAK;gBACf,SAAS,EAAE;oBACT,EAAE,QAAQ,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE;oBAChD,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE;iBACpC;aACF,CAAC,CAAA;YACF,MAAK;QACP,KAAK,OAAO;YACV,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAA;YACrF,MAAK;QACP,KAAK,OAAO,CAAC;QACb,KAAK,MAAM;YACT,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,CAAA;YAC5E,MAAK;QACP;YACE,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAA;IACpD,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,CAAA;AAC5C,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,QAAgB;IACtC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAA;IACvC,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC,CAAA;IAC1D,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAA;IAC3D,OAAO,MAAM,CAAC,IAAI,CAAA;AACpB,CAAC;AAED,SAAS,aAAa,CAAC,KAAa;IAClC,OAAO,KAAK;SACT,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC;SACtB,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC;SAC1B,OAAO,CAAC,SAAS,EAAE,QAAQ,CAAC;SAC5B,IAAI,EAAE,CAAA;AACX,CAAC"} \ No newline at end of file diff --git a/dist/query.js.map b/dist/query.js.map deleted file mode 100644 index 43be57a..0000000 --- a/dist/query.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"query.js","sourceRoot":"","sources":["../src/query.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAA;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAC3C,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAA;AAClD,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAW1C,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,KAAa,EAAE,UAAyB,EAAE;IACrE,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;IACrE,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAA;IACzC,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,EAAE,CAAA;IACX,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;IAC7C,MAAM,IAAI,GAAG,CAAC,MAAM,KAAK;SACtB,YAAY,CAAC,MAAM,CAAC;SACpB,KAAK,CAAC,OAAO,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC;SAClC,OAAO,EAAE,CAAgB,CAAA;IAE5B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACjC,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,YAAY,EAAE,GAAG,CAAC,YAAY;QAC9B,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,QAAQ,EAAE,OAAO,GAAG,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;KACnE,CAAC,CAAC,CAAA;IACH,MAAM,YAAY,CAAC,MAAM,EAAE;QACzB,MAAM,EAAE,QAAQ;QAChB,KAAK;QACL,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI;QACjC,WAAW,EAAE,OAAO,CAAC,MAAM;KAC5B,CAAC,CAAA;IACF,OAAO,OAAO,CAAA;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,GAAG,CAAC,KAAa,EAAE,UAAyB,EAAE;IAClE,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;IACrE,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;IAE5C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO;YACL,MAAM,EAAE,2EAA2E;YACnF,OAAO;SACR,CAAA;IACH,CAAC;IAED,MAAM,OAAO,GAAG,OAAO;SACpB,GAAG,CACF,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAChB,IAAI,KAAK,GAAG,CAAC,KAAK,MAAM,CAAC,YAAY,IAAI,MAAM,CAAC,UAAU,KAAK,MAAM,CAAC,IAAI,EAAE,CAC/E;SACA,IAAI,CAAC,aAAa,CAAC,CAAA;IAEtB,mBAAmB,CAAC,MAAM,CAAC,CAAA;IAC3B,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC,CAAA;IACtD,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC;QACjC,KAAK,EAAE,MAAM,CAAC,QAAQ;QACtB,QAAQ,EAAE;YACR;gBACE,IAAI,EAAE,QAAQ;gBACd,OAAO,EACL,8HAA8H;aACjI;YACD;gBACE,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE,cAAc,KAAK,iBAAiB,OAAO,EAAE;aACvD;SACF;QACD,MAAM,EAAE,KAAK;KACd,CAAC,CAAA;IAEF,MAAM,YAAY,CAAC,MAAM,EAAE;QACzB,MAAM,EAAE,KAAK;QACb,KAAK;QACL,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI;QACjC,WAAW,EAAE,OAAO,CAAC,MAAM;KAC5B,CAAC,CAAA;IAEF,OAAO;QACL,MAAM,EAAE,QAAQ,CAAC,OAAO,CAAC,OAAO;QAChC,OAAO;KACR,CAAA;AACH,CAAC"} \ No newline at end of file diff --git a/dist/security.js.map b/dist/security.js.map deleted file mode 100644 index b75784e..0000000 --- a/dist/security.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"security.js","sourceRoot":"","sources":["../src/security.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AAG3C,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE;IACrD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAA;IACpC,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;IACzD,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,UAAU,CAAC,CAAA;IAC/C,MAAM,QAAQ,GAAa,EAAE,CAAA;IAE7B,MAAM,SAAS,GAAG,iBAAiB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;IACtD,MAAM,YAAY,GAAG,iBAAiB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAA;IAC5D,MAAM,cAAc,GAAG,iBAAiB,CAAC,SAAS,EAAE,YAAY,CAAC,CAAA;IAEjE,IAAI,MAAM,CAAC,aAAa,KAAK,WAAW,EAAE,CAAC;QACzC,QAAQ,CAAC,IAAI,CAAC,gFAAgF,CAAC,CAAA;IACjG,CAAC;IACD,IAAI,MAAM,CAAC,aAAa,KAAK,YAAY,IAAI,OAAO,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;QACzE,QAAQ,CAAC,IAAI,CAAC,6DAA6D,CAAC,CAAA;IAC9E,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;QAC9B,QAAQ,CAAC,IAAI,CAAC,8EAA8E,CAAC,CAAA;IAC/F,CAAC;IACD,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,QAAQ,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAA;IAC9C,CAAC;IACD,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,QAAQ,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAA;IACjD,CAAC;IACD,IAAI,CAAC,cAAc,EAAE,CAAC;QACpB,QAAQ,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAA;IACpD,CAAC;IAED,OAAO;QACL,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,aAAa,EAAE,IAAI;QACnB,OAAO,EAAE;YACP,MAAM,EAAE,MAAM,CAAC,aAAa;YAC5B,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,cAAc,EAAE,OAAO,CAAC,IAAI;SAC7B;QACD,SAAS,EAAE;YACT,OAAO,EAAE,MAAM,CAAC,SAAS,CAAC,OAAO;YACjC,OAAO,EAAE,MAAM,CAAC,SAAS,CAAC,OAAO;YACjC,cAAc,EAAE,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;SACzE;QACD,SAAS,EAAE;YACT,OAAO,EAAE,MAAM,CAAC,SAAS;YACzB,IAAI,EAAE,MAAM,CAAC,aAAa;YAC1B,gBAAgB,EAAE,KAAK;SACxB;QACD,OAAO,EAAE;YACP,IAAI,EAAE,MAAM,CAAC,UAAU;YACvB,UAAU,EAAE,SAAS;YACrB,eAAe,EAAE,mBAAmB;SACrC;QACD,GAAG,EAAE;YACH,OAAO,EAAE,MAAM,CAAC,UAAU;YAC1B,uBAAuB,EAAE,KAAK;SAC/B;QACD,SAAS,EAAE;YACT,SAAS;YACT,YAAY;YACZ,cAAc;SACf;QACD,eAAe,EAAE;YACf,qFAAqF;YACrF,yFAAyF;YACzF,2FAA2F;SAC5F;QACD,QAAQ;KACT,CAAA;AACH,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,WAAmB;IAC9C,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,YAAY,CAAC,CAAA;IAC1D,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;QAC/B,OAAO,IAAI,GAAG,EAAE,CAAA;IAClB,CAAC;IAED,OAAO,IAAI,GAAG,CACZ,CAAC,MAAM,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;SACpC,KAAK,CAAC,OAAO,CAAC;SACd,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;SAC1B,MAAM,CAAC,OAAO,CAAC,CACnB,CAAA;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAkB,EAAE,KAAa;IAC1D,OAAO,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;AACzB,CAAC"} \ No newline at end of file diff --git a/dist/skill.d.ts.map b/dist/skill.d.ts.map deleted file mode 100644 index c44432e..0000000 --- a/dist/skill.d.ts.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"skill.d.ts","sourceRoot":"","sources":["../src/skill.ts"],"names":[],"mappings":"AAKA,MAAM,WAAW,mBAAmB;IAClC,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,kBAAkB;IACjC,SAAS,EAAE,MAAM,CAAA;IACjB,aAAa,EAAE,MAAM,CAAA;IACrB,UAAU,EAAE,MAAM,CAAA;IAClB,OAAO,EAAE,MAAM,EAAE,CAAA;CAClB;AAKD,wBAAgB,gBAAgB,IAAI,MAAM,CAEzC;AAED,wBAAsB,YAAY,CAAC,OAAO,GAAE,mBAAwB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAgCjG"} \ No newline at end of file diff --git a/dist/skill.js.map b/dist/skill.js.map deleted file mode 100644 index bf6541d..0000000 --- a/dist/skill.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"skill.js","sourceRoot":"","sources":["../src/skill.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACvD,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AACxC,OAAO,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAA;AAcrD,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;AAC/E,MAAM,UAAU,GAAG,OAAO,CAAA;AAE1B,MAAM,UAAU,gBAAgB;IAC9B,OAAO,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAA;AACtD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,UAA+B,EAAE;IAClE,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;IACtD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,SAAS,IAAI,eAAe,CAAC,CAAA;IACzE,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,UAAU,CAAC,CAAA;IAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAA;IAC5C,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAA;IACrD,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAA;IAEnD,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAC3C,MAAM,KAAK,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAC1C,MAAM,EAAE,CAAC,gBAAgB,EAAE,EAAE,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;IAEzE,MAAM,SAAS,CAAC,aAAa,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IACtF,MAAM,SAAS,CAAC,UAAU,EAAE,cAAc,CAAC,SAAS,EAAE,aAAa,CAAC,EAAE,MAAM,CAAC,CAAA;IAC7E,MAAM,cAAc,GAAG,MAAM,oBAAoB,CAAC,GAAG,CAAC,CAAA;IAEtD,MAAM,OAAO,GAAG;QACd,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,SAAS,CAAC;QAC7B,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,aAAa,CAAC;QACjC,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,UAAU,CAAC;KAC/B,CAAA;IAED,IAAI,cAAc,EAAE,CAAC;QACnB,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;IAC5B,CAAC;IAED,OAAO;QACL,SAAS;QACT,aAAa;QACb,UAAU;QACV,OAAO;KACR,CAAA;AACH,CAAC;AAED,SAAS,SAAS,CAAC,GAAW;IAC5B,OAAO;QACL,UAAU,EAAE;YACV,KAAK,EAAE;gBACL,OAAO,EAAE,MAAM;gBACf,IAAI,EAAE,CAAC,MAAM,EAAE,IAAI,EAAE,WAAW,CAAC;gBACjC,GAAG;aACJ;SACF;KACF,CAAA;AACH,CAAC;AAED,SAAS,cAAc,CAAC,SAAiB,EAAE,aAAqB;IAC9D,OAAO;;;;;;;;;EASP,SAAS;;;;;;;;;;EAUT,aAAa;;;;;;;;;CASd,CAAA;AACD,CAAC"} \ No newline at end of file diff --git a/dist/store.js.map b/dist/store.js.map deleted file mode 100644 index 8b14dd3..0000000 --- a/dist/store.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"store.js","sourceRoot":"","sources":["../src/store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAA;AACxC,OAAO,KAAK,OAAO,MAAM,kBAAkB,CAAA;AAG3C,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAiB,EAAE,MAAc;IAC/D,MAAM,KAAK,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IACnD,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAA;IAEnD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,UAAU,EAAE,CAAA;QACxC,IAAI,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;YAC1C,MAAM,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;QACtC,CAAC;QACD,OAAM;IACR,CAAC;IAED,MAAM,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,EAAE,IAA4C,EAAE;QACnF,IAAI,EAAE,WAAW;KAClB,CAAC,CAAA;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,MAAc;IAChD,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAA;IACnD,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,UAAU,EAAE,CAAA;IACxC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;QAC3C,OAAO,IAAI,CAAA;IACb,CAAC;IACD,OAAO,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;AACvC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,MAAc;IAC5C,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAA;IACzC,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;AACtC,CAAC"} \ No newline at end of file diff --git a/dist/types.d.ts.map b/dist/types.d.ts.map deleted file mode 100644 index e04e208..0000000 --- a/dist/types.d.ts.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AAEvC,MAAM,WAAW,MAAM;IACrB,WAAW,EAAE,MAAM,CAAA;IACnB,MAAM,EAAE,MAAM,CAAA;IACd,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,EAAE,MAAM,CAAA;IACnB,aAAa,EAAE,MAAM,CAAA;IACrB,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,aAAa,EAAE,aAAa,CAAA;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,eAAe,CAAA;IAC1B,SAAS,EAAE,OAAO,CAAA;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,SAAS,EAAE,MAAM,CAAA;IACjB,YAAY,EAAE,MAAM,CAAA;CACrB;AAED,MAAM,MAAM,aAAa,GAAG,YAAY,GAAG,eAAe,GAAG,WAAW,CAAA;AAExE,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,OAAO,CAAA;IAChB,OAAO,EAAE,OAAO,CAAA;IAChB,QAAQ,EAAE,gBAAgB,EAAE,CAAA;CAC7B;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IAC1B,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CACjC;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,UAAU,GAAG,SAAS,GAAG,QAAQ,GAAG,SAAS,CAAA;IACnD,IAAI,EAAE,MAAM,CAAA;CACb;AAED,MAAM,WAAW,UAAU;IACzB,YAAY,EAAE,MAAM,CAAA;IACpB,YAAY,EAAE,MAAM,CAAA;IACpB,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,KAAK,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,MAAM,CAAA;CACjB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,UAAU,CAAA;IAChB,IAAI,EAAE,MAAM,CAAA;CACb;AAED,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAA;IACV,MAAM,EAAE,MAAM,CAAA;IACd,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,SAAU,SAAQ,SAAS;IAC1C,MAAM,EAAE,MAAM,EAAE,CAAA;CACjB;AAED,MAAM,WAAW,aAAa;IAC5B,GAAG,CAAC,EAAE,QAAQ,CAAA;IACd,OAAO,CAAC,EAAE,OAAO,CAAA;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,MAAM,CAAA;IACpB,MAAM,EAAE,MAAM,CAAA;IACd,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,MAAM,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;CACjD;AAED,MAAM,WAAW,aAAa;IAC5B,GAAG,CAAC,EAAE,QAAQ,CAAA;IACd,IAAI,CAAC,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAA;IACd,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;CACxB;AAED,MAAM,WAAW,SAAS;IACxB,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,YAAY,EAAE,CAAA;CACxB;AAED,MAAM,WAAW,WAAW;IAC1B,YAAY,EAAE,KAAK,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IACvD,cAAc,EAAE,MAAM,EAAE,CAAA;IACxB,gBAAgB,EAAE,MAAM,EAAE,CAAA;IAC1B,YAAY,EAAE,MAAM,EAAE,CAAA;IACtB,WAAW,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,kBAAkB;IACjC,UAAU,EAAE,MAAM,CAAA;IAClB,OAAO,EAAE,OAAO,CAAA;IAChB,IAAI,EAAE,MAAM,CAAA;CACb;AAED,MAAM,WAAW,mBAAmB;IAClC,WAAW,EAAE,MAAM,CAAA;IACnB,aAAa,EAAE,IAAI,CAAA;IACnB,OAAO,EAAE;QACP,MAAM,EAAE,aAAa,CAAA;QACrB,UAAU,EAAE,MAAM,CAAA;QAClB,IAAI,EAAE,MAAM,CAAA;QACZ,cAAc,EAAE,kBAAkB,CAAC,MAAM,CAAC,CAAA;KAC3C,CAAA;IACD,SAAS,EAAE;QACT,OAAO,EAAE,OAAO,CAAA;QAChB,OAAO,EAAE,OAAO,CAAA;QAChB,cAAc,EAAE,MAAM,EAAE,CAAA;KACzB,CAAA;IACD,SAAS,EAAE;QACT,OAAO,EAAE,OAAO,CAAA;QAChB,IAAI,EAAE,MAAM,CAAA;QACZ,gBAAgB,EAAE,KAAK,CAAA;KACxB,CAAA;IACD,OAAO,EAAE;QACP,IAAI,EAAE,MAAM,CAAA;QACZ,UAAU,EAAE,OAAO,CAAA;QACnB,eAAe,EAAE,mBAAmB,CAAA;KACrC,CAAA;IACD,GAAG,EAAE;QACH,OAAO,EAAE,MAAM,CAAA;QACf,uBAAuB,EAAE,KAAK,CAAA;KAC/B,CAAA;IACD,SAAS,EAAE;QACT,SAAS,EAAE,OAAO,CAAA;QAClB,YAAY,EAAE,OAAO,CAAA;QACrB,cAAc,EAAE,OAAO,CAAA;KACxB,CAAA;IACD,eAAe,EAAE,MAAM,EAAE,CAAA;IACzB,QAAQ,EAAE,MAAM,EAAE,CAAA;CACnB"} \ No newline at end of file diff --git a/dist/version.d.ts b/dist/version.d.ts deleted file mode 100644 index f7aae6a..0000000 --- a/dist/version.d.ts +++ /dev/null @@ -1,2 +0,0 @@ -export declare const VERSION = "0.3.0"; -//# sourceMappingURL=version.d.ts.map \ No newline at end of file diff --git a/package.json b/package.json index 58fbb62..ae99e4e 100644 --- a/package.json +++ b/package.json @@ -1,89 +1,28 @@ { - "name": "@jcode.labs/mimir", - "version": "0.3.0", - "description": "Mimir: open-source local-first memory and retrieval for private project knowledge.", + "name": "jcode-mimir", + "version": "0.4.0", + "private": true, + "description": "Monorepo for the Mimir open-source local RAG packages.", "type": "module", "license": "MIT", - "author": { - "name": "Jean-Baptiste Thery", - "url": "https://github.com/jb-thery" - }, - "keywords": [ - "jcode", - "mimir", - "open-source", - "rag", - "knowledge-base", - "local-first", - "mcp", - "ollama", - "lancedb" - ], - "repository": { - "type": "git", - "url": "git+https://github.com/jcode-works/jcode-mimir.git" - }, - "homepage": "https://github.com/jcode-works/jcode-mimir#readme", - "bugs": { - "url": "https://github.com/jcode-works/jcode-mimir/issues" - }, - "engines": { - "node": ">=20" - }, - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, - "bin": { - "kb": "dist/cli.js" - }, - "files": [ - "dist", - "skills", - "README.md", - "SECURITY-HARDENING.md", - "CHANGELOG.md", - "SECURITY.md" - ], - "publishConfig": { - "access": "public" - }, + "packageManager": "pnpm@11.9.0", "scripts": { - "build": "tsc -p tsconfig.json", - "check": "tsc -p tsconfig.json --noEmit", + "build": "pnpm --filter @jcode.labs/mimir-tts build && pnpm --filter @jcode.labs/mimir build", + "check": "pnpm --filter @jcode.labs/mimir-tts check && pnpm --filter @jcode.labs/mimir check", "commitlint": "commitlint --from=HEAD~1 --to=HEAD --verbose", "format": "biome format --write .", "lint": "biome ci .", "lint:fix": "biome check --write .", - "package:check": "publint", + "package:check": "pnpm --filter @jcode.labs/mimir-tts package:check && pnpm --filter @jcode.labs/mimir package:check", "release:artifacts": "node scripts/release-artifacts.mjs", - "smoke": "node scripts/smoke.mjs", - "test": "vitest run", + "smoke": "node scripts/smoke.mjs && pnpm --filter @jcode.labs/mimir-tts smoke", + "test": "pnpm --filter @jcode.labs/mimir-tts test && pnpm --filter @jcode.labs/mimir test", "validate": "pnpm lint && pnpm check && pnpm test && pnpm build && pnpm smoke && pnpm package:check && pnpm release:artifacts" }, - "dependencies": { - "@lancedb/lancedb": "^0.30.0", - "@modelcontextprotocol/sdk": "^1.29.0", - "commander": "^14.0.2", - "fast-glob": "^3.3.3", - "html-to-text": "^9.0.5", - "ollama": "^0.6.3", - "picocolors": "^1.1.1", - "unpdf": "^1.4.0", - "yaml": "^2.8.1", - "zod": "^4.1.13" - }, "devDependencies": { "@biomejs/biome": "^2.5.1", "@commitlint/cli": "^21.1.0", "@commitlint/config-conventional": "^21.1.0", - "@types/html-to-text": "^9.0.4", - "@types/node": "^24.10.1", - "publint": "^0.3.21", - "typescript": "^5.9.3", - "vitest": "^4.0.15" - }, - "packageManager": "pnpm@11.9.0" + "yaml": "^2.8.1" + } } diff --git a/packages/mimir-tts/LICENSE b/packages/mimir-tts/LICENSE new file mode 100644 index 0000000..09c4e55 --- /dev/null +++ b/packages/mimir-tts/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Jean-Baptiste Thery + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/mimir-tts/README.md b/packages/mimir-tts/README.md new file mode 100644 index 0000000..2fd8dc0 --- /dev/null +++ b/packages/mimir-tts/README.md @@ -0,0 +1,33 @@ +# Mimir TTS + +Plug-and-play local text-to-speech for Mimir audio summaries. + +`@jcode.labs/mimir-tts` renders narration text to WAV with Transformers.js. It does not require +Python, ffmpeg, Piper, XTTS, or a local server. The first render can download a public ONNX model +from Hugging Face into `.mimir/models/tts`; the source text is processed locally. + +## Install + +```bash +pnpm add -D @jcode.labs/mimir-tts +``` + +## Render + +```bash +pnpm exec mimir-tts render /tmp/MIMIR-SUMMARY-tax.txt --out .mimir/audio/tax-summary.wav +``` + +For offline or air-gapped use, preload the model files and run: + +```bash +pnpm exec mimir-tts render summary.txt --offline --model-path .mimir/models/tts +``` + +## Doctor + +```bash +pnpm exec mimir-tts doctor --json +``` + +The default model is `Xenova/mms-tts-fra`. Override it with `--model` or `MIMIR_TTS_MODEL`. diff --git a/dist/cli.d.ts b/packages/mimir-tts/dist/cli.d.ts similarity index 100% rename from dist/cli.d.ts rename to packages/mimir-tts/dist/cli.d.ts diff --git a/dist/cli.d.ts.map b/packages/mimir-tts/dist/cli.d.ts.map similarity index 100% rename from dist/cli.d.ts.map rename to packages/mimir-tts/dist/cli.d.ts.map diff --git a/packages/mimir-tts/dist/cli.js b/packages/mimir-tts/dist/cli.js new file mode 100755 index 0000000..4414ce8 --- /dev/null +++ b/packages/mimir-tts/dist/cli.js @@ -0,0 +1,140 @@ +#!/usr/bin/env node +import { parseArgs } from "node:util"; +import { doctor, renderSpeech } from "./index.js"; +const command = process.argv[2]; +try { + if (command === "doctor") { + await runDoctor(process.argv.slice(3)); + } + else if (command === "render") { + await runRender(process.argv.slice(3)); + } + else { + printHelp(); + process.exitCode = command ? 1 : 0; + } +} +catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +} +async function runDoctor(args) { + const { values } = parseArgs({ + args, + options: { + json: { type: "boolean" }, + }, + }); + const report = await doctor(); + if (values.json) { + console.log(JSON.stringify(report, null, 2)); + return; + } + printKeyValue("node", report.node); + printKeyValue("defaultModel", report.defaultModel); + printKeyValue("defaultModelPath", report.defaultModelPath); + printKeyValue("transformersAvailable", String(report.transformersAvailable)); + printKeyValue("pythonRequired", String(report.pythonRequired)); + printKeyValue("ffmpegRequired", String(report.ffmpegRequired)); + printKeyValue("outputFormat", report.outputFormat); +} +async function runRender(args) { + const { values, positionals } = parseArgs({ + args, + allowPositionals: true, + options: { + out: { type: "string", short: "o" }, + model: { type: "string" }, + "model-path": { type: "string" }, + offline: { type: "boolean" }, + "allow-remote-models": { type: "boolean" }, + "speaker-embeddings": { type: "string" }, + speed: { type: "string" }, + json: { type: "boolean" }, + }, + }); + const textFile = positionals[0]; + if (!textFile) { + throw new Error("usage: mimir-tts render [--out output.wav]"); + } + const renderOptions = { + textFile, + }; + addStringOption(renderOptions, "outputPath", stringValue(values, "out")); + addStringOption(renderOptions, "model", stringValue(values, "model")); + addStringOption(renderOptions, "modelPath", stringValue(values, "model-path")); + addBooleanOption(renderOptions, "allowRemoteModels", allowRemoteModels(values)); + addStringOption(renderOptions, "speakerEmbeddings", stringValue(values, "speaker-embeddings")); + addNumberOption(renderOptions, "speed", numberValue(values, "speed")); + const result = await renderSpeech(renderOptions); + if (values.json) { + console.log(JSON.stringify(result, null, 2)); + return; + } + printKeyValue("outputPath", result.outputPath); + printKeyValue("model", result.model); + printKeyValue("modelPath", result.modelPath); + printKeyValue("allowRemoteModels", String(result.allowRemoteModels)); + printKeyValue("samplingRate", String(result.samplingRate ?? "unknown")); + printKeyValue("samples", String(result.samples ?? "unknown")); +} +function allowRemoteModels(values) { + if (values.offline === true) { + return false; + } + if (values["allow-remote-models"] === true) { + return true; + } + return undefined; +} +function stringValue(values, key) { + const value = values[key]; + return typeof value === "string" ? value : undefined; +} +function addStringOption(target, key, value) { + if (value !== undefined) { + target[key] = value; + } +} +function addBooleanOption(target, key, value) { + if (value !== undefined) { + target[key] = value; + } +} +function addNumberOption(target, key, value) { + if (value !== undefined) { + target[key] = value; + } +} +function numberValue(values, key) { + const value = stringValue(values, key); + if (!value) { + return undefined; + } + const parsed = Number.parseFloat(value); + if (!Number.isFinite(parsed)) { + throw new Error(`Expected a number for --${key}.`); + } + return parsed; +} +function printKeyValue(key, value) { + console.log(`${key}=${value}`); +} +function printHelp() { + console.log(`mimir-tts + +Usage: + mimir-tts doctor [--json] + mimir-tts render [--out output.wav] [--offline] + +Options: + --model Transformers.js TTS model ID. + --model-path Local model/cache path. Defaults to .mimir/models/tts. + --offline Disable remote model downloads. + --allow-remote-models Explicitly allow remote model downloads. + --speaker-embeddings Optional model-specific speaker embedding path or URL. + --speed Optional model-specific speech speed. + --json Print JSON output. +`); +} +//# sourceMappingURL=cli.js.map \ No newline at end of file diff --git a/packages/mimir-tts/dist/cli.js.map b/packages/mimir-tts/dist/cli.js.map new file mode 100644 index 0000000..0d63bc1 --- /dev/null +++ b/packages/mimir-tts/dist/cli.js.map @@ -0,0 +1 @@ +{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AACrC,OAAO,EAAE,MAAM,EAA4B,YAAY,EAAE,MAAM,YAAY,CAAA;AAI3E,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AAE/B,IAAI,CAAC;IACH,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;QACzB,MAAM,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;IACxC,CAAC;SAAM,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;QAChC,MAAM,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;IACxC,CAAC;SAAM,CAAC;QACN,SAAS,EAAE,CAAA;QACX,OAAO,CAAC,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IACpC,CAAC;AACH,CAAC;AAAC,OAAO,KAAK,EAAE,CAAC;IACf,OAAO,CAAC,KAAK,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAA;IACrE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAA;AACtB,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,IAAc;IACrC,MAAM,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;QAC3B,IAAI;QACJ,OAAO,EAAE;YACP,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;SAC1B;KACF,CAAC,CAAA;IACF,MAAM,MAAM,GAAG,MAAM,MAAM,EAAE,CAAA;IAC7B,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;QAChB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAA;QAC5C,OAAM;IACR,CAAC;IAED,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,CAAA;IAClC,aAAa,CAAC,cAAc,EAAE,MAAM,CAAC,YAAY,CAAC,CAAA;IAClD,aAAa,CAAC,kBAAkB,EAAE,MAAM,CAAC,gBAAgB,CAAC,CAAA;IAC1D,aAAa,CAAC,uBAAuB,EAAE,MAAM,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC,CAAA;IAC5E,aAAa,CAAC,gBAAgB,EAAE,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAA;IAC9D,aAAa,CAAC,gBAAgB,EAAE,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAA;IAC9D,aAAa,CAAC,cAAc,EAAE,MAAM,CAAC,YAAY,CAAC,CAAA;AACpD,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,IAAc;IACrC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,SAAS,CAAC;QACxC,IAAI;QACJ,gBAAgB,EAAE,IAAI;QACtB,OAAO,EAAE;YACP,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE;YACnC,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACzB,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAChC,OAAO,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;YAC5B,qBAAqB,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;YAC1C,oBAAoB,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACxC,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACzB,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;SAC1B;KACF,CAAC,CAAA;IACF,MAAM,QAAQ,GAAG,WAAW,CAAC,CAAC,CAAC,CAAA;IAC/B,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAA;IAC3E,CAAC;IAED,MAAM,aAAa,GAAwB;QACzC,QAAQ;KACT,CAAA;IACD,eAAe,CAAC,aAAa,EAAE,YAAY,EAAE,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAA;IACxE,eAAe,CAAC,aAAa,EAAE,OAAO,EAAE,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAA;IACrE,eAAe,CAAC,aAAa,EAAE,WAAW,EAAE,WAAW,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,CAAA;IAC9E,gBAAgB,CAAC,aAAa,EAAE,mBAAmB,EAAE,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAA;IAC/E,eAAe,CAAC,aAAa,EAAE,mBAAmB,EAAE,WAAW,CAAC,MAAM,EAAE,oBAAoB,CAAC,CAAC,CAAA;IAC9F,eAAe,CAAC,aAAa,EAAE,OAAO,EAAE,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAA;IAErE,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,aAAa,CAAC,CAAA;IAEhD,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;QAChB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAA;QAC5C,OAAM;IACR,CAAC;IACD,aAAa,CAAC,YAAY,EAAE,MAAM,CAAC,UAAU,CAAC,CAAA;IAC9C,aAAa,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,CAAA;IACpC,aAAa,CAAC,WAAW,EAAE,MAAM,CAAC,SAAS,CAAC,CAAA;IAC5C,aAAa,CAAC,mBAAmB,EAAE,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAA;IACpE,aAAa,CAAC,cAAc,EAAE,MAAM,CAAC,MAAM,CAAC,YAAY,IAAI,SAAS,CAAC,CAAC,CAAA;IACvE,aAAa,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO,IAAI,SAAS,CAAC,CAAC,CAAA;AAC/D,CAAC;AAED,SAAS,iBAAiB,CAAC,MAAiB;IAC1C,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;QAC5B,OAAO,KAAK,CAAA;IACd,CAAC;IACD,IAAI,MAAM,CAAC,qBAAqB,CAAC,KAAK,IAAI,EAAE,CAAC;QAC3C,OAAO,IAAI,CAAA;IACb,CAAC;IACD,OAAO,SAAS,CAAA;AAClB,CAAC;AAED,SAAS,WAAW,CAAC,MAAiB,EAAE,GAAW;IACjD,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAA;IACzB,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAA;AACtD,CAAC;AAED,SAAS,eAAe,CACtB,MAA2B,EAC3B,GAA+D,EAC/D,KAAyB;IAEzB,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,CAAA;IACrB,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CACvB,MAA2B,EAC3B,GAAwB,EACxB,KAA0B;IAE1B,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,CAAA;IACrB,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CACtB,MAA2B,EAC3B,GAAY,EACZ,KAAyB;IAEzB,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,CAAA;IACrB,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,MAAiB,EAAE,GAAW;IACjD,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;IACtC,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,SAAS,CAAA;IAClB,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAA;IACvC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,2BAA2B,GAAG,GAAG,CAAC,CAAA;IACpD,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC;AAED,SAAS,aAAa,CAAC,GAAW,EAAE,KAAa;IAC/C,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,IAAI,KAAK,EAAE,CAAC,CAAA;AAChC,CAAC;AAED,SAAS,SAAS;IAChB,OAAO,CAAC,GAAG,CAAC;;;;;;;;;;;;;;CAcb,CAAC,CAAA;AACF,CAAC"} \ No newline at end of file diff --git a/packages/mimir-tts/dist/index.d.ts b/packages/mimir-tts/dist/index.d.ts new file mode 100644 index 0000000..0322e1b --- /dev/null +++ b/packages/mimir-tts/dist/index.d.ts @@ -0,0 +1,46 @@ +export declare const DEFAULT_TTS_MODEL = "Xenova/mms-tts-fra"; +export declare const DEFAULT_TTS_MODEL_PATH = ".mimir/models/tts"; +export declare const DEFAULT_AUDIO_DIR = ".mimir/audio"; +export interface TextToAudioOutputLike { + save(path: string): Promise; + sampling_rate?: number; + data?: Float32Array; +} +export type TextToAudioSynthesizer = (text: string, options?: TextToAudioOptions) => Promise; +export interface TextToAudioOptions { + speaker_embeddings?: string; + speed?: number; +} +export interface RenderSpeechOptions { + cwd?: string; + text?: string; + textFile?: string; + outputPath?: string; + model?: string; + modelPath?: string; + allowRemoteModels?: boolean; + speakerEmbeddings?: string; + speed?: number; + synthesizer?: TextToAudioSynthesizer; +} +export interface RenderSpeechResult { + outputPath: string; + model: string; + modelPath: string; + allowRemoteModels: boolean; + samplingRate: number | null; + samples: number | null; +} +export interface DoctorReport { + node: string; + defaultModel: string; + defaultModelPath: string; + transformersAvailable: boolean; + pythonRequired: false; + ffmpegRequired: false; + outputFormat: "wav"; +} +export declare function renderSpeech(options: RenderSpeechOptions): Promise; +export declare function doctor(): Promise; +export declare function modelCacheExists(cwd?: string): boolean; +//# sourceMappingURL=index.d.ts.map \ No newline at end of file diff --git a/packages/mimir-tts/dist/index.d.ts.map b/packages/mimir-tts/dist/index.d.ts.map new file mode 100644 index 0000000..ad43cda --- /dev/null +++ b/packages/mimir-tts/dist/index.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,iBAAiB,uBAAuB,CAAA;AACrD,eAAO,MAAM,sBAAsB,sBAAsB,CAAA;AACzD,eAAO,MAAM,iBAAiB,iBAAiB,CAAA;AAE/C,MAAM,WAAW,qBAAqB;IACpC,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACjC,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,IAAI,CAAC,EAAE,YAAY,CAAA;CACpB;AAED,MAAM,MAAM,sBAAsB,GAAG,CACnC,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,kBAAkB,KACzB,OAAO,CAAC,qBAAqB,CAAC,CAAA;AAEnC,MAAM,WAAW,kBAAkB;IACjC,kBAAkB,CAAC,EAAE,MAAM,CAAA;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,mBAAmB;IAClC,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,iBAAiB,CAAC,EAAE,OAAO,CAAA;IAC3B,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,WAAW,CAAC,EAAE,sBAAsB,CAAA;CACrC;AAED,MAAM,WAAW,kBAAkB;IACjC,UAAU,EAAE,MAAM,CAAA;IAClB,KAAK,EAAE,MAAM,CAAA;IACb,SAAS,EAAE,MAAM,CAAA;IACjB,iBAAiB,EAAE,OAAO,CAAA;IAC1B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;IAC3B,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;CACvB;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAA;IACZ,YAAY,EAAE,MAAM,CAAA;IACpB,gBAAgB,EAAE,MAAM,CAAA;IACxB,qBAAqB,EAAE,OAAO,CAAA;IAC9B,cAAc,EAAE,KAAK,CAAA;IACrB,cAAc,EAAE,KAAK,CAAA;IACrB,YAAY,EAAE,KAAK,CAAA;CACpB;AAED,wBAAsB,YAAY,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CA6B5F;AAED,wBAAsB,MAAM,IAAI,OAAO,CAAC,YAAY,CAAC,CAUpD;AAgED,wBAAgB,gBAAgB,CAAC,GAAG,SAAgB,GAAG,OAAO,CAE7D"} \ No newline at end of file diff --git a/packages/mimir-tts/dist/index.js b/packages/mimir-tts/dist/index.js new file mode 100644 index 0000000..c77bbf1 --- /dev/null +++ b/packages/mimir-tts/dist/index.js @@ -0,0 +1,92 @@ +import { existsSync } from "node:fs"; +import { mkdir, readFile } from "node:fs/promises"; +import path from "node:path"; +export const DEFAULT_TTS_MODEL = "Xenova/mms-tts-fra"; +export const DEFAULT_TTS_MODEL_PATH = ".mimir/models/tts"; +export const DEFAULT_AUDIO_DIR = ".mimir/audio"; +export async function renderSpeech(options) { + const cwd = path.resolve(options.cwd ?? process.cwd()); + const text = await readInputText(options); + const model = options.model ?? process.env.MIMIR_TTS_MODEL ?? DEFAULT_TTS_MODEL; + const modelPath = resolveFromCwd(cwd, options.modelPath ?? process.env.MIMIR_TTS_MODEL_PATH ?? DEFAULT_TTS_MODEL_PATH); + const outputPath = resolveFromCwd(cwd, options.outputPath ?? defaultOutputPath(cwd, options.textFile)); + const allowRemoteModels = options.allowRemoteModels ?? readBooleanEnv("MIMIR_TTS_ALLOW_REMOTE_MODELS", true); + await mkdir(path.dirname(outputPath), { recursive: true }); + const synthesizer = options.synthesizer ?? (await transformerSynthesizer(model, modelPath, allowRemoteModels)); + const output = await synthesizer(text, textToAudioOptions(options)); + await output.save(outputPath); + return { + outputPath, + model, + modelPath, + allowRemoteModels, + samplingRate: typeof output.sampling_rate === "number" ? output.sampling_rate : null, + samples: output.data instanceof Float32Array ? output.data.length : null, + }; +} +export async function doctor() { + return { + node: process.versions.node, + defaultModel: DEFAULT_TTS_MODEL, + defaultModelPath: DEFAULT_TTS_MODEL_PATH, + transformersAvailable: await canImportTransformers(), + pythonRequired: false, + ffmpegRequired: false, + outputFormat: "wav", + }; +} +async function readInputText(options) { + const text = options.text ?? (options.textFile ? await readFile(options.textFile, "utf8") : ""); + const trimmed = text.trim(); + if (!trimmed) { + throw new Error("A non-empty text input or text file is required."); + } + return trimmed; +} +function defaultOutputPath(cwd, textFile) { + const name = textFile ? path.basename(textFile, path.extname(textFile)) : "mimir-summary"; + return path.join(cwd, DEFAULT_AUDIO_DIR, `${name}.wav`); +} +function resolveFromCwd(cwd, input) { + return path.isAbsolute(input) ? input : path.resolve(cwd, input); +} +function textToAudioOptions(options) { + const output = {}; + if (options.speakerEmbeddings) { + output.speaker_embeddings = options.speakerEmbeddings; + } + if (typeof options.speed === "number") { + output.speed = options.speed; + } + return Object.keys(output).length > 0 ? output : undefined; +} +async function transformerSynthesizer(model, modelPath, allowRemoteModels) { + const transformers = await import("@huggingface/transformers"); + transformers.env.localModelPath = modelPath; + transformers.env.cacheDir = modelPath; + transformers.env.allowRemoteModels = allowRemoteModels; + return (await transformers.pipeline("text-to-speech", model)); +} +async function canImportTransformers() { + try { + await import("@huggingface/transformers"); + return true; + } + catch { + return false; + } +} +function readBooleanEnv(name, fallback) { + const raw = process.env[name]?.toLowerCase(); + if (raw === "1" || raw === "true" || raw === "yes") { + return true; + } + if (raw === "0" || raw === "false" || raw === "no") { + return false; + } + return fallback; +} +export function modelCacheExists(cwd = process.cwd()) { + return existsSync(path.resolve(cwd, process.env.MIMIR_TTS_MODEL_PATH ?? DEFAULT_TTS_MODEL_PATH)); +} +//# sourceMappingURL=index.js.map \ No newline at end of file diff --git a/packages/mimir-tts/dist/index.js.map b/packages/mimir-tts/dist/index.js.map new file mode 100644 index 0000000..469d3d4 --- /dev/null +++ b/packages/mimir-tts/dist/index.js.map @@ -0,0 +1 @@ +{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAClD,OAAO,IAAI,MAAM,WAAW,CAAA;AAE5B,MAAM,CAAC,MAAM,iBAAiB,GAAG,oBAAoB,CAAA;AACrD,MAAM,CAAC,MAAM,sBAAsB,GAAG,mBAAmB,CAAA;AACzD,MAAM,CAAC,MAAM,iBAAiB,GAAG,cAAc,CAAA;AAkD/C,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAA4B;IAC7D,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;IACtD,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,CAAA;IACzC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,iBAAiB,CAAA;IAC/E,MAAM,SAAS,GAAG,cAAc,CAC9B,GAAG,EACH,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,sBAAsB,CAChF,CAAA;IACD,MAAM,UAAU,GAAG,cAAc,CAC/B,GAAG,EACH,OAAO,CAAC,UAAU,IAAI,iBAAiB,CAAC,GAAG,EAAE,OAAO,CAAC,QAAQ,CAAC,CAC/D,CAAA;IACD,MAAM,iBAAiB,GACrB,OAAO,CAAC,iBAAiB,IAAI,cAAc,CAAC,+BAA+B,EAAE,IAAI,CAAC,CAAA;IAEpF,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAC1D,MAAM,WAAW,GACf,OAAO,CAAC,WAAW,IAAI,CAAC,MAAM,sBAAsB,CAAC,KAAK,EAAE,SAAS,EAAE,iBAAiB,CAAC,CAAC,CAAA;IAC5F,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAA;IACnE,MAAM,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;IAE7B,OAAO;QACL,UAAU;QACV,KAAK;QACL,SAAS;QACT,iBAAiB;QACjB,YAAY,EAAE,OAAO,MAAM,CAAC,aAAa,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI;QACpF,OAAO,EAAE,MAAM,CAAC,IAAI,YAAY,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;KACzE,CAAA;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,MAAM;IAC1B,OAAO;QACL,IAAI,EAAE,OAAO,CAAC,QAAQ,CAAC,IAAI;QAC3B,YAAY,EAAE,iBAAiB;QAC/B,gBAAgB,EAAE,sBAAsB;QACxC,qBAAqB,EAAE,MAAM,qBAAqB,EAAE;QACpD,cAAc,EAAE,KAAK;QACrB,cAAc,EAAE,KAAK;QACrB,YAAY,EAAE,KAAK;KACpB,CAAA;AACH,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,OAA4B;IACvD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;IAC/F,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;IAC3B,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAA;IACrE,CAAC;IACD,OAAO,OAAO,CAAA;AAChB,CAAC;AAED,SAAS,iBAAiB,CAAC,GAAW,EAAE,QAA4B;IAClE,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,eAAe,CAAA;IACzF,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,iBAAiB,EAAE,GAAG,IAAI,MAAM,CAAC,CAAA;AACzD,CAAC;AAED,SAAS,cAAc,CAAC,GAAW,EAAE,KAAa;IAChD,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;AAClE,CAAC;AAED,SAAS,kBAAkB,CAAC,OAA4B;IACtD,MAAM,MAAM,GAAuB,EAAE,CAAA;IACrC,IAAI,OAAO,CAAC,iBAAiB,EAAE,CAAC;QAC9B,MAAM,CAAC,kBAAkB,GAAG,OAAO,CAAC,iBAAiB,CAAA;IACvD,CAAC;IACD,IAAI,OAAO,OAAO,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;QACtC,MAAM,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAA;IAC9B,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAA;AAC5D,CAAC;AAED,KAAK,UAAU,sBAAsB,CACnC,KAAa,EACb,SAAiB,EACjB,iBAA0B;IAE1B,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAA;IAC9D,YAAY,CAAC,GAAG,CAAC,cAAc,GAAG,SAAS,CAAA;IAC3C,YAAY,CAAC,GAAG,CAAC,QAAQ,GAAG,SAAS,CAAA;IACrC,YAAY,CAAC,GAAG,CAAC,iBAAiB,GAAG,iBAAiB,CAAA;IAEtD,OAAO,CAAC,MAAM,YAAY,CAAC,QAAQ,CAAC,gBAAgB,EAAE,KAAK,CAAC,CAA2B,CAAA;AACzF,CAAC;AAED,KAAK,UAAU,qBAAqB;IAClC,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAA;QACzC,OAAO,IAAI,CAAA;IACb,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAA;IACd,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,IAAY,EAAE,QAAiB;IACrD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,CAAA;IAC5C,IAAI,GAAG,KAAK,GAAG,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;QACnD,OAAO,IAAI,CAAA;IACb,CAAC;IACD,IAAI,GAAG,KAAK,GAAG,IAAI,GAAG,KAAK,OAAO,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;QACnD,OAAO,KAAK,CAAA;IACd,CAAC;IACD,OAAO,QAAQ,CAAA;AACjB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE;IAClD,OAAO,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,sBAAsB,CAAC,CAAC,CAAA;AAClG,CAAC"} \ No newline at end of file diff --git a/packages/mimir-tts/package.json b/packages/mimir-tts/package.json new file mode 100644 index 0000000..330ef1e --- /dev/null +++ b/packages/mimir-tts/package.json @@ -0,0 +1,66 @@ +{ + "name": "@jcode.labs/mimir-tts", + "version": "0.4.0", + "description": "Plug-and-play local text-to-speech for Mimir audio summaries.", + "type": "module", + "license": "MIT", + "author": { + "name": "Jean-Baptiste Thery", + "url": "https://github.com/jb-thery" + }, + "keywords": [ + "jcode", + "mimir", + "text-to-speech", + "tts", + "local-first", + "transformers", + "onnx", + "audio" + ], + "repository": { + "type": "git", + "url": "git+https://github.com/jcode-works/jcode-mimir.git", + "directory": "packages/mimir-tts" + }, + "homepage": "https://github.com/jcode-works/jcode-mimir/tree/main/packages/mimir-tts#readme", + "bugs": { + "url": "https://github.com/jcode-works/jcode-mimir/issues" + }, + "engines": { + "node": ">=20" + }, + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "bin": { + "mimir-tts": "dist/cli.js" + }, + "files": [ + "dist", + "README.md", + "LICENSE" + ], + "publishConfig": { + "access": "public" + }, + "scripts": { + "build": "tsc -p tsconfig.json", + "check": "tsc -p tsconfig.json --noEmit", + "package:check": "publint", + "smoke": "node dist/cli.js doctor --json", + "test": "vitest run" + }, + "dependencies": { + "@huggingface/transformers": "^4.2.0" + }, + "devDependencies": { + "@types/node": "^24.10.1", + "publint": "^0.3.21", + "typescript": "^5.9.3", + "vitest": "^4.0.15" + } +} diff --git a/packages/mimir-tts/src/cli.ts b/packages/mimir-tts/src/cli.ts new file mode 100644 index 0000000..92fe65a --- /dev/null +++ b/packages/mimir-tts/src/cli.ts @@ -0,0 +1,166 @@ +#!/usr/bin/env node +import { parseArgs } from "node:util" +import { doctor, type RenderSpeechOptions, renderSpeech } from "./index.js" + +type CliValues = Record + +const command = process.argv[2] + +try { + if (command === "doctor") { + await runDoctor(process.argv.slice(3)) + } else if (command === "render") { + await runRender(process.argv.slice(3)) + } else { + printHelp() + process.exitCode = command ? 1 : 0 + } +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)) + process.exitCode = 1 +} + +async function runDoctor(args: string[]): Promise { + const { values } = parseArgs({ + args, + options: { + json: { type: "boolean" }, + }, + }) + const report = await doctor() + if (values.json) { + console.log(JSON.stringify(report, null, 2)) + return + } + + printKeyValue("node", report.node) + printKeyValue("defaultModel", report.defaultModel) + printKeyValue("defaultModelPath", report.defaultModelPath) + printKeyValue("transformersAvailable", String(report.transformersAvailable)) + printKeyValue("pythonRequired", String(report.pythonRequired)) + printKeyValue("ffmpegRequired", String(report.ffmpegRequired)) + printKeyValue("outputFormat", report.outputFormat) +} + +async function runRender(args: string[]): Promise { + const { values, positionals } = parseArgs({ + args, + allowPositionals: true, + options: { + out: { type: "string", short: "o" }, + model: { type: "string" }, + "model-path": { type: "string" }, + offline: { type: "boolean" }, + "allow-remote-models": { type: "boolean" }, + "speaker-embeddings": { type: "string" }, + speed: { type: "string" }, + json: { type: "boolean" }, + }, + }) + const textFile = positionals[0] + if (!textFile) { + throw new Error("usage: mimir-tts render [--out output.wav]") + } + + const renderOptions: RenderSpeechOptions = { + textFile, + } + addStringOption(renderOptions, "outputPath", stringValue(values, "out")) + addStringOption(renderOptions, "model", stringValue(values, "model")) + addStringOption(renderOptions, "modelPath", stringValue(values, "model-path")) + addBooleanOption(renderOptions, "allowRemoteModels", allowRemoteModels(values)) + addStringOption(renderOptions, "speakerEmbeddings", stringValue(values, "speaker-embeddings")) + addNumberOption(renderOptions, "speed", numberValue(values, "speed")) + + const result = await renderSpeech(renderOptions) + + if (values.json) { + console.log(JSON.stringify(result, null, 2)) + return + } + printKeyValue("outputPath", result.outputPath) + printKeyValue("model", result.model) + printKeyValue("modelPath", result.modelPath) + printKeyValue("allowRemoteModels", String(result.allowRemoteModels)) + printKeyValue("samplingRate", String(result.samplingRate ?? "unknown")) + printKeyValue("samples", String(result.samples ?? "unknown")) +} + +function allowRemoteModels(values: CliValues): boolean | undefined { + if (values.offline === true) { + return false + } + if (values["allow-remote-models"] === true) { + return true + } + return undefined +} + +function stringValue(values: CliValues, key: string): string | undefined { + const value = values[key] + return typeof value === "string" ? value : undefined +} + +function addStringOption( + target: RenderSpeechOptions, + key: "outputPath" | "model" | "modelPath" | "speakerEmbeddings", + value: string | undefined, +): void { + if (value !== undefined) { + target[key] = value + } +} + +function addBooleanOption( + target: RenderSpeechOptions, + key: "allowRemoteModels", + value: boolean | undefined, +): void { + if (value !== undefined) { + target[key] = value + } +} + +function addNumberOption( + target: RenderSpeechOptions, + key: "speed", + value: number | undefined, +): void { + if (value !== undefined) { + target[key] = value + } +} + +function numberValue(values: CliValues, key: string): number | undefined { + const value = stringValue(values, key) + if (!value) { + return undefined + } + const parsed = Number.parseFloat(value) + if (!Number.isFinite(parsed)) { + throw new Error(`Expected a number for --${key}.`) + } + return parsed +} + +function printKeyValue(key: string, value: string): void { + console.log(`${key}=${value}`) +} + +function printHelp(): void { + console.log(`mimir-tts + +Usage: + mimir-tts doctor [--json] + mimir-tts render [--out output.wav] [--offline] + +Options: + --model Transformers.js TTS model ID. + --model-path Local model/cache path. Defaults to .mimir/models/tts. + --offline Disable remote model downloads. + --allow-remote-models Explicitly allow remote model downloads. + --speaker-embeddings Optional model-specific speaker embedding path or URL. + --speed Optional model-specific speech speed. + --json Print JSON output. +`) +} diff --git a/packages/mimir-tts/src/index.test.ts b/packages/mimir-tts/src/index.test.ts new file mode 100644 index 0000000..d7b90f1 --- /dev/null +++ b/packages/mimir-tts/src/index.test.ts @@ -0,0 +1,54 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { afterEach, describe, expect, it } from "vitest" +import { doctor, renderSpeech, type TextToAudioSynthesizer } from "./index.js" + +const tempDirs: string[] = [] + +afterEach(async () => { + for (const dir of tempDirs.splice(0)) { + await rm(dir, { recursive: true, force: true }) + } +}) + +describe("renderSpeech", () => { + it("renders a text file to the requested wav path through an injected synthesizer", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "mimir-tts-")) + tempDirs.push(root) + const textFile = path.join(root, "summary.txt") + const outputPath = path.join(root, ".mimir/audio/summary.wav") + await writeFile(textFile, "Bonjour depuis Mimir.", "utf8") + + const synthesizer: TextToAudioSynthesizer = async () => ({ + sampling_rate: 16_000, + data: new Float32Array([0, 0.5, -0.5]), + save: async (target) => { + await writeFile(target, "RIFF fake wav", "utf8") + }, + }) + + const result = await renderSpeech({ + cwd: root, + textFile, + outputPath, + allowRemoteModels: false, + synthesizer, + }) + + expect(result.outputPath).toBe(outputPath) + expect(result.allowRemoteModels).toBe(false) + expect(result.samplingRate).toBe(16_000) + expect(await readFile(outputPath, "utf8")).toBe("RIFF fake wav") + }) +}) + +describe("doctor", () => { + it("reports a Python-free wav renderer", async () => { + await expect(doctor()).resolves.toMatchObject({ + pythonRequired: false, + ffmpegRequired: false, + outputFormat: "wav", + }) + }) +}) diff --git a/packages/mimir-tts/src/index.ts b/packages/mimir-tts/src/index.ts new file mode 100644 index 0000000..1a66e08 --- /dev/null +++ b/packages/mimir-tts/src/index.ts @@ -0,0 +1,164 @@ +import { existsSync } from "node:fs" +import { mkdir, readFile } from "node:fs/promises" +import path from "node:path" + +export const DEFAULT_TTS_MODEL = "Xenova/mms-tts-fra" +export const DEFAULT_TTS_MODEL_PATH = ".mimir/models/tts" +export const DEFAULT_AUDIO_DIR = ".mimir/audio" + +export interface TextToAudioOutputLike { + save(path: string): Promise + sampling_rate?: number + data?: Float32Array +} + +export type TextToAudioSynthesizer = ( + text: string, + options?: TextToAudioOptions, +) => Promise + +export interface TextToAudioOptions { + speaker_embeddings?: string + speed?: number +} + +export interface RenderSpeechOptions { + cwd?: string + text?: string + textFile?: string + outputPath?: string + model?: string + modelPath?: string + allowRemoteModels?: boolean + speakerEmbeddings?: string + speed?: number + synthesizer?: TextToAudioSynthesizer +} + +export interface RenderSpeechResult { + outputPath: string + model: string + modelPath: string + allowRemoteModels: boolean + samplingRate: number | null + samples: number | null +} + +export interface DoctorReport { + node: string + defaultModel: string + defaultModelPath: string + transformersAvailable: boolean + pythonRequired: false + ffmpegRequired: false + outputFormat: "wav" +} + +export async function renderSpeech(options: RenderSpeechOptions): Promise { + const cwd = path.resolve(options.cwd ?? process.cwd()) + const text = await readInputText(options) + const model = options.model ?? process.env.MIMIR_TTS_MODEL ?? DEFAULT_TTS_MODEL + const modelPath = resolveFromCwd( + cwd, + options.modelPath ?? process.env.MIMIR_TTS_MODEL_PATH ?? DEFAULT_TTS_MODEL_PATH, + ) + const outputPath = resolveFromCwd( + cwd, + options.outputPath ?? defaultOutputPath(cwd, options.textFile), + ) + const allowRemoteModels = + options.allowRemoteModels ?? readBooleanEnv("MIMIR_TTS_ALLOW_REMOTE_MODELS", true) + + await mkdir(path.dirname(outputPath), { recursive: true }) + const synthesizer = + options.synthesizer ?? (await transformerSynthesizer(model, modelPath, allowRemoteModels)) + const output = await synthesizer(text, textToAudioOptions(options)) + await output.save(outputPath) + + return { + outputPath, + model, + modelPath, + allowRemoteModels, + samplingRate: typeof output.sampling_rate === "number" ? output.sampling_rate : null, + samples: output.data instanceof Float32Array ? output.data.length : null, + } +} + +export async function doctor(): Promise { + return { + node: process.versions.node, + defaultModel: DEFAULT_TTS_MODEL, + defaultModelPath: DEFAULT_TTS_MODEL_PATH, + transformersAvailable: await canImportTransformers(), + pythonRequired: false, + ffmpegRequired: false, + outputFormat: "wav", + } +} + +async function readInputText(options: RenderSpeechOptions): Promise { + const text = options.text ?? (options.textFile ? await readFile(options.textFile, "utf8") : "") + const trimmed = text.trim() + if (!trimmed) { + throw new Error("A non-empty text input or text file is required.") + } + return trimmed +} + +function defaultOutputPath(cwd: string, textFile: string | undefined): string { + const name = textFile ? path.basename(textFile, path.extname(textFile)) : "mimir-summary" + return path.join(cwd, DEFAULT_AUDIO_DIR, `${name}.wav`) +} + +function resolveFromCwd(cwd: string, input: string): string { + return path.isAbsolute(input) ? input : path.resolve(cwd, input) +} + +function textToAudioOptions(options: RenderSpeechOptions): TextToAudioOptions | undefined { + const output: TextToAudioOptions = {} + if (options.speakerEmbeddings) { + output.speaker_embeddings = options.speakerEmbeddings + } + if (typeof options.speed === "number") { + output.speed = options.speed + } + return Object.keys(output).length > 0 ? output : undefined +} + +async function transformerSynthesizer( + model: string, + modelPath: string, + allowRemoteModels: boolean, +): Promise { + const transformers = await import("@huggingface/transformers") + transformers.env.localModelPath = modelPath + transformers.env.cacheDir = modelPath + transformers.env.allowRemoteModels = allowRemoteModels + + return (await transformers.pipeline("text-to-speech", model)) as TextToAudioSynthesizer +} + +async function canImportTransformers(): Promise { + try { + await import("@huggingface/transformers") + return true + } catch { + return false + } +} + +function readBooleanEnv(name: string, fallback: boolean): boolean { + const raw = process.env[name]?.toLowerCase() + if (raw === "1" || raw === "true" || raw === "yes") { + return true + } + if (raw === "0" || raw === "false" || raw === "no") { + return false + } + return fallback +} + +export function modelCacheExists(cwd = process.cwd()): boolean { + return existsSync(path.resolve(cwd, process.env.MIMIR_TTS_MODEL_PATH ?? DEFAULT_TTS_MODEL_PATH)) +} diff --git a/packages/mimir-tts/tsconfig.json b/packages/mimir-tts/tsconfig.json new file mode 100644 index 0000000..8635ab2 --- /dev/null +++ b/packages/mimir-tts/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src/**/*.ts"], + "exclude": ["src/**/*.test.ts"] +} diff --git a/packages/mimir/CHANGELOG.md b/packages/mimir/CHANGELOG.md new file mode 100644 index 0000000..6d4ed26 --- /dev/null +++ b/packages/mimir/CHANGELOG.md @@ -0,0 +1,47 @@ +# Changelog + +## 0.4.0 - 2026-06-28 + +- Reposition Mimir as sovereign local RAG for confidential datasets and AI agents. +- Expand default ingestion to common text, Office/OpenDocument, data, config, log, and source-code + file types. +- Add `includeExtensions` / `KB_INCLUDE_EXTENSIONS` for custom UTF-8 text file extensions. +- Add the optional `mimir-audio-summary` bundled skill for confidential audio summaries. +- Install both the main Mimir skill and optional audio-summary skill with `kb install-skill`. +- Improve agent guidance for deep multi-query retrieval before synthesis. +- Make Mimir core retrieval-only: `kb ask` now returns cited context for external agents or LLMs + instead of generating answers internally. +- Add optional Transformers.js semantic embeddings through `embeddingProvider: "transformers"`. +- Remove Ollama providers and keep `embeddingProvider: "local-hash"` as the no-model default. +- Move the repository to a simple pnpm workspace monorepo without adding Turbo. +- Move the core `@jcode.labs/mimir` package into `packages/mimir`. +- Add `@jcode.labs/mimir-tts` for plug-and-play JS/ONNX WAV rendering without Python or ffmpeg. +- Add `kb audio` and update the audio-summary skill to use Mimir TTS before advanced fallback + engines. + +## 0.3.0 - 2026-06-28 + +- Add confidentiality hardening defaults: built-in redaction before indexing, metadata-only access + logs, and bounded MCP retrieval. +- Add `kb security-audit` for zero-telemetry, provider, redaction, gitignore, storage, and + MCP posture checks. +- Add `kb destroy-index --yes` to remove generated vector indexes. +- Add release verification artifacts: npm tarball, SHA256 checksums, SBOM, and manifest. +- Document air-gapped operation, threat model, MCP hardening, and secure deletion limits. + +## 0.2.1 - 2026-06-28 + +- Add GitHub Sponsors funding metadata and document suggested sponsor tiers. +- Add maintainer positioning for Jean-Baptiste Thery and JCode Labs in the README. +- Make `kb init` and `kb install-skill` automatically keep `.kb/` and `.mimir/` + ignored by Git. + +## 0.2.0 - 2026-06-28 + +- Rename public product branding to Mimir while keeping the JCode Labs npm scope. +- Add the bundled portable `mimir` agent skill. +- Add the MCP stdio server with `mimir_status`, `mimir_search`, `mimir_ask`, and + `mimir_audit`. +- Add production smoke coverage for the built CLI and MCP server. +- Add Biome, commitlint, publint, CodeQL, Dependabot grouping, protected npm publishing, + and open-source contribution/security documentation. diff --git a/packages/mimir/CONTRIBUTING.md b/packages/mimir/CONTRIBUTING.md new file mode 100644 index 0000000..3a14d25 --- /dev/null +++ b/packages/mimir/CONTRIBUTING.md @@ -0,0 +1,28 @@ +# Contributing + +Mimir is an open-source project under the MIT License. Issues and pull requests are welcome. + +## Development + +Use Node.js 20+ and pnpm: + +```bash +pnpm install +pnpm validate +``` + +`pnpm validate` runs Biome, TypeScript, Vitest, the production CLI/MCP smoke test, and npm +package metadata checks. + +## Pull Requests + +- Open pull requests against `main`. +- Keep changes focused and include tests or smoke coverage for behavior changes. +- Do not commit private documents, generated vector stores, environment files, tokens, or + credentials. +- Use conventional commit messages such as `feat: add source parser` or + `fix: handle empty index`. + +## Security + +Do not report vulnerabilities through public issues. Follow [`SECURITY.md`](./SECURITY.md). diff --git a/packages/mimir/LICENSE b/packages/mimir/LICENSE new file mode 100644 index 0000000..2e2d13f --- /dev/null +++ b/packages/mimir/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Jean-Baptiste Thery and JCode Labs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/mimir/README.md b/packages/mimir/README.md new file mode 100644 index 0000000..f965200 --- /dev/null +++ b/packages/mimir/README.md @@ -0,0 +1,519 @@ +# Mimir + +[![CI](https://github.com/jcode-works/jcode-mimir/actions/workflows/ci.yml/badge.svg)](https://github.com/jcode-works/jcode-mimir/actions/workflows/ci.yml) +[![CodeQL](https://github.com/jcode-works/jcode-mimir/actions/workflows/codeql.yml/badge.svg)](https://github.com/jcode-works/jcode-mimir/actions/workflows/codeql.yml) +[![npm](https://img.shields.io/npm/v/@jcode.labs/mimir)](https://www.npmjs.com/package/@jcode.labs/mimir) +[![license: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](./LICENSE) + +Open-source, sovereign local RAG for confidential datasets and AI agents. + +Mimir provides a TypeScript CLI, library, MCP server, and portable agent skills that can be +installed in any Node.js repository. It indexes local files from the target repository, stores +vectors locally with LanceDB, and can use either built-in local-hash retrieval or optional +Transformers.js semantic embeddings. Mimir core returns cited retrieval context; answer synthesis +belongs to the AI agent, LLM, or local model runtime you choose around it. + +The intended use case is simple: put confidential company, institutional, legal, operational, or +research documents in a private local folder, index them locally, then let any compatible AI agent or +LLM workflow retrieve grounded context for summaries, briefs, audits, and decision support without +shipping the dataset to a hosted RAG service. + +Created by Jean-Baptiste Thery and published under the JCode Labs npm scope. + +Built by Jean-Baptiste Thery, freelance full-stack/AI tooling engineer at JCode Labs. + +## Open Source + +Mimir is a public open-source project under the MIT License. It is designed to be +inspectable, forkable, and usable without a JCode Labs account. + +Contributions are welcome through pull requests. Start with [`CONTRIBUTING.md`](./CONTRIBUTING.md). +Security reports should stay private and follow the policy in [`SECURITY.md`](./SECURITY.md). + +## Sponsors + +Mimir stays MIT open source. Sponsorship helps fund maintenance, issue triage, +documentation, and practical agent-workflow improvements. + +Sponsor the project through [GitHub Sponsors](https://github.com/sponsors/jb-thery). + +Suggested GitHub Sponsors tiers: + +- EUR 5/month: support the project. +- EUR 15/month: active sponsor. +- EUR 49/month: priority on issues and questions. +- EUR 199/month: company sponsor and light advisory support. + +## Status + +Early public package. APIs may evolve before `1.0.0`. + +## What Mimir Is For + +- Build a local RAG knowledge base inside any repository. +- Analyze confidential datasets while keeping raw files and generated indexes local. +- Give Claude, Codex, Cursor, internal assistants, or other MCP-compatible tools the same private + retrieval layer. +- Retrieve grounded local evidence through CLI, library calls, MCP tools, or the bundled agent + skills so your chosen AI agent can produce cited summaries. +- Optionally create listenable WAV summaries with `kb audio`, `@jcode.labs/mimir-tts`, and the + bundled `mimir-audio-summary` skill. + +Mimir is not a hosted SaaS, not a remote vector database, and not a certified high-assurance system. +For regulated or state-grade environments, pair it with encrypted disks, controlled machines, release +verification, and an external security review. + +## Use Cases + +Mimir is useful whenever the source material should stay local but an AI agent still needs grounded +context. + +| Use case | Example questions | +| --- | --- | +| Understand a code repository | "Where is authentication implemented?", "What depends on this module?", "Summarize the payment flow." | +| Understand architecture | "What services exist?", "What are the data boundaries?", "Which components are risky to change?" | +| Analyze specifications | "What does the technical spec require?", "Which requirements are still unclear?", "Generate an implementation checklist." | +| Work through a request for proposal or tender | "What are the mandatory constraints?", "Which documents prove compliance?", "What risks should be clarified?" | +| Study courses and training material | "Summarize chapter three.", "Create revision questions.", "Compare these two concepts." | +| Analyze a book or long report | "Extract the main thesis.", "Find recurring arguments.", "Create a chapter-by-chapter brief." | +| Build an internal knowledge base | "What is the policy for incident review?", "Who owns this process?", "Which source says that?" | +| Prepare meetings or decisions | "Give me a one-page briefing.", "What is missing before deciding?", "List action items and evidence." | +| Ask questions over offline documents | "Which files mention local-only operation?", "What evidence supports this claim?" | +| Generate audio briefings | "Create a listenable summary of the current dossier using offline TTS." | + +## Requirements + +- Node.js 20+ +- pnpm, npm, yarn or bun +- No model runtime is required for the default `embeddingProvider: "local-hash"` mode. +- Optional semantic embeddings use Transformers.js with local model files under `.mimir/models` by + default. +- Generated answers are intentionally outside Mimir core. Use Claude, Codex, OpenAI, a local model + MCP server, or another trusted model runtime to synthesize from Mimir's cited context. +- Optional audio summaries use the separate `@jcode.labs/mimir-tts` workspace package. It renders + WAV files with Transformers.js and does not require Python, ffmpeg, Piper, XTTS, or a local server. + +## Install From npm + +The package is public. Users do not need a JCode Labs account or npm token to install it. + +With pnpm: + +```bash +pnpm add -D @jcode.labs/mimir +``` + +With npm: + +```bash +npm install --save-dev @jcode.labs/mimir +``` + +Maintainer tokens are only needed to publish new versions. + +## Install From Source Checkout + +```bash +git clone git@github.com:jcode-works/jcode-mimir.git +cd jcode-mimir +pnpm install +pnpm build +``` + +For local development: + +```bash +pnpm add -D file:../jcode-mimir/packages/mimir +``` + +Before creating an npm tarball later, run: + +```bash +pnpm build +pnpm --dir packages/mimir pack +``` + +## Use In Any Repository + +Initialize the local project config: + +```bash +pnpm exec kb init +``` + +Add private documents under `private/`, then run: + +```bash +pnpm exec kb ingest +pnpm exec kb search "vendor invoice status" +pnpm exec kb ask "What do the documents prove?" +pnpm exec kb audit +pnpm exec kb security-audit +pnpm exec kb status +``` + +With npm, use `npx` after installing the package: + +```bash +npx kb init +npx kb ingest +npx kb search "vendor invoice status" +npx kb ask "What do the documents prove?" +npx kb audit +npx kb security-audit +npx kb status +``` + +## Choose A Retrieval Mode + +Mimir has two embedding modes. + +### Default Local Hash Retrieval + +Use this when you want a fully local, no-model smoke test or a dependency-light setup. Retrieval is +lexical/hash-based, not semantic. + +`.kb/config.json`: + +```json +{ + "embeddingProvider": "local-hash" +} +``` + +Commands: + +```bash +pnpm exec kb ingest +pnpm exec kb search "offline retrieval approval" +pnpm exec kb ask "What evidence supports offline operation?" +``` + +`kb ask` always returns cited retrieved passages instead of a generated synthesis. You can pass those +passages to any LLM or agent you trust. + +### Optional Semantic Embeddings With Transformers.js + +Use this when you want better semantic retrieval while keeping Mimir core free of an LLM server. + +`.kb/config.json`: + +```json +{ + "embeddingProvider": "transformers", + "embeddingModel": "mixedbread-ai/mxbai-embed-xsmall-v1", + "embeddingModelPath": ".mimir/models", + "transformersAllowRemoteModels": false +} +``` + +Commands: + +```bash +pnpm exec kb ingest +pnpm exec kb ask "Which passages support offline operation?" +``` + +Keep `transformersAllowRemoteModels` false for confidential or air-gapped work and preload model +files into `embeddingModelPath`. Set it to true only when you explicitly allow Transformers.js to +download model files from Hugging Face. + +## Dependency Footprint + +Mimir can run retrieval without a model runtime. Some runtime dependencies remain because they own +core features: + +| Dependency | Why it remains | +| --- | --- | +| @huggingface/transformers | optional local semantic embeddings | +| LanceDB | local vector storage and nearest-neighbor retrieval | +| MCP SDK | MCP server for compatible agents | +| fast-glob | safe source-file discovery | +| unpdf, html-to-text, yaml, fflate | document parsing for PDF, HTML, YAML, Office/OpenDocument ZIP files | +| commander, zod, picocolors | CLI, config validation, readable terminal output | + +Removing more dependencies is possible only by dropping features or replacing them with smaller +internal implementations. The current low-friction path is dependency-light at runtime for users who +choose `local-hash`, while preserving richer parsing, MCP support, and optional semantic embeddings. + +## Example Test Workspace + +This repository includes a synthetic example under +[`examples/sovereign-rag-demo`](./examples/sovereign-rag-demo). It can be used to test ingestion, +retrieval, `security-audit`, and custom text extensions without using private documents. + +From a local checkout: + +```bash +pnpm build +cd examples/sovereign-rag-demo +node ../../dist/cli.js security-audit +node ../../dist/cli.js ingest +node ../../dist/cli.js search "offline retrieval approval" +node ../../dist/cli.js audit +``` + +The example uses the default local-hash retrieval mode, so it can run without downloading an +embedding or chat model. + +## Typical Workflows + +### Understand A Codebase + +```bash +pnpm exec kb init +printf "src\nREADME.md\ndocs\n" >> .kb/sources.txt +pnpm exec kb ingest +pnpm exec kb search "authentication flow" +pnpm exec kb ask "Explain the architecture and cite the relevant files." +``` + +### Analyze Specifications Or A Course + +```bash +pnpm exec kb ingest +pnpm exec kb ask "Summarize the requirements and list open questions." +pnpm exec kb ask "Create revision questions from the indexed course material." +``` + +### Work Offline + +```bash +pnpm exec kb security-audit --strict +pnpm exec kb ingest +pnpm exec kb search "incident review policy" +pnpm exec kb ask "What does the local evidence prove?" +``` + +Use `embeddingProvider: "local-hash"` for a no-model offline workflow. Use +`embeddingProvider: "transformers"` with preloaded model files for semantic offline retrieval. +Generated answers should come from a trusted external agent or model runtime. + +### Generate A Local Audio Briefing + +Mimir includes a plug-and-play JS text-to-speech path for listenable summaries: + +```bash +pnpm exec kb audio --doctor +pnpm exec kb audio /tmp/MIMIR-SUMMARY-project.txt --out .mimir/audio/project-summary.wav +``` + +The command writes WAV output locally and does not require Python or ffmpeg. The first render can +download a public Transformers.js-compatible model into `.mimir/models/tts`; the narration text is +processed locally. For confidential air-gapped work, preload model files and run: + +```bash +pnpm exec kb audio /tmp/MIMIR-SUMMARY-project.txt --out .mimir/audio/project-summary.wav --offline +``` + +The standalone package can also be installed directly: + +```bash +pnpm add -D @jcode.labs/mimir-tts +pnpm exec mimir-tts render /tmp/MIMIR-SUMMARY-project.txt --out .mimir/audio/project-summary.wav +``` + +## Agent Skills And MCP + +Mimir ships with portable agent skills and a standard MCP server. + +Install the agent kit into a repository: + +```bash +pnpm exec kb install-skill +``` + +This creates: + +```plain text +.mimir/skills/mimir/SKILL.md +.mimir/skills/mimir-audio-summary/SKILL.md +.mimir/mcp.json +.mimir/README.md +``` + +Agents that support skill folders can load `.mimir/skills/mimir/` for deep local RAG usage. +Load `.mimir/skills/mimir-audio-summary/` only when an optional spoken summary is needed. +Other agents can read the generated `.mimir/README.md` and use the MCP config snippet. + +Start the MCP server from the repository root: + +```bash +pnpm exec kb serve-mcp +``` + +MCP tools exposed: + +- `mimir_status` +- `mimir_search` +- `mimir_ask` +- `mimir_audit` +- `mimir_security_audit` + +This MCP layer is the recommended way to let any compatible LLM or agent query the same local +knowledge base. The LLM does not need to know about LanceDB or the raw file layout; it asks Mimir for +ranked passages or cited context and uses the returned citations. + +Print the bundled skill path from the installed package: + +```bash +pnpm exec kb skill-path +``` + +## Data Boundary + +The package code lives in `node_modules` or in this repository. Project data stays in the +repository where you run the CLI: + +```plain text +your-project/ + private/ # raw documents to ingest + .kb/config.json # local config + .kb/sources.txt # optional extra source paths + .kb/storage/ # generated LanceDB index + .kb/access.log # metadata-only access log +``` + +The package never ships project documents. `kb init` adds gitignore entries for `.kb/` +and `private/**`, and `kb install-skill` keeps `.mimir/` ignored as generated local agent +state. + +## Confidentiality Defaults + +Mimir is designed for private repositories and sensitive local evidence. + +- Zero telemetry: no analytics or document content is sent to JCode Labs. +- No LLM generation in core: Mimir returns cited context for the agent/runtime you choose. +- Local-hash by default: no model runtime is required for the default retrieval path. +- Transformers.js remote model loading is disabled by default. +- Redaction before indexing: common secrets and identifiers are redacted before chunks are + embedded and stored. +- Metadata-only access logs: query hashes and action metadata are logged, not raw queries. +- MCP is read-focused and bounded by `mcpMaxTopK`. +- Generated local state is ignored by Git. + +Run: + +```bash +pnpm exec kb security-audit --strict +``` + +Remove the generated vector index: + +```bash +pnpm exec kb destroy-index --yes +``` + +For air-gapped operation, release verification, secure deletion limits, and threat model details, +read [`SECURITY-HARDENING.md`](./SECURITY-HARDENING.md). + +## Supported Files + +Mimir supports common text, document, data, config, log, and source-code files out of the box: + +- Markdown: `.md`, `.mdx` +- Text: `.txt`, `.text` +- JSON: `.json` +- YAML: `.yaml`, `.yml` +- CSV/TSV: `.csv`, `.tsv` +- HTML: `.html`, `.htm` +- PDF: `.pdf` +- Office/OpenDocument: `.docx`, `.pptx`, `.xlsx`, `.odt`, `.ods`, `.odp` +- Rich text: `.rtf` +- Line data and logs: `.jsonl`, `.ndjson`, `.log` +- XML feeds and documents: `.xml`, `.rss`, `.atom` +- Config and data files: `.toml`, `.ini`, `.conf`, `.cfg`, `.properties`, `.sql` +- Source code: `.ts`, `.tsx`, `.js`, `.jsx`, `.py`, `.go`, `.rs`, `.java`, `.rb`, `.php`, + `.cs`, `.c`, `.cpp`, `.h`, `.css` + +Custom UTF-8 text extensions can be enabled without changing code: + +```json +{ + "includeExtensions": [".transcript", ".evidence"] +} +``` + +Or through: + +```bash +KB_INCLUDE_EXTENSIONS=".transcript,.evidence" pnpm exec kb ingest +``` + +Images, scans, audio/video files, old proprietary Office binaries such as `.doc`, and other formats +that are not listed should be OCRed, transcribed, converted, or exported to text/PDF/HTML first. +Mimir intentionally avoids pretending that every binary format can be indexed safely without +extraction logic. + +## Config + +`.kb/config.json`: + +```json +{ + "rawDir": "private", + "storageDir": ".kb/storage", + "sourcesFile": ".kb/sources.txt", + "accessLogPath": ".kb/access.log", + "embeddingModelPath": ".mimir/models", + "tableName": "chunks", + "embeddingProvider": "local-hash", + "embeddingModel": "mixedbread-ai/mxbai-embed-xsmall-v1", + "transformersAllowRemoteModels": false, + "redaction": { + "enabled": true, + "builtIn": true, + "patterns": [] + }, + "accessLog": true, + "mcpMaxTopK": 10, + "topK": 5, + "chunkSize": 1200, + "chunkOverlap": 150, + "includeExtensions": [] +} +``` + +Environment overrides: + +- `KB_RAW_DIR` +- `KB_STORAGE_DIR` +- `KB_SOURCES_FILE` +- `KB_ACCESS_LOG_PATH` +- `KB_EMBEDDING_PROVIDER` +- `KB_EMBEDDING_MODEL` +- `KB_EMBEDDING_MODEL_PATH` +- `KB_TRANSFORMERS_ALLOW_REMOTE_MODELS` +- `KB_REDACTION_ENABLED` +- `KB_REDACTION_BUILT_IN` +- `KB_ACCESS_LOG` +- `KB_MCP_MAX_TOP_K` +- `KB_TOP_K` +- `KB_CHUNK_SIZE` +- `KB_CHUNK_OVERLAP` +- `KB_INCLUDE_EXTENSIONS` + +## Library API + +```ts +import { ingest, search, ask } from "@jcode.labs/mimir" + +await ingest({ rebuild: true }) +const results = await search("vendor invoice status") +const answer = await ask("What documents support the project timeline?") +``` + +## Privacy + +- Mimir core does not generate answers or call a chat model. +- `local-hash` can run ingestion, search, and cited retrieval without a model runtime. +- Transformers.js remote model loading is disabled by default. +- Built-in redaction runs before indexing by default. +- Access logs store query hashes, not raw queries. +- The vector index is stored locally. +- Raw private documents should stay in the target repository's ignored `private/` folder. +- Do not put secrets or scans inside this package repository. + +## License + +MIT © Jean-Baptiste Thery. diff --git a/packages/mimir/SECURITY-HARDENING.md b/packages/mimir/SECURITY-HARDENING.md new file mode 100644 index 0000000..f3873b9 --- /dev/null +++ b/packages/mimir/SECURITY-HARDENING.md @@ -0,0 +1,194 @@ +# Mimir Security Hardening + +Mimir is a sovereign local RAG knowledge base for confidential project documents and datasets. It is +built to minimize data movement, but it is not a certified high-assurance system. + +## Current Guarantees + +- Zero telemetry: Mimir does not send usage analytics or document content to JCode Labs. +- Retrieval-only core: Mimir does not call a chat model or generate LLM answers. +- No-model retrieval mode: `embeddingProvider: "local-hash"` can ingest, search, and return cited + passages without a model server. +- Optional semantic embeddings: `embeddingProvider: "transformers"` uses Transformers.js, with + remote model loading disabled by default through `transformersAllowRemoteModels: false`. +- Redaction before indexing: built-in DLP patterns redact common secrets and identifiers before + chunks are embedded and stored. +- Metadata-only access logs: access logs contain action metadata and query hashes, not raw + queries or retrieved text. +- Generated local state is ignored by Git: `.kb/`, `.mimir/`, and `private/**` are ignored by + default. +- MCP is read-focused: destructive tools are not exposed over MCP, and MCP retrieval is capped by + `mcpMaxTopK`. +- Optional audio summaries use `kb audio` / `@jcode.labs/mimir-tts` for local WAV rendering with + Transformers.js. They do not require Python, ffmpeg, Piper, XTTS, or a local TTS server. +- npm releases are published with provenance from the protected GitHub Actions workflow. +- Release artifacts include a package tarball, SHA256 checksums, SBOM, and manifest. + +## Threat Model + +Mimir protects against accidental repository leaks, accidental built-in LLM usage, accidental online +TTS usage for generated summaries, accidental secret indexing, and weak release traceability. + +Mimir does not protect against a compromised local machine, malicious dependencies already present +in the runtime, a user with filesystem access to the same checkout, or forensic recovery from an +unencrypted disk. + +## At-Rest Encryption + +Native encrypted LanceDB storage is not implemented yet. For sensitive environments, put the +repository and `.kb/` on an encrypted volume: + +- macOS: FileVault or an encrypted APFS volume. +- Linux: LUKS, fscrypt, or an encrypted VM disk. +- Containers/VMs: mount `.kb/` on an encrypted host volume. + +`kb destroy-index --yes` removes generated index files, but secure deletion on SSDs and copy-on-write +filesystems cannot be guaranteed without encrypted storage and key destruction. + +## Air-Gapped Operation + +Prepare artifacts on an internet-connected build machine: + +```bash +pnpm install --frozen-lockfile +pnpm build +pnpm release:artifacts +``` + +Move the generated tarballs from `release-artifacts/` into the offline environment and install them: + +```bash +pnpm add -D ./jcode.labs-mimir-tts-.tgz ./jcode.labs-mimir-.tgz +pnpm exec kb init +pnpm exec kb ingest +``` + +For semantic embeddings, preload the Transformers.js-compatible embedding model files inside the +offline environment under the configured `embeddingModelPath`. For audio, preload the TTS model +files under `.mimir/models/tts` and render with `pnpm exec kb audio --offline`. + +## Zero Network Posture + +Default no-model config: + +```json +{ + "embeddingProvider": "local-hash" +} +``` + +Optional semantic config: + +```json +{ + "embeddingProvider": "transformers", + "embeddingModel": "mixedbread-ai/mxbai-embed-xsmall-v1", + "embeddingModelPath": ".mimir/models", + "transformersAllowRemoteModels": false +} +``` + +The local-hash mode performs lexical/hash retrieval only. It is useful for smoke tests, +dependency-light offline workflows, and handing cited passages to another trusted LLM. It is not +equivalent to model semantic retrieval. + +Keep `transformersAllowRemoteModels` false for confidential or air-gapped work. If it is true, +Transformers.js may download model files from Hugging Face during model loading. + +Run: + +```bash +pnpm exec kb security-audit --strict +``` + +## DLP Redaction + +Built-in redaction is enabled by default for common secret and identifier shapes: private keys, +JWTs, API tokens, emails, IBANs, and card-like numbers. + +Custom patterns can be added in `.kb/config.json`: + +```json +{ + "redaction": { + "enabled": true, + "builtIn": true, + "patterns": [ + { + "name": "internal_case_id", + "pattern": "CASE-[0-9]+", + "replacement": "[CASE]" + } + ] + } +} +``` + +Redaction changes the indexed text, not the raw files under `private/`. + +## Optional Audio Summaries + +`kb install-skill` installs an optional `mimir-audio-summary` skill. It is designed for listenable +briefings from a local Mimir index. The default renderer is `kb audio`, backed by +`@jcode.labs/mimir-tts` and Transformers.js. + +Confidentiality defaults: + +- narration text is written to a temp file outside the repository; +- generated WAV audio should be written under `.mimir/audio/`; +- `.mimir/` is ignored by Git; +- Python, ffmpeg, Piper, XTTS, and local TTS servers are not required for the default path; +- the first online-enabled render may download public model weights into `.mimir/models/tts`, but + the narration text is processed locally; +- `--offline` disables remote model loading and requires preloaded model files. + +Generated audio can still contain sensitive information. Treat it like a derived confidential +document. + +## MCP Hardening + +MCP gives an agent access to retrieved private context. Use it only for agents running under the +same trust boundary as the repository. + +Mimir MCP defaults: + +- read-focused tools only; +- no index deletion tool exposed over MCP; +- bounded retrieval through `mcpMaxTopK`; +- metadata-only access logging. + +For team use, prefer one checkout per user or per role. Mimir does not implement RBAC. + +## Release Verification + +The protected npm workflow runs validation, generates release artifacts, and publishes both +workspace packages with provenance: + +```bash +pnpm --dir packages/mimir-tts publish --access public --provenance --no-git-checks +pnpm --dir packages/mimir publish --access public --provenance --no-git-checks +``` + +Release artifacts include: + +- npm tarballs for `@jcode.labs/mimir-tts` and `@jcode.labs/mimir`; +- `SHA256SUMS`; +- CycloneDX SBOM; +- `release-manifest.json`. + +Verify checksums offline with: + +```bash +sha256sum -c SHA256SUMS +``` + +On macOS: + +```bash +shasum -a 256 -c SHA256SUMS +``` + +## External Audit Status + +No external security audit has been completed yet. Treat Mimir as useful hardening for private +developer workflows, not as military-grade certified software. diff --git a/packages/mimir/SECURITY.md b/packages/mimir/SECURITY.md new file mode 100644 index 0000000..4472845 --- /dev/null +++ b/packages/mimir/SECURITY.md @@ -0,0 +1,21 @@ +# Security Policy + +## Supported Versions + +Only the latest published version of `@jcode.labs/mimir` receives security fixes. + +## Reporting A Vulnerability + +Please report vulnerabilities privately by email: + +```plain text +contact@jcode.works +``` + +Do not open public issues for vulnerabilities, leaked secrets, credential exposure, +or private document disclosure. + +## Data Boundary + +Mimir is designed to index local project documents. Raw project documents, +`.kb/storage/`, environment files, and credentials must remain outside commits. diff --git a/dist/access-log.d.ts b/packages/mimir/dist/access-log.d.ts similarity index 100% rename from dist/access-log.d.ts rename to packages/mimir/dist/access-log.d.ts diff --git a/dist/access-log.d.ts.map b/packages/mimir/dist/access-log.d.ts.map similarity index 100% rename from dist/access-log.d.ts.map rename to packages/mimir/dist/access-log.d.ts.map diff --git a/dist/access-log.js b/packages/mimir/dist/access-log.js similarity index 100% rename from dist/access-log.js rename to packages/mimir/dist/access-log.js diff --git a/dist/access-log.js.map b/packages/mimir/dist/access-log.js.map similarity index 100% rename from dist/access-log.js.map rename to packages/mimir/dist/access-log.js.map diff --git a/dist/chunking.d.ts b/packages/mimir/dist/chunking.d.ts similarity index 100% rename from dist/chunking.d.ts rename to packages/mimir/dist/chunking.d.ts diff --git a/dist/chunking.d.ts.map b/packages/mimir/dist/chunking.d.ts.map similarity index 76% rename from dist/chunking.d.ts.map rename to packages/mimir/dist/chunking.d.ts.map index bfa5786..9c95f0d 100644 --- a/dist/chunking.d.ts.map +++ b/packages/mimir/dist/chunking.d.ts.map @@ -1 +1 @@ -{"version":3,"file":"chunking.d.ts","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAE3D,wBAAgB,aAAa,CAC3B,QAAQ,EAAE,cAAc,EACxB,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,MAAM,GACnB,SAAS,EAAE,CAqCb"} \ No newline at end of file +{"version":3,"file":"chunking.d.ts","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAM3D,wBAAgB,aAAa,CAC3B,QAAQ,EAAE,cAAc,EACxB,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,MAAM,GACnB,SAAS,EAAE,CAqCb"} \ No newline at end of file diff --git a/dist/chunking.js b/packages/mimir/dist/chunking.js similarity index 84% rename from dist/chunking.js rename to packages/mimir/dist/chunking.js index 1cd4cf0..4b92d0a 100644 --- a/dist/chunking.js +++ b/packages/mimir/dist/chunking.js @@ -1,4 +1,7 @@ import { createHash } from "node:crypto"; +const PARAGRAPH_BREAK_MIN_RATIO = 0.45; +const SENTENCE_BREAK_MIN_RATIO = 0.55; +const WHITESPACE_BREAK_MIN_RATIO = 0.75; export function chunkDocument(document, chunkSize, chunkOverlap) { if (!document.text) { return []; @@ -39,15 +42,15 @@ function chooseChunkEnd(text, cursor, chunkSize) { } const window = text.slice(cursor, hardEnd); const paragraphBreak = window.lastIndexOf("\n\n"); - if (paragraphBreak > chunkSize * 0.45) { + if (paragraphBreak > chunkSize * PARAGRAPH_BREAK_MIN_RATIO) { return cursor + paragraphBreak; } const sentenceBreak = Math.max(window.lastIndexOf(". "), window.lastIndexOf("? "), window.lastIndexOf("! ")); - if (sentenceBreak > chunkSize * 0.55) { + if (sentenceBreak > chunkSize * SENTENCE_BREAK_MIN_RATIO) { return cursor + sentenceBreak + 1; } const whitespace = window.lastIndexOf(" "); - if (whitespace > chunkSize * 0.75) { + if (whitespace > chunkSize * WHITESPACE_BREAK_MIN_RATIO) { return cursor + whitespace; } return hardEnd; diff --git a/packages/mimir/dist/chunking.js.map b/packages/mimir/dist/chunking.js.map new file mode 100644 index 0000000..864588a --- /dev/null +++ b/packages/mimir/dist/chunking.js.map @@ -0,0 +1 @@ +{"version":3,"file":"chunking.js","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AAGxC,MAAM,yBAAyB,GAAG,IAAI,CAAA;AACtC,MAAM,wBAAwB,GAAG,IAAI,CAAA;AACrC,MAAM,0BAA0B,GAAG,IAAI,CAAA;AAEvC,MAAM,UAAU,aAAa,CAC3B,QAAwB,EACxB,SAAiB,EACjB,YAAoB;IAEpB,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnB,OAAO,EAAE,CAAA;IACX,CAAC;IAED,MAAM,MAAM,GAAgB,EAAE,CAAA;IAC9B,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,UAAU,GAAG,CAAC,CAAA;IAElB,OAAO,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,cAAc,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,CAAC,CAAA;QAC5D,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;QAEpD,IAAI,IAAI,EAAE,CAAC;YACT,MAAM,EAAE,GAAG,UAAU,CAAC,QAAQ,CAAC;iBAC5B,MAAM,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,YAAY,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;iBAC7D,MAAM,CAAC,KAAK,CAAC,CAAA;YAChB,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE;gBACF,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM;gBAC5B,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,YAAY;gBACxC,UAAU;gBACV,IAAI;gBACJ,QAAQ,EAAE,QAAQ,CAAC,IAAI,CAAC,QAAQ;gBAChC,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK;gBAC1B,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,OAAO;aAC/B,CAAC,CAAA;YACF,UAAU,IAAI,CAAC,CAAA;QACjB,CAAC;QAED,IAAI,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAChC,MAAK;QACP,CAAC;QACD,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,YAAY,EAAE,MAAM,GAAG,CAAC,CAAC,CAAA;IACnD,CAAC;IAED,OAAO,MAAM,CAAA;AACf,CAAC;AAED,SAAS,cAAc,CAAC,IAAY,EAAE,MAAc,EAAE,SAAiB;IACrE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;IACzD,IAAI,OAAO,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;QAC5B,OAAO,OAAO,CAAA;IAChB,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAC1C,MAAM,cAAc,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAA;IACjD,IAAI,cAAc,GAAG,SAAS,GAAG,yBAAyB,EAAE,CAAC;QAC3D,OAAO,MAAM,GAAG,cAAc,CAAA;IAChC,CAAC;IAED,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAC5B,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,EACxB,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,EACxB,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CACzB,CAAA;IACD,IAAI,aAAa,GAAG,SAAS,GAAG,wBAAwB,EAAE,CAAC;QACzD,OAAO,MAAM,GAAG,aAAa,GAAG,CAAC,CAAA;IACnC,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAA;IAC1C,IAAI,UAAU,GAAG,SAAS,GAAG,0BAA0B,EAAE,CAAC;QACxD,OAAO,MAAM,GAAG,UAAU,CAAA;IAC5B,CAAC;IAED,OAAO,OAAO,CAAA;AAChB,CAAC"} \ No newline at end of file diff --git a/packages/mimir/dist/cli.d.ts b/packages/mimir/dist/cli.d.ts new file mode 100644 index 0000000..faaadd5 --- /dev/null +++ b/packages/mimir/dist/cli.d.ts @@ -0,0 +1,3 @@ +#!/usr/bin/env node +export {}; +//# sourceMappingURL=cli.d.ts.map \ No newline at end of file diff --git a/packages/mimir/dist/cli.d.ts.map b/packages/mimir/dist/cli.d.ts.map new file mode 100644 index 0000000..f022439 --- /dev/null +++ b/packages/mimir/dist/cli.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/dist/cli.js b/packages/mimir/dist/cli.js similarity index 64% rename from dist/cli.js rename to packages/mimir/dist/cli.js index 3bd4eea..6879ed8 100755 --- a/dist/cli.js +++ b/packages/mimir/dist/cli.js @@ -11,6 +11,8 @@ import { securityAudit } from "./security.js"; import { bundledSkillPath, installSkill } from "./skill.js"; import { countRows } from "./store.js"; import { VERSION } from "./version.js"; +const SEARCH_TEXT_PREVIEW_LENGTH = 900; +const TTS_PACKAGE_NAME = "@jcode.labs/mimir-tts"; const program = new Command(); program .name("kb") @@ -59,12 +61,12 @@ program for (const [index, result] of results.entries()) { const distance = result.distance === null ? "n/a" : result.distance.toFixed(4); console.log(`\n${pc.cyan(`[${index + 1}] ${result.relativePath}`)} chunk=${result.chunkIndex} distance=${distance}`); - console.log(result.text.slice(0, 900)); + console.log(result.text.slice(0, SEARCH_TEXT_PREVIEW_LENGTH)); } }); program .command("ask") - .description("Answer a question using retrieved passages and a local Ollama model.") + .description("Return cited retrieval context for a question without calling an LLM.") .argument("", "Question to answer.") .option("-k, --top-k ", "Number of passages to use.", parsePositiveInt) .action(async (query, options) => { @@ -108,17 +110,19 @@ program console.log(`storageDir=${config.storageDir}`); console.log(`sourcesFile=${config.sourcesFile}`); console.log(`accessLogPath=${config.accessLogPath}`); - console.log(`networkPolicy=${config.networkPolicy}`); - console.log(`embedModel=${config.embedModel}`); - console.log(`llmModel=${config.llmModel}`); + console.log(`embeddingModelPath=${config.embeddingModelPath}`); + console.log(`embeddingProvider=${config.embeddingProvider}`); + console.log(`embeddingModel=${config.embeddingModel}`); + console.log(`transformersAllowRemoteModels=${config.transformersAllowRemoteModels}`); console.log(`redactionEnabled=${config.redaction.enabled}`); console.log(`accessLog=${config.accessLog}`); console.log(`mcpMaxTopK=${config.mcpMaxTopK}`); + console.log(`includeExtensions=${config.includeExtensions.join(",")}`); console.log(`chunksIndexed=${rows}`); }); program .command("security-audit") - .description("Show local privacy, network, redaction, MCP, and gitignore posture.") + .description("Show local privacy, provider, redaction, MCP, and gitignore posture.") .option("--json", "Print machine-readable JSON.") .option("--strict", "Exit with code 1 when warnings are present.") .action(async (options) => { @@ -128,9 +132,11 @@ program } else { console.log(`zeroTelemetry=${report.zeroTelemetry}`); - console.log(`networkPolicy=${report.network.policy}`); - console.log(`ollamaHost=${report.network.ollamaHost}`); - console.log(`ollamaHostClassification=${report.network.classification}`); + console.log(`embeddingProvider=${report.providers.embedding}`); + console.log(`embeddingModel=${report.providers.embeddingModel}`); + console.log(`embeddingModelPath=${report.providers.embeddingModelPath}`); + console.log(`transformersAllowRemoteModels=${report.providers.transformersAllowRemoteModels}`); + console.log(`llmGeneration=${report.providers.llmGeneration}`); console.log(`redactionEnabled=${report.redaction.enabled}`); console.log(`redactionBuiltIn=${report.redaction.builtIn}`); console.log(`accessLog=${report.accessLog.enabled}`); @@ -161,6 +167,44 @@ program console.log(`removed=${result.removed}`); console.log(result.note); }); +program + .command("audio") + .description("Render a narration text file to local speech audio with Mimir TTS.") + .argument("[text-file]", "Narration text file to render.") + .option("-o, --out ", "Output WAV path.") + .option("--model ", "Transformers.js TTS model ID.") + .option("--model-path ", "Local model/cache path.") + .option("--offline", "Disable remote model downloads.") + .option("--allow-remote-models", "Explicitly allow remote model downloads.") + .option("--speaker-embeddings ", "Optional model-specific speaker embedding path or URL.") + .option("--speed ", "Optional model-specific speech speed.", parseNumber) + .option("--doctor", "Show TTS runtime readiness instead of rendering.") + .option("--json", "Print machine-readable JSON.") + .action(async (textFile, options) => { + const tts = await loadTts(); + if (options.doctor) { + const report = await tts.doctor(); + printMaybeJson(report, options.json); + return; + } + if (!textFile) { + console.error(pc.red("Missing text file. Use `kb audio `.")); + process.exitCode = 1; + return; + } + const renderOptions = { + cwd: process.cwd(), + textFile, + }; + addOption(renderOptions, "outputPath", options.out); + addOption(renderOptions, "model", options.model); + addOption(renderOptions, "modelPath", options.modelPath); + addOption(renderOptions, "allowRemoteModels", audioAllowRemoteModels(options)); + addOption(renderOptions, "speakerEmbeddings", options.speakerEmbeddings); + addOption(renderOptions, "speed", options.speed); + const result = await tts.renderSpeech(renderOptions); + printMaybeJson(result, options.json); +}); program .command("serve-mcp") .description("Start the MCP server over stdio for Claude, Codex, and other MCP-compatible agents.") @@ -184,6 +228,7 @@ program console.log(` - ${file}`); } console.log(`Skill path: ${result.skillPath}`); + console.log(`Optional audio skill path: ${result.audioSkillPath}`); console.log(`MCP config example: ${result.mcpConfigPath}`); }); await program.parseAsync(process.argv); @@ -194,7 +239,56 @@ function parsePositiveInt(value) { } return parsed; } +function parseNumber(value) { + const parsed = Number.parseFloat(value); + if (!Number.isFinite(parsed)) { + throw new Error("Expected a number."); + } + return parsed; +} function withTopK(topK) { return topK === undefined ? { cwd: process.cwd() } : { cwd: process.cwd(), topK }; } +async function loadTts() { + const module = await import(TTS_PACKAGE_NAME); + if (!isTtsModule(module)) { + throw new Error(`${TTS_PACKAGE_NAME} did not expose the expected TTS API.`); + } + return module; +} +function isTtsModule(value) { + return (typeof value === "object" && + value !== null && + "doctor" in value && + typeof value.doctor === "function" && + "renderSpeech" in value && + typeof value.renderSpeech === "function"); +} +function audioAllowRemoteModels(options) { + if (options.offline) { + return false; + } + if (options.allowRemoteModels) { + return true; + } + return undefined; +} +function printMaybeJson(value, json) { + if (json) { + console.log(JSON.stringify(value, null, 2)); + return; + } + if (typeof value === "object" && value !== null) { + for (const [key, entry] of Object.entries(value)) { + console.log(`${key}=${String(entry)}`); + } + return; + } + console.log(String(value)); +} +function addOption(target, key, value) { + if (value !== undefined) { + target[key] = value; + } +} //# sourceMappingURL=cli.js.map \ No newline at end of file diff --git a/packages/mimir/dist/cli.js.map b/packages/mimir/dist/cli.js.map new file mode 100644 index 0000000..637d2ab --- /dev/null +++ b/packages/mimir/dist/cli.js.map @@ -0,0 +1 @@ +{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AACnC,OAAO,EAAE,MAAM,YAAY,CAAA;AAC3B,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AAC3C,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAA;AACvC,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAA;AACnC,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;AACxC,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC7C,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAC3D,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AACtC,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAA;AAEtC,MAAM,0BAA0B,GAAG,GAAG,CAAA;AACtC,MAAM,gBAAgB,GAAG,uBAAuB,CAAA;AAEhD,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAA;AAE7B,OAAO;KACJ,IAAI,CAAC,IAAI,CAAC;KACV,WAAW,CAAC,+DAA+D,CAAC;KAC5E,OAAO,CAAC,OAAO,CAAC,CAAA;AAEnB,OAAO;KACJ,OAAO,CAAC,MAAM,CAAC;KACf,WAAW,CAAC,iFAAiF,CAAC;KAC9F,MAAM,CAAC,KAAK,IAAI,EAAE;IACjB,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;IAChD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAA;QAC7C,OAAM;IACR,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAA;IACjC,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QAC3B,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,CAAA;IAC5B,CAAC;AACH,CAAC,CAAC,CAAA;AAEJ,OAAO;KACJ,OAAO,CAAC,QAAQ,CAAC;KACjB,WAAW,CAAC,oFAAoF,CAAC;KACjG,MAAM,CAAC,WAAW,EAAE,qEAAqE,CAAC;KAC1F,MAAM,CAAC,KAAK,IAAI,EAAE;IACjB,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAA;IAClE,OAAO,CAAC,GAAG,CACT,EAAE,CAAC,KAAK,CACN,sBAAsB,MAAM,CAAC,YAAY,WAAW,MAAM,CAAC,MAAM,iBAAiB,MAAM,CAAC,YAAY,eAAe,MAAM,CAAC,UAAU,WAAW,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CACvK,CACF,CAAA;IACD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG,CAAC,OAAO,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;IAC9D,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAA;IACtB,CAAC;AACH,CAAC,CAAC,CAAA;AAEJ,OAAO;KACJ,OAAO,CAAC,QAAQ,CAAC;KACjB,WAAW,CAAC,6DAA6D,CAAC;KAC1E,QAAQ,CAAC,SAAS,EAAE,eAAe,CAAC;KACpC,MAAM,CAAC,sBAAsB,EAAE,+BAA+B,EAAE,gBAAgB,CAAC;KACjF,MAAM,CAAC,KAAK,EAAE,KAAa,EAAE,OAA0B,EAAE,EAAE;IAC1D,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAA;IAC3D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,sDAAsD,CAAC,CAAC,CAAA;QAChF,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAA;QACpB,OAAM;IACR,CAAC;IAED,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;QAChD,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;QAC9E,OAAO,CAAC,GAAG,CACT,KAAK,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,GAAG,CAAC,KAAK,MAAM,CAAC,YAAY,EAAE,CAAC,UAAU,MAAM,CAAC,UAAU,aAAa,QAAQ,EAAE,CACxG,CAAA;QACD,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,0BAA0B,CAAC,CAAC,CAAA;IAC/D,CAAC;AACH,CAAC,CAAC,CAAA;AAEJ,OAAO;KACJ,OAAO,CAAC,KAAK,CAAC;KACd,WAAW,CAAC,uEAAuE,CAAC;KACpF,QAAQ,CAAC,SAAS,EAAE,qBAAqB,CAAC;KAC1C,MAAM,CAAC,sBAAsB,EAAE,4BAA4B,EAAE,gBAAgB,CAAC;KAC9E,MAAM,CAAC,KAAK,EAAE,KAAa,EAAE,OAA0B,EAAE,EAAE;IAC1D,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAA;IACvD,OAAO,CAAC,GAAG,CAAC,KAAK,MAAM,CAAC,MAAM,IAAI,CAAC,CAAA;IACnC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAA;QAC/B,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;YACvD,OAAO,CAAC,GAAG,CAAC,MAAM,KAAK,GAAG,CAAC,KAAK,MAAM,CAAC,YAAY,UAAU,MAAM,CAAC,UAAU,EAAE,CAAC,CAAA;QACnF,CAAC;IACH,CAAC;AACH,CAAC,CAAC,CAAA;AAEJ,OAAO;KACJ,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,gEAAgE,CAAC;KAC7E,MAAM,CAAC,KAAK,IAAI,EAAE;IACjB,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;IACzC,OAAO,CAAC,GAAG,CAAC,kBAAkB,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE,CAAC,CAAA;IAC7D,OAAO,CAAC,GAAG,CAAC,gBAAgB,MAAM,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAA;IACzD,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,WAAW,EAAE,CAAC,CAAA;IAChD,OAAO,CAAC,GAAG,CAAC,oBAAoB,MAAM,CAAC,gBAAgB,CAAC,MAAM,EAAE,CAAC,CAAA;IACjE,OAAO,CAAC,GAAG,CAAC,gBAAgB,MAAM,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAA;IAEzD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,CAAA;IAC5C,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;QACvC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,CAAA;IACvC,CAAC;IAED,IAAI,MAAM,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAA;IACtB,CAAC;AACH,CAAC,CAAC,CAAA;AAEJ,OAAO;KACJ,OAAO,CAAC,QAAQ,CAAC;KACjB,WAAW,CAAC,gDAAgD,CAAC;KAC7D,MAAM,CAAC,KAAK,IAAI,EAAE;IACjB,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;IAC9C,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,CAAA;IACpC,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,WAAW,EAAE,CAAC,CAAA;IAChD,OAAO,CAAC,GAAG,CAAC,UAAU,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;IACtC,OAAO,CAAC,GAAG,CAAC,cAAc,MAAM,CAAC,UAAU,EAAE,CAAC,CAAA;IAC9C,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,WAAW,EAAE,CAAC,CAAA;IAChD,OAAO,CAAC,GAAG,CAAC,iBAAiB,MAAM,CAAC,aAAa,EAAE,CAAC,CAAA;IACpD,OAAO,CAAC,GAAG,CAAC,sBAAsB,MAAM,CAAC,kBAAkB,EAAE,CAAC,CAAA;IAC9D,OAAO,CAAC,GAAG,CAAC,qBAAqB,MAAM,CAAC,iBAAiB,EAAE,CAAC,CAAA;IAC5D,OAAO,CAAC,GAAG,CAAC,kBAAkB,MAAM,CAAC,cAAc,EAAE,CAAC,CAAA;IACtD,OAAO,CAAC,GAAG,CAAC,iCAAiC,MAAM,CAAC,6BAA6B,EAAE,CAAC,CAAA;IACpF,OAAO,CAAC,GAAG,CAAC,oBAAoB,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,CAAA;IAC3D,OAAO,CAAC,GAAG,CAAC,aAAa,MAAM,CAAC,SAAS,EAAE,CAAC,CAAA;IAC5C,OAAO,CAAC,GAAG,CAAC,cAAc,MAAM,CAAC,UAAU,EAAE,CAAC,CAAA;IAC9C,OAAO,CAAC,GAAG,CAAC,qBAAqB,MAAM,CAAC,iBAAiB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IACtE,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAA;AACtC,CAAC,CAAC,CAAA;AAEJ,OAAO;KACJ,OAAO,CAAC,gBAAgB,CAAC;KACzB,WAAW,CAAC,sEAAsE,CAAC;KACnF,MAAM,CAAC,QAAQ,EAAE,8BAA8B,CAAC;KAChD,MAAM,CAAC,UAAU,EAAE,6CAA6C,CAAC;KACjE,MAAM,CAAC,KAAK,EAAE,OAA6C,EAAE,EAAE;IAC9D,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;IACjD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAA;IAC9C,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,iBAAiB,MAAM,CAAC,aAAa,EAAE,CAAC,CAAA;QACpD,OAAO,CAAC,GAAG,CAAC,qBAAqB,MAAM,CAAC,SAAS,CAAC,SAAS,EAAE,CAAC,CAAA;QAC9D,OAAO,CAAC,GAAG,CAAC,kBAAkB,MAAM,CAAC,SAAS,CAAC,cAAc,EAAE,CAAC,CAAA;QAChE,OAAO,CAAC,GAAG,CAAC,sBAAsB,MAAM,CAAC,SAAS,CAAC,kBAAkB,EAAE,CAAC,CAAA;QACxE,OAAO,CAAC,GAAG,CAAC,iCAAiC,MAAM,CAAC,SAAS,CAAC,6BAA6B,EAAE,CAAC,CAAA;QAC9F,OAAO,CAAC,GAAG,CAAC,iBAAiB,MAAM,CAAC,SAAS,CAAC,aAAa,EAAE,CAAC,CAAA;QAC9D,OAAO,CAAC,GAAG,CAAC,oBAAoB,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,CAAA;QAC3D,OAAO,CAAC,GAAG,CAAC,oBAAoB,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,CAAA;QAC3D,OAAO,CAAC,GAAG,CAAC,aAAa,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,CAAA;QACpD,OAAO,CAAC,GAAG,CAAC,6BAA6B,MAAM,CAAC,SAAS,CAAC,gBAAgB,EAAE,CAAC,CAAA;QAC7E,OAAO,CAAC,GAAG,CAAC,qBAAqB,MAAM,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAA;QAC7D,OAAO,CAAC,GAAG,CAAC,cAAc,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAA;QAC/C,OAAO,CAAC,GAAG,CAAC,8BAA8B,MAAM,CAAC,GAAG,CAAC,uBAAuB,EAAE,CAAC,CAAA;QAC/E,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACtC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,YAAY,OAAO,EAAE,CAAC,CAAC,CAAA;QAC/C,CAAC;IACH,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjD,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAA;IACtB,CAAC;AACH,CAAC,CAAC,CAAA;AAEJ,OAAO;KACJ,OAAO,CAAC,eAAe,CAAC;KACxB,WAAW,CAAC,2DAA2D,CAAC;KACxE,MAAM,CAAC,OAAO,EAAE,iDAAiD,CAAC;KAClE,MAAM,CAAC,KAAK,EAAE,OAA0B,EAAE,EAAE;IAC3C,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;QACjB,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG,CAAC,6CAA6C,CAAC,CAAC,CAAA;QACpE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAA;QACpB,OAAM;IACR,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;IAChD,OAAO,CAAC,GAAG,CAAC,cAAc,MAAM,CAAC,UAAU,EAAE,CAAC,CAAA;IAC9C,OAAO,CAAC,GAAG,CAAC,WAAW,MAAM,CAAC,OAAO,EAAE,CAAC,CAAA;IACxC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;AAC1B,CAAC,CAAC,CAAA;AAEJ,OAAO;KACJ,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,oEAAoE,CAAC;KACjF,QAAQ,CAAC,aAAa,EAAE,gCAAgC,CAAC;KACzD,MAAM,CAAC,kBAAkB,EAAE,kBAAkB,CAAC;KAC9C,MAAM,CAAC,cAAc,EAAE,+BAA+B,CAAC;KACvD,MAAM,CAAC,qBAAqB,EAAE,yBAAyB,CAAC;KACxD,MAAM,CAAC,WAAW,EAAE,iCAAiC,CAAC;KACtD,MAAM,CAAC,uBAAuB,EAAE,0CAA0C,CAAC;KAC3E,MAAM,CAAC,6BAA6B,EAAE,wDAAwD,CAAC;KAC/F,MAAM,CAAC,kBAAkB,EAAE,uCAAuC,EAAE,WAAW,CAAC;KAChF,MAAM,CAAC,UAAU,EAAE,kDAAkD,CAAC;KACtE,MAAM,CAAC,QAAQ,EAAE,8BAA8B,CAAC;KAChD,MAAM,CAAC,KAAK,EAAE,QAA4B,EAAE,OAAqB,EAAE,EAAE;IACpE,MAAM,GAAG,GAAG,MAAM,OAAO,EAAE,CAAA;IAE3B,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,MAAM,EAAE,CAAA;QACjC,cAAc,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,CAAA;QACpC,OAAM;IACR,CAAC;IAED,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC,CAAA;QACvE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAA;QACpB,OAAM;IACR,CAAC;IAED,MAAM,aAAa,GAAqB;QACtC,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE;QAClB,QAAQ;KACT,CAAA;IACD,SAAS,CAAC,aAAa,EAAE,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,CAAA;IACnD,SAAS,CAAC,aAAa,EAAE,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,CAAA;IAChD,SAAS,CAAC,aAAa,EAAE,WAAW,EAAE,OAAO,CAAC,SAAS,CAAC,CAAA;IACxD,SAAS,CAAC,aAAa,EAAE,mBAAmB,EAAE,sBAAsB,CAAC,OAAO,CAAC,CAAC,CAAA;IAC9E,SAAS,CAAC,aAAa,EAAE,mBAAmB,EAAE,OAAO,CAAC,iBAAiB,CAAC,CAAA;IACxE,SAAS,CAAC,aAAa,EAAE,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,CAAA;IAEhD,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,YAAY,CAAC,aAAa,CAAC,CAAA;IACpD,cAAc,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,CAAA;AACtC,CAAC,CAAC,CAAA;AAEJ,OAAO;KACJ,OAAO,CAAC,WAAW,CAAC;KACpB,WAAW,CACV,qFAAqF,CACtF;KACA,MAAM,CAAC,KAAK,IAAI,EAAE;IACjB,MAAM,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;AAC/B,CAAC,CAAC,CAAA;AAEJ,OAAO;KACJ,OAAO,CAAC,YAAY,CAAC;KACrB,WAAW,CAAC,+EAA+E,CAAC;KAC5F,MAAM,CAAC,GAAG,EAAE;IACX,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC,CAAA;AACjC,CAAC,CAAC,CAAA;AAEJ,OAAO;KACJ,OAAO,CAAC,eAAe,CAAC;KACxB,WAAW,CAAC,kFAAkF,CAAC;KAC/F,MAAM,CACL,qBAAqB,EACrB,oDAAoD,EACpD,eAAe,CAChB;KACA,MAAM,CAAC,KAAK,EAAE,OAA8B,EAAE,EAAE;IAC/C,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAA;IACvF,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAA;IACzC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QAClC,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,CAAA;IAC5B,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,SAAS,EAAE,CAAC,CAAA;IAC9C,OAAO,CAAC,GAAG,CAAC,8BAA8B,MAAM,CAAC,cAAc,EAAE,CAAC,CAAA;IAClE,OAAO,CAAC,GAAG,CAAC,uBAAuB,MAAM,CAAC,aAAa,EAAE,CAAC,CAAA;AAC5D,CAAC,CAAC,CAAA;AAEJ,MAAM,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;AAEtC,SAAS,gBAAgB,CAAC,KAAa;IACrC,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAA;IACzC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAA;IACjD,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC;AAED,SAAS,WAAW,CAAC,KAAa;IAChC,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAA;IACvC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAA;IACvC,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC;AAED,SAAS,QAAQ,CAAC,IAAwB;IACxC,OAAO,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,CAAA;AACnF,CAAC;AA8BD,KAAK,UAAU,OAAO;IACpB,MAAM,MAAM,GAAY,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAA;IACtD,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,GAAG,gBAAgB,uCAAuC,CAAC,CAAA;IAC7E,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,QAAQ,IAAI,KAAK;QACjB,OAAO,KAAK,CAAC,MAAM,KAAK,UAAU;QAClC,cAAc,IAAI,KAAK;QACvB,OAAO,KAAK,CAAC,YAAY,KAAK,UAAU,CACzC,CAAA;AACH,CAAC;AAED,SAAS,sBAAsB,CAAC,OAAqB;IACnD,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACpB,OAAO,KAAK,CAAA;IACd,CAAC;IACD,IAAI,OAAO,CAAC,iBAAiB,EAAE,CAAC;QAC9B,OAAO,IAAI,CAAA;IACb,CAAC;IACD,OAAO,SAAS,CAAA;AAClB,CAAC;AAED,SAAS,cAAc,CAAC,KAAc,EAAE,IAAyB;IAC/D,IAAI,IAAI,EAAE,CAAC;QACT,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAA;QAC3C,OAAM;IACR,CAAC;IACD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QAChD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACjD,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QACxC,CAAC;QACD,OAAM;IACR,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAA;AAC5B,CAAC;AAED,SAAS,SAAS,CAChB,MAAS,EACT,GAAM,EACN,KAAuB;IAEvB,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,CAAA;IACrB,CAAC;AACH,CAAC"} \ No newline at end of file diff --git a/dist/config.d.ts b/packages/mimir/dist/config.d.ts similarity index 100% rename from dist/config.d.ts rename to packages/mimir/dist/config.d.ts diff --git a/packages/mimir/dist/config.d.ts.map b/packages/mimir/dist/config.d.ts.map new file mode 100644 index 0000000..042b817 --- /dev/null +++ b/packages/mimir/dist/config.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;AAwCxC,wBAAgB,eAAe,CAAC,KAAK,SAAgB,GAAG,MAAM,CAc7D;AAED,wBAAsB,UAAU,CAAC,KAAK,SAAgB,GAAG,OAAO,CAAC,MAAM,CAAC,CA+BvE"} \ No newline at end of file diff --git a/dist/config.js b/packages/mimir/dist/config.js similarity index 55% rename from dist/config.js rename to packages/mimir/dist/config.js index 64a485f..ba7e839 100644 --- a/dist/config.js +++ b/packages/mimir/dist/config.js @@ -2,20 +2,22 @@ import { existsSync } from "node:fs"; import { readFile } from "node:fs/promises"; import path from "node:path"; import { z } from "zod"; +import { CONFIG_PATH, DEFAULT_CONFIG } from "./defaults.js"; +const embeddingProviderSchema = z.enum(["local-hash", "transformers"]); const rawConfigSchema = z.object({ - rawDir: z.string().default("private"), - storageDir: z.string().default(".kb/storage"), - sourcesFile: z.string().default(".kb/sources.txt"), - accessLogPath: z.string().default(".kb/access.log"), - tableName: z.string().default("chunks"), - ollamaHost: z.string().default("http://localhost:11434"), - networkPolicy: z.enum(["local-only", "allow-private", "allow-any"]).default("local-only"), - embedModel: z.string().default("nomic-embed-text"), - llmModel: z.string().default("gemma4:latest"), + rawDir: z.string().default(DEFAULT_CONFIG.rawDir), + storageDir: z.string().default(DEFAULT_CONFIG.storageDir), + sourcesFile: z.string().default(DEFAULT_CONFIG.sourcesFile), + accessLogPath: z.string().default(DEFAULT_CONFIG.accessLogPath), + embeddingModelPath: z.string().default(DEFAULT_CONFIG.embeddingModelPath), + tableName: z.string().default(DEFAULT_CONFIG.tableName), + embeddingProvider: embeddingProviderSchema.default(DEFAULT_CONFIG.embeddingProvider), + embeddingModel: z.string().default(DEFAULT_CONFIG.embeddingModel), + transformersAllowRemoteModels: z.boolean().default(DEFAULT_CONFIG.transformersAllowRemoteModels), redaction: z .object({ - enabled: z.boolean().default(true), - builtIn: z.boolean().default(true), + enabled: z.boolean().default(DEFAULT_CONFIG.redaction.enabled), + builtIn: z.boolean().default(DEFAULT_CONFIG.redaction.builtIn), patterns: z .array(z.object({ name: z.string().min(1), @@ -23,16 +25,16 @@ const rawConfigSchema = z.object({ flags: z.string().optional(), replacement: z.string().optional(), })) - .default([]), + .default(DEFAULT_CONFIG.redaction.patterns), }) - .default({ enabled: true, builtIn: true, patterns: [] }), - accessLog: z.boolean().default(true), - mcpMaxTopK: z.number().int().positive().default(10), - topK: z.number().int().positive().default(5), - chunkSize: z.number().int().positive().default(1200), - chunkOverlap: z.number().int().nonnegative().default(150), + .default(DEFAULT_CONFIG.redaction), + accessLog: z.boolean().default(DEFAULT_CONFIG.accessLog), + mcpMaxTopK: z.number().int().positive().default(DEFAULT_CONFIG.mcpMaxTopK), + topK: z.number().int().positive().default(DEFAULT_CONFIG.topK), + chunkSize: z.number().int().positive().default(DEFAULT_CONFIG.chunkSize), + chunkOverlap: z.number().int().nonnegative().default(DEFAULT_CONFIG.chunkOverlap), + includeExtensions: z.array(z.string().min(1)).default(DEFAULT_CONFIG.includeExtensions), }); -const CONFIG_PATH = ".kb/config.json"; export function findProjectRoot(start = process.cwd()) { let current = path.resolve(start); while (true) { @@ -49,9 +51,7 @@ export function findProjectRoot(start = process.cwd()) { export async function loadConfig(start = process.cwd()) { const projectRoot = findProjectRoot(start); const configFile = path.join(projectRoot, CONFIG_PATH); - const raw = existsSync(configFile) - ? JSON.parse(await readFile(configFile, "utf8")) - : {}; + const raw = existsSync(configFile) ? JSON.parse(await readFile(configFile, "utf8")) : {}; const parsed = rawConfigSchema.parse(raw); const withEnv = applyEnv(parsed); if (withEnv.chunkOverlap >= withEnv.chunkSize) { @@ -63,17 +63,18 @@ export async function loadConfig(start = process.cwd()) { storageDir: resolveFromRoot(projectRoot, withEnv.storageDir), sourcesFile: resolveFromRoot(projectRoot, withEnv.sourcesFile), accessLogPath: resolveFromRoot(projectRoot, withEnv.accessLogPath), + embeddingModelPath: resolveFromRoot(projectRoot, withEnv.embeddingModelPath), tableName: withEnv.tableName, - ollamaHost: withEnv.ollamaHost, - networkPolicy: withEnv.networkPolicy, - embedModel: withEnv.embedModel, - llmModel: withEnv.llmModel, + embeddingProvider: withEnv.embeddingProvider, + embeddingModel: withEnv.embeddingModel, + transformersAllowRemoteModels: withEnv.transformersAllowRemoteModels, redaction: withEnv.redaction, accessLog: withEnv.accessLog, mcpMaxTopK: withEnv.mcpMaxTopK, topK: withEnv.topK, chunkSize: withEnv.chunkSize, chunkOverlap: withEnv.chunkOverlap, + includeExtensions: normalizeExtensions(withEnv.includeExtensions), }; } function resolveFromRoot(projectRoot, input) { @@ -86,10 +87,10 @@ function applyEnv(config) { storageDir: process.env.KB_STORAGE_DIR ?? config.storageDir, sourcesFile: process.env.KB_SOURCES_FILE ?? config.sourcesFile, accessLogPath: process.env.KB_ACCESS_LOG_PATH ?? config.accessLogPath, - ollamaHost: process.env.KB_OLLAMA_HOST ?? config.ollamaHost, - networkPolicy: readNetworkPolicyEnv("KB_NETWORK_POLICY", config.networkPolicy), - embedModel: process.env.KB_EMBED_MODEL ?? config.embedModel, - llmModel: process.env.KB_LLM_MODEL ?? config.llmModel, + embeddingProvider: readEmbeddingProviderEnv("KB_EMBEDDING_PROVIDER", config.embeddingProvider), + embeddingModel: process.env.KB_EMBEDDING_MODEL ?? config.embeddingModel, + embeddingModelPath: process.env.KB_EMBEDDING_MODEL_PATH ?? config.embeddingModelPath, + transformersAllowRemoteModels: readBooleanEnv("KB_TRANSFORMERS_ALLOW_REMOTE_MODELS", config.transformersAllowRemoteModels), redaction: { ...config.redaction, enabled: readBooleanEnv("KB_REDACTION_ENABLED", config.redaction.enabled), @@ -100,14 +101,20 @@ function applyEnv(config) { topK: readPositiveIntEnv("KB_TOP_K", config.topK), chunkSize: readPositiveIntEnv("KB_CHUNK_SIZE", config.chunkSize), chunkOverlap: readNonNegativeIntEnv("KB_CHUNK_OVERLAP", config.chunkOverlap), + includeExtensions: readExtensionsEnv("KB_INCLUDE_EXTENSIONS", config.includeExtensions), }; } -function readNetworkPolicyEnv(name, fallback) { - const raw = process.env[name]; - if (raw === "local-only" || raw === "allow-private" || raw === "allow-any") { - return raw; - } - return fallback; +function normalizeExtensions(extensions) { + return [ + ...new Set(extensions + .map((extension) => extension.trim().toLowerCase()) + .filter(Boolean) + .map((extension) => (extension.startsWith(".") ? extension : `.${extension}`))), + ].sort(); +} +function readEmbeddingProviderEnv(name, fallback) { + const parsed = embeddingProviderSchema.safeParse(process.env[name]); + return parsed.success ? parsed.data : fallback; } function readBooleanEnv(name, fallback) { const raw = process.env[name]?.toLowerCase(); @@ -135,4 +142,11 @@ function readNonNegativeIntEnv(name, fallback) { const value = Number.parseInt(raw, 10); return Number.isInteger(value) && value >= 0 ? value : fallback; } +function readExtensionsEnv(name, fallback) { + const raw = process.env[name]; + if (!raw) { + return fallback; + } + return raw.split(","); +} //# sourceMappingURL=config.js.map \ No newline at end of file diff --git a/packages/mimir/dist/config.js.map b/packages/mimir/dist/config.js.map new file mode 100644 index 0000000..99e3282 --- /dev/null +++ b/packages/mimir/dist/config.js.map @@ -0,0 +1 @@ +{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AACvB,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AAG3D,MAAM,uBAAuB,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC,CAAA;AAEtE,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC;IACjD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC;IACzD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,cAAc,CAAC,WAAW,CAAC;IAC3D,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC;IAC/D,kBAAkB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,cAAc,CAAC,kBAAkB,CAAC;IACzE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC;IACvD,iBAAiB,EAAE,uBAAuB,CAAC,OAAO,CAAC,cAAc,CAAC,iBAAiB,CAAC;IACpF,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,cAAc,CAAC,cAAc,CAAC;IACjE,6BAA6B,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,cAAc,CAAC,6BAA6B,CAAC;IAChG,SAAS,EAAE,CAAC;SACT,MAAM,CAAC;QACN,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC,OAAO,CAAC;QAC9D,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC,OAAO,CAAC;QAC9D,QAAQ,EAAE,CAAC;aACR,KAAK,CACJ,CAAC,CAAC,MAAM,CAAC;YACP,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;YACvB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1B,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YAC5B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;SACnC,CAAC,CACH;aACA,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC,QAAQ,CAAC;KAC9C,CAAC;SACD,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC;IACpC,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC;IACxD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC;IAC1E,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC;IAC9D,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC;IACxE,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,cAAc,CAAC,YAAY,CAAC;IACjF,iBAAiB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,iBAAiB,CAAC;CACxF,CAAC,CAAA;AAIF,MAAM,UAAU,eAAe,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,EAAE;IACnD,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IAEjC,OAAO,IAAI,EAAE,CAAC;QACZ,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,EAAE,CAAC;YAChD,OAAO,OAAO,CAAA;QAChB,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;QACpC,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;QAC5B,CAAC;QACD,OAAO,GAAG,MAAM,CAAA;IAClB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,EAAE;IACpD,MAAM,WAAW,GAAG,eAAe,CAAC,KAAK,CAAC,CAAA;IAC1C,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAA;IACtD,MAAM,GAAG,GAAY,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;IAEjG,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IACzC,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAA;IAEhC,IAAI,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAA;IAC/D,CAAC;IAED,OAAO;QACL,WAAW;QACX,MAAM,EAAE,eAAe,CAAC,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC;QACpD,UAAU,EAAE,eAAe,CAAC,WAAW,EAAE,OAAO,CAAC,UAAU,CAAC;QAC5D,WAAW,EAAE,eAAe,CAAC,WAAW,EAAE,OAAO,CAAC,WAAW,CAAC;QAC9D,aAAa,EAAE,eAAe,CAAC,WAAW,EAAE,OAAO,CAAC,aAAa,CAAC;QAClE,kBAAkB,EAAE,eAAe,CAAC,WAAW,EAAE,OAAO,CAAC,kBAAkB,CAAC;QAC5E,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;QAC5C,cAAc,EAAE,OAAO,CAAC,cAAc;QACtC,6BAA6B,EAAE,OAAO,CAAC,6BAA6B;QACpE,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,iBAAiB,EAAE,mBAAmB,CAAC,OAAO,CAAC,iBAAiB,CAAC;KAClE,CAAA;AACH,CAAC;AAED,SAAS,eAAe,CAAC,WAAmB,EAAE,KAAa;IACzD,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,KAAK,CAAC,CAAA;AAC1E,CAAC;AAED,SAAS,QAAQ,CAAC,MAAiB;IACjC,OAAO;QACL,GAAG,MAAM;QACT,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,MAAM,CAAC,MAAM;QAC/C,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,MAAM,CAAC,UAAU;QAC3D,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,MAAM,CAAC,WAAW;QAC9D,aAAa,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,MAAM,CAAC,aAAa;QACrE,iBAAiB,EAAE,wBAAwB,CAAC,uBAAuB,EAAE,MAAM,CAAC,iBAAiB,CAAC;QAC9F,cAAc,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,MAAM,CAAC,cAAc;QACvE,kBAAkB,EAAE,OAAO,CAAC,GAAG,CAAC,uBAAuB,IAAI,MAAM,CAAC,kBAAkB;QACpF,6BAA6B,EAAE,cAAc,CAC3C,qCAAqC,EACrC,MAAM,CAAC,6BAA6B,CACrC;QACD,SAAS,EAAE;YACT,GAAG,MAAM,CAAC,SAAS;YACnB,OAAO,EAAE,cAAc,CAAC,sBAAsB,EAAE,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC;YACzE,OAAO,EAAE,cAAc,CAAC,uBAAuB,EAAE,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC;SAC3E;QACD,SAAS,EAAE,cAAc,CAAC,eAAe,EAAE,MAAM,CAAC,SAAS,CAAC;QAC5D,UAAU,EAAE,kBAAkB,CAAC,kBAAkB,EAAE,MAAM,CAAC,UAAU,CAAC;QACrE,IAAI,EAAE,kBAAkB,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC;QACjD,SAAS,EAAE,kBAAkB,CAAC,eAAe,EAAE,MAAM,CAAC,SAAS,CAAC;QAChE,YAAY,EAAE,qBAAqB,CAAC,kBAAkB,EAAE,MAAM,CAAC,YAAY,CAAC;QAC5E,iBAAiB,EAAE,iBAAiB,CAAC,uBAAuB,EAAE,MAAM,CAAC,iBAAiB,CAAC;KACxF,CAAA;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,UAAoB;IAC/C,OAAO;QACL,GAAG,IAAI,GAAG,CACR,UAAU;aACP,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;aAClD,MAAM,CAAC,OAAO,CAAC;aACf,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,SAAS,EAAE,CAAC,CAAC,CACjF;KACF,CAAC,IAAI,EAAE,CAAA;AACV,CAAC;AAED,SAAS,wBAAwB,CAC/B,IAAY,EACZ,QAAwC;IAExC,MAAM,MAAM,GAAG,uBAAuB,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAA;IACnE,OAAO,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAA;AAChD,CAAC;AAED,SAAS,cAAc,CAAC,IAAY,EAAE,QAAiB;IACrD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,CAAA;IAC5C,IAAI,GAAG,KAAK,GAAG,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;QACnD,OAAO,IAAI,CAAA;IACb,CAAC;IACD,IAAI,GAAG,KAAK,GAAG,IAAI,GAAG,KAAK,OAAO,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;QACnD,OAAO,KAAK,CAAA;IACd,CAAC;IACD,OAAO,QAAQ,CAAA;AACjB,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY,EAAE,QAAgB;IACxD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAC7B,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO,QAAQ,CAAA;IACjB,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAA;IACtC,OAAO,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAA;AAChE,CAAC;AAED,SAAS,qBAAqB,CAAC,IAAY,EAAE,QAAgB;IAC3D,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAC7B,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO,QAAQ,CAAA;IACjB,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAA;IACtC,OAAO,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAA;AACjE,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY,EAAE,QAAkB;IACzD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAC7B,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO,QAAQ,CAAA;IACjB,CAAC;IACD,OAAO,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;AACvB,CAAC"} \ No newline at end of file diff --git a/packages/mimir/dist/defaults.d.ts b/packages/mimir/dist/defaults.d.ts new file mode 100644 index 0000000..db085c7 --- /dev/null +++ b/packages/mimir/dist/defaults.d.ts @@ -0,0 +1,11 @@ +import type { Config } from "./types.js"; +export declare const KB_DIR = ".kb"; +export declare const MIMIR_DIR = ".mimir"; +export declare const PRIVATE_DIR = "private"; +export declare const CONFIG_PATH = ".kb/config.json"; +export declare const DEFAULT_SKILL_TARGET_DIR = ".mimir/skills"; +export declare const KB_GITIGNORE_ENTRY = ".kb/"; +export declare const MIMIR_GITIGNORE_ENTRY = ".mimir/"; +export declare const PRIVATE_GITIGNORE_ENTRY = "private/**"; +export declare const DEFAULT_CONFIG: Omit; +//# sourceMappingURL=defaults.d.ts.map \ No newline at end of file diff --git a/packages/mimir/dist/defaults.d.ts.map b/packages/mimir/dist/defaults.d.ts.map new file mode 100644 index 0000000..c25f3ca --- /dev/null +++ b/packages/mimir/dist/defaults.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"defaults.d.ts","sourceRoot":"","sources":["../src/defaults.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;AAExC,eAAO,MAAM,MAAM,QAAQ,CAAA;AAC3B,eAAO,MAAM,SAAS,WAAW,CAAA;AACjC,eAAO,MAAM,WAAW,YAAY,CAAA;AACpC,eAAO,MAAM,WAAW,oBAA0B,CAAA;AAClD,eAAO,MAAM,wBAAwB,kBAAwB,CAAA;AAE7D,eAAO,MAAM,kBAAkB,SAAe,CAAA;AAC9C,eAAO,MAAM,qBAAqB,YAAkB,CAAA;AACpD,eAAO,MAAM,uBAAuB,eAAsB,CAAA;AAE1D,eAAO,MAAM,cAAc,EAAE,IAAI,CAAC,MAAM,EAAE,aAAa,CAqBtD,CAAA"} \ No newline at end of file diff --git a/packages/mimir/dist/defaults.js b/packages/mimir/dist/defaults.js new file mode 100644 index 0000000..62d3016 --- /dev/null +++ b/packages/mimir/dist/defaults.js @@ -0,0 +1,31 @@ +export const KB_DIR = ".kb"; +export const MIMIR_DIR = ".mimir"; +export const PRIVATE_DIR = "private"; +export const CONFIG_PATH = `${KB_DIR}/config.json`; +export const DEFAULT_SKILL_TARGET_DIR = `${MIMIR_DIR}/skills`; +export const KB_GITIGNORE_ENTRY = `${KB_DIR}/`; +export const MIMIR_GITIGNORE_ENTRY = `${MIMIR_DIR}/`; +export const PRIVATE_GITIGNORE_ENTRY = `${PRIVATE_DIR}/**`; +export const DEFAULT_CONFIG = { + rawDir: PRIVATE_DIR, + storageDir: `${KB_DIR}/storage`, + sourcesFile: `${KB_DIR}/sources.txt`, + accessLogPath: `${KB_DIR}/access.log`, + embeddingModelPath: `${MIMIR_DIR}/models`, + tableName: "chunks", + embeddingProvider: "local-hash", + embeddingModel: "mixedbread-ai/mxbai-embed-xsmall-v1", + transformersAllowRemoteModels: false, + redaction: { + enabled: true, + builtIn: true, + patterns: [], + }, + accessLog: true, + mcpMaxTopK: 10, + topK: 5, + chunkSize: 1200, + chunkOverlap: 150, + includeExtensions: [], +}; +//# sourceMappingURL=defaults.js.map \ No newline at end of file diff --git a/packages/mimir/dist/defaults.js.map b/packages/mimir/dist/defaults.js.map new file mode 100644 index 0000000..5ad7352 --- /dev/null +++ b/packages/mimir/dist/defaults.js.map @@ -0,0 +1 @@ +{"version":3,"file":"defaults.js","sourceRoot":"","sources":["../src/defaults.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,MAAM,MAAM,GAAG,KAAK,CAAA;AAC3B,MAAM,CAAC,MAAM,SAAS,GAAG,QAAQ,CAAA;AACjC,MAAM,CAAC,MAAM,WAAW,GAAG,SAAS,CAAA;AACpC,MAAM,CAAC,MAAM,WAAW,GAAG,GAAG,MAAM,cAAc,CAAA;AAClD,MAAM,CAAC,MAAM,wBAAwB,GAAG,GAAG,SAAS,SAAS,CAAA;AAE7D,MAAM,CAAC,MAAM,kBAAkB,GAAG,GAAG,MAAM,GAAG,CAAA;AAC9C,MAAM,CAAC,MAAM,qBAAqB,GAAG,GAAG,SAAS,GAAG,CAAA;AACpD,MAAM,CAAC,MAAM,uBAAuB,GAAG,GAAG,WAAW,KAAK,CAAA;AAE1D,MAAM,CAAC,MAAM,cAAc,GAAgC;IACzD,MAAM,EAAE,WAAW;IACnB,UAAU,EAAE,GAAG,MAAM,UAAU;IAC/B,WAAW,EAAE,GAAG,MAAM,cAAc;IACpC,aAAa,EAAE,GAAG,MAAM,aAAa;IACrC,kBAAkB,EAAE,GAAG,SAAS,SAAS;IACzC,SAAS,EAAE,QAAQ;IACnB,iBAAiB,EAAE,YAAY;IAC/B,cAAc,EAAE,qCAAqC;IACrD,6BAA6B,EAAE,KAAK;IACpC,SAAS,EAAE;QACT,OAAO,EAAE,IAAI;QACb,OAAO,EAAE,IAAI;QACb,QAAQ,EAAE,EAAE;KACb;IACD,SAAS,EAAE,IAAI;IACf,UAAU,EAAE,EAAE;IACd,IAAI,EAAE,CAAC;IACP,SAAS,EAAE,IAAI;IACf,YAAY,EAAE,GAAG;IACjB,iBAAiB,EAAE,EAAE;CACtB,CAAA"} \ No newline at end of file diff --git a/dist/destroy.d.ts b/packages/mimir/dist/destroy.d.ts similarity index 100% rename from dist/destroy.d.ts rename to packages/mimir/dist/destroy.d.ts diff --git a/dist/destroy.d.ts.map b/packages/mimir/dist/destroy.d.ts.map similarity index 100% rename from dist/destroy.d.ts.map rename to packages/mimir/dist/destroy.d.ts.map diff --git a/dist/destroy.js b/packages/mimir/dist/destroy.js similarity index 100% rename from dist/destroy.js rename to packages/mimir/dist/destroy.js diff --git a/dist/destroy.js.map b/packages/mimir/dist/destroy.js.map similarity index 100% rename from dist/destroy.js.map rename to packages/mimir/dist/destroy.js.map diff --git a/dist/embeddings.d.ts b/packages/mimir/dist/embeddings.d.ts similarity index 100% rename from dist/embeddings.d.ts rename to packages/mimir/dist/embeddings.d.ts diff --git a/packages/mimir/dist/embeddings.d.ts.map b/packages/mimir/dist/embeddings.d.ts.map new file mode 100644 index 0000000..5b5470b --- /dev/null +++ b/packages/mimir/dist/embeddings.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"embeddings.d.ts","sourceRoot":"","sources":["../src/embeddings.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;AAYxC,wBAAsB,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAUrF;AAED,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAM/E"} \ No newline at end of file diff --git a/packages/mimir/dist/embeddings.js b/packages/mimir/dist/embeddings.js new file mode 100644 index 0000000..a90d738 --- /dev/null +++ b/packages/mimir/dist/embeddings.js @@ -0,0 +1,99 @@ +import { createHash } from "node:crypto"; +const LOCAL_HASH_DIMENSIONS = 384; +const LONG_TOKEN_MIN_LENGTH = 6; +const LONG_TOKEN_WEIGHT = 1.4; +const transformersPipelines = new Map(); +export async function embedTexts(texts, config) { + if (texts.length === 0) { + return []; + } + if (config.embeddingProvider === "local-hash") { + return texts.map(localHashEmbedding); + } + return embedWithTransformers(texts, config); +} +export async function embedText(text, config) { + const [embedding] = await embedTexts([text], config); + if (!embedding) { + throw new Error("No embedding returned for query."); + } + return embedding; +} +async function embedWithTransformers(texts, config) { + const extractor = await transformersExtractor(config); + const output = await extractor(texts, { pooling: "mean", normalize: true }); + const rows = tensorToEmbeddingRows(output); + if (rows.length !== texts.length) { + throw new Error(`Expected ${texts.length} embeddings, received ${rows.length}.`); + } + return rows; +} +async function transformersExtractor(config) { + const key = [ + config.embeddingModel, + config.embeddingModelPath, + String(config.transformersAllowRemoteModels), + ].join("\n"); + const cached = transformersPipelines.get(key); + if (cached) { + return cached; + } + const transformers = await import("@huggingface/transformers"); + transformers.env.localModelPath = config.embeddingModelPath; + transformers.env.cacheDir = config.embeddingModelPath; + transformers.env.allowRemoteModels = config.transformersAllowRemoteModels; + const extractor = (await transformers.pipeline("feature-extraction", config.embeddingModel)); + transformersPipelines.set(key, extractor); + return extractor; +} +function localHashEmbedding(text) { + const vector = Array.from({ length: LOCAL_HASH_DIMENSIONS }, () => 0); + const tokens = tokenize(text); + for (const token of tokens) { + const hash = createHash("sha256").update(token).digest(); + const index = hash.readUInt32BE(0) % LOCAL_HASH_DIMENSIONS; + const sign = (hash.at(4) ?? 0) % 2 === 0 ? 1 : -1; + vector[index] = (vector[index] ?? 0) + sign * tokenWeight(token); + } + const magnitude = Math.sqrt(vector.reduce((sum, value) => sum + value * value, 0)); + if (magnitude === 0) { + return vector; + } + return vector.map((value) => value / magnitude); +} +function tokenize(text) { + return (text + .toLowerCase() + .normalize("NFKD") + .replace(/\p{Diacritic}/gu, "") + .match(/[\p{L}\p{N}]{2,}/gu) ?? []); +} +function tokenWeight(token) { + return token.length >= LONG_TOKEN_MIN_LENGTH ? LONG_TOKEN_WEIGHT : 1; +} +function tensorToEmbeddingRows(output) { + if (!hasToList(output)) { + throw new Error("Transformers embedding output does not expose tolist()."); + } + const value = output.tolist(); + if (isNumberMatrix(value)) { + return value; + } + if (isNumberArray(value)) { + return [value]; + } + throw new Error("Transformers embedding output is not a numeric vector matrix."); +} +function hasToList(value) { + return (typeof value === "object" && + value !== null && + "tolist" in value && + typeof value.tolist === "function"); +} +function isNumberArray(value) { + return Array.isArray(value) && value.every((item) => typeof item === "number"); +} +function isNumberMatrix(value) { + return Array.isArray(value) && value.every(isNumberArray); +} +//# sourceMappingURL=embeddings.js.map \ No newline at end of file diff --git a/packages/mimir/dist/embeddings.js.map b/packages/mimir/dist/embeddings.js.map new file mode 100644 index 0000000..f57f4b4 --- /dev/null +++ b/packages/mimir/dist/embeddings.js.map @@ -0,0 +1 @@ +{"version":3,"file":"embeddings.js","sourceRoot":"","sources":["../src/embeddings.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AAGxC,MAAM,qBAAqB,GAAG,GAAG,CAAA;AACjC,MAAM,qBAAqB,GAAG,CAAC,CAAA;AAC/B,MAAM,iBAAiB,GAAG,GAAG,CAAA;AAC7B,MAAM,qBAAqB,GAAG,IAAI,GAAG,EAAiC,CAAA;AAOtE,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,KAAe,EAAE,MAAc;IAC9D,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,EAAE,CAAA;IACX,CAAC;IAED,IAAI,MAAM,CAAC,iBAAiB,KAAK,YAAY,EAAE,CAAC;QAC9C,OAAO,KAAK,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAA;IACtC,CAAC;IAED,OAAO,qBAAqB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;AAC7C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAY,EAAE,MAAc;IAC1D,MAAM,CAAC,SAAS,CAAC,GAAG,MAAM,UAAU,CAAC,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,CAAA;IACpD,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAA;IACrD,CAAC;IACD,OAAO,SAAS,CAAA;AAClB,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,KAAe,EAAE,MAAc;IAClE,MAAM,SAAS,GAAG,MAAM,qBAAqB,CAAC,MAAM,CAAC,CAAA;IACrD,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAC3E,MAAM,IAAI,GAAG,qBAAqB,CAAC,MAAM,CAAC,CAAA;IAE1C,IAAI,IAAI,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;QACjC,MAAM,IAAI,KAAK,CAAC,YAAY,KAAK,CAAC,MAAM,yBAAyB,IAAI,CAAC,MAAM,GAAG,CAAC,CAAA;IAClF,CAAC;IAED,OAAO,IAAI,CAAA;AACb,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,MAAc;IACjD,MAAM,GAAG,GAAG;QACV,MAAM,CAAC,cAAc;QACrB,MAAM,CAAC,kBAAkB;QACzB,MAAM,CAAC,MAAM,CAAC,6BAA6B,CAAC;KAC7C,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACZ,MAAM,MAAM,GAAG,qBAAqB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IAC7C,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,MAAM,CAAA;IACf,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAA;IAC9D,YAAY,CAAC,GAAG,CAAC,cAAc,GAAG,MAAM,CAAC,kBAAkB,CAAA;IAC3D,YAAY,CAAC,GAAG,CAAC,QAAQ,GAAG,MAAM,CAAC,kBAAkB,CAAA;IACrD,YAAY,CAAC,GAAG,CAAC,iBAAiB,GAAG,MAAM,CAAC,6BAA6B,CAAA;IAEzE,MAAM,SAAS,GAAG,CAAC,MAAM,YAAY,CAAC,QAAQ,CAC5C,oBAAoB,EACpB,MAAM,CAAC,cAAc,CACtB,CAA0B,CAAA;IAC3B,qBAAqB,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,CAAC,CAAA;IACzC,OAAO,SAAS,CAAA;AAClB,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY;IACtC,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,qBAAqB,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAA;IACrE,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAA;IAE7B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,CAAA;QACxD,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,qBAAqB,CAAA;QAC1D,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QACjD,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,GAAG,WAAW,CAAC,KAAK,CAAC,CAAA;IAClE,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,CAAA;IAClF,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;QACpB,OAAO,MAAM,CAAA;IACf,CAAC;IACD,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,SAAS,CAAC,CAAA;AACjD,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,CACL,IAAI;SACD,WAAW,EAAE;SACb,SAAS,CAAC,MAAM,CAAC;SACjB,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC;SAC9B,KAAK,CAAC,oBAAoB,CAAC,IAAI,EAAE,CACrC,CAAA;AACH,CAAC;AAED,SAAS,WAAW,CAAC,KAAa;IAChC,OAAO,KAAK,CAAC,MAAM,IAAI,qBAAqB,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAA;AACtE,CAAC;AAED,SAAS,qBAAqB,CAAC,MAAe;IAC5C,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAA;IAC5E,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,CAAA;IAC7B,IAAI,cAAc,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,KAAK,CAAA;IACd,CAAC;IACD,IAAI,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,CAAC,KAAK,CAAC,CAAA;IAChB,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,+DAA+D,CAAC,CAAA;AAClF,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,QAAQ,IAAI,KAAK;QACjB,OAAO,KAAK,CAAC,MAAM,KAAK,UAAU,CACnC,CAAA;AACH,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAA;AAChF,CAAC;AAED,SAAS,cAAc,CAAC,KAAc;IACpC,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,aAAa,CAAC,CAAA;AAC3D,CAAC"} \ No newline at end of file diff --git a/dist/files.d.ts b/packages/mimir/dist/files.d.ts similarity index 55% rename from dist/files.d.ts rename to packages/mimir/dist/files.d.ts index 91e2cb9..20807eb 100644 --- a/dist/files.d.ts +++ b/packages/mimir/dist/files.d.ts @@ -1,4 +1,5 @@ import type { Config, SourceFile } from "./types.js"; -export declare const SUPPORTED_EXTENSIONS: Set; +export declare const DEFAULT_SUPPORTED_EXTENSIONS: Set; export declare function listSourceFiles(config: Config): Promise; +export declare function supportedExtensions(config: Config): Set; //# sourceMappingURL=files.d.ts.map \ No newline at end of file diff --git a/packages/mimir/dist/files.d.ts.map b/packages/mimir/dist/files.d.ts.map new file mode 100644 index 0000000..1e8fc82 --- /dev/null +++ b/packages/mimir/dist/files.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"files.d.ts","sourceRoot":"","sources":["../src/files.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,YAAY,CAAA;AAEpD,eAAO,MAAM,4BAA4B,aA+CvC,CAAA;AAEF,wBAAsB,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAuC3E;AAED,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CAE/D"} \ No newline at end of file diff --git a/dist/files.js b/packages/mimir/dist/files.js similarity index 76% rename from dist/files.js rename to packages/mimir/dist/files.js index bcad32b..ea3712f 100644 --- a/dist/files.js +++ b/packages/mimir/dist/files.js @@ -3,17 +3,51 @@ import { existsSync } from "node:fs"; import { readFile, stat } from "node:fs/promises"; import path from "node:path"; import fg from "fast-glob"; -export const SUPPORTED_EXTENSIONS = new Set([ +export const DEFAULT_SUPPORTED_EXTENSIONS = new Set([ + ".atom", + ".c", + ".cfg", + ".conf", + ".cpp", + ".cs", + ".css", ".csv", + ".docx", + ".go", + ".h", ".htm", ".html", + ".ini", + ".java", + ".js", ".json", + ".jsonl", + ".jsx", + ".log", ".md", ".mdx", + ".ndjson", + ".odp", + ".ods", + ".odt", ".pdf", + ".php", + ".pptx", + ".properties", + ".py", + ".rb", + ".rs", + ".rss", + ".rtf", + ".sql", ".text", + ".toml", + ".ts", ".tsv", + ".tsx", ".txt", + ".xml", + ".xlsx", ".yaml", ".yml", ]); @@ -34,7 +68,7 @@ export async function listSourceFiles(config) { }); for (const absolutePath of entries) { const extension = path.extname(absolutePath).toLowerCase(); - if (!SUPPORTED_EXTENSIONS.has(extension)) { + if (!supportedExtensions(config).has(extension)) { continue; } const info = await stat(absolutePath); @@ -52,6 +86,9 @@ export async function listSourceFiles(config) { } return [...files.values()].sort((a, b) => a.relativePath.localeCompare(b.relativePath)); } +export function supportedExtensions(config) { + return new Set([...DEFAULT_SUPPORTED_EXTENSIONS, ...config.includeExtensions]); +} async function sourceRoots(config) { const roots = [config.rawDir]; if (!existsSync(config.sourcesFile)) { diff --git a/packages/mimir/dist/files.js.map b/packages/mimir/dist/files.js.map new file mode 100644 index 0000000..01efba1 --- /dev/null +++ b/packages/mimir/dist/files.js.map @@ -0,0 +1 @@ +{"version":3,"file":"files.js","sourceRoot":"","sources":["../src/files.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAA;AACjD,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,MAAM,WAAW,CAAA;AAG1B,MAAM,CAAC,MAAM,4BAA4B,GAAG,IAAI,GAAG,CAAC;IAClD,OAAO;IACP,IAAI;IACJ,MAAM;IACN,OAAO;IACP,MAAM;IACN,KAAK;IACL,MAAM;IACN,MAAM;IACN,OAAO;IACP,KAAK;IACL,IAAI;IACJ,MAAM;IACN,OAAO;IACP,MAAM;IACN,OAAO;IACP,KAAK;IACL,OAAO;IACP,QAAQ;IACR,MAAM;IACN,MAAM;IACN,KAAK;IACL,MAAM;IACN,SAAS;IACT,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,OAAO;IACP,aAAa;IACb,KAAK;IACL,KAAK;IACL,KAAK;IACL,MAAM;IACN,MAAM;IACN,MAAM;IACN,OAAO;IACP,OAAO;IACP,KAAK;IACL,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,OAAO;IACP,OAAO;IACP,MAAM;CACP,CAAC,CAAA;AAEF,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,MAAc;IAClD,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,CAAA;IACvC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAsB,CAAA;IAE3C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACtB,SAAQ;QACV,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,MAAM,EAAE;YAC/B,GAAG,EAAE,IAAI;YACT,QAAQ,EAAE,IAAI;YACd,SAAS,EAAE,IAAI;YACf,GAAG,EAAE,KAAK;YACV,mBAAmB,EAAE,KAAK;YAC1B,MAAM,EAAE,CAAC,YAAY,EAAE,oBAAoB,EAAE,WAAW,EAAE,cAAc,CAAC;SAC1E,CAAC,CAAA;QAEF,KAAK,MAAM,YAAY,IAAI,OAAO,EAAE,CAAC;YACnC,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,WAAW,EAAE,CAAA;YAC1D,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBAChD,SAAQ;YACV,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,CAAA;YACrC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,YAAY,CAAC,CAAA;YAC3C,KAAK,CAAC,GAAG,CAAC,YAAY,EAAE;gBACtB,YAAY;gBACZ,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,YAAY,CAAC;gBAC7D,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC;gBACxE,SAAS;gBACT,KAAK,EAAE,IAAI,CAAC,IAAI;gBAChB,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAC5D,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAA;AACzF,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,MAAc;IAChD,OAAO,IAAI,GAAG,CAAC,CAAC,GAAG,4BAA4B,EAAE,GAAG,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAA;AAChF,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,MAAc;IACvC,MAAM,KAAK,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;IAC7B,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC;QACpC,OAAO,KAAK,CAAA;IACd,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,CAAA;IAC1D,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;QAC3B,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACxC,SAAQ;QACV,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,CAAA;IAC5F,CAAC;IAED,OAAO,KAAK,CAAA;AACd,CAAC"} \ No newline at end of file diff --git a/packages/mimir/dist/gitignore.d.ts b/packages/mimir/dist/gitignore.d.ts new file mode 100644 index 0000000..3731e9d --- /dev/null +++ b/packages/mimir/dist/gitignore.d.ts @@ -0,0 +1,3 @@ +export declare const MIMIR_GITIGNORE_ENTRIES: string[]; +export declare function ensureMimirGitignore(cwd?: string): Promise; +//# sourceMappingURL=gitignore.d.ts.map \ No newline at end of file diff --git a/packages/mimir/dist/gitignore.d.ts.map b/packages/mimir/dist/gitignore.d.ts.map new file mode 100644 index 0000000..e2a01cb --- /dev/null +++ b/packages/mimir/dist/gitignore.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"gitignore.d.ts","sourceRoot":"","sources":["../src/gitignore.ts"],"names":[],"mappings":"AAUA,eAAO,MAAM,uBAAuB,UAQnC,CAAA;AAED,wBAAsB,oBAAoB,CAAC,GAAG,SAAgB,GAAG,OAAO,CAAC,OAAO,CAAC,CAyBhF"} \ No newline at end of file diff --git a/dist/gitignore.js b/packages/mimir/dist/gitignore.js similarity index 78% rename from dist/gitignore.js rename to packages/mimir/dist/gitignore.js index 7cd4746..6ca0d58 100644 --- a/dist/gitignore.js +++ b/packages/mimir/dist/gitignore.js @@ -1,14 +1,15 @@ import { existsSync } from "node:fs"; import { readFile, writeFile } from "node:fs/promises"; import path from "node:path"; +import { KB_GITIGNORE_ENTRY, MIMIR_GITIGNORE_ENTRY, PRIVATE_DIR, PRIVATE_GITIGNORE_ENTRY, } from "./defaults.js"; export const MIMIR_GITIGNORE_ENTRIES = [ - ".kb/", - ".mimir/", - "private/**", - "!private/", - "!private/README.md", - "!private/**/", - "!private/**/.gitkeep", + KB_GITIGNORE_ENTRY, + MIMIR_GITIGNORE_ENTRY, + PRIVATE_GITIGNORE_ENTRY, + `!${PRIVATE_DIR}/`, + `!${PRIVATE_DIR}/README.md`, + `!${PRIVATE_DIR}/**/`, + `!${PRIVATE_DIR}/**/.gitkeep`, ]; export async function ensureMimirGitignore(cwd = process.cwd()) { const root = path.resolve(cwd); diff --git a/dist/gitignore.js.map b/packages/mimir/dist/gitignore.js.map similarity index 80% rename from dist/gitignore.js.map rename to packages/mimir/dist/gitignore.js.map index 0460f9c..0b2df3b 100644 --- a/dist/gitignore.js.map +++ b/packages/mimir/dist/gitignore.js.map @@ -1 +1 @@ -{"version":3,"file":"gitignore.js","sourceRoot":"","sources":["../src/gitignore.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACtD,OAAO,IAAI,MAAM,WAAW,CAAA;AAE5B,MAAM,CAAC,MAAM,uBAAuB,GAAG;IACrC,MAAM;IACN,SAAS;IACT,YAAY;IACZ,WAAW;IACX,oBAAoB;IACpB,cAAc;IACd,sBAAsB;CACd,CAAA;AAEV,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE;IAC5D,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;IAC9B,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,YAAY,CAAC,CAAA;IACnD,MAAM,OAAO,GAAG,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;IACtF,MAAM,YAAY,GAAG,IAAI,GAAG,CAC1B,OAAO;SACJ,KAAK,CAAC,OAAO,CAAC;SACd,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;SAC1B,MAAM,CAAC,OAAO,CAAC,CACnB,CAAA;IACD,MAAM,cAAc,GAAG,uBAAuB,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAA;IAE1F,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,KAAK,CAAA;IACd,CAAC;IAED,MAAM,cAAc,GAAG,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,YAAY,CAAC,GAAG,CAAC,eAAe,CAAC,CAAA;IACvF,MAAM,KAAK,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,EAAE,GAAG,cAAc,CAAC;SACtE,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,SAAS,CAAC;SACpC,IAAI,CAAC,IAAI,CAAC,CAAA;IACb,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,EAAE,CAAA;IAChC,MAAM,IAAI,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,IAAI,CAAA;IAEzD,MAAM,SAAS,CAAC,aAAa,EAAE,IAAI,EAAE,MAAM,CAAC,CAAA;IAC5C,OAAO,IAAI,CAAA;AACb,CAAC"} \ No newline at end of file +{"version":3,"file":"gitignore.js","sourceRoot":"","sources":["../src/gitignore.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACtD,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EACL,kBAAkB,EAClB,qBAAqB,EACrB,WAAW,EACX,uBAAuB,GACxB,MAAM,eAAe,CAAA;AAEtB,MAAM,CAAC,MAAM,uBAAuB,GAAG;IACrC,kBAAkB;IAClB,qBAAqB;IACrB,uBAAuB;IACvB,IAAI,WAAW,GAAG;IAClB,IAAI,WAAW,YAAY;IAC3B,IAAI,WAAW,MAAM;IACrB,IAAI,WAAW,cAAc;CAC9B,CAAA;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE;IAC5D,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;IAC9B,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,YAAY,CAAC,CAAA;IACnD,MAAM,OAAO,GAAG,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;IACtF,MAAM,YAAY,GAAG,IAAI,GAAG,CAC1B,OAAO;SACJ,KAAK,CAAC,OAAO,CAAC;SACd,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;SAC1B,MAAM,CAAC,OAAO,CAAC,CACnB,CAAA;IACD,MAAM,cAAc,GAAG,uBAAuB,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAA;IAE1F,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,KAAK,CAAA;IACd,CAAC;IAED,MAAM,cAAc,GAAG,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,YAAY,CAAC,GAAG,CAAC,eAAe,CAAC,CAAA;IACvF,MAAM,KAAK,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,EAAE,GAAG,cAAc,CAAC;SACtE,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,SAAS,CAAC;SACpC,IAAI,CAAC,IAAI,CAAC,CAAA;IACb,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,EAAE,CAAA;IAChC,MAAM,IAAI,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,IAAI,CAAA;IAEzD,MAAM,SAAS,CAAC,aAAa,EAAE,IAAI,EAAE,MAAM,CAAC,CAAA;IAC5C,OAAO,IAAI,CAAA;AACb,CAAC"} \ No newline at end of file diff --git a/dist/index.d.ts b/packages/mimir/dist/index.d.ts similarity index 100% rename from dist/index.d.ts rename to packages/mimir/dist/index.d.ts diff --git a/dist/index.d.ts.map b/packages/mimir/dist/index.d.ts.map similarity index 100% rename from dist/index.d.ts.map rename to packages/mimir/dist/index.d.ts.map diff --git a/dist/index.js b/packages/mimir/dist/index.js similarity index 100% rename from dist/index.js rename to packages/mimir/dist/index.js diff --git a/dist/index.js.map b/packages/mimir/dist/index.js.map similarity index 100% rename from dist/index.js.map rename to packages/mimir/dist/index.js.map diff --git a/dist/ingest.d.ts b/packages/mimir/dist/ingest.d.ts similarity index 100% rename from dist/ingest.d.ts rename to packages/mimir/dist/ingest.d.ts diff --git a/dist/ingest.d.ts.map b/packages/mimir/dist/ingest.d.ts.map similarity index 62% rename from dist/ingest.d.ts.map rename to packages/mimir/dist/ingest.d.ts.map index a845844..b46d11b 100644 --- a/dist/ingest.d.ts.map +++ b/packages/mimir/dist/ingest.d.ts.map @@ -1 +1 @@ -{"version":3,"file":"ingest.d.ts","sourceRoot":"","sources":["../src/ingest.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EACV,WAAW,EACX,aAAa,EACb,YAAY,EAIb,MAAM,YAAY,CAAA;AAInB,wBAAsB,MAAM,CAAC,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,YAAY,CAAC,CA4D/E;AAED,wBAAsB,KAAK,CAAC,GAAG,SAAgB,GAAG,OAAO,CAAC,WAAW,CAAC,CAkCrE"} \ No newline at end of file +{"version":3,"file":"ingest.d.ts","sourceRoot":"","sources":["../src/ingest.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EACV,WAAW,EACX,aAAa,EACb,YAAY,EAIb,MAAM,YAAY,CAAA;AAKnB,wBAAsB,MAAM,CAAC,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,YAAY,CAAC,CA4D/E;AAED,wBAAsB,KAAK,CAAC,GAAG,SAAgB,GAAG,OAAO,CAAC,WAAW,CAAC,CAoCrE"} \ No newline at end of file diff --git a/dist/ingest.js b/packages/mimir/dist/ingest.js similarity index 97% rename from dist/ingest.js rename to packages/mimir/dist/ingest.js index 670defe..d9ed010 100644 --- a/dist/ingest.js +++ b/packages/mimir/dist/ingest.js @@ -7,6 +7,7 @@ import { parseFile } from "./parsing.js"; import { redactText, totalRedactions } from "./redaction.js"; import { openRowsTable, writeRows } from "./store.js"; const EMBED_BATCH_SIZE = 32; +const MAX_AUDIT_ROWS = 100_000; export async function ingest(options = {}) { const config = await loadConfig(String(options.cwd ?? process.cwd())); const files = await listSourceFiles(config); @@ -72,7 +73,7 @@ export async function audit(cwd = process.cwd()) { totalChunks: 0, }; } - const rows = (await table.query().limit(100000).toArray()); + const rows = (await table.query().limit(MAX_AUDIT_ROWS).toArray()); const counts = new Map(); for (const row of rows) { counts.set(row.relativePath, (counts.get(row.relativePath) ?? 0) + 1); diff --git a/packages/mimir/dist/ingest.js.map b/packages/mimir/dist/ingest.js.map new file mode 100644 index 0000000..0192595 --- /dev/null +++ b/packages/mimir/dist/ingest.js.map @@ -0,0 +1 @@ +{"version":3,"file":"ingest.js","sourceRoot":"","sources":["../src/ingest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAC9C,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;AACxC,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAA;AAC5D,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAUrD,MAAM,gBAAgB,GAAG,EAAE,CAAA;AAC3B,MAAM,cAAc,GAAG,OAAO,CAAA;AAE9B,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,UAAyB,EAAE;IACtD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;IACrE,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,CAAA;IAC3C,MAAM,SAAS,GAAgB,EAAE,CAAA;IACjC,MAAM,MAAM,GAA2B,EAAE,CAAA;IACzC,MAAM,eAAe,GAAqB,EAAE,CAAA;IAC5C,IAAI,YAAY,GAAG,CAAC,CAAA;IAEpB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,CAAA;YACpC,MAAM,QAAQ,GAAG,UAAU,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;YAChD,eAAe,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAA;YACxC,MAAM,MAAM,GAAG,aAAa,CAC1B,EAAE,GAAG,MAAM,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,EAClC,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,YAAY,CACpB,CAAA;YACD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACxB,YAAY,IAAI,CAAC,CAAA;YACnB,CAAC;YACD,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAA;QAC3B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,IAAI,CAAC,YAAY;gBACvB,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAChE,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAgB,EAAE,CAAA;IAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,gBAAgB,EAAE,CAAC;QAC5D,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,CAAA;QACtD,MAAM,UAAU,GAAG,MAAM,UAAU,CACjC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,EAChC,MAAM,CACP,CAAA;QACD,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;YAC7C,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,CAAA;YAChC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,+BAA+B,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,UAAU,GAAG,CAAC,CAAA;YAC3F,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,KAAK,EAAE,MAAM,EAAE,CAAC,CAAA;QACjC,CAAC;IACH,CAAC;IAED,MAAM,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IAC7B,MAAM,YAAY,CAAC,MAAM,EAAE;QACzB,MAAM,EAAE,QAAQ;QAChB,WAAW,EAAE,IAAI,CAAC,MAAM;QACxB,UAAU,EAAE,eAAe,CAAC,eAAe,CAAC;KAC7C,CAAC,CAAA;IAEF,OAAO;QACL,YAAY,EAAE,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI;QAC/D,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,YAAY;QACZ,UAAU,EAAE,eAAe,CAAC,eAAe,CAAC;QAC5C,MAAM;KACP,CAAA;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE;IAC7C,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAA;IACpC,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,CAAA;IAC3C,MAAM,cAAc,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;IAC7D,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAA;IAEzC,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO;YACL,YAAY,EAAE,EAAE;YAChB,cAAc;YACd,gBAAgB,EAAE,cAAc;YAChC,YAAY,EAAE,EAAE;YAChB,WAAW,EAAE,CAAC;SACf,CAAA;IACH,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,OAAO,EAAE,CAE/D,CAAA;IACF,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAA;IACxC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IACvE,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,CAAA;IAC5C,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAA;IAEzC,OAAO;QACL,YAAY,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;aAChC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;aACtC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QAClD,cAAc;QACd,gBAAgB,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACxE,YAAY,EAAE,CAAC,GAAG,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE;QAC9E,WAAW,EAAE,IAAI,CAAC,MAAM;KACzB,CAAA;AACH,CAAC"} \ No newline at end of file diff --git a/dist/init.d.ts b/packages/mimir/dist/init.d.ts similarity index 100% rename from dist/init.d.ts rename to packages/mimir/dist/init.d.ts diff --git a/packages/mimir/dist/init.d.ts.map b/packages/mimir/dist/init.d.ts.map new file mode 100644 index 0000000..ba4e123 --- /dev/null +++ b/packages/mimir/dist/init.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":"AAMA,wBAAsB,WAAW,CAAC,GAAG,SAAgB,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAwCxE"} \ No newline at end of file diff --git a/dist/init.js b/packages/mimir/dist/init.js similarity index 66% rename from dist/init.js rename to packages/mimir/dist/init.js index 5418e0a..589949a 100644 --- a/dist/init.js +++ b/packages/mimir/dist/init.js @@ -1,36 +1,16 @@ import { existsSync } from "node:fs"; import { mkdir, writeFile } from "node:fs/promises"; import path from "node:path"; +import { CONFIG_PATH, DEFAULT_CONFIG, KB_DIR, PRIVATE_DIR } from "./defaults.js"; import { ensureMimirGitignore } from "./gitignore.js"; -const DEFAULT_CONFIG = { - rawDir: "private", - storageDir: ".kb/storage", - sourcesFile: ".kb/sources.txt", - accessLogPath: ".kb/access.log", - tableName: "chunks", - ollamaHost: "http://localhost:11434", - networkPolicy: "local-only", - embedModel: "nomic-embed-text", - llmModel: "gemma4:latest", - redaction: { - enabled: true, - builtIn: true, - patterns: [], - }, - accessLog: true, - mcpMaxTopK: 10, - topK: 5, - chunkSize: 1200, - chunkOverlap: 150, -}; export async function initProject(cwd = process.cwd()) { const root = path.resolve(cwd); - const kbDir = path.join(root, ".kb"); - const privateDir = path.join(root, "private"); + const kbDir = path.join(root, KB_DIR); + const privateDir = path.join(root, PRIVATE_DIR); const created = []; await mkdir(kbDir, { recursive: true }); await mkdir(privateDir, { recursive: true }); - const configPath = path.join(kbDir, "config.json"); + const configPath = path.join(root, CONFIG_PATH); if (!existsSync(configPath)) { await writeFile(configPath, `${JSON.stringify(DEFAULT_CONFIG, null, 2)}\n`, "utf8"); created.push(path.relative(root, configPath)); diff --git a/packages/mimir/dist/init.js.map b/packages/mimir/dist/init.js.map new file mode 100644 index 0000000..4b47228 --- /dev/null +++ b/packages/mimir/dist/init.js.map @@ -0,0 +1 @@ +{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAChF,OAAO,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAA;AAErD,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE;IACnD,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;IAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IACrC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;IAC/C,MAAM,OAAO,GAAa,EAAE,CAAA;IAE5B,MAAM,KAAK,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IACvC,MAAM,KAAK,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAE5C,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;IAC/C,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC5B,MAAM,SAAS,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;QACnF,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,CAAA;IAC/C,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,aAAa,CAAC,CAAA;IACnD,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC7B,MAAM,SAAS,CACb,WAAW,EACX,8FAA8F,EAC9F,MAAM,CACP,CAAA;QACD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,CAAA;IAChD,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,CAAA;IACrD,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC5B,MAAM,SAAS,CACb,UAAU,EACV,6FAA6F,EAC7F,MAAM,CACP,CAAA;QACD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,CAAA;IAC/C,CAAC;IAED,IAAI,MAAM,oBAAoB,CAAC,IAAI,CAAC,EAAE,CAAC;QACrC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;IAC5B,CAAC;IAED,OAAO,OAAO,CAAA;AAChB,CAAC"} \ No newline at end of file diff --git a/dist/mcp.d.ts b/packages/mimir/dist/mcp.d.ts similarity index 100% rename from dist/mcp.d.ts rename to packages/mimir/dist/mcp.d.ts diff --git a/packages/mimir/dist/mcp.d.ts.map b/packages/mimir/dist/mcp.d.ts.map new file mode 100644 index 0000000..281a76a --- /dev/null +++ b/packages/mimir/dist/mcp.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"mcp.d.ts","sourceRoot":"","sources":["../src/mcp.ts"],"names":[],"mappings":"AAeA,wBAAsB,QAAQ,CAAC,GAAG,SAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CA6EjE"} \ No newline at end of file diff --git a/dist/mcp.js b/packages/mimir/dist/mcp.js similarity index 77% rename from dist/mcp.js rename to packages/mimir/dist/mcp.js index 043c30f..aa88f56 100644 --- a/dist/mcp.js +++ b/packages/mimir/dist/mcp.js @@ -7,6 +7,10 @@ import { ask, search } from "./query.js"; import { securityAudit } from "./security.js"; import { countRows } from "./store.js"; import { VERSION } from "./version.js"; +const queryToolInputSchema = z.object({ + query: z.string().min(1), + topK: z.number().int().positive().optional(), +}); export async function serveMcp(cwd = process.cwd()) { const server = new McpServer({ name: "mimir", @@ -24,11 +28,14 @@ export async function serveMcp(cwd = process.cwd()) { rawDir: config.rawDir, storageDir: config.storageDir, sourcesFile: config.sourcesFile, - embedModel: config.embedModel, - llmModel: config.llmModel, - networkPolicy: config.networkPolicy, + embeddingProvider: config.embeddingProvider, + embeddingModel: config.embeddingModel, + embeddingModelPath: config.embeddingModelPath, + transformersAllowRemoteModels: config.transformersAllowRemoteModels, + llmGeneration: false, redactionEnabled: config.redaction.enabled, mcpMaxTopK: config.mcpMaxTopK, + includeExtensions: config.includeExtensions, chunksIndexed, }; return textResult(output); @@ -36,18 +43,12 @@ export async function serveMcp(cwd = process.cwd()) { server.registerTool("mimir_search", { title: "Mimir Search", description: "Retrieve relevant passages from the local Mimir knowledge base.", - inputSchema: z.object({ - query: z.string().min(1), - topK: z.number().int().positive().optional(), - }), + inputSchema: queryToolInputSchema, }, async ({ query, topK }) => textResult(await search(query, await searchOptions(cwd, topK)))); server.registerTool("mimir_ask", { title: "Mimir Ask", - description: "Answer a question using local retrieved passages and the configured Ollama model.", - inputSchema: z.object({ - query: z.string().min(1), - topK: z.number().int().positive().optional(), - }), + description: "Return cited retrieval context for a question without calling an LLM.", + inputSchema: queryToolInputSchema, }, async ({ query, topK }) => textResult(await ask(query, await searchOptions(cwd, topK)))); server.registerTool("mimir_audit", { title: "Mimir Audit", @@ -56,7 +57,7 @@ export async function serveMcp(cwd = process.cwd()) { }, async () => textResult(await audit(cwd))); server.registerTool("mimir_security_audit", { title: "Mimir Security Audit", - description: "Show local privacy, network, redaction, MCP, and gitignore posture.", + description: "Show local privacy, provider, redaction, MCP, and gitignore posture.", inputSchema: z.object({}), }, async () => textResult(await securityAudit(cwd))); await server.connect(new StdioServerTransport()); diff --git a/packages/mimir/dist/mcp.js.map b/packages/mimir/dist/mcp.js.map new file mode 100644 index 0000000..6158aed --- /dev/null +++ b/packages/mimir/dist/mcp.js.map @@ -0,0 +1 @@ +{"version":3,"file":"mcp.js","sourceRoot":"","sources":["../src/mcp.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAA;AACnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAA;AAChF,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AACvB,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAA;AACnC,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;AACxC,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC7C,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AACtC,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAA;AAEtC,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IACpC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACxB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CAC7C,CAAC,CAAA;AAEF,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE;IAChD,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;QAC3B,IAAI,EAAE,OAAO;QACb,OAAO,EAAE,OAAO;KACjB,CAAC,CAAA;IAEF,MAAM,CAAC,YAAY,CACjB,cAAc,EACd;QACE,KAAK,EAAE,cAAc;QACrB,WAAW,EAAE,0DAA0D;QACvE,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;KAC1B,EACD,KAAK,IAAI,EAAE;QACT,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAA;QACpC,MAAM,aAAa,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,CAAA;QAC7C,MAAM,MAAM,GAAG;YACb,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;YAC3C,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,kBAAkB,EAAE,MAAM,CAAC,kBAAkB;YAC7C,6BAA6B,EAAE,MAAM,CAAC,6BAA6B;YACnE,aAAa,EAAE,KAAK;YACpB,gBAAgB,EAAE,MAAM,CAAC,SAAS,CAAC,OAAO;YAC1C,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;YAC3C,aAAa;SACd,CAAA;QAED,OAAO,UAAU,CAAC,MAAM,CAAC,CAAA;IAC3B,CAAC,CACF,CAAA;IAED,MAAM,CAAC,YAAY,CACjB,cAAc,EACd;QACE,KAAK,EAAE,cAAc;QACrB,WAAW,EAAE,iEAAiE;QAC9E,WAAW,EAAE,oBAAoB;KAClC,EACD,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,UAAU,CAAC,MAAM,MAAM,CAAC,KAAK,EAAE,MAAM,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CAC3F,CAAA;IAED,MAAM,CAAC,YAAY,CACjB,WAAW,EACX;QACE,KAAK,EAAE,WAAW;QAClB,WAAW,EAAE,uEAAuE;QACpF,WAAW,EAAE,oBAAoB;KAClC,EACD,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,KAAK,EAAE,MAAM,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CACxF,CAAA;IAED,MAAM,CAAC,YAAY,CACjB,aAAa,EACb;QACE,KAAK,EAAE,aAAa;QACpB,WAAW,EAAE,uEAAuE;QACpF,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;KAC1B,EACD,KAAK,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC,CACzC,CAAA;IAED,MAAM,CAAC,YAAY,CACjB,sBAAsB,EACtB;QACE,KAAK,EAAE,sBAAsB;QAC7B,WAAW,EAAE,sEAAsE;QACnF,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;KAC1B,EACD,KAAK,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,aAAa,CAAC,GAAG,CAAC,CAAC,CACjD,CAAA;IAED,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,oBAAoB,EAAE,CAAC,CAAA;AAClD,CAAC;AAED,SAAS,UAAU,CAAC,KAAc;IAChC,OAAO;QACL,OAAO,EAAE;YACP;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;aACrC;SACF;KACF,CAAA;AACH,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,GAAW,EACX,IAAwB;IAExB,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAA;IACpC,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,UAAU,CAAC,CAAA;IACpE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,CAAA;AACnC,CAAC"} \ No newline at end of file diff --git a/dist/parsing.d.ts b/packages/mimir/dist/parsing.d.ts similarity index 100% rename from dist/parsing.d.ts rename to packages/mimir/dist/parsing.d.ts diff --git a/packages/mimir/dist/parsing.d.ts.map b/packages/mimir/dist/parsing.d.ts.map new file mode 100644 index 0000000..91d92ba --- /dev/null +++ b/packages/mimir/dist/parsing.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"parsing.d.ts","sourceRoot":"","sources":["../src/parsing.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,YAAY,CAAA;AAI5D,wBAAsB,SAAS,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,cAAc,CAAC,CA8CzE"} \ No newline at end of file diff --git a/packages/mimir/dist/parsing.js b/packages/mimir/dist/parsing.js new file mode 100644 index 0000000..9ac1219 --- /dev/null +++ b/packages/mimir/dist/parsing.js @@ -0,0 +1,184 @@ +import { readFile } from "node:fs/promises"; +import { strFromU8, unzipSync } from "fflate"; +import { htmlToText } from "html-to-text"; +import { extractText, getDocumentProxy } from "unpdf"; +import YAML from "yaml"; +const MAX_OFFICE_XML_ENTRY_BYTES = 25_000_000; +export async function parseFile(file) { + let text; + switch (file.extension) { + case ".pdf": + text = await parsePdf(file.absolutePath); + break; + case ".docx": + text = await parseDocx(file.absolutePath); + break; + case ".pptx": + text = await parsePptx(file.absolutePath); + break; + case ".xlsx": + text = await parseXlsx(file.absolutePath); + break; + case ".odt": + case ".ods": + case ".odp": + text = await parseOpenDocument(file.absolutePath); + break; + case ".html": + case ".htm": + text = htmlToText(await readFile(file.absolutePath, "utf8"), { + wordwrap: false, + selectors: [ + { selector: "a", options: { ignoreHref: true } }, + { selector: "img", format: "skip" }, + ], + }); + break; + case ".json": + text = JSON.stringify(JSON.parse(await readFile(file.absolutePath, "utf8")), null, 2); + break; + case ".yaml": + case ".yml": + text = YAML.stringify(YAML.parse(await readFile(file.absolutePath, "utf8"))); + break; + case ".rtf": + text = stripRtf(await readFile(file.absolutePath, "utf8")); + break; + default: + text = await readFile(file.absolutePath, "utf8"); + } + return { file, text: normalizeText(text) }; +} +async function parseDocx(filePath) { + const entries = unzipOfficeFile(await readFile(filePath)); + return xmlEntriesToText(entries, [ + /^word\/document\.xml$/u, + /^word\/header\d*\.xml$/u, + /^word\/footer\d*\.xml$/u, + /^word\/footnotes\.xml$/u, + /^word\/endnotes\.xml$/u, + /^word\/comments\.xml$/u, + ]); +} +async function parsePptx(filePath) { + const entries = unzipOfficeFile(await readFile(filePath)); + return xmlEntriesToText(entries, [ + /^ppt\/slides\/slide\d+\.xml$/u, + /^ppt\/notesSlides\/notesSlide\d+\.xml$/u, + ]); +} +async function parseXlsx(filePath) { + const entries = unzipOfficeFile(await readFile(filePath)); + const sharedStrings = parseSharedStrings(entries.get("xl/sharedStrings.xml") ?? ""); + const sheets = [...entries.entries()] + .filter(([name]) => /^xl\/worksheets\/sheet\d+\.xml$/u.test(name)) + .sort(([a], [b]) => a.localeCompare(b)); + const rows = []; + for (const [name, xml] of sheets) { + const values = parseSheetValues(xml, sharedStrings); + if (values.length > 0) { + rows.push(`# ${name}`, values.join("\n")); + } + } + return rows.join("\n\n"); +} +async function parseOpenDocument(filePath) { + const entries = unzipOfficeFile(await readFile(filePath)); + return xmlEntriesToText(entries, [/^content\.xml$/u, /^meta\.xml$/u]); +} +function unzipOfficeFile(buffer) { + const unzipped = unzipSync(new Uint8Array(buffer), { + filter: (file) => file.originalSize <= MAX_OFFICE_XML_ENTRY_BYTES, + }); + const entries = new Map(); + for (const [name, content] of Object.entries(unzipped)) { + if (name.endsWith(".xml")) { + entries.set(name, strFromU8(content)); + } + } + return entries; +} +function xmlEntriesToText(entries, patterns) { + const parts = []; + for (const [name, xml] of [...entries.entries()].sort(([a], [b]) => a.localeCompare(b))) { + if (patterns.some((pattern) => pattern.test(name))) { + const text = xmlToText(xml); + if (text) { + parts.push(text); + } + } + } + return parts.join("\n\n"); +} +function parseSharedStrings(xml) { + return [...xml.matchAll(//gu)].map(([item]) => xmlToText(item)); +} +function parseSheetValues(xml, sharedStrings) { + const rows = []; + for (const rowMatch of xml.matchAll(//gu)) { + const rowXml = rowMatch[0]; + const values = [...rowXml.matchAll(/]*)>([\s\S]*?)<\/c>/gu)] + .map((cellMatch) => { + const attrs = cellMatch[1] ?? ""; + const cellXml = cellMatch[2] ?? ""; + const inline = firstMatch(cellXml, //u); + if (inline) { + return xmlToText(inline); + } + const rawValue = firstMatch(cellXml, /([\s\S]*?)<\/v>/u); + if (!rawValue) { + return ""; + } + if (/\bt="s"/u.test(attrs)) { + return sharedStrings[Number.parseInt(rawValue, 10)] ?? ""; + } + return decodeXmlEntities(rawValue); + }) + .filter(Boolean); + if (values.length > 0) { + rows.push(values.join("\t")); + } + } + return rows; +} +function firstMatch(input, pattern) { + const match = input.match(pattern); + return match?.[1] ?? match?.[0] ?? ""; +} +function xmlToText(xml) { + return normalizeText(decodeXmlEntities(xml + .replace(//gu, " ") + .replace(//gu, "\n") + .replace(/<\/(?:w:p|a:p|text:p|text:h|table:table-row)>/gu, "\n") + .replace(/<[^>]+>/gu, " ") + .replace(/[ \t]{2,}/gu, " "))); +} +function stripRtf(input) { + return input + .replace(/\\par[d]?/gu, "\n") + .replace(/\\'[0-9a-fA-F]{2}/gu, " ") + .replace(/\\[a-zA-Z]+-?\d* ?/gu, " ") + .replace(/[{}]/gu, " "); +} +function decodeXmlEntities(input) { + return input + .replace(/</gu, "<") + .replace(/>/gu, ">") + .replace(/"/gu, '"') + .replace(/'/gu, "'") + .replace(/&/gu, "&"); +} +async function parsePdf(filePath) { + const buffer = await readFile(filePath); + const pdf = await getDocumentProxy(new Uint8Array(buffer)); + const result = await extractText(pdf, { mergePages: true }); + return result.text; +} +function normalizeText(input) { + return input + .replace(/\r\n/g, "\n") + .replace(/[ \t]+\n/g, "\n") + .replace(/\n{4,}/g, "\n\n\n") + .trim(); +} +//# sourceMappingURL=parsing.js.map \ No newline at end of file diff --git a/packages/mimir/dist/parsing.js.map b/packages/mimir/dist/parsing.js.map new file mode 100644 index 0000000..08091dc --- /dev/null +++ b/packages/mimir/dist/parsing.js.map @@ -0,0 +1 @@ +{"version":3,"file":"parsing.js","sourceRoot":"","sources":["../src/parsing.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAA;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAA;AACzC,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,OAAO,CAAA;AACrD,OAAO,IAAI,MAAM,MAAM,CAAA;AAGvB,MAAM,0BAA0B,GAAG,UAAU,CAAA;AAE7C,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAgB;IAC9C,IAAI,IAAY,CAAA;IAEhB,QAAQ,IAAI,CAAC,SAAS,EAAE,CAAC;QACvB,KAAK,MAAM;YACT,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;YACxC,MAAK;QACP,KAAK,OAAO;YACV,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;YACzC,MAAK;QACP,KAAK,OAAO;YACV,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;YACzC,MAAK;QACP,KAAK,OAAO;YACV,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;YACzC,MAAK;QACP,KAAK,MAAM,CAAC;QACZ,KAAK,MAAM,CAAC;QACZ,KAAK,MAAM;YACT,IAAI,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;YACjD,MAAK;QACP,KAAK,OAAO,CAAC;QACb,KAAK,MAAM;YACT,IAAI,GAAG,UAAU,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,EAAE;gBAC3D,QAAQ,EAAE,KAAK;gBACf,SAAS,EAAE;oBACT,EAAE,QAAQ,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE;oBAChD,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE;iBACpC;aACF,CAAC,CAAA;YACF,MAAK;QACP,KAAK,OAAO;YACV,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAA;YACrF,MAAK;QACP,KAAK,OAAO,CAAC;QACb,KAAK,MAAM;YACT,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,CAAA;YAC5E,MAAK;QACP,KAAK,MAAM;YACT,IAAI,GAAG,QAAQ,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAA;YAC1D,MAAK;QACP;YACE,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAA;IACpD,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,CAAA;AAC5C,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,QAAgB;IACvC,MAAM,OAAO,GAAG,eAAe,CAAC,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAA;IACzD,OAAO,gBAAgB,CAAC,OAAO,EAAE;QAC/B,wBAAwB;QACxB,yBAAyB;QACzB,yBAAyB;QACzB,yBAAyB;QACzB,wBAAwB;QACxB,wBAAwB;KACzB,CAAC,CAAA;AACJ,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,QAAgB;IACvC,MAAM,OAAO,GAAG,eAAe,CAAC,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAA;IACzD,OAAO,gBAAgB,CAAC,OAAO,EAAE;QAC/B,+BAA+B;QAC/B,yCAAyC;KAC1C,CAAC,CAAA;AACJ,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,QAAgB;IACvC,MAAM,OAAO,GAAG,eAAe,CAAC,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAA;IACzD,MAAM,aAAa,GAAG,kBAAkB,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,IAAI,EAAE,CAAC,CAAA;IACnF,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;SAClC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,kCAAkC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SACjE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAA;IAEzC,MAAM,IAAI,GAAa,EAAE,CAAA;IACzB,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,MAAM,EAAE,CAAC;QACjC,MAAM,MAAM,GAAG,gBAAgB,CAAC,GAAG,EAAE,aAAa,CAAC,CAAA;QACnD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAA;QAC3C,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;AAC1B,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,QAAgB;IAC/C,MAAM,OAAO,GAAG,eAAe,CAAC,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAA;IACzD,OAAO,gBAAgB,CAAC,OAAO,EAAE,CAAC,iBAAiB,EAAE,cAAc,CAAC,CAAC,CAAA;AACvE,CAAC;AAED,SAAS,eAAe,CAAC,MAAc;IACrC,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,EAAE;QACjD,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,IAAI,0BAA0B;KAClE,CAAC,CAAA;IACF,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAA;IACzC,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QACvD,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1B,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC,CAAA;QACvC,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAA;AAChB,CAAC;AAED,SAAS,gBAAgB,CAAC,OAA4B,EAAE,QAAkB;IACxE,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACxF,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YACnD,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,CAAA;YAC3B,IAAI,IAAI,EAAE,CAAC;gBACT,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAClB,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;AAC3B,CAAC;AAED,SAAS,kBAAkB,CAAC,GAAW;IACrC,OAAO,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,uBAAuB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAA;AACpF,CAAC;AAED,SAAS,gBAAgB,CAAC,GAAW,EAAE,aAAuB;IAC5D,MAAM,IAAI,GAAa,EAAE,CAAA;IACzB,KAAK,MAAM,QAAQ,IAAI,GAAG,CAAC,QAAQ,CAAC,yBAAyB,CAAC,EAAE,CAAC;QAC/D,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAA;QAC1B,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,+BAA+B,CAAC,CAAC;aACjE,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE;YACjB,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;YAChC,MAAM,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;YAClC,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,EAAE,sBAAsB,CAAC,CAAA;YAC1D,IAAI,MAAM,EAAE,CAAC;gBACX,OAAO,SAAS,CAAC,MAAM,CAAC,CAAA;YAC1B,CAAC;YAED,MAAM,QAAQ,GAAG,UAAU,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAA;YAC3D,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,OAAO,EAAE,CAAA;YACX,CAAC;YAED,IAAI,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC3B,OAAO,aAAa,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;YAC3D,CAAC;YACD,OAAO,iBAAiB,CAAC,QAAQ,CAAC,CAAA;QACpC,CAAC,CAAC;aACD,MAAM,CAAC,OAAO,CAAC,CAAA;QAElB,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAA;QAC9B,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC;AAED,SAAS,UAAU,CAAC,KAAa,EAAE,OAAe;IAChD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;IAClC,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;AACvC,CAAC;AAED,SAAS,SAAS,CAAC,GAAW;IAC5B,OAAO,aAAa,CAClB,iBAAiB,CACf,GAAG;SACA,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC;SAC3B,OAAO,CAAC,iDAAiD,EAAE,IAAI,CAAC;SAChE,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC;SACzB,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC,CAC/B,CACF,CAAA;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,KAAa;IAC7B,OAAO,KAAK;SACT,OAAO,CAAC,aAAa,EAAE,IAAI,CAAC;SAC5B,OAAO,CAAC,qBAAqB,EAAE,GAAG,CAAC;SACnC,OAAO,CAAC,sBAAsB,EAAE,GAAG,CAAC;SACpC,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAA;AAC3B,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAa;IACtC,OAAO,KAAK;SACT,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;SACxB,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;SACxB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;AAC5B,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,QAAgB;IACtC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAA;IACvC,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC,CAAA;IAC1D,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAA;IAC3D,OAAO,MAAM,CAAC,IAAI,CAAA;AACpB,CAAC;AAED,SAAS,aAAa,CAAC,KAAa;IAClC,OAAO,KAAK;SACT,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC;SACtB,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC;SAC1B,OAAO,CAAC,SAAS,EAAE,QAAQ,CAAC;SAC5B,IAAI,EAAE,CAAA;AACX,CAAC"} \ No newline at end of file diff --git a/dist/query.d.ts b/packages/mimir/dist/query.d.ts similarity index 100% rename from dist/query.d.ts rename to packages/mimir/dist/query.d.ts diff --git a/dist/query.d.ts.map b/packages/mimir/dist/query.d.ts.map similarity index 72% rename from dist/query.d.ts.map rename to packages/mimir/dist/query.d.ts.map index d2f9275..9ccb479 100644 --- a/dist/query.d.ts.map +++ b/packages/mimir/dist/query.d.ts.map @@ -1 +1 @@ -{"version":3,"file":"query.d.ts","sourceRoot":"","sources":["../src/query.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,SAAS,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAUxE,wBAAsB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CA2BhG;AAED,wBAAsB,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,SAAS,CAAC,CA+CxF"} \ No newline at end of file +{"version":3,"file":"query.d.ts","sourceRoot":"","sources":["../src/query.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,SAAS,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAUxE,wBAAsB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CA2BhG;AAED,wBAAsB,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,SAAS,CAAC,CAsBxF"} \ No newline at end of file diff --git a/dist/query.js b/packages/mimir/dist/query.js similarity index 64% rename from dist/query.js rename to packages/mimir/dist/query.js index 723ea27..e1793b3 100644 --- a/dist/query.js +++ b/packages/mimir/dist/query.js @@ -1,8 +1,6 @@ -import { Ollama } from "ollama"; import { recordAccess } from "./access-log.js"; import { loadConfig } from "./config.js"; import { embedText } from "./embeddings.js"; -import { assertNetworkPolicy } from "./network.js"; import { openRowsTable } from "./store.js"; export async function search(query, options = {}) { const config = await loadConfig(String(options.cwd ?? process.cwd())); @@ -39,25 +37,6 @@ export async function ask(query, options = {}) { sources, }; } - const context = sources - .map((source, index) => `[${index + 1}] ${source.relativePath}#${source.chunkIndex}\n${source.text}`) - .join("\n\n---\n\n"); - assertNetworkPolicy(config); - const client = new Ollama({ host: config.ollamaHost }); - const response = await client.chat({ - model: config.llmModel, - messages: [ - { - role: "system", - content: "Answer only from the provided context. If the context is insufficient, say what is missing. Cite sources with [1], [2], etc.", - }, - { - role: "user", - content: `Question:\n${query}\n\nContext:\n${context}`, - }, - ], - stream: false, - }); await recordAccess(config, { action: "ask", query, @@ -65,8 +44,21 @@ export async function ask(query, options = {}) { resultCount: sources.length, }); return { - answer: response.message.content, + answer: retrievalOnlyAnswer(sources), sources, }; } +function retrievalOnlyAnswer(sources) { + const snippets = sources + .map((source, index) => { + const text = source.text.replace(/\s+/gu, " ").trim(); + return `[${index + 1}] ${source.relativePath}#${source.chunkIndex}: ${text}`; + }) + .join("\n\n"); + return [ + "Mimir returns retrieval context only. Use these passages as grounded context for your agent or LLM:", + "", + snippets, + ].join("\n"); +} //# sourceMappingURL=query.js.map \ No newline at end of file diff --git a/packages/mimir/dist/query.js.map b/packages/mimir/dist/query.js.map new file mode 100644 index 0000000..b3bcc78 --- /dev/null +++ b/packages/mimir/dist/query.js.map @@ -0,0 +1 @@ +{"version":3,"file":"query.js","sourceRoot":"","sources":["../src/query.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAW1C,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,KAAa,EAAE,UAAyB,EAAE;IACrE,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;IACrE,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAA;IACzC,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,EAAE,CAAA;IACX,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;IAC7C,MAAM,IAAI,GAAG,CAAC,MAAM,KAAK;SACtB,YAAY,CAAC,MAAM,CAAC;SACpB,KAAK,CAAC,OAAO,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC;SAClC,OAAO,EAAE,CAAgB,CAAA;IAE5B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACjC,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,YAAY,EAAE,GAAG,CAAC,YAAY;QAC9B,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,QAAQ,EAAE,OAAO,GAAG,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;KACnE,CAAC,CAAC,CAAA;IACH,MAAM,YAAY,CAAC,MAAM,EAAE;QACzB,MAAM,EAAE,QAAQ;QAChB,KAAK;QACL,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI;QACjC,WAAW,EAAE,OAAO,CAAC,MAAM;KAC5B,CAAC,CAAA;IACF,OAAO,OAAO,CAAA;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,GAAG,CAAC,KAAa,EAAE,UAAyB,EAAE;IAClE,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;IACrE,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;IAE5C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO;YACL,MAAM,EAAE,2EAA2E;YACnF,OAAO;SACR,CAAA;IACH,CAAC;IAED,MAAM,YAAY,CAAC,MAAM,EAAE;QACzB,MAAM,EAAE,KAAK;QACb,KAAK;QACL,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI;QACjC,WAAW,EAAE,OAAO,CAAC,MAAM;KAC5B,CAAC,CAAA;IAEF,OAAO;QACL,MAAM,EAAE,mBAAmB,CAAC,OAAO,CAAC;QACpC,OAAO;KACR,CAAA;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,OAAuB;IAClD,MAAM,QAAQ,GAAG,OAAO;SACrB,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;QACrB,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;QACrD,OAAO,IAAI,KAAK,GAAG,CAAC,KAAK,MAAM,CAAC,YAAY,IAAI,MAAM,CAAC,UAAU,KAAK,IAAI,EAAE,CAAA;IAC9E,CAAC,CAAC;SACD,IAAI,CAAC,MAAM,CAAC,CAAA;IAEf,OAAO;QACL,qGAAqG;QACrG,EAAE;QACF,QAAQ;KACT,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACd,CAAC"} \ No newline at end of file diff --git a/dist/redaction.d.ts b/packages/mimir/dist/redaction.d.ts similarity index 100% rename from dist/redaction.d.ts rename to packages/mimir/dist/redaction.d.ts diff --git a/dist/redaction.d.ts.map b/packages/mimir/dist/redaction.d.ts.map similarity index 100% rename from dist/redaction.d.ts.map rename to packages/mimir/dist/redaction.d.ts.map diff --git a/dist/redaction.js b/packages/mimir/dist/redaction.js similarity index 100% rename from dist/redaction.js rename to packages/mimir/dist/redaction.js diff --git a/dist/redaction.js.map b/packages/mimir/dist/redaction.js.map similarity index 100% rename from dist/redaction.js.map rename to packages/mimir/dist/redaction.js.map diff --git a/dist/security.d.ts b/packages/mimir/dist/security.d.ts similarity index 100% rename from dist/security.d.ts rename to packages/mimir/dist/security.d.ts diff --git a/dist/security.d.ts.map b/packages/mimir/dist/security.d.ts.map similarity index 100% rename from dist/security.d.ts.map rename to packages/mimir/dist/security.d.ts.map diff --git a/dist/security.js b/packages/mimir/dist/security.js similarity index 65% rename from dist/security.js rename to packages/mimir/dist/security.js index a54a20f..033b6c6 100644 --- a/dist/security.js +++ b/packages/mimir/dist/security.js @@ -2,41 +2,38 @@ import { existsSync } from "node:fs"; import { readFile } from "node:fs/promises"; import path from "node:path"; import { loadConfig } from "./config.js"; -import { classifyHost } from "./network.js"; +import { KB_GITIGNORE_ENTRY, MIMIR_GITIGNORE_ENTRY, PRIVATE_GITIGNORE_ENTRY } from "./defaults.js"; export async function securityAudit(cwd = process.cwd()) { const config = await loadConfig(cwd); const gitignore = await readGitignore(config.projectRoot); - const network = classifyHost(config.ollamaHost); const warnings = []; - const kbIgnored = hasGitignoreEntry(gitignore, ".kb/"); - const mimirIgnored = hasGitignoreEntry(gitignore, ".mimir/"); - const privateIgnored = hasGitignoreEntry(gitignore, "private/**"); - if (config.networkPolicy === "allow-any") { - warnings.push("networkPolicy is allow-any; document text can be sent to a remote Ollama host."); - } - if (config.networkPolicy === "local-only" && network.kind !== "loopback") { - warnings.push("networkPolicy is local-only but ollamaHost is not loopback."); + const kbIgnored = hasGitignoreEntry(gitignore, KB_GITIGNORE_ENTRY); + const mimirIgnored = hasGitignoreEntry(gitignore, MIMIR_GITIGNORE_ENTRY); + const privateIgnored = hasGitignoreEntry(gitignore, PRIVATE_GITIGNORE_ENTRY); + if (config.embeddingProvider === "transformers" && config.transformersAllowRemoteModels) { + warnings.push("Transformers remote model loading is enabled; model files can be downloaded from Hugging Face."); } if (!config.redaction.enabled) { warnings.push("Redaction is disabled; secrets and identifiers may be embedded in the index."); } if (!kbIgnored) { - warnings.push(".kb/ is not ignored by Git."); + warnings.push(`${KB_GITIGNORE_ENTRY} is not ignored by Git.`); } if (!mimirIgnored) { - warnings.push(".mimir/ is not ignored by Git."); + warnings.push(`${MIMIR_GITIGNORE_ENTRY} is not ignored by Git.`); } if (!privateIgnored) { - warnings.push("private/** is not ignored by Git."); + warnings.push(`${PRIVATE_GITIGNORE_ENTRY} is not ignored by Git.`); } return { projectRoot: config.projectRoot, zeroTelemetry: true, - network: { - policy: config.networkPolicy, - ollamaHost: config.ollamaHost, - host: network.host, - classification: network.kind, + providers: { + embedding: config.embeddingProvider, + embeddingModel: config.embeddingModel, + embeddingModelPath: config.embeddingModelPath, + transformersAllowRemoteModels: config.transformersAllowRemoteModels, + llmGeneration: false, }, redaction: { enabled: config.redaction.enabled, @@ -66,6 +63,7 @@ export async function securityAudit(cwd = process.cwd()) { "Run Mimir inside an encrypted disk, VM, or container volume for at-rest encryption.", "Use npm provenance, release checksums, and the generated SBOM for release verification.", "Use one repository checkout per trust boundary; Mimir does not implement multi-user RBAC.", + "Use an external agent, MCP server, or local model runtime for LLM synthesis.", ], warnings, }; diff --git a/packages/mimir/dist/security.js.map b/packages/mimir/dist/security.js.map new file mode 100644 index 0000000..8870fce --- /dev/null +++ b/packages/mimir/dist/security.js.map @@ -0,0 +1 @@ +{"version":3,"file":"security.js","sourceRoot":"","sources":["../src/security.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAA;AAGlG,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE;IACrD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAA;IACpC,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;IACzD,MAAM,QAAQ,GAAa,EAAE,CAAA;IAE7B,MAAM,SAAS,GAAG,iBAAiB,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAA;IAClE,MAAM,YAAY,GAAG,iBAAiB,CAAC,SAAS,EAAE,qBAAqB,CAAC,CAAA;IACxE,MAAM,cAAc,GAAG,iBAAiB,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAA;IAE5E,IAAI,MAAM,CAAC,iBAAiB,KAAK,cAAc,IAAI,MAAM,CAAC,6BAA6B,EAAE,CAAC;QACxF,QAAQ,CAAC,IAAI,CACX,gGAAgG,CACjG,CAAA;IACH,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;QAC9B,QAAQ,CAAC,IAAI,CAAC,8EAA8E,CAAC,CAAA;IAC/F,CAAC;IACD,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,QAAQ,CAAC,IAAI,CAAC,GAAG,kBAAkB,yBAAyB,CAAC,CAAA;IAC/D,CAAC;IACD,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,QAAQ,CAAC,IAAI,CAAC,GAAG,qBAAqB,yBAAyB,CAAC,CAAA;IAClE,CAAC;IACD,IAAI,CAAC,cAAc,EAAE,CAAC;QACpB,QAAQ,CAAC,IAAI,CAAC,GAAG,uBAAuB,yBAAyB,CAAC,CAAA;IACpE,CAAC;IAED,OAAO;QACL,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,aAAa,EAAE,IAAI;QACnB,SAAS,EAAE;YACT,SAAS,EAAE,MAAM,CAAC,iBAAiB;YACnC,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,kBAAkB,EAAE,MAAM,CAAC,kBAAkB;YAC7C,6BAA6B,EAAE,MAAM,CAAC,6BAA6B;YACnE,aAAa,EAAE,KAAK;SACrB;QACD,SAAS,EAAE;YACT,OAAO,EAAE,MAAM,CAAC,SAAS,CAAC,OAAO;YACjC,OAAO,EAAE,MAAM,CAAC,SAAS,CAAC,OAAO;YACjC,cAAc,EAAE,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;SACzE;QACD,SAAS,EAAE;YACT,OAAO,EAAE,MAAM,CAAC,SAAS;YACzB,IAAI,EAAE,MAAM,CAAC,aAAa;YAC1B,gBAAgB,EAAE,KAAK;SACxB;QACD,OAAO,EAAE;YACP,IAAI,EAAE,MAAM,CAAC,UAAU;YACvB,UAAU,EAAE,SAAS;YACrB,eAAe,EAAE,mBAAmB;SACrC;QACD,GAAG,EAAE;YACH,OAAO,EAAE,MAAM,CAAC,UAAU;YAC1B,uBAAuB,EAAE,KAAK;SAC/B;QACD,SAAS,EAAE;YACT,SAAS;YACT,YAAY;YACZ,cAAc;SACf;QACD,eAAe,EAAE;YACf,qFAAqF;YACrF,yFAAyF;YACzF,2FAA2F;YAC3F,8EAA8E;SAC/E;QACD,QAAQ;KACT,CAAA;AACH,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,WAAmB;IAC9C,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,YAAY,CAAC,CAAA;IAC1D,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;QAC/B,OAAO,IAAI,GAAG,EAAE,CAAA;IAClB,CAAC;IAED,OAAO,IAAI,GAAG,CACZ,CAAC,MAAM,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;SACpC,KAAK,CAAC,OAAO,CAAC;SACd,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;SAC1B,MAAM,CAAC,OAAO,CAAC,CACnB,CAAA;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAkB,EAAE,KAAa;IAC1D,OAAO,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;AACzB,CAAC"} \ No newline at end of file diff --git a/dist/skill.d.ts b/packages/mimir/dist/skill.d.ts similarity index 78% rename from dist/skill.d.ts rename to packages/mimir/dist/skill.d.ts index ded3377..fc7d323 100644 --- a/dist/skill.d.ts +++ b/packages/mimir/dist/skill.d.ts @@ -4,10 +4,11 @@ export interface InstallSkillOptions { } export interface InstallSkillResult { skillPath: string; + audioSkillPath: string; mcpConfigPath: string; readmePath: string; written: string[]; } -export declare function bundledSkillPath(): string; +export declare function bundledSkillPath(skillName?: string): string; export declare function installSkill(options?: InstallSkillOptions): Promise; //# sourceMappingURL=skill.d.ts.map \ No newline at end of file diff --git a/packages/mimir/dist/skill.d.ts.map b/packages/mimir/dist/skill.d.ts.map new file mode 100644 index 0000000..7dc9278 --- /dev/null +++ b/packages/mimir/dist/skill.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"skill.d.ts","sourceRoot":"","sources":["../src/skill.ts"],"names":[],"mappings":"AAMA,MAAM,WAAW,mBAAmB;IAClC,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,kBAAkB;IACjC,SAAS,EAAE,MAAM,CAAA;IACjB,cAAc,EAAE,MAAM,CAAA;IACtB,aAAa,EAAE,MAAM,CAAA;IACrB,UAAU,EAAE,MAAM,CAAA;IAClB,OAAO,EAAE,MAAM,EAAE,CAAA;CAClB;AAMD,wBAAgB,gBAAgB,CAAC,SAAS,SAAqB,GAAG,MAAM,CAEvE;AAED,wBAAsB,YAAY,CAAC,OAAO,GAAE,mBAAwB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAoCjG"} \ No newline at end of file diff --git a/dist/skill.js b/packages/mimir/dist/skill.js similarity index 58% rename from dist/skill.js rename to packages/mimir/dist/skill.js index 6d19270..4e9d3a1 100644 --- a/dist/skill.js +++ b/packages/mimir/dist/skill.js @@ -1,27 +1,32 @@ import { cp, mkdir, writeFile } from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; +import { DEFAULT_SKILL_TARGET_DIR, MIMIR_DIR } from "./defaults.js"; import { ensureMimirGitignore } from "./gitignore.js"; const PACKAGE_ROOT = path.dirname(path.dirname(fileURLToPath(import.meta.url))); -const SKILL_NAME = "mimir"; -export function bundledSkillPath() { - return path.join(PACKAGE_ROOT, "skills", SKILL_NAME); +const PRIMARY_SKILL_NAME = "mimir"; +const AUDIO_SKILL_NAME = "mimir-audio-summary"; +export function bundledSkillPath(skillName = PRIMARY_SKILL_NAME) { + return path.join(PACKAGE_ROOT, "skills", skillName); } export async function installSkill(options = {}) { const cwd = path.resolve(options.cwd ?? process.cwd()); - const targetDir = path.resolve(cwd, options.targetDir ?? ".mimir/skills"); - const skillPath = path.join(targetDir, SKILL_NAME); - const mimirDir = path.resolve(cwd, ".mimir"); + const targetDir = path.resolve(cwd, options.targetDir ?? DEFAULT_SKILL_TARGET_DIR); + const skillPath = path.join(targetDir, PRIMARY_SKILL_NAME); + const audioSkillPath = path.join(targetDir, AUDIO_SKILL_NAME); + const mimirDir = path.resolve(cwd, MIMIR_DIR); const mcpConfigPath = path.join(mimirDir, "mcp.json"); const readmePath = path.join(mimirDir, "README.md"); await mkdir(targetDir, { recursive: true }); await mkdir(mimirDir, { recursive: true }); - await cp(bundledSkillPath(), skillPath, { recursive: true, force: true }); + await cp(bundledSkillPath(PRIMARY_SKILL_NAME), skillPath, { recursive: true, force: true }); + await cp(bundledSkillPath(AUDIO_SKILL_NAME), audioSkillPath, { recursive: true, force: true }); await writeFile(mcpConfigPath, `${JSON.stringify(mcpConfig(cwd), null, 2)}\n`, "utf8"); - await writeFile(readmePath, agentKitReadme(skillPath, mcpConfigPath), "utf8"); + await writeFile(readmePath, agentKitReadme(skillPath, audioSkillPath, mcpConfigPath), "utf8"); const wroteGitignore = await ensureMimirGitignore(cwd); const written = [ path.relative(cwd, skillPath), + path.relative(cwd, audioSkillPath), path.relative(cwd, mcpConfigPath), path.relative(cwd, readmePath), ]; @@ -30,6 +35,7 @@ export async function installSkill(options = {}) { } return { skillPath, + audioSkillPath, mcpConfigPath, readmePath, written, @@ -46,7 +52,7 @@ function mcpConfig(cwd) { }, }; } -function agentKitReadme(skillPath, mcpConfigPath) { +function agentKitReadme(skillPath, audioSkillPath, mcpConfigPath) { return `# Mimir Agent Kit This folder contains portable agent instructions for Mimir. @@ -61,6 +67,15 @@ ${skillPath} Agents that support skill folders can load that folder directly. +Optional audio-summary skill folder: + +\`\`\`plain text +${audioSkillPath} +\`\`\` + +Use it only when the user asks for a listenable summary. It renders generated audio under ignored +local Mimir state by default and prefers offline TTS engines for confidential content. + ## MCP MCP config example: diff --git a/packages/mimir/dist/skill.js.map b/packages/mimir/dist/skill.js.map new file mode 100644 index 0000000..f3ad7ff --- /dev/null +++ b/packages/mimir/dist/skill.js.map @@ -0,0 +1 @@ +{"version":3,"file":"skill.js","sourceRoot":"","sources":["../src/skill.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACvD,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AACxC,OAAO,EAAE,wBAAwB,EAAE,SAAS,EAAE,MAAM,eAAe,CAAA;AACnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAA;AAerD,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;AAC/E,MAAM,kBAAkB,GAAG,OAAO,CAAA;AAClC,MAAM,gBAAgB,GAAG,qBAAqB,CAAA;AAE9C,MAAM,UAAU,gBAAgB,CAAC,SAAS,GAAG,kBAAkB;IAC7D,OAAO,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAA;AACrD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,UAA+B,EAAE;IAClE,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;IACtD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,SAAS,IAAI,wBAAwB,CAAC,CAAA;IAClF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAA;IAC1D,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAA;IAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,SAAS,CAAC,CAAA;IAC7C,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAA;IACrD,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAA;IAEnD,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAC3C,MAAM,KAAK,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAC1C,MAAM,EAAE,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,EAAE,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;IAC3F,MAAM,EAAE,CAAC,gBAAgB,CAAC,gBAAgB,CAAC,EAAE,cAAc,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;IAE9F,MAAM,SAAS,CAAC,aAAa,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IACtF,MAAM,SAAS,CAAC,UAAU,EAAE,cAAc,CAAC,SAAS,EAAE,cAAc,EAAE,aAAa,CAAC,EAAE,MAAM,CAAC,CAAA;IAC7F,MAAM,cAAc,GAAG,MAAM,oBAAoB,CAAC,GAAG,CAAC,CAAA;IAEtD,MAAM,OAAO,GAAG;QACd,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,SAAS,CAAC;QAC7B,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,cAAc,CAAC;QAClC,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,aAAa,CAAC;QACjC,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,UAAU,CAAC;KAC/B,CAAA;IAED,IAAI,cAAc,EAAE,CAAC;QACnB,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;IAC5B,CAAC;IAED,OAAO;QACL,SAAS;QACT,cAAc;QACd,aAAa;QACb,UAAU;QACV,OAAO;KACR,CAAA;AACH,CAAC;AAED,SAAS,SAAS,CAAC,GAAW;IAC5B,OAAO;QACL,UAAU,EAAE;YACV,KAAK,EAAE;gBACL,OAAO,EAAE,MAAM;gBACf,IAAI,EAAE,CAAC,MAAM,EAAE,IAAI,EAAE,WAAW,CAAC;gBACjC,GAAG;aACJ;SACF;KACF,CAAA;AACH,CAAC;AAED,SAAS,cAAc,CAAC,SAAiB,EAAE,cAAsB,EAAE,aAAqB;IACtF,OAAO;;;;;;;;;EASP,SAAS;;;;;;;;EAQT,cAAc;;;;;;;;;;;EAWd,aAAa;;;;;;;;;CASd,CAAA;AACD,CAAC"} \ No newline at end of file diff --git a/dist/store.d.ts b/packages/mimir/dist/store.d.ts similarity index 100% rename from dist/store.d.ts rename to packages/mimir/dist/store.d.ts diff --git a/dist/store.d.ts.map b/packages/mimir/dist/store.d.ts.map similarity index 57% rename from dist/store.d.ts.map rename to packages/mimir/dist/store.d.ts.map index 6f3f732..61ddb06 100644 --- a/dist/store.d.ts.map +++ b/packages/mimir/dist/store.d.ts.map @@ -1 +1 @@ -{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../src/store.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,OAAO,MAAM,kBAAkB,CAAA;AAC3C,OAAO,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAEnD,wBAAsB,SAAS,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAehF;AAED,wBAAsB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,CAOjF;AAED,wBAAsB,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAG/D"} \ No newline at end of file +{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../src/store.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,OAAO,MAAM,kBAAkB,CAAA;AAC3C,OAAO,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAEnD,wBAAsB,SAAS,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAgBhF;AAED,wBAAsB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,CAOjF;AAED,wBAAsB,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAG/D"} \ No newline at end of file diff --git a/dist/store.js b/packages/mimir/dist/store.js similarity index 89% rename from dist/store.js rename to packages/mimir/dist/store.js index b8670c1..a6531c5 100644 --- a/dist/store.js +++ b/packages/mimir/dist/store.js @@ -10,7 +10,8 @@ export async function writeRows(rows, config) { } return; } - await db.createTable(config.tableName, rows, { + const records = rows.map((row) => ({ ...row })); + await db.createTable(config.tableName, records, { mode: "overwrite", }); } diff --git a/packages/mimir/dist/store.js.map b/packages/mimir/dist/store.js.map new file mode 100644 index 0000000..55315c3 --- /dev/null +++ b/packages/mimir/dist/store.js.map @@ -0,0 +1 @@ +{"version":3,"file":"store.js","sourceRoot":"","sources":["../src/store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAA;AACxC,OAAO,KAAK,OAAO,MAAM,kBAAkB,CAAA;AAG3C,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAiB,EAAE,MAAc;IAC/D,MAAM,KAAK,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IACnD,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAA;IAEnD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,UAAU,EAAE,CAAA;QACxC,IAAI,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;YAC1C,MAAM,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;QACtC,CAAC;QACD,OAAM;IACR,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,GAAG,EAAE,CAAC,CAAC,CAAA;IAC/C,MAAM,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,EAAE,OAAO,EAAE;QAC9C,IAAI,EAAE,WAAW;KAClB,CAAC,CAAA;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,MAAc;IAChD,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAA;IACnD,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,UAAU,EAAE,CAAA;IACxC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;QAC3C,OAAO,IAAI,CAAA;IACb,CAAC;IACD,OAAO,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;AACvC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,MAAc;IAC5C,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAA;IACzC,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;AACtC,CAAC"} \ No newline at end of file diff --git a/dist/types.d.ts b/packages/mimir/dist/types.d.ts similarity index 86% rename from dist/types.d.ts rename to packages/mimir/dist/types.d.ts index 616f552..f9b07a1 100644 --- a/dist/types.d.ts +++ b/packages/mimir/dist/types.d.ts @@ -5,19 +5,20 @@ export interface Config { storageDir: string; sourcesFile: string; accessLogPath: string; + embeddingModelPath: string; tableName: string; - ollamaHost: string; - networkPolicy: NetworkPolicy; - embedModel: string; - llmModel: string; + embeddingProvider: EmbeddingProvider; + embeddingModel: string; + transformersAllowRemoteModels: boolean; redaction: RedactionConfig; accessLog: boolean; mcpMaxTopK: number; topK: number; chunkSize: number; chunkOverlap: number; + includeExtensions: string[]; } -export type NetworkPolicy = "local-only" | "allow-private" | "allow-any"; +export type EmbeddingProvider = "local-hash" | "transformers"; export interface RedactionConfig { enabled: boolean; builtIn: boolean; @@ -33,10 +34,6 @@ export interface RedactionCount { name: string; count: number; } -export interface HostClassification { - kind: "loopback" | "private" | "remote" | "invalid"; - host: string; -} export interface SourceFile { absolutePath: string; relativePath: string; @@ -110,11 +107,12 @@ export interface DestroyIndexResult { export interface SecurityAuditReport { projectRoot: string; zeroTelemetry: true; - network: { - policy: NetworkPolicy; - ollamaHost: string; - host: string; - classification: HostClassification["kind"]; + providers: { + embedding: EmbeddingProvider; + embeddingModel: string; + embeddingModelPath: string; + transformersAllowRemoteModels: boolean; + llmGeneration: false; }; redaction: { enabled: boolean; diff --git a/packages/mimir/dist/types.d.ts.map b/packages/mimir/dist/types.d.ts.map new file mode 100644 index 0000000..84d2f33 --- /dev/null +++ b/packages/mimir/dist/types.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AAEvC,MAAM,WAAW,MAAM;IACrB,WAAW,EAAE,MAAM,CAAA;IACnB,MAAM,EAAE,MAAM,CAAA;IACd,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,EAAE,MAAM,CAAA;IACnB,aAAa,EAAE,MAAM,CAAA;IACrB,kBAAkB,EAAE,MAAM,CAAA;IAC1B,SAAS,EAAE,MAAM,CAAA;IACjB,iBAAiB,EAAE,iBAAiB,CAAA;IACpC,cAAc,EAAE,MAAM,CAAA;IACtB,6BAA6B,EAAE,OAAO,CAAA;IACtC,SAAS,EAAE,eAAe,CAAA;IAC1B,SAAS,EAAE,OAAO,CAAA;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,SAAS,EAAE,MAAM,CAAA;IACjB,YAAY,EAAE,MAAM,CAAA;IACpB,iBAAiB,EAAE,MAAM,EAAE,CAAA;CAC5B;AAED,MAAM,MAAM,iBAAiB,GAAG,YAAY,GAAG,cAAc,CAAA;AAE7D,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,OAAO,CAAA;IAChB,OAAO,EAAE,OAAO,CAAA;IAChB,QAAQ,EAAE,gBAAgB,EAAE,CAAA;CAC7B;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IAC1B,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CACjC;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,UAAU;IACzB,YAAY,EAAE,MAAM,CAAA;IACpB,YAAY,EAAE,MAAM,CAAA;IACpB,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,KAAK,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,MAAM,CAAA;CACjB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,UAAU,CAAA;IAChB,IAAI,EAAE,MAAM,CAAA;CACb;AAED,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAA;IACV,MAAM,EAAE,MAAM,CAAA;IACd,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,SAAU,SAAQ,SAAS;IAC1C,MAAM,EAAE,MAAM,EAAE,CAAA;CACjB;AAED,MAAM,WAAW,aAAa;IAC5B,GAAG,CAAC,EAAE,QAAQ,CAAA;IACd,OAAO,CAAC,EAAE,OAAO,CAAA;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,MAAM,CAAA;IACpB,MAAM,EAAE,MAAM,CAAA;IACd,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,MAAM,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;CACjD;AAED,MAAM,WAAW,aAAa;IAC5B,GAAG,CAAC,EAAE,QAAQ,CAAA;IACd,IAAI,CAAC,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAA;IACd,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;CACxB;AAED,MAAM,WAAW,SAAS;IACxB,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,YAAY,EAAE,CAAA;CACxB;AAED,MAAM,WAAW,WAAW;IAC1B,YAAY,EAAE,KAAK,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IACvD,cAAc,EAAE,MAAM,EAAE,CAAA;IACxB,gBAAgB,EAAE,MAAM,EAAE,CAAA;IAC1B,YAAY,EAAE,MAAM,EAAE,CAAA;IACtB,WAAW,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,kBAAkB;IACjC,UAAU,EAAE,MAAM,CAAA;IAClB,OAAO,EAAE,OAAO,CAAA;IAChB,IAAI,EAAE,MAAM,CAAA;CACb;AAED,MAAM,WAAW,mBAAmB;IAClC,WAAW,EAAE,MAAM,CAAA;IACnB,aAAa,EAAE,IAAI,CAAA;IACnB,SAAS,EAAE;QACT,SAAS,EAAE,iBAAiB,CAAA;QAC5B,cAAc,EAAE,MAAM,CAAA;QACtB,kBAAkB,EAAE,MAAM,CAAA;QAC1B,6BAA6B,EAAE,OAAO,CAAA;QACtC,aAAa,EAAE,KAAK,CAAA;KACrB,CAAA;IACD,SAAS,EAAE;QACT,OAAO,EAAE,OAAO,CAAA;QAChB,OAAO,EAAE,OAAO,CAAA;QAChB,cAAc,EAAE,MAAM,EAAE,CAAA;KACzB,CAAA;IACD,SAAS,EAAE;QACT,OAAO,EAAE,OAAO,CAAA;QAChB,IAAI,EAAE,MAAM,CAAA;QACZ,gBAAgB,EAAE,KAAK,CAAA;KACxB,CAAA;IACD,OAAO,EAAE;QACP,IAAI,EAAE,MAAM,CAAA;QACZ,UAAU,EAAE,OAAO,CAAA;QACnB,eAAe,EAAE,mBAAmB,CAAA;KACrC,CAAA;IACD,GAAG,EAAE;QACH,OAAO,EAAE,MAAM,CAAA;QACf,uBAAuB,EAAE,KAAK,CAAA;KAC/B,CAAA;IACD,SAAS,EAAE;QACT,SAAS,EAAE,OAAO,CAAA;QAClB,YAAY,EAAE,OAAO,CAAA;QACrB,cAAc,EAAE,OAAO,CAAA;KACxB,CAAA;IACD,eAAe,EAAE,MAAM,EAAE,CAAA;IACzB,QAAQ,EAAE,MAAM,EAAE,CAAA;CACnB"} \ No newline at end of file diff --git a/dist/types.js b/packages/mimir/dist/types.js similarity index 100% rename from dist/types.js rename to packages/mimir/dist/types.js diff --git a/dist/types.js.map b/packages/mimir/dist/types.js.map similarity index 100% rename from dist/types.js.map rename to packages/mimir/dist/types.js.map diff --git a/packages/mimir/dist/version.d.ts b/packages/mimir/dist/version.d.ts new file mode 100644 index 0000000..72b1961 --- /dev/null +++ b/packages/mimir/dist/version.d.ts @@ -0,0 +1,2 @@ +export declare const VERSION = "0.4.0"; +//# sourceMappingURL=version.d.ts.map \ No newline at end of file diff --git a/dist/version.d.ts.map b/packages/mimir/dist/version.d.ts.map similarity index 100% rename from dist/version.d.ts.map rename to packages/mimir/dist/version.d.ts.map diff --git a/dist/version.js b/packages/mimir/dist/version.js similarity index 52% rename from dist/version.js rename to packages/mimir/dist/version.js index b7312b3..112a255 100644 --- a/dist/version.js +++ b/packages/mimir/dist/version.js @@ -1,2 +1,2 @@ -export const VERSION = "0.3.0"; +export const VERSION = "0.4.0"; //# sourceMappingURL=version.js.map \ No newline at end of file diff --git a/dist/version.js.map b/packages/mimir/dist/version.js.map similarity index 100% rename from dist/version.js.map rename to packages/mimir/dist/version.js.map diff --git a/packages/mimir/examples/sovereign-rag-demo/.gitignore b/packages/mimir/examples/sovereign-rag-demo/.gitignore new file mode 100644 index 0000000..f79f9f2 --- /dev/null +++ b/packages/mimir/examples/sovereign-rag-demo/.gitignore @@ -0,0 +1,9 @@ +.kb/ +!.kb/ +!.kb/config.json +!.kb/sources.txt +.kb/storage/ +.kb/access.log +.mimir/ +node_modules/ +private/** diff --git a/packages/mimir/examples/sovereign-rag-demo/.kb/config.json b/packages/mimir/examples/sovereign-rag-demo/.kb/config.json new file mode 100644 index 0000000..bc93253 --- /dev/null +++ b/packages/mimir/examples/sovereign-rag-demo/.kb/config.json @@ -0,0 +1,22 @@ +{ + "rawDir": "raw", + "storageDir": ".kb/storage", + "sourcesFile": ".kb/sources.txt", + "accessLogPath": ".kb/access.log", + "embeddingModelPath": ".mimir/models", + "tableName": "chunks", + "embeddingProvider": "local-hash", + "embeddingModel": "mixedbread-ai/mxbai-embed-xsmall-v1", + "transformersAllowRemoteModels": false, + "redaction": { + "enabled": true, + "builtIn": true, + "patterns": [] + }, + "accessLog": true, + "mcpMaxTopK": 10, + "topK": 5, + "chunkSize": 900, + "chunkOverlap": 120, + "includeExtensions": [".evidence"] +} diff --git a/packages/mimir/examples/sovereign-rag-demo/.kb/sources.txt b/packages/mimir/examples/sovereign-rag-demo/.kb/sources.txt new file mode 100644 index 0000000..bb36a89 --- /dev/null +++ b/packages/mimir/examples/sovereign-rag-demo/.kb/sources.txt @@ -0,0 +1,2 @@ +# Optional extra source paths, one per line. +# This demo keeps all indexed files under ./raw. diff --git a/packages/mimir/examples/sovereign-rag-demo/README.md b/packages/mimir/examples/sovereign-rag-demo/README.md new file mode 100644 index 0000000..9dc9f0c --- /dev/null +++ b/packages/mimir/examples/sovereign-rag-demo/README.md @@ -0,0 +1,80 @@ +# Sovereign RAG Demo + +Synthetic test workspace for Mimir. It is intentionally safe to commit: every document is fictional, +generic, and designed only to exercise local ingestion, retrieval, redaction, custom extensions, and +security-audit flows. + +This folder must never contain real-world sensitive, regulated, or production documents. + +## What It Covers + +- Markdown operational briefs. +- CSV dataset inventories. +- JSONL incident timelines. +- YAML policy metadata. +- A custom `.evidence` text extension enabled through `.kb/config.json`. + +## Run From This Repository Checkout + +Build Mimir once from the repository root: + +```bash +pnpm build +``` + +Then run the CLI from this folder: + +```bash +cd examples/sovereign-rag-demo +node ../../dist/cli.js security-audit +node ../../dist/cli.js ingest +node ../../dist/cli.js search "offline retrieval approval" +node ../../dist/cli.js search "dataset residency" +node ../../dist/cli.js ask "What evidence supports offline operation?" +node ../../dist/cli.js audit +node ../../dist/cli.js status +``` + +This example uses `embeddingProvider: "local-hash"`, so it does not require a model runtime. +Retrieval is lexical/hash-based rather than model-semantic. + +## Useful Test Queries + +- `offline retrieval approval` +- `dataset residency` +- `incident containment evidence` +- `who owns the usage review` +- `what documents support sovereign deployment` + +## Switch To Transformers Semantic Mode + +To compare no-model retrieval with semantic local retrieval, change `.kb/config.json`: + +```json +{ + "embeddingProvider": "transformers", + "embeddingModel": "mixedbread-ai/mxbai-embed-xsmall-v1", + "embeddingModelPath": ".mimir/models", + "transformersAllowRemoteModels": false +} +``` + +Preload the model files under `.mimir/models` for offline work, then rebuild the index: + +```bash +node ../../dist/cli.js ingest +node ../../dist/cli.js ask "What documents support sovereign deployment?" +``` + +## Generated State + +Generated state stays local and ignored: + +```plain text +.kb/storage/ +.kb/access.log +.mimir/ +``` + +Do not replace these synthetic documents with real confidential files inside the Mimir package +repository. diff --git a/packages/mimir/examples/sovereign-rag-demo/raw/dataset-inventory.csv b/packages/mimir/examples/sovereign-rag-demo/raw/dataset-inventory.csv new file mode 100644 index 0000000..559f60d --- /dev/null +++ b/packages/mimir/examples/sovereign-rag-demo/raw/dataset-inventory.csv @@ -0,0 +1,5 @@ +dataset,source_type,data_residency,network_access,review_status +Example Notes,markdown notes,local only,none,approved +Example Model Card,model metadata,local repository,none,approved +Example Audio Draft,generated summary,local only,none,approved with offline TTS +Example Remote Transcript,transcription sample,remote processing,internet,rejected for confidential tests diff --git a/packages/mimir/examples/sovereign-rag-demo/raw/incident-timeline.jsonl b/packages/mimir/examples/sovereign-rag-demo/raw/incident-timeline.jsonl new file mode 100644 index 0000000..e701290 --- /dev/null +++ b/packages/mimir/examples/sovereign-rag-demo/raw/incident-timeline.jsonl @@ -0,0 +1,4 @@ +{"date":"2026-02-12","event":"synthetic dataset imported","owner":"example knowledge team","impact":"none","evidence":"fictional files only"} +{"date":"2026-02-13","event":"remote model loading disabled","owner":"example review team","impact":"offline policy enforced","evidence":"transformersAllowRemoteModels false"} +{"date":"2026-02-14","event":"offline retrieval approval completed","owner":"example platform team","impact":"local cited retrieval allowed","evidence":"local-hash provider"} +{"date":"2026-02-15","event":"audio summary test requested","owner":"example operations role","impact":"offline TTS required","evidence":"mimir-audio-summary skill"} diff --git a/packages/mimir/examples/sovereign-rag-demo/raw/operations-brief.md b/packages/mimir/examples/sovereign-rag-demo/raw/operations-brief.md new file mode 100644 index 0000000..b7a03a6 --- /dev/null +++ b/packages/mimir/examples/sovereign-rag-demo/raw/operations-brief.md @@ -0,0 +1,16 @@ +# Offline Analytics Brief + +Example Workspace is a synthetic local knowledge-base deployment used to test Mimir. + +The demo goal is to let a test team summarize fictional operational notes without shipping source +files to a hosted RAG service. The approved runtime is a local Linux workstation with an encrypted +disk, local retrieval, and no telemetry. + +The initial approval covers three workflows: + +- summarize daily operational briefs; +- compare dataset handling notes against internal policy; +- prepare audio briefings only with an offline text-to-speech engine. + +The usage review is owned by the Example Review Team. Any remote model endpoint requires a written +exception before use. diff --git a/packages/mimir/examples/sovereign-rag-demo/raw/review-notes.evidence b/packages/mimir/examples/sovereign-rag-demo/raw/review-notes.evidence new file mode 100644 index 0000000..74fd730 --- /dev/null +++ b/packages/mimir/examples/sovereign-rag-demo/raw/review-notes.evidence @@ -0,0 +1,11 @@ +Evidence memo for the synthetic usage review. + +The Example Review Team approved local-only retrieval because the source documents remain under the +repository raw folder and the generated vector store remains under .kb/storage. + +The review rejected remote transcription and online text-to-speech for fictional confidential test +datasets. The approved audio path is a generated briefing under .mimir/audio created with an offline +engine. + +The test is considered complete when Mimir can retrieve the offline retrieval approval, dataset +residency requirements, and incident containment evidence from separate file formats. diff --git a/packages/mimir/examples/sovereign-rag-demo/raw/security-policy.yaml b/packages/mimir/examples/sovereign-rag-demo/raw/security-policy.yaml new file mode 100644 index 0000000..2baace2 --- /dev/null +++ b/packages/mimir/examples/sovereign-rag-demo/raw/security-policy.yaml @@ -0,0 +1,14 @@ +classification: synthetic +purpose: local-rag-test +controls: + remote_model_loading: disabled + storage: encrypted-volume-recommended + telemetry: none + access_log: metadata-only + mcp: + destructive_tools: false + max_top_k: 10 +review: + owner: Example Review Team + required_before_remote_model_loading: true + required_before_online_tts: true diff --git a/packages/mimir/package.json b/packages/mimir/package.json new file mode 100644 index 0000000..94faa7e --- /dev/null +++ b/packages/mimir/package.json @@ -0,0 +1,88 @@ +{ + "name": "@jcode.labs/mimir", + "version": "0.4.0", + "description": "Mimir: open-source sovereign local RAG for confidential datasets and AI agents.", + "type": "module", + "license": "MIT", + "author": { + "name": "Jean-Baptiste Thery", + "url": "https://github.com/jb-thery" + }, + "keywords": [ + "jcode", + "mimir", + "open-source", + "rag", + "sovereign-rag", + "knowledge-base", + "confidential-data", + "private-ai", + "local-first", + "mcp", + "transformers", + "semantic-search", + "lancedb" + ], + "repository": { + "type": "git", + "url": "git+https://github.com/jcode-works/jcode-mimir.git", + "directory": "packages/mimir" + }, + "homepage": "https://github.com/jcode-works/jcode-mimir/tree/main/packages/mimir#readme", + "bugs": { + "url": "https://github.com/jcode-works/jcode-mimir/issues" + }, + "engines": { + "node": ">=20" + }, + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "bin": { + "kb": "dist/cli.js" + }, + "files": [ + "dist", + "skills", + "examples", + "README.md", + "LICENSE", + "CONTRIBUTING.md", + "SECURITY-HARDENING.md", + "CHANGELOG.md", + "SECURITY.md" + ], + "publishConfig": { + "access": "public" + }, + "scripts": { + "build": "tsc -p tsconfig.json", + "check": "tsc -p tsconfig.json --noEmit", + "package:check": "publint", + "test": "vitest run" + }, + "dependencies": { + "@jcode.labs/mimir-tts": "workspace:*", + "@huggingface/transformers": "^4.2.0", + "@lancedb/lancedb": "^0.30.0", + "@modelcontextprotocol/sdk": "^1.29.0", + "commander": "^14.0.2", + "fast-glob": "^3.3.3", + "fflate": "^0.8.3", + "html-to-text": "^9.0.5", + "picocolors": "^1.1.1", + "unpdf": "^1.4.0", + "yaml": "^2.8.1", + "zod": "^4.1.13" + }, + "devDependencies": { + "@types/html-to-text": "^9.0.4", + "@types/node": "^24.10.1", + "publint": "^0.3.21", + "typescript": "^5.9.3", + "vitest": "^4.0.15" + } +} diff --git a/packages/mimir/skills/mimir-audio-summary/SKILL.md b/packages/mimir/skills/mimir-audio-summary/SKILL.md new file mode 100644 index 0000000..8d4661f --- /dev/null +++ b/packages/mimir/skills/mimir-audio-summary/SKILL.md @@ -0,0 +1,134 @@ +--- +name: mimir-audio-summary +description: >- + Create an optional spoken audio summary from a Mimir local knowledge base. Use when the user asks + for an audio, TTS, spoken brief, briefing, narration, or listenable summary based on private + repository documents indexed by Mimir. The skill is confidentiality-first: gather facts through + Mimir, write only a temporary narration text file outside the repository, and render the final + audio under ignored local Mimir state unless the user explicitly chooses another output path. +--- + +# Mimir Audio Summary + +Use this skill to turn a confidential local Mimir knowledge base into an optional audio summary. +The knowledge base stays local; the final audio is a generated artifact and must not be committed. + +## Confidentiality Rules + +- Treat the source documents, retrieved passages, generated narration, and final audio as sensitive. +- Do not use online TTS for confidential content unless the user explicitly allows it. +- Prefer `pnpm exec kb audio` or `pnpm exec mimir-tts render` for plug-and-play local WAV output. +- Use `--offline` when model files are already present and remote model loading is not allowed. +- Write the narration text to a temp file outside the repository, such as `/tmp/MIMIR-SUMMARY-topic.txt`. +- Render audio under `.mimir/audio/` by default. This directory is ignored by Git when Mimir is installed. +- Never stage or commit generated audio, temporary text, WAV, AIFF, or intermediate files. + +## 1. Verify The Knowledge Base + +From the repository root, run: + +```bash +pnpm exec kb status +pnpm exec kb audit +pnpm exec kb security-audit +``` + +If the audit reports missing or stale files, run: + +```bash +pnpm exec kb ingest +pnpm exec kb audit +``` + +Do not create an audio summary from stale or incomplete evidence unless the user explicitly accepts +that limitation. + +## 2. Search Deeply Before Writing + +Use Mimir search or MCP tools to gather evidence before drafting the narration. + +For a broad summary, run multiple searches: + +```bash +pnpm exec kb search "
" --top-k 8 +pnpm exec kb search "" --top-k 8 +pnpm exec kb ask "" --top-k 8 +``` + +When MCP is available, prefer `mimir_search`, `mimir_ask`, `mimir_audit`, and +`mimir_security_audit` over shell commands. + +Keep citations in your working notes, but do not read long raw passages aloud. The audio should be a +clear synthesis, not a dump of source text. + +## 3. Write For Listening + +Write one flowing narration in the user's working language. Do not use markdown, headings, bullets, +tables, SSML, XML tags, or stage directions in the spoken text. + +Good audio structure: + +1. Start with the purpose of the summary and the two-to-four ideas to retain. +2. Explain the current evidence in plain language. +3. Separate proven facts from uncertainty. +4. Highlight decisions, risks, deadlines, and missing documents. +5. End with a concise recap and two or three self-check questions. + +Use short sentences and natural punctuation. Spell acronyms and symbols in a way a TTS engine can +pronounce. + +## 4. Render The Audio + +Create the output directory and write the narration to a temp file outside the repo: + +```bash +mkdir -p .mimir/audio +``` + +Then render with the default Mimir TTS path: + +```bash +pnpm exec kb audio /tmp/MIMIR-SUMMARY-.txt \ + --out .mimir/audio/MIMIR-SUMMARY-.wav +``` + +For air-gapped operation, preload the model files under `.mimir/models/tts` and run: + +```bash +pnpm exec kb audio /tmp/MIMIR-SUMMARY-.txt \ + --out .mimir/audio/MIMIR-SUMMARY-.wav \ + --offline +``` + +The default renderer uses `@jcode.labs/mimir-tts` and Transformers.js. It does not require Python, +ffmpeg, Piper, XTTS, or a local TTS server. The first non-offline render can download public model +files into `.mimir/models/tts`, but the narration text is processed locally. + +Use the legacy voice-forge helper only when the user explicitly wants MP3 output or an engine not +covered by Mimir TTS: + +```bash +OUT_MP3="/.mimir/audio/MIMIR-SUMMARY-.mp3" \ + TTS_ENGINE=auto \ + bash /forge-voice.sh /tmp/MIMIR-SUMMARY-.txt +``` + +Engine selection: + +- `auto`: confidentiality-first order: XTTS, macOS `say`, Piper, then online Edge only when + `MIMIR_ALLOW_ONLINE_TTS=1`. +- `xtts`: local Coqui XTTS-v2. +- `say`: local macOS speech engine, converted to MP3 with `ffmpeg`. +- `piper`: local neural TTS, converted to MP3 with `ffmpeg`. +- `edge`: online Edge TTS. Use only with explicit user approval for non-sensitive text. + +Voice can be selected with `TTS_VOICE`. Speed for Edge can be selected with `TTS_RATE`. + +## 5. Report The Result + +After rendering, report: + +- the audio path; +- which renderer/model was used or requested; +- whether remote model downloads or online TTS were allowed; +- any evidence limitation, such as stale index, missing documents, or weak search results. diff --git a/packages/mimir/skills/mimir-audio-summary/forge-voice.sh b/packages/mimir/skills/mimir-audio-summary/forge-voice.sh new file mode 100755 index 0000000..36c5bda --- /dev/null +++ b/packages/mimir/skills/mimir-audio-summary/forge-voice.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash +# Render a Mimir audio-summary text file to MP3. The text is a throwaway intermediate written +# outside the repository by the skill. The final audio should normally be written under .mimir/audio. +# +# Engine via TTS_ENGINE (auto|xtts|say|piper|edge); default "auto" is confidentiality-first: +# xtts - local Coqui XTTS-v2 when installed. +# say - macOS built-in, offline, clean but robotic. +# piper - local neural TTS. +# edge - online Edge TTS only when TTS_ENGINE=edge or MIMIR_ALLOW_ONLINE_TTS=1. +# +# Usage: forge-voice.sh [voice] +# Env: OUT_MP3 explicit mp3 output path +# TTS_ENGINE force an engine (default: auto) +# TTS_VOICE voice/speaker (engine-specific); the [voice] arg overrides it +# TTS_RATE edge-tts speed delta, default +0% +# TTS_SEGMENT=1 edge: render sentence-by-sentence +# XTTS_SPEAKER xtts preset speaker (default Ana Florence) +# PIPER_MODEL piper onnx model path +# MIMIR_ALLOW_ONLINE_TTS=1 allow edge in auto mode +# KEEP_TEXT=1 keep the source text file after a successful render +set -euo pipefail + +TXT="${1:?usage: forge-voice.sh [voice]}" +[ -f "$TXT" ] || { echo "error: file not found: $TXT" >&2; exit 1; } +VOICE="${2:-${TTS_VOICE:-}}" +ENGINE="${TTS_ENGINE:-auto}" +OUT_FINAL="${OUT_MP3:-${TXT%.txt}.mp3}" +OUTBASE="${OUT_FINAL%.mp3}" + +cleanup() { + [ "${KEEP_TEXT:-0}" = "1" ] || rm -f "$TXT" + rm -f "${OUTBASE}.wav" "${OUTBASE}.aiff" +} +trap cleanup EXIT + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +XTTS_PY="${XTTS_PY:-$HOME/.local/share/voice-forge/xtts/bin/python}" +XTTS_HELPER="$SCRIPT_DIR/xtts-voice.py" + +xtts_ready() { [ -x "$XTTS_PY" ] && [ -f "$XTTS_HELPER" ]; } + +finish() { + echo "$1" + exit 0 +} + +to_mp3() { + local src="$1" out="${OUTBASE}.mp3" + if ! command -v ffmpeg >/dev/null 2>&1; then + rm -f "$src" + echo "error: ffmpeg is required to convert local TTS output to mp3" >&2 + exit 1 + fi + if ffmpeg -y -loglevel error -i "$src" -ac 1 -c:a libmp3lame -q:a 4 "$out"; then + rm -f "$src" + finish "$out" + fi + rm -f "$src" "$out" + echo "error: failed to convert local TTS output to mp3" >&2 + exit 1 +} + +if [ "$ENGINE" = "xtts" ] || { [ "$ENGINE" = "auto" ] && xtts_ready; }; then + if ! xtts_ready; then + echo "error: TTS_ENGINE=xtts but venv/helper missing ($XTTS_PY)" >&2 + exit 1 + fi + WAV="${OUTBASE}.wav" + if [ -n "$VOICE" ]; then + COQUI_TOS_AGREED=1 "$XTTS_PY" "$XTTS_HELPER" "$TXT" "$WAV" "$VOICE" >&2 + else + COQUI_TOS_AGREED=1 "$XTTS_PY" "$XTTS_HELPER" "$TXT" "$WAV" >&2 + fi + to_mp3 "$WAV" +fi + +if [ "$ENGINE" = "say" ] || [ "$ENGINE" = "auto" ]; then + if command -v say >/dev/null 2>&1; then + AIFF="${OUTBASE}.aiff" + say -v "${VOICE:-Jacques}" -f "$TXT" -o "$AIFF" 2>/dev/null || say -f "$TXT" -o "$AIFF" + to_mp3 "$AIFF" + fi + if [ "$ENGINE" = "say" ]; then + echo "error: TTS_ENGINE=say but 'say' not available" >&2 + exit 1 + fi +fi + +if [ "$ENGINE" = "piper" ] || [ "$ENGINE" = "auto" ]; then + if command -v piper >/dev/null 2>&1; then + WAV="${OUTBASE}.wav" + piper -m "${PIPER_MODEL:-fr_FR-siwis-medium.onnx}" -f "$WAV" < "$TXT" + to_mp3 "$WAV" + fi + if [ "$ENGINE" = "piper" ]; then + echo "error: TTS_ENGINE=piper but piper not installed (pip install piper-tts)" >&2 + exit 1 + fi +fi + +if [ "$ENGINE" = "edge" ] || { [ "$ENGINE" = "auto" ] && [ "${MIMIR_ALLOW_ONLINE_TTS:-0}" = "1" ]; }; then + if command -v edge-tts >/dev/null 2>&1; then + OUT="${OUTBASE}.mp3" + SPLIT="$SCRIPT_DIR/split-lines.py" + voice="${VOICE:-fr-FR-DeniseNeural}" + rate="${TTS_RATE:-+0%}" + if [ "${TTS_SEGMENT:-0}" = "1" ] && [ -f "$SPLIT" ] \ + && command -v ffmpeg >/dev/null 2>&1 && command -v python3 >/dev/null 2>&1; then + TMP="$(mktemp -d)" + ffmpeg -y -loglevel error -f lavfi -i anullsrc=r=24000:cl=mono -t 0.28 \ + -c:a libmp3lame -q:a 4 "$TMP/sil.mp3" + i=0 + : > "$TMP/list.txt" + while IFS= read -r line; do + [ -z "$line" ] && continue + i=$((i + 1)) + if edge-tts --text "$line" --voice "$voice" --rate="$rate" \ + --write-media "$TMP/seg_$i.mp3" >/dev/null 2>&1; then + printf "file '%s'\n" "$TMP/seg_$i.mp3" >> "$TMP/list.txt" + printf "file '%s'\n" "$TMP/sil.mp3" >> "$TMP/list.txt" + else + echo "warn: edge-tts failed on a sentence, skipping it" >&2 + fi + done < <(python3 "$SPLIT" "$TXT") + ffmpeg -y -loglevel error -f concat -safe 0 -i "$TMP/list.txt" \ + -ac 1 -c:a libmp3lame -q:a 4 "$OUT" + rm -rf "$TMP" + finish "$OUT" + fi + edge-tts --file "$TXT" --voice "$voice" --rate="$rate" --write-media "$OUT" >/dev/null + finish "$OUT" + fi + if [ "$ENGINE" = "edge" ]; then + echo "error: TTS_ENGINE=edge but edge-tts not installed (pipx install edge-tts)" >&2 + exit 1 + fi +fi + +cat >&2 <<'EOF' +error: no offline TTS engine available. Install one: + XTTS-v2 (local): + uv venv --python 3.11 ~/.local/share/voice-forge/xtts + uv pip install --python ~/.local/share/voice-forge/xtts/bin/python \ + coqui-tts 'transformers>=4.57,<5' 'torch==2.8.*' 'torchaudio==2.8.*' + piper (local): pip install piper-tts + ffmpeg: required for local engine MP3 conversion + +Online fallback: + pipx install edge-tts + MIMIR_ALLOW_ONLINE_TTS=1 TTS_ENGINE=auto ... + or explicitly set TTS_ENGINE=edge for non-sensitive text. +EOF +exit 127 diff --git a/packages/mimir/skills/mimir-audio-summary/split-lines.py b/packages/mimir/skills/mimir-audio-summary/split-lines.py new file mode 100644 index 0000000..9b05540 --- /dev/null +++ b/packages/mimir/skills/mimir-audio-summary/split-lines.py @@ -0,0 +1,13 @@ +"""Print one sentence per line from a text file. + +Used to render long narrations sentence-by-sentence when an engine benefits from segmentation. +""" +import re +import sys + +raw = open(sys.argv[1], encoding="utf-8").read() +text = re.sub(r"\s+", " ", raw).strip() +for part in re.split(r"(?<=[.!?…])\s+", text): + part = part.strip() + if part: + print(part) diff --git a/packages/mimir/skills/mimir-audio-summary/xtts-voice.py b/packages/mimir/skills/mimir-audio-summary/xtts-voice.py new file mode 100755 index 0000000..e41c3d6 --- /dev/null +++ b/packages/mimir/skills/mimir-audio-summary/xtts-voice.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Render a text file to WAV with Coqui XTTS-v2. + +Run with the dedicated venv interpreter: + ~/.local/share/voice-forge/xtts/bin/python xtts-voice.py [speaker] + +Env: XTTS_SPEAKER (preset name), XTTS_LANG (default "fr"). +""" +import os +import sys + +os.environ.setdefault("COQUI_TOS_AGREED", "1") + + +def main() -> None: + if len(sys.argv) < 3: + sys.exit("usage: xtts-voice.py [speaker]") + text_file, out_wav = sys.argv[1], sys.argv[2] + speaker = sys.argv[3] if len(sys.argv) > 3 else os.environ.get("XTTS_SPEAKER", "Ana Florence") + language = os.environ.get("XTTS_LANG", "fr") + + with open(text_file, encoding="utf-8") as handle: + text = handle.read().strip() + if not text: + sys.exit("error: empty text file") + + from TTS.api import TTS + + tts = TTS("tts_models/multilingual/multi-dataset/xtts_v2") + speakers = list(getattr(tts, "speakers", None) or []) + if speakers and speaker not in speakers: + sys.stderr.write(f"speaker '{speaker}' not found; falling back to '{speakers[0]}'\n") + speaker = speakers[0] + + tts.tts_to_file( + text=text, + speaker=speaker, + language=language, + file_path=out_wav, + split_sentences=True, + ) + print(out_wav) + + +if __name__ == "__main__": + main() diff --git a/skills/mimir/SKILL.md b/packages/mimir/skills/mimir/SKILL.md similarity index 60% rename from skills/mimir/SKILL.md rename to packages/mimir/skills/mimir/SKILL.md index 6868fcc..9551f3f 100644 --- a/skills/mimir/SKILL.md +++ b/packages/mimir/skills/mimir/SKILL.md @@ -5,7 +5,9 @@ description: Use this skill whenever a repository uses or should use Mimir, loca # Mimir -Mimir is a local-first knowledge base for project documents. It indexes files from the current repository, stores vectors locally, and exposes both a CLI and an MCP server. +Mimir is a sovereign local RAG knowledge base for confidential project documents and datasets. It +indexes files from the current repository, stores vectors locally, and exposes both a CLI and an MCP +server. Use this skill to help an AI agent work with a Mimir-enabled repository without leaking private documents or relying on stale memory. @@ -55,6 +57,34 @@ npm install --save-dev @jcode.labs/mimir npx kb init ``` +## Provider Modes + +Default retrieval mode: + +```json +{ + "embeddingProvider": "local-hash" +} +``` + +This supports ingestion, search, MCP retrieval, and `kb ask` with cited passages without a model +server. It is lexical/hash retrieval, not model-semantic search. Do not present it as equivalent to +semantic embeddings. + +Optional semantic embedding mode: + +```json +{ + "embeddingProvider": "transformers", + "embeddingModel": "mixedbread-ai/mxbai-embed-xsmall-v1", + "embeddingModelPath": ".mimir/models", + "transformersAllowRemoteModels": false +} +``` + +This uses Transformers.js for embeddings only. Keep `transformersAllowRemoteModels` false for +air-gapped or confidential work and preload model files under `embeddingModelPath`. + ## Ingestion Workflow After documents are added or changed: @@ -77,13 +107,32 @@ Use search when you need exact source passages: pnpm exec kb search "your query" ``` -Use ask when you need a synthesized answer with citations: +Use ask when you need cited context for the current agent or an external LLM: ```bash pnpm exec kb ask "your question" ``` -Ground answers in returned sources. If search results are weak, say that the current index does not prove the point and ask for the missing document. +Ground answers in returned sources. If search results are weak, say that the current index does not +prove the point and ask for the missing document. `kb ask` returns cited passages rather than LLM +synthesis. Use those passages as context for the current agent, or tell the user that generative +synthesis needs a trusted external LLM or model runtime. + +## Deep Research Workflow + +For broad summaries, audits, planning, or institutional dossiers, do not rely on one query. Build a +small retrieval plan first: + +- check `kb audit` and `kb security-audit`; +- query the main topic; +- query names, dates, amounts, obligations, risks, decisions, and missing evidence separately; +- compare the strongest passages across files; +- ask a synthesis question only after search has found enough grounded context; +- cite source paths and chunk numbers in the answer when useful; +- explicitly say when the index does not prove a claim. + +For sensitive work, prefer the smallest useful `topK`; raise it only when the first results are too +thin. Do not dump large raw passages into the chat unless the user explicitly asks for extracts. ## MCP Usage @@ -105,15 +154,27 @@ Available MCP tools: - `mimir_status`: show config and chunk count. - `mimir_search`: retrieve source passages. -- `mimir_ask`: synthesize an answer with local citations. +- `mimir_ask`: return cited retrieval context. - `mimir_audit`: compare source files with the current index. -- `mimir_security_audit`: inspect local privacy, network, redaction, MCP, and gitignore posture. +- `mimir_security_audit`: inspect local privacy, provider, redaction, MCP, and gitignore posture. Prefer MCP tools over shell commands when the agent runtime provides them. Use shell commands when MCP is unavailable. MCP is read-focused and intentionally does not expose index deletion. Use `pnpm exec kb destroy-index --yes` from the shell when the user explicitly wants to remove the generated index. +## Optional Audio Summaries + +If the user asks for a listenable or TTS summary, load the optional +`.mimir/skills/mimir-audio-summary/` skill installed by `pnpm exec kb install-skill`. + +That skill should: + +- gather evidence through Mimir first; +- write narration text only to a temp file outside the repository; +- render generated audio under `.mimir/audio/` by default; +- prefer offline TTS engines for confidential content. + ## Installing This Skill Into A Repository Run: diff --git a/src/access-log.ts b/packages/mimir/src/access-log.ts similarity index 100% rename from src/access-log.ts rename to packages/mimir/src/access-log.ts diff --git a/src/chunking.test.ts b/packages/mimir/src/chunking.test.ts similarity index 100% rename from src/chunking.test.ts rename to packages/mimir/src/chunking.test.ts diff --git a/src/chunking.ts b/packages/mimir/src/chunking.ts similarity index 84% rename from src/chunking.ts rename to packages/mimir/src/chunking.ts index fcf0331..54babb5 100644 --- a/src/chunking.ts +++ b/packages/mimir/src/chunking.ts @@ -1,6 +1,10 @@ import { createHash } from "node:crypto" import type { ParsedDocument, TextChunk } from "./types.js" +const PARAGRAPH_BREAK_MIN_RATIO = 0.45 +const SENTENCE_BREAK_MIN_RATIO = 0.55 +const WHITESPACE_BREAK_MIN_RATIO = 0.75 + export function chunkDocument( document: ParsedDocument, chunkSize: number, @@ -52,7 +56,7 @@ function chooseChunkEnd(text: string, cursor: number, chunkSize: number): number const window = text.slice(cursor, hardEnd) const paragraphBreak = window.lastIndexOf("\n\n") - if (paragraphBreak > chunkSize * 0.45) { + if (paragraphBreak > chunkSize * PARAGRAPH_BREAK_MIN_RATIO) { return cursor + paragraphBreak } @@ -61,12 +65,12 @@ function chooseChunkEnd(text: string, cursor: number, chunkSize: number): number window.lastIndexOf("? "), window.lastIndexOf("! "), ) - if (sentenceBreak > chunkSize * 0.55) { + if (sentenceBreak > chunkSize * SENTENCE_BREAK_MIN_RATIO) { return cursor + sentenceBreak + 1 } const whitespace = window.lastIndexOf(" ") - if (whitespace > chunkSize * 0.75) { + if (whitespace > chunkSize * WHITESPACE_BREAK_MIN_RATIO) { return cursor + whitespace } diff --git a/src/cli.ts b/packages/mimir/src/cli.ts similarity index 61% rename from src/cli.ts rename to packages/mimir/src/cli.ts index 23720a7..531320c 100644 --- a/src/cli.ts +++ b/packages/mimir/src/cli.ts @@ -12,6 +12,9 @@ import { bundledSkillPath, installSkill } from "./skill.js" import { countRows } from "./store.js" import { VERSION } from "./version.js" +const SEARCH_TEXT_PREVIEW_LENGTH = 900 +const TTS_PACKAGE_NAME = "@jcode.labs/mimir-tts" + const program = new Command() program @@ -71,13 +74,13 @@ program console.log( `\n${pc.cyan(`[${index + 1}] ${result.relativePath}`)} chunk=${result.chunkIndex} distance=${distance}`, ) - console.log(result.text.slice(0, 900)) + console.log(result.text.slice(0, SEARCH_TEXT_PREVIEW_LENGTH)) } }) program .command("ask") - .description("Answer a question using retrieved passages and a local Ollama model.") + .description("Return cited retrieval context for a question without calling an LLM.") .argument("", "Question to answer.") .option("-k, --top-k ", "Number of passages to use.", parsePositiveInt) .action(async (query: string, options: { topK?: number }) => { @@ -125,18 +128,20 @@ program console.log(`storageDir=${config.storageDir}`) console.log(`sourcesFile=${config.sourcesFile}`) console.log(`accessLogPath=${config.accessLogPath}`) - console.log(`networkPolicy=${config.networkPolicy}`) - console.log(`embedModel=${config.embedModel}`) - console.log(`llmModel=${config.llmModel}`) + console.log(`embeddingModelPath=${config.embeddingModelPath}`) + console.log(`embeddingProvider=${config.embeddingProvider}`) + console.log(`embeddingModel=${config.embeddingModel}`) + console.log(`transformersAllowRemoteModels=${config.transformersAllowRemoteModels}`) console.log(`redactionEnabled=${config.redaction.enabled}`) console.log(`accessLog=${config.accessLog}`) console.log(`mcpMaxTopK=${config.mcpMaxTopK}`) + console.log(`includeExtensions=${config.includeExtensions.join(",")}`) console.log(`chunksIndexed=${rows}`) }) program .command("security-audit") - .description("Show local privacy, network, redaction, MCP, and gitignore posture.") + .description("Show local privacy, provider, redaction, MCP, and gitignore posture.") .option("--json", "Print machine-readable JSON.") .option("--strict", "Exit with code 1 when warnings are present.") .action(async (options: { json?: boolean; strict?: boolean }) => { @@ -145,9 +150,11 @@ program console.log(JSON.stringify(report, null, 2)) } else { console.log(`zeroTelemetry=${report.zeroTelemetry}`) - console.log(`networkPolicy=${report.network.policy}`) - console.log(`ollamaHost=${report.network.ollamaHost}`) - console.log(`ollamaHostClassification=${report.network.classification}`) + console.log(`embeddingProvider=${report.providers.embedding}`) + console.log(`embeddingModel=${report.providers.embeddingModel}`) + console.log(`embeddingModelPath=${report.providers.embeddingModelPath}`) + console.log(`transformersAllowRemoteModels=${report.providers.transformersAllowRemoteModels}`) + console.log(`llmGeneration=${report.providers.llmGeneration}`) console.log(`redactionEnabled=${report.redaction.enabled}`) console.log(`redactionBuiltIn=${report.redaction.builtIn}`) console.log(`accessLog=${report.accessLog.enabled}`) @@ -181,6 +188,49 @@ program console.log(result.note) }) +program + .command("audio") + .description("Render a narration text file to local speech audio with Mimir TTS.") + .argument("[text-file]", "Narration text file to render.") + .option("-o, --out ", "Output WAV path.") + .option("--model ", "Transformers.js TTS model ID.") + .option("--model-path ", "Local model/cache path.") + .option("--offline", "Disable remote model downloads.") + .option("--allow-remote-models", "Explicitly allow remote model downloads.") + .option("--speaker-embeddings ", "Optional model-specific speaker embedding path or URL.") + .option("--speed ", "Optional model-specific speech speed.", parseNumber) + .option("--doctor", "Show TTS runtime readiness instead of rendering.") + .option("--json", "Print machine-readable JSON.") + .action(async (textFile: string | undefined, options: AudioOptions) => { + const tts = await loadTts() + + if (options.doctor) { + const report = await tts.doctor() + printMaybeJson(report, options.json) + return + } + + if (!textFile) { + console.error(pc.red("Missing text file. Use `kb audio `.")) + process.exitCode = 1 + return + } + + const renderOptions: TtsRenderOptions = { + cwd: process.cwd(), + textFile, + } + addOption(renderOptions, "outputPath", options.out) + addOption(renderOptions, "model", options.model) + addOption(renderOptions, "modelPath", options.modelPath) + addOption(renderOptions, "allowRemoteModels", audioAllowRemoteModels(options)) + addOption(renderOptions, "speakerEmbeddings", options.speakerEmbeddings) + addOption(renderOptions, "speed", options.speed) + + const result = await tts.renderSpeech(renderOptions) + printMaybeJson(result, options.json) + }) + program .command("serve-mcp") .description( @@ -212,6 +262,7 @@ program console.log(` - ${file}`) } console.log(`Skill path: ${result.skillPath}`) + console.log(`Optional audio skill path: ${result.audioSkillPath}`) console.log(`MCP config example: ${result.mcpConfigPath}`) }) @@ -225,6 +276,95 @@ function parsePositiveInt(value: string): number { return parsed } +function parseNumber(value: string): number { + const parsed = Number.parseFloat(value) + if (!Number.isFinite(parsed)) { + throw new Error("Expected a number.") + } + return parsed +} + function withTopK(topK: number | undefined): { cwd: string; topK?: number } { return topK === undefined ? { cwd: process.cwd() } : { cwd: process.cwd(), topK } } + +interface AudioOptions { + out?: string + model?: string + modelPath?: string + offline?: boolean + allowRemoteModels?: boolean + speakerEmbeddings?: string + speed?: number + doctor?: boolean + json?: boolean +} + +interface TtsModule { + doctor: () => Promise + renderSpeech: (options: TtsRenderOptions) => Promise +} + +interface TtsRenderOptions { + cwd: string + textFile: string + outputPath?: string + model?: string + modelPath?: string + allowRemoteModels?: boolean + speakerEmbeddings?: string + speed?: number +} + +async function loadTts(): Promise { + const module: unknown = await import(TTS_PACKAGE_NAME) + if (!isTtsModule(module)) { + throw new Error(`${TTS_PACKAGE_NAME} did not expose the expected TTS API.`) + } + return module +} + +function isTtsModule(value: unknown): value is TtsModule { + return ( + typeof value === "object" && + value !== null && + "doctor" in value && + typeof value.doctor === "function" && + "renderSpeech" in value && + typeof value.renderSpeech === "function" + ) +} + +function audioAllowRemoteModels(options: AudioOptions): boolean | undefined { + if (options.offline) { + return false + } + if (options.allowRemoteModels) { + return true + } + return undefined +} + +function printMaybeJson(value: unknown, json: boolean | undefined): void { + if (json) { + console.log(JSON.stringify(value, null, 2)) + return + } + if (typeof value === "object" && value !== null) { + for (const [key, entry] of Object.entries(value)) { + console.log(`${key}=${String(entry)}`) + } + return + } + console.log(String(value)) +} + +function addOption( + target: T, + key: K, + value: T[K] | undefined, +): void { + if (value !== undefined) { + target[key] = value + } +} diff --git a/packages/mimir/src/config.test.ts b/packages/mimir/src/config.test.ts new file mode 100644 index 0000000..1b8f737 --- /dev/null +++ b/packages/mimir/src/config.test.ts @@ -0,0 +1,121 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { afterEach, describe, expect, it } from "vitest" +import { findProjectRoot, loadConfig } from "./config.js" + +const tempDirs: string[] = [] + +afterEach(async () => { + for (const dir of tempDirs.splice(0)) { + await rm(dir, { recursive: true, force: true }) + } +}) + +describe("loadConfig", () => { + it("resolves project config upward and paths from the project root", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "jcode-kb-")) + tempDirs.push(root) + await writeFile(path.join(root, ".kb-config-placeholder"), "", "utf8") + await mkdir(path.join(root, ".kb"), { recursive: true }) + await writeFile( + path.join(root, ".kb/config.json"), + JSON.stringify({ rawDir: "docs", storageDir: ".kb/index" }), + "utf8", + ) + const nested = path.join(root, "packages/app") + await mkdir(nested, { recursive: true }) + + expect(findProjectRoot(nested)).toBe(root) + + const config = await loadConfig(nested) + expect(config.rawDir).toBe(path.join(root, "docs")) + expect(config.storageDir).toBe(path.join(root, ".kb/index")) + expect(config.accessLogPath).toBe(path.join(root, ".kb/access.log")) + expect(config.embeddingModelPath).toBe(path.join(root, ".mimir/models")) + expect(config.embeddingProvider).toBe("local-hash") + expect(config.embeddingModel).toBe("mixedbread-ai/mxbai-embed-xsmall-v1") + expect(config.transformersAllowRemoteModels).toBe(false) + expect(config.redaction.enabled).toBe(true) + expect(config.accessLog).toBe(true) + expect(config.mcpMaxTopK).toBe(10) + expect(config.includeExtensions).toEqual([]) + }) + + it("normalizes custom text extensions from config and env", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "jcode-kb-")) + tempDirs.push(root) + await mkdir(path.join(root, ".kb"), { recursive: true }) + await writeFile( + path.join(root, ".kb/config.json"), + JSON.stringify({ includeExtensions: ["transcript", ".Custom"] }), + "utf8", + ) + + const original = process.env.KB_INCLUDE_EXTENSIONS + process.env.KB_INCLUDE_EXTENSIONS = "logbook,.evidence" + try { + const config = await loadConfig(root) + expect(config.includeExtensions).toEqual([".evidence", ".logbook"]) + } finally { + if (original === undefined) { + delete process.env.KB_INCLUDE_EXTENSIONS + } else { + process.env.KB_INCLUDE_EXTENSIONS = original + } + } + }) + + it("loads provider overrides from config and env", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "jcode-kb-")) + tempDirs.push(root) + await mkdir(path.join(root, ".kb"), { recursive: true }) + await writeFile( + path.join(root, ".kb/config.json"), + JSON.stringify({ + embeddingProvider: "transformers", + embeddingModel: "example/local-embedder", + embeddingModelPath: ".mimir/custom-models", + transformersAllowRemoteModels: false, + }), + "utf8", + ) + + const originalEmbedding = process.env.KB_EMBEDDING_PROVIDER + const originalModel = process.env.KB_EMBEDDING_MODEL + const originalModelPath = process.env.KB_EMBEDDING_MODEL_PATH + const originalRemoteModels = process.env.KB_TRANSFORMERS_ALLOW_REMOTE_MODELS + process.env.KB_EMBEDDING_PROVIDER = "local-hash" + process.env.KB_EMBEDDING_MODEL = "example/env-embedder" + process.env.KB_EMBEDDING_MODEL_PATH = ".mimir/env-models" + process.env.KB_TRANSFORMERS_ALLOW_REMOTE_MODELS = "true" + try { + const config = await loadConfig(root) + expect(config.embeddingProvider).toBe("local-hash") + expect(config.embeddingModel).toBe("example/env-embedder") + expect(config.embeddingModelPath).toBe(path.join(root, ".mimir/env-models")) + expect(config.transformersAllowRemoteModels).toBe(true) + } finally { + if (originalEmbedding === undefined) { + delete process.env.KB_EMBEDDING_PROVIDER + } else { + process.env.KB_EMBEDDING_PROVIDER = originalEmbedding + } + if (originalModel === undefined) { + delete process.env.KB_EMBEDDING_MODEL + } else { + process.env.KB_EMBEDDING_MODEL = originalModel + } + if (originalModelPath === undefined) { + delete process.env.KB_EMBEDDING_MODEL_PATH + } else { + process.env.KB_EMBEDDING_MODEL_PATH = originalModelPath + } + if (originalRemoteModels === undefined) { + delete process.env.KB_TRANSFORMERS_ALLOW_REMOTE_MODELS + } else { + process.env.KB_TRANSFORMERS_ALLOW_REMOTE_MODELS = originalRemoteModels + } + } + }) +}) diff --git a/src/config.ts b/packages/mimir/src/config.ts similarity index 54% rename from src/config.ts rename to packages/mimir/src/config.ts index 0af5f4d..70e8efe 100644 --- a/src/config.ts +++ b/packages/mimir/src/config.ts @@ -2,22 +2,25 @@ import { existsSync } from "node:fs" import { readFile } from "node:fs/promises" import path from "node:path" import { z } from "zod" +import { CONFIG_PATH, DEFAULT_CONFIG } from "./defaults.js" import type { Config } from "./types.js" +const embeddingProviderSchema = z.enum(["local-hash", "transformers"]) + const rawConfigSchema = z.object({ - rawDir: z.string().default("private"), - storageDir: z.string().default(".kb/storage"), - sourcesFile: z.string().default(".kb/sources.txt"), - accessLogPath: z.string().default(".kb/access.log"), - tableName: z.string().default("chunks"), - ollamaHost: z.string().default("http://localhost:11434"), - networkPolicy: z.enum(["local-only", "allow-private", "allow-any"]).default("local-only"), - embedModel: z.string().default("nomic-embed-text"), - llmModel: z.string().default("gemma4:latest"), + rawDir: z.string().default(DEFAULT_CONFIG.rawDir), + storageDir: z.string().default(DEFAULT_CONFIG.storageDir), + sourcesFile: z.string().default(DEFAULT_CONFIG.sourcesFile), + accessLogPath: z.string().default(DEFAULT_CONFIG.accessLogPath), + embeddingModelPath: z.string().default(DEFAULT_CONFIG.embeddingModelPath), + tableName: z.string().default(DEFAULT_CONFIG.tableName), + embeddingProvider: embeddingProviderSchema.default(DEFAULT_CONFIG.embeddingProvider), + embeddingModel: z.string().default(DEFAULT_CONFIG.embeddingModel), + transformersAllowRemoteModels: z.boolean().default(DEFAULT_CONFIG.transformersAllowRemoteModels), redaction: z .object({ - enabled: z.boolean().default(true), - builtIn: z.boolean().default(true), + enabled: z.boolean().default(DEFAULT_CONFIG.redaction.enabled), + builtIn: z.boolean().default(DEFAULT_CONFIG.redaction.builtIn), patterns: z .array( z.object({ @@ -27,20 +30,19 @@ const rawConfigSchema = z.object({ replacement: z.string().optional(), }), ) - .default([]), + .default(DEFAULT_CONFIG.redaction.patterns), }) - .default({ enabled: true, builtIn: true, patterns: [] }), - accessLog: z.boolean().default(true), - mcpMaxTopK: z.number().int().positive().default(10), - topK: z.number().int().positive().default(5), - chunkSize: z.number().int().positive().default(1200), - chunkOverlap: z.number().int().nonnegative().default(150), + .default(DEFAULT_CONFIG.redaction), + accessLog: z.boolean().default(DEFAULT_CONFIG.accessLog), + mcpMaxTopK: z.number().int().positive().default(DEFAULT_CONFIG.mcpMaxTopK), + topK: z.number().int().positive().default(DEFAULT_CONFIG.topK), + chunkSize: z.number().int().positive().default(DEFAULT_CONFIG.chunkSize), + chunkOverlap: z.number().int().nonnegative().default(DEFAULT_CONFIG.chunkOverlap), + includeExtensions: z.array(z.string().min(1)).default(DEFAULT_CONFIG.includeExtensions), }) type RawConfig = z.infer -const CONFIG_PATH = ".kb/config.json" - export function findProjectRoot(start = process.cwd()): string { let current = path.resolve(start) @@ -60,9 +62,7 @@ export function findProjectRoot(start = process.cwd()): string { export async function loadConfig(start = process.cwd()): Promise { const projectRoot = findProjectRoot(start) const configFile = path.join(projectRoot, CONFIG_PATH) - const raw = existsSync(configFile) - ? (JSON.parse(await readFile(configFile, "utf8")) as unknown) - : {} + const raw: unknown = existsSync(configFile) ? JSON.parse(await readFile(configFile, "utf8")) : {} const parsed = rawConfigSchema.parse(raw) const withEnv = applyEnv(parsed) @@ -77,17 +77,18 @@ export async function loadConfig(start = process.cwd()): Promise { storageDir: resolveFromRoot(projectRoot, withEnv.storageDir), sourcesFile: resolveFromRoot(projectRoot, withEnv.sourcesFile), accessLogPath: resolveFromRoot(projectRoot, withEnv.accessLogPath), + embeddingModelPath: resolveFromRoot(projectRoot, withEnv.embeddingModelPath), tableName: withEnv.tableName, - ollamaHost: withEnv.ollamaHost, - networkPolicy: withEnv.networkPolicy, - embedModel: withEnv.embedModel, - llmModel: withEnv.llmModel, + embeddingProvider: withEnv.embeddingProvider, + embeddingModel: withEnv.embeddingModel, + transformersAllowRemoteModels: withEnv.transformersAllowRemoteModels, redaction: withEnv.redaction, accessLog: withEnv.accessLog, mcpMaxTopK: withEnv.mcpMaxTopK, topK: withEnv.topK, chunkSize: withEnv.chunkSize, chunkOverlap: withEnv.chunkOverlap, + includeExtensions: normalizeExtensions(withEnv.includeExtensions), } } @@ -102,10 +103,13 @@ function applyEnv(config: RawConfig): RawConfig { storageDir: process.env.KB_STORAGE_DIR ?? config.storageDir, sourcesFile: process.env.KB_SOURCES_FILE ?? config.sourcesFile, accessLogPath: process.env.KB_ACCESS_LOG_PATH ?? config.accessLogPath, - ollamaHost: process.env.KB_OLLAMA_HOST ?? config.ollamaHost, - networkPolicy: readNetworkPolicyEnv("KB_NETWORK_POLICY", config.networkPolicy), - embedModel: process.env.KB_EMBED_MODEL ?? config.embedModel, - llmModel: process.env.KB_LLM_MODEL ?? config.llmModel, + embeddingProvider: readEmbeddingProviderEnv("KB_EMBEDDING_PROVIDER", config.embeddingProvider), + embeddingModel: process.env.KB_EMBEDDING_MODEL ?? config.embeddingModel, + embeddingModelPath: process.env.KB_EMBEDDING_MODEL_PATH ?? config.embeddingModelPath, + transformersAllowRemoteModels: readBooleanEnv( + "KB_TRANSFORMERS_ALLOW_REMOTE_MODELS", + config.transformersAllowRemoteModels, + ), redaction: { ...config.redaction, enabled: readBooleanEnv("KB_REDACTION_ENABLED", config.redaction.enabled), @@ -116,18 +120,27 @@ function applyEnv(config: RawConfig): RawConfig { topK: readPositiveIntEnv("KB_TOP_K", config.topK), chunkSize: readPositiveIntEnv("KB_CHUNK_SIZE", config.chunkSize), chunkOverlap: readNonNegativeIntEnv("KB_CHUNK_OVERLAP", config.chunkOverlap), + includeExtensions: readExtensionsEnv("KB_INCLUDE_EXTENSIONS", config.includeExtensions), } } -function readNetworkPolicyEnv( +function normalizeExtensions(extensions: string[]): string[] { + return [ + ...new Set( + extensions + .map((extension) => extension.trim().toLowerCase()) + .filter(Boolean) + .map((extension) => (extension.startsWith(".") ? extension : `.${extension}`)), + ), + ].sort() +} + +function readEmbeddingProviderEnv( name: string, - fallback: RawConfig["networkPolicy"], -): RawConfig["networkPolicy"] { - const raw = process.env[name] - if (raw === "local-only" || raw === "allow-private" || raw === "allow-any") { - return raw - } - return fallback + fallback: RawConfig["embeddingProvider"], +): RawConfig["embeddingProvider"] { + const parsed = embeddingProviderSchema.safeParse(process.env[name]) + return parsed.success ? parsed.data : fallback } function readBooleanEnv(name: string, fallback: boolean): boolean { @@ -158,3 +171,11 @@ function readNonNegativeIntEnv(name: string, fallback: number): number { const value = Number.parseInt(raw, 10) return Number.isInteger(value) && value >= 0 ? value : fallback } + +function readExtensionsEnv(name: string, fallback: string[]): string[] { + const raw = process.env[name] + if (!raw) { + return fallback + } + return raw.split(",") +} diff --git a/packages/mimir/src/defaults.ts b/packages/mimir/src/defaults.ts new file mode 100644 index 0000000..784f521 --- /dev/null +++ b/packages/mimir/src/defaults.ts @@ -0,0 +1,34 @@ +import type { Config } from "./types.js" + +export const KB_DIR = ".kb" +export const MIMIR_DIR = ".mimir" +export const PRIVATE_DIR = "private" +export const CONFIG_PATH = `${KB_DIR}/config.json` +export const DEFAULT_SKILL_TARGET_DIR = `${MIMIR_DIR}/skills` + +export const KB_GITIGNORE_ENTRY = `${KB_DIR}/` +export const MIMIR_GITIGNORE_ENTRY = `${MIMIR_DIR}/` +export const PRIVATE_GITIGNORE_ENTRY = `${PRIVATE_DIR}/**` + +export const DEFAULT_CONFIG: Omit = { + rawDir: PRIVATE_DIR, + storageDir: `${KB_DIR}/storage`, + sourcesFile: `${KB_DIR}/sources.txt`, + accessLogPath: `${KB_DIR}/access.log`, + embeddingModelPath: `${MIMIR_DIR}/models`, + tableName: "chunks", + embeddingProvider: "local-hash", + embeddingModel: "mixedbread-ai/mxbai-embed-xsmall-v1", + transformersAllowRemoteModels: false, + redaction: { + enabled: true, + builtIn: true, + patterns: [], + }, + accessLog: true, + mcpMaxTopK: 10, + topK: 5, + chunkSize: 1200, + chunkOverlap: 150, + includeExtensions: [], +} diff --git a/src/destroy.ts b/packages/mimir/src/destroy.ts similarity index 100% rename from src/destroy.ts rename to packages/mimir/src/destroy.ts diff --git a/packages/mimir/src/embeddings.test.ts b/packages/mimir/src/embeddings.test.ts new file mode 100644 index 0000000..7dc33d2 --- /dev/null +++ b/packages/mimir/src/embeddings.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from "vitest" +import { embedText, embedTexts } from "./embeddings.js" +import { testConfig } from "./test-support/config.js" + +describe("local hash embeddings", () => { + it("creates deterministic normalized embeddings without a model runtime", async () => { + const config = testConfig() + + const first = await embedText("offline model approval", config) + const second = await embedText("offline model approval", config) + const batch = await embedTexts(["offline model approval", "dataset residency"], config) + + expect(first).toHaveLength(384) + expect(second).toEqual(first) + expect(batch).toHaveLength(2) + expect(Math.round(vectorMagnitude(first) * 1000) / 1000).toBe(1) + }) +}) + +function vectorMagnitude(vector: number[]): number { + return Math.sqrt(vector.reduce((sum, value) => sum + value * value, 0)) +} diff --git a/packages/mimir/src/embeddings.ts b/packages/mimir/src/embeddings.ts new file mode 100644 index 0000000..a1705fb --- /dev/null +++ b/packages/mimir/src/embeddings.ts @@ -0,0 +1,133 @@ +import { createHash } from "node:crypto" +import type { Config } from "./types.js" + +const LOCAL_HASH_DIMENSIONS = 384 +const LONG_TOKEN_MIN_LENGTH = 6 +const LONG_TOKEN_WEIGHT = 1.4 +const transformersPipelines = new Map() + +type TransformersExtractor = ( + texts: string[], + options: { pooling: "mean"; normalize: true }, +) => Promise + +export async function embedTexts(texts: string[], config: Config): Promise { + if (texts.length === 0) { + return [] + } + + if (config.embeddingProvider === "local-hash") { + return texts.map(localHashEmbedding) + } + + return embedWithTransformers(texts, config) +} + +export async function embedText(text: string, config: Config): Promise { + const [embedding] = await embedTexts([text], config) + if (!embedding) { + throw new Error("No embedding returned for query.") + } + return embedding +} + +async function embedWithTransformers(texts: string[], config: Config): Promise { + const extractor = await transformersExtractor(config) + const output = await extractor(texts, { pooling: "mean", normalize: true }) + const rows = tensorToEmbeddingRows(output) + + if (rows.length !== texts.length) { + throw new Error(`Expected ${texts.length} embeddings, received ${rows.length}.`) + } + + return rows +} + +async function transformersExtractor(config: Config): Promise { + const key = [ + config.embeddingModel, + config.embeddingModelPath, + String(config.transformersAllowRemoteModels), + ].join("\n") + const cached = transformersPipelines.get(key) + if (cached) { + return cached + } + + const transformers = await import("@huggingface/transformers") + transformers.env.localModelPath = config.embeddingModelPath + transformers.env.cacheDir = config.embeddingModelPath + transformers.env.allowRemoteModels = config.transformersAllowRemoteModels + + const extractor = (await transformers.pipeline( + "feature-extraction", + config.embeddingModel, + )) as TransformersExtractor + transformersPipelines.set(key, extractor) + return extractor +} + +function localHashEmbedding(text: string): number[] { + const vector = Array.from({ length: LOCAL_HASH_DIMENSIONS }, () => 0) + const tokens = tokenize(text) + + for (const token of tokens) { + const hash = createHash("sha256").update(token).digest() + const index = hash.readUInt32BE(0) % LOCAL_HASH_DIMENSIONS + const sign = (hash.at(4) ?? 0) % 2 === 0 ? 1 : -1 + vector[index] = (vector[index] ?? 0) + sign * tokenWeight(token) + } + + const magnitude = Math.sqrt(vector.reduce((sum, value) => sum + value * value, 0)) + if (magnitude === 0) { + return vector + } + return vector.map((value) => value / magnitude) +} + +function tokenize(text: string): string[] { + return ( + text + .toLowerCase() + .normalize("NFKD") + .replace(/\p{Diacritic}/gu, "") + .match(/[\p{L}\p{N}]{2,}/gu) ?? [] + ) +} + +function tokenWeight(token: string): number { + return token.length >= LONG_TOKEN_MIN_LENGTH ? LONG_TOKEN_WEIGHT : 1 +} + +function tensorToEmbeddingRows(output: unknown): number[][] { + if (!hasToList(output)) { + throw new Error("Transformers embedding output does not expose tolist().") + } + + const value = output.tolist() + if (isNumberMatrix(value)) { + return value + } + if (isNumberArray(value)) { + return [value] + } + + throw new Error("Transformers embedding output is not a numeric vector matrix.") +} + +function hasToList(value: unknown): value is { tolist: () => unknown } { + return ( + typeof value === "object" && + value !== null && + "tolist" in value && + typeof value.tolist === "function" + ) +} + +function isNumberArray(value: unknown): value is number[] { + return Array.isArray(value) && value.every((item) => typeof item === "number") +} + +function isNumberMatrix(value: unknown): value is number[][] { + return Array.isArray(value) && value.every(isNumberArray) +} diff --git a/packages/mimir/src/files.test.ts b/packages/mimir/src/files.test.ts new file mode 100644 index 0000000..0ba89a8 --- /dev/null +++ b/packages/mimir/src/files.test.ts @@ -0,0 +1,39 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { afterEach, describe, expect, it } from "vitest" +import { listSourceFiles } from "./files.js" +import { testConfig } from "./test-support/config.js" + +const tempDirs: string[] = [] + +afterEach(async () => { + for (const dir of tempDirs.splice(0)) { + await rm(dir, { recursive: true, force: true }) + } +}) + +describe("listSourceFiles", () => { + it("indexes broad text-like formats and custom extensions while ignoring runtime folders", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "mimir-files-")) + tempDirs.push(root) + + await mkdir(path.join(root, "private", "nested"), { recursive: true }) + await mkdir(path.join(root, "private", ".kb"), { recursive: true }) + await mkdir(path.join(root, "private", ".mimir"), { recursive: true }) + await writeFile(path.join(root, "private", "events.jsonl"), '{"event":"login"}\n', "utf8") + await writeFile(path.join(root, "private", "schema.sql"), "select 1;\n", "utf8") + await writeFile(path.join(root, "private", "notes.transcript"), "call notes\n", "utf8") + await writeFile(path.join(root, "private", "image.png"), "not indexed\n", "utf8") + await writeFile(path.join(root, "private", ".kb", "index.md"), "ignored\n", "utf8") + await writeFile(path.join(root, "private", ".mimir", "agent.md"), "ignored\n", "utf8") + + const files = await listSourceFiles(testConfig(root, { includeExtensions: [".transcript"] })) + + expect(files.map((file) => file.relativePath)).toEqual([ + "private/events.jsonl", + "private/notes.transcript", + "private/schema.sql", + ]) + }) +}) diff --git a/src/files.ts b/packages/mimir/src/files.ts similarity index 76% rename from src/files.ts rename to packages/mimir/src/files.ts index 3960611..39e165b 100644 --- a/src/files.ts +++ b/packages/mimir/src/files.ts @@ -5,17 +5,51 @@ import path from "node:path" import fg from "fast-glob" import type { Config, SourceFile } from "./types.js" -export const SUPPORTED_EXTENSIONS = new Set([ +export const DEFAULT_SUPPORTED_EXTENSIONS = new Set([ + ".atom", + ".c", + ".cfg", + ".conf", + ".cpp", + ".cs", + ".css", ".csv", + ".docx", + ".go", + ".h", ".htm", ".html", + ".ini", + ".java", + ".js", ".json", + ".jsonl", + ".jsx", + ".log", ".md", ".mdx", + ".ndjson", + ".odp", + ".ods", + ".odt", ".pdf", + ".php", + ".pptx", + ".properties", + ".py", + ".rb", + ".rs", + ".rss", + ".rtf", + ".sql", ".text", + ".toml", + ".ts", ".tsv", + ".tsx", ".txt", + ".xml", + ".xlsx", ".yaml", ".yml", ]) @@ -40,7 +74,7 @@ export async function listSourceFiles(config: Config): Promise { for (const absolutePath of entries) { const extension = path.extname(absolutePath).toLowerCase() - if (!SUPPORTED_EXTENSIONS.has(extension)) { + if (!supportedExtensions(config).has(extension)) { continue } @@ -61,6 +95,10 @@ export async function listSourceFiles(config: Config): Promise { return [...files.values()].sort((a, b) => a.relativePath.localeCompare(b.relativePath)) } +export function supportedExtensions(config: Config): Set { + return new Set([...DEFAULT_SUPPORTED_EXTENSIONS, ...config.includeExtensions]) +} + async function sourceRoots(config: Config): Promise { const roots = [config.rawDir] if (!existsSync(config.sourcesFile)) { diff --git a/src/gitignore.ts b/packages/mimir/src/gitignore.ts similarity index 77% rename from src/gitignore.ts rename to packages/mimir/src/gitignore.ts index 46c8ab5..5998952 100644 --- a/src/gitignore.ts +++ b/packages/mimir/src/gitignore.ts @@ -1,16 +1,22 @@ import { existsSync } from "node:fs" import { readFile, writeFile } from "node:fs/promises" import path from "node:path" +import { + KB_GITIGNORE_ENTRY, + MIMIR_GITIGNORE_ENTRY, + PRIVATE_DIR, + PRIVATE_GITIGNORE_ENTRY, +} from "./defaults.js" export const MIMIR_GITIGNORE_ENTRIES = [ - ".kb/", - ".mimir/", - "private/**", - "!private/", - "!private/README.md", - "!private/**/", - "!private/**/.gitkeep", -] as const + KB_GITIGNORE_ENTRY, + MIMIR_GITIGNORE_ENTRY, + PRIVATE_GITIGNORE_ENTRY, + `!${PRIVATE_DIR}/`, + `!${PRIVATE_DIR}/README.md`, + `!${PRIVATE_DIR}/**/`, + `!${PRIVATE_DIR}/**/.gitkeep`, +] export async function ensureMimirGitignore(cwd = process.cwd()): Promise { const root = path.resolve(cwd) diff --git a/src/index.ts b/packages/mimir/src/index.ts similarity index 100% rename from src/index.ts rename to packages/mimir/src/index.ts diff --git a/src/ingest.ts b/packages/mimir/src/ingest.ts similarity index 96% rename from src/ingest.ts rename to packages/mimir/src/ingest.ts index e936a4b..419051a 100644 --- a/src/ingest.ts +++ b/packages/mimir/src/ingest.ts @@ -16,6 +16,7 @@ import type { } from "./types.js" const EMBED_BATCH_SIZE = 32 +const MAX_AUDIT_ROWS = 100_000 export async function ingest(options: IngestOptions = {}): Promise { const config = await loadConfig(String(options.cwd ?? process.cwd())) @@ -95,7 +96,9 @@ export async function audit(cwd = process.cwd()): Promise { } } - const rows = (await table.query().limit(100000).toArray()) as Array<{ relativePath: string }> + const rows = (await table.query().limit(MAX_AUDIT_ROWS).toArray()) as Array<{ + relativePath: string + }> const counts = new Map() for (const row of rows) { counts.set(row.relativePath, (counts.get(row.relativePath) ?? 0) + 1) diff --git a/src/init.ts b/packages/mimir/src/init.ts similarity index 67% rename from src/init.ts rename to packages/mimir/src/init.ts index 7d76c81..3ad2a69 100644 --- a/src/init.ts +++ b/packages/mimir/src/init.ts @@ -1,40 +1,19 @@ import { existsSync } from "node:fs" import { mkdir, writeFile } from "node:fs/promises" import path from "node:path" +import { CONFIG_PATH, DEFAULT_CONFIG, KB_DIR, PRIVATE_DIR } from "./defaults.js" import { ensureMimirGitignore } from "./gitignore.js" -const DEFAULT_CONFIG = { - rawDir: "private", - storageDir: ".kb/storage", - sourcesFile: ".kb/sources.txt", - accessLogPath: ".kb/access.log", - tableName: "chunks", - ollamaHost: "http://localhost:11434", - networkPolicy: "local-only", - embedModel: "nomic-embed-text", - llmModel: "gemma4:latest", - redaction: { - enabled: true, - builtIn: true, - patterns: [], - }, - accessLog: true, - mcpMaxTopK: 10, - topK: 5, - chunkSize: 1200, - chunkOverlap: 150, -} - export async function initProject(cwd = process.cwd()): Promise { const root = path.resolve(cwd) - const kbDir = path.join(root, ".kb") - const privateDir = path.join(root, "private") + const kbDir = path.join(root, KB_DIR) + const privateDir = path.join(root, PRIVATE_DIR) const created: string[] = [] await mkdir(kbDir, { recursive: true }) await mkdir(privateDir, { recursive: true }) - const configPath = path.join(kbDir, "config.json") + const configPath = path.join(root, CONFIG_PATH) if (!existsSync(configPath)) { await writeFile(configPath, `${JSON.stringify(DEFAULT_CONFIG, null, 2)}\n`, "utf8") created.push(path.relative(root, configPath)) diff --git a/src/mcp.ts b/packages/mimir/src/mcp.ts similarity index 79% rename from src/mcp.ts rename to packages/mimir/src/mcp.ts index 3bf67b8..0e4fd96 100644 --- a/src/mcp.ts +++ b/packages/mimir/src/mcp.ts @@ -8,6 +8,11 @@ import { securityAudit } from "./security.js" import { countRows } from "./store.js" import { VERSION } from "./version.js" +const queryToolInputSchema = z.object({ + query: z.string().min(1), + topK: z.number().int().positive().optional(), +}) + export async function serveMcp(cwd = process.cwd()): Promise { const server = new McpServer({ name: "mimir", @@ -29,11 +34,14 @@ export async function serveMcp(cwd = process.cwd()): Promise { rawDir: config.rawDir, storageDir: config.storageDir, sourcesFile: config.sourcesFile, - embedModel: config.embedModel, - llmModel: config.llmModel, - networkPolicy: config.networkPolicy, + embeddingProvider: config.embeddingProvider, + embeddingModel: config.embeddingModel, + embeddingModelPath: config.embeddingModelPath, + transformersAllowRemoteModels: config.transformersAllowRemoteModels, + llmGeneration: false, redactionEnabled: config.redaction.enabled, mcpMaxTopK: config.mcpMaxTopK, + includeExtensions: config.includeExtensions, chunksIndexed, } @@ -46,10 +54,7 @@ export async function serveMcp(cwd = process.cwd()): Promise { { title: "Mimir Search", description: "Retrieve relevant passages from the local Mimir knowledge base.", - inputSchema: z.object({ - query: z.string().min(1), - topK: z.number().int().positive().optional(), - }), + inputSchema: queryToolInputSchema, }, async ({ query, topK }) => textResult(await search(query, await searchOptions(cwd, topK))), ) @@ -58,12 +63,8 @@ export async function serveMcp(cwd = process.cwd()): Promise { "mimir_ask", { title: "Mimir Ask", - description: - "Answer a question using local retrieved passages and the configured Ollama model.", - inputSchema: z.object({ - query: z.string().min(1), - topK: z.number().int().positive().optional(), - }), + description: "Return cited retrieval context for a question without calling an LLM.", + inputSchema: queryToolInputSchema, }, async ({ query, topK }) => textResult(await ask(query, await searchOptions(cwd, topK))), ) @@ -82,7 +83,7 @@ export async function serveMcp(cwd = process.cwd()): Promise { "mimir_security_audit", { title: "Mimir Security Audit", - description: "Show local privacy, network, redaction, MCP, and gitignore posture.", + description: "Show local privacy, provider, redaction, MCP, and gitignore posture.", inputSchema: z.object({}), }, async () => textResult(await securityAudit(cwd)), diff --git a/packages/mimir/src/parsing.test.ts b/packages/mimir/src/parsing.test.ts new file mode 100644 index 0000000..5a64c6e --- /dev/null +++ b/packages/mimir/src/parsing.test.ts @@ -0,0 +1,66 @@ +import { mkdtemp, rm, writeFile } from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { strToU8, zipSync } from "fflate" +import { afterEach, describe, expect, it } from "vitest" +import { parseFile } from "./parsing.js" +import type { SourceFile } from "./types.js" + +const tempDirs: string[] = [] + +afterEach(async () => { + for (const dir of tempDirs.splice(0)) { + await rm(dir, { recursive: true, force: true }) + } +}) + +describe("parseFile", () => { + it("extracts text from docx files", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "mimir-docx-")) + tempDirs.push(root) + const filePath = path.join(root, "brief.docx") + await writeFile( + filePath, + zipSync({ + "word/document.xml": strToU8( + "Confidential briefing", + ), + }), + ) + + const parsed = await parseFile(sourceFile(root, filePath, ".docx")) + + expect(parsed.text).toContain("Confidential briefing") + }) + + it("extracts shared strings and values from xlsx files", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "mimir-xlsx-")) + tempDirs.push(root) + const filePath = path.join(root, "dataset.xlsx") + await writeFile( + filePath, + zipSync({ + "xl/sharedStrings.xml": strToU8("InvoicePaid"), + "xl/worksheets/sheet1.xml": strToU8( + '0240001', + ), + }), + ) + + const parsed = await parseFile(sourceFile(root, filePath, ".xlsx")) + + expect(parsed.text).toContain("Invoice\t24000\tPaid") + }) +}) + +function sourceFile(root: string, absolutePath: string, extension: string): SourceFile { + return { + absolutePath, + relativePath: path.relative(root, absolutePath), + source: path.basename(absolutePath), + extension, + bytes: 0, + mtimeMs: 0, + checksum: "test", + } +} diff --git a/packages/mimir/src/parsing.ts b/packages/mimir/src/parsing.ts new file mode 100644 index 0000000..a735656 --- /dev/null +++ b/packages/mimir/src/parsing.ts @@ -0,0 +1,210 @@ +import { readFile } from "node:fs/promises" +import { strFromU8, unzipSync } from "fflate" +import { htmlToText } from "html-to-text" +import { extractText, getDocumentProxy } from "unpdf" +import YAML from "yaml" +import type { ParsedDocument, SourceFile } from "./types.js" + +const MAX_OFFICE_XML_ENTRY_BYTES = 25_000_000 + +export async function parseFile(file: SourceFile): Promise { + let text: string + + switch (file.extension) { + case ".pdf": + text = await parsePdf(file.absolutePath) + break + case ".docx": + text = await parseDocx(file.absolutePath) + break + case ".pptx": + text = await parsePptx(file.absolutePath) + break + case ".xlsx": + text = await parseXlsx(file.absolutePath) + break + case ".odt": + case ".ods": + case ".odp": + text = await parseOpenDocument(file.absolutePath) + break + case ".html": + case ".htm": + text = htmlToText(await readFile(file.absolutePath, "utf8"), { + wordwrap: false, + selectors: [ + { selector: "a", options: { ignoreHref: true } }, + { selector: "img", format: "skip" }, + ], + }) + break + case ".json": + text = JSON.stringify(JSON.parse(await readFile(file.absolutePath, "utf8")), null, 2) + break + case ".yaml": + case ".yml": + text = YAML.stringify(YAML.parse(await readFile(file.absolutePath, "utf8"))) + break + case ".rtf": + text = stripRtf(await readFile(file.absolutePath, "utf8")) + break + default: + text = await readFile(file.absolutePath, "utf8") + } + + return { file, text: normalizeText(text) } +} + +async function parseDocx(filePath: string): Promise { + const entries = unzipOfficeFile(await readFile(filePath)) + return xmlEntriesToText(entries, [ + /^word\/document\.xml$/u, + /^word\/header\d*\.xml$/u, + /^word\/footer\d*\.xml$/u, + /^word\/footnotes\.xml$/u, + /^word\/endnotes\.xml$/u, + /^word\/comments\.xml$/u, + ]) +} + +async function parsePptx(filePath: string): Promise { + const entries = unzipOfficeFile(await readFile(filePath)) + return xmlEntriesToText(entries, [ + /^ppt\/slides\/slide\d+\.xml$/u, + /^ppt\/notesSlides\/notesSlide\d+\.xml$/u, + ]) +} + +async function parseXlsx(filePath: string): Promise { + const entries = unzipOfficeFile(await readFile(filePath)) + const sharedStrings = parseSharedStrings(entries.get("xl/sharedStrings.xml") ?? "") + const sheets = [...entries.entries()] + .filter(([name]) => /^xl\/worksheets\/sheet\d+\.xml$/u.test(name)) + .sort(([a], [b]) => a.localeCompare(b)) + + const rows: string[] = [] + for (const [name, xml] of sheets) { + const values = parseSheetValues(xml, sharedStrings) + if (values.length > 0) { + rows.push(`# ${name}`, values.join("\n")) + } + } + return rows.join("\n\n") +} + +async function parseOpenDocument(filePath: string): Promise { + const entries = unzipOfficeFile(await readFile(filePath)) + return xmlEntriesToText(entries, [/^content\.xml$/u, /^meta\.xml$/u]) +} + +function unzipOfficeFile(buffer: Buffer): Map { + const unzipped = unzipSync(new Uint8Array(buffer), { + filter: (file) => file.originalSize <= MAX_OFFICE_XML_ENTRY_BYTES, + }) + const entries = new Map() + for (const [name, content] of Object.entries(unzipped)) { + if (name.endsWith(".xml")) { + entries.set(name, strFromU8(content)) + } + } + return entries +} + +function xmlEntriesToText(entries: Map, patterns: RegExp[]): string { + const parts: string[] = [] + for (const [name, xml] of [...entries.entries()].sort(([a], [b]) => a.localeCompare(b))) { + if (patterns.some((pattern) => pattern.test(name))) { + const text = xmlToText(xml) + if (text) { + parts.push(text) + } + } + } + return parts.join("\n\n") +} + +function parseSharedStrings(xml: string): string[] { + return [...xml.matchAll(//gu)].map(([item]) => xmlToText(item)) +} + +function parseSheetValues(xml: string, sharedStrings: string[]): string[] { + const rows: string[] = [] + for (const rowMatch of xml.matchAll(//gu)) { + const rowXml = rowMatch[0] + const values = [...rowXml.matchAll(/]*)>([\s\S]*?)<\/c>/gu)] + .map((cellMatch) => { + const attrs = cellMatch[1] ?? "" + const cellXml = cellMatch[2] ?? "" + const inline = firstMatch(cellXml, //u) + if (inline) { + return xmlToText(inline) + } + + const rawValue = firstMatch(cellXml, /([\s\S]*?)<\/v>/u) + if (!rawValue) { + return "" + } + + if (/\bt="s"/u.test(attrs)) { + return sharedStrings[Number.parseInt(rawValue, 10)] ?? "" + } + return decodeXmlEntities(rawValue) + }) + .filter(Boolean) + + if (values.length > 0) { + rows.push(values.join("\t")) + } + } + return rows +} + +function firstMatch(input: string, pattern: RegExp): string { + const match = input.match(pattern) + return match?.[1] ?? match?.[0] ?? "" +} + +function xmlToText(xml: string): string { + return normalizeText( + decodeXmlEntities( + xml + .replace(//gu, " ") + .replace(//gu, "\n") + .replace(/<\/(?:w:p|a:p|text:p|text:h|table:table-row)>/gu, "\n") + .replace(/<[^>]+>/gu, " ") + .replace(/[ \t]{2,}/gu, " "), + ), + ) +} + +function stripRtf(input: string): string { + return input + .replace(/\\par[d]?/gu, "\n") + .replace(/\\'[0-9a-fA-F]{2}/gu, " ") + .replace(/\\[a-zA-Z]+-?\d* ?/gu, " ") + .replace(/[{}]/gu, " ") +} + +function decodeXmlEntities(input: string): string { + return input + .replace(/</gu, "<") + .replace(/>/gu, ">") + .replace(/"/gu, '"') + .replace(/'/gu, "'") + .replace(/&/gu, "&") +} + +async function parsePdf(filePath: string): Promise { + const buffer = await readFile(filePath) + const pdf = await getDocumentProxy(new Uint8Array(buffer)) + const result = await extractText(pdf, { mergePages: true }) + return result.text +} + +function normalizeText(input: string): string { + return input + .replace(/\r\n/g, "\n") + .replace(/[ \t]+\n/g, "\n") + .replace(/\n{4,}/g, "\n\n\n") + .trim() +} diff --git a/src/query.ts b/packages/mimir/src/query.ts similarity index 68% rename from src/query.ts rename to packages/mimir/src/query.ts index 93fb9ce..61eb596 100644 --- a/src/query.ts +++ b/packages/mimir/src/query.ts @@ -1,8 +1,6 @@ -import { Ollama } from "ollama" import { recordAccess } from "./access-log.js" import { loadConfig } from "./config.js" import { embedText } from "./embeddings.js" -import { assertNetworkPolicy } from "./network.js" import { openRowsTable } from "./store.js" import type { AskResult, SearchOptions, SearchResult } from "./types.js" @@ -54,31 +52,6 @@ export async function ask(query: string, options: SearchOptions = {}): Promise - `[${index + 1}] ${source.relativePath}#${source.chunkIndex}\n${source.text}`, - ) - .join("\n\n---\n\n") - - assertNetworkPolicy(config) - const client = new Ollama({ host: config.ollamaHost }) - const response = await client.chat({ - model: config.llmModel, - messages: [ - { - role: "system", - content: - "Answer only from the provided context. If the context is insufficient, say what is missing. Cite sources with [1], [2], etc.", - }, - { - role: "user", - content: `Question:\n${query}\n\nContext:\n${context}`, - }, - ], - stream: false, - }) - await recordAccess(config, { action: "ask", query, @@ -87,7 +60,22 @@ export async function ask(query: string, options: SearchOptions = {}): Promise { + const text = source.text.replace(/\s+/gu, " ").trim() + return `[${index + 1}] ${source.relativePath}#${source.chunkIndex}: ${text}` + }) + .join("\n\n") + + return [ + "Mimir returns retrieval context only. Use these passages as grounded context for your agent or LLM:", + "", + snippets, + ].join("\n") +} diff --git a/src/redaction.test.ts b/packages/mimir/src/redaction.test.ts similarity index 54% rename from src/redaction.test.ts rename to packages/mimir/src/redaction.test.ts index 421a137..325500b 100644 --- a/src/redaction.test.ts +++ b/packages/mimir/src/redaction.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest" import { redactText } from "./redaction.js" -import type { Config } from "./types.js" +import { testConfig } from "./test-support/config.js" describe("redactText", () => { it("redacts built-in sensitive identifiers before indexing", () => { @@ -24,25 +24,3 @@ describe("redactText", () => { expect(redactText("CLIENT-12345", config).text).toBe("[CLIENT]") }) }) - -function testConfig(overrides: Partial = {}): Config { - return { - projectRoot: "/tmp/project", - rawDir: "/tmp/project/private", - storageDir: "/tmp/project/.kb/storage", - sourcesFile: "/tmp/project/.kb/sources.txt", - accessLogPath: "/tmp/project/.kb/access.log", - tableName: "chunks", - ollamaHost: "http://localhost:11434", - networkPolicy: "local-only", - embedModel: "nomic-embed-text", - llmModel: "gemma4:latest", - redaction: { enabled: true, builtIn: true, patterns: [] }, - accessLog: true, - mcpMaxTopK: 10, - topK: 5, - chunkSize: 1200, - chunkOverlap: 150, - ...overrides, - } -} diff --git a/src/redaction.ts b/packages/mimir/src/redaction.ts similarity index 100% rename from src/redaction.ts rename to packages/mimir/src/redaction.ts diff --git a/src/security.ts b/packages/mimir/src/security.ts similarity index 65% rename from src/security.ts rename to packages/mimir/src/security.ts index 7346d9e..2ba4737 100644 --- a/src/security.ts +++ b/packages/mimir/src/security.ts @@ -2,46 +2,45 @@ import { existsSync } from "node:fs" import { readFile } from "node:fs/promises" import path from "node:path" import { loadConfig } from "./config.js" -import { classifyHost } from "./network.js" +import { KB_GITIGNORE_ENTRY, MIMIR_GITIGNORE_ENTRY, PRIVATE_GITIGNORE_ENTRY } from "./defaults.js" import type { SecurityAuditReport } from "./types.js" export async function securityAudit(cwd = process.cwd()): Promise { const config = await loadConfig(cwd) const gitignore = await readGitignore(config.projectRoot) - const network = classifyHost(config.ollamaHost) const warnings: string[] = [] - const kbIgnored = hasGitignoreEntry(gitignore, ".kb/") - const mimirIgnored = hasGitignoreEntry(gitignore, ".mimir/") - const privateIgnored = hasGitignoreEntry(gitignore, "private/**") + const kbIgnored = hasGitignoreEntry(gitignore, KB_GITIGNORE_ENTRY) + const mimirIgnored = hasGitignoreEntry(gitignore, MIMIR_GITIGNORE_ENTRY) + const privateIgnored = hasGitignoreEntry(gitignore, PRIVATE_GITIGNORE_ENTRY) - if (config.networkPolicy === "allow-any") { - warnings.push("networkPolicy is allow-any; document text can be sent to a remote Ollama host.") - } - if (config.networkPolicy === "local-only" && network.kind !== "loopback") { - warnings.push("networkPolicy is local-only but ollamaHost is not loopback.") + if (config.embeddingProvider === "transformers" && config.transformersAllowRemoteModels) { + warnings.push( + "Transformers remote model loading is enabled; model files can be downloaded from Hugging Face.", + ) } if (!config.redaction.enabled) { warnings.push("Redaction is disabled; secrets and identifiers may be embedded in the index.") } if (!kbIgnored) { - warnings.push(".kb/ is not ignored by Git.") + warnings.push(`${KB_GITIGNORE_ENTRY} is not ignored by Git.`) } if (!mimirIgnored) { - warnings.push(".mimir/ is not ignored by Git.") + warnings.push(`${MIMIR_GITIGNORE_ENTRY} is not ignored by Git.`) } if (!privateIgnored) { - warnings.push("private/** is not ignored by Git.") + warnings.push(`${PRIVATE_GITIGNORE_ENTRY} is not ignored by Git.`) } return { projectRoot: config.projectRoot, zeroTelemetry: true, - network: { - policy: config.networkPolicy, - ollamaHost: config.ollamaHost, - host: network.host, - classification: network.kind, + providers: { + embedding: config.embeddingProvider, + embeddingModel: config.embeddingModel, + embeddingModelPath: config.embeddingModelPath, + transformersAllowRemoteModels: config.transformersAllowRemoteModels, + llmGeneration: false, }, redaction: { enabled: config.redaction.enabled, @@ -71,6 +70,7 @@ export async function securityAudit(cwd = process.cwd()): Promise { const result = await installSkill({ cwd: root }) const skill = await readFile(path.join(result.skillPath, "SKILL.md"), "utf8") + const audioSkill = await readFile(path.join(result.audioSkillPath, "SKILL.md"), "utf8") const mcpConfig = JSON.parse(await readFile(result.mcpConfigPath, "utf8")) as { mcpServers: { mimir: { command: string; args: string[]; cwd: string } } } expect(skill).toContain("name: mimir") + expect(audioSkill).toContain("name: mimir-audio-summary") expect(mcpConfig.mcpServers.mimir.command).toBe("pnpm") expect(mcpConfig.mcpServers.mimir.args).toEqual(["exec", "kb", "serve-mcp"]) expect(mcpConfig.mcpServers.mimir.cwd).toBe(root) @@ -39,6 +41,7 @@ describe("installSkill", () => { expect(first.written).toContain(".gitignore") expect(second.written).not.toContain(".gitignore") + expect(first.written).toContain(path.join(".mimir", "skills", "mimir-audio-summary")) expect(gitignore.match(/^\.kb\/$/gm)).toHaveLength(1) expect(gitignore.match(/^\.mimir\/$/gm)).toHaveLength(1) }) diff --git a/src/skill.ts b/packages/mimir/src/skill.ts similarity index 59% rename from src/skill.ts rename to packages/mimir/src/skill.ts index c88c45a..822c7f4 100644 --- a/src/skill.ts +++ b/packages/mimir/src/skill.ts @@ -1,6 +1,7 @@ import { cp, mkdir, writeFile } from "node:fs/promises" import path from "node:path" import { fileURLToPath } from "node:url" +import { DEFAULT_SKILL_TARGET_DIR, MIMIR_DIR } from "./defaults.js" import { ensureMimirGitignore } from "./gitignore.js" export interface InstallSkillOptions { @@ -10,36 +11,41 @@ export interface InstallSkillOptions { export interface InstallSkillResult { skillPath: string + audioSkillPath: string mcpConfigPath: string readmePath: string written: string[] } const PACKAGE_ROOT = path.dirname(path.dirname(fileURLToPath(import.meta.url))) -const SKILL_NAME = "mimir" +const PRIMARY_SKILL_NAME = "mimir" +const AUDIO_SKILL_NAME = "mimir-audio-summary" -export function bundledSkillPath(): string { - return path.join(PACKAGE_ROOT, "skills", SKILL_NAME) +export function bundledSkillPath(skillName = PRIMARY_SKILL_NAME): string { + return path.join(PACKAGE_ROOT, "skills", skillName) } export async function installSkill(options: InstallSkillOptions = {}): Promise { const cwd = path.resolve(options.cwd ?? process.cwd()) - const targetDir = path.resolve(cwd, options.targetDir ?? ".mimir/skills") - const skillPath = path.join(targetDir, SKILL_NAME) - const mimirDir = path.resolve(cwd, ".mimir") + const targetDir = path.resolve(cwd, options.targetDir ?? DEFAULT_SKILL_TARGET_DIR) + const skillPath = path.join(targetDir, PRIMARY_SKILL_NAME) + const audioSkillPath = path.join(targetDir, AUDIO_SKILL_NAME) + const mimirDir = path.resolve(cwd, MIMIR_DIR) const mcpConfigPath = path.join(mimirDir, "mcp.json") const readmePath = path.join(mimirDir, "README.md") await mkdir(targetDir, { recursive: true }) await mkdir(mimirDir, { recursive: true }) - await cp(bundledSkillPath(), skillPath, { recursive: true, force: true }) + await cp(bundledSkillPath(PRIMARY_SKILL_NAME), skillPath, { recursive: true, force: true }) + await cp(bundledSkillPath(AUDIO_SKILL_NAME), audioSkillPath, { recursive: true, force: true }) await writeFile(mcpConfigPath, `${JSON.stringify(mcpConfig(cwd), null, 2)}\n`, "utf8") - await writeFile(readmePath, agentKitReadme(skillPath, mcpConfigPath), "utf8") + await writeFile(readmePath, agentKitReadme(skillPath, audioSkillPath, mcpConfigPath), "utf8") const wroteGitignore = await ensureMimirGitignore(cwd) const written = [ path.relative(cwd, skillPath), + path.relative(cwd, audioSkillPath), path.relative(cwd, mcpConfigPath), path.relative(cwd, readmePath), ] @@ -50,6 +56,7 @@ export async function installSkill(options: InstallSkillOptions = {}): Promise[], { + const records = rows.map((row) => ({ ...row })) + await db.createTable(config.tableName, records, { mode: "overwrite", }) } diff --git a/packages/mimir/src/test-support/config.ts b/packages/mimir/src/test-support/config.ts new file mode 100644 index 0000000..4025b7c --- /dev/null +++ b/packages/mimir/src/test-support/config.ts @@ -0,0 +1,40 @@ +import path from "node:path" +import { DEFAULT_CONFIG } from "../defaults.js" +import type { Config } from "../types.js" + +export function testConfig(overrides?: Partial): Config +export function testConfig(projectRoot: string, overrides?: Partial): Config +export function testConfig( + projectRootOrOverrides: string | Partial = "/tmp/mimir", + overrides: Partial = {}, +): Config { + const projectRoot = + typeof projectRootOrOverrides === "string" ? projectRootOrOverrides : "/tmp/mimir" + const configOverrides = + typeof projectRootOrOverrides === "string" ? overrides : projectRootOrOverrides + + const config: Config = { + projectRoot, + rawDir: path.join(projectRoot, DEFAULT_CONFIG.rawDir), + storageDir: path.join(projectRoot, DEFAULT_CONFIG.storageDir), + sourcesFile: path.join(projectRoot, DEFAULT_CONFIG.sourcesFile), + accessLogPath: path.join(projectRoot, DEFAULT_CONFIG.accessLogPath), + embeddingModelPath: path.join(projectRoot, DEFAULT_CONFIG.embeddingModelPath), + tableName: DEFAULT_CONFIG.tableName, + embeddingProvider: DEFAULT_CONFIG.embeddingProvider, + embeddingModel: DEFAULT_CONFIG.embeddingModel, + transformersAllowRemoteModels: DEFAULT_CONFIG.transformersAllowRemoteModels, + redaction: { + ...DEFAULT_CONFIG.redaction, + patterns: [...DEFAULT_CONFIG.redaction.patterns], + }, + accessLog: DEFAULT_CONFIG.accessLog, + mcpMaxTopK: DEFAULT_CONFIG.mcpMaxTopK, + topK: DEFAULT_CONFIG.topK, + chunkSize: DEFAULT_CONFIG.chunkSize, + chunkOverlap: DEFAULT_CONFIG.chunkOverlap, + includeExtensions: [...DEFAULT_CONFIG.includeExtensions], + } + + return { ...config, ...configOverrides } +} diff --git a/src/types.ts b/packages/mimir/src/types.ts similarity index 86% rename from src/types.ts rename to packages/mimir/src/types.ts index 27cd55a..77ff70a 100644 --- a/src/types.ts +++ b/packages/mimir/src/types.ts @@ -6,20 +6,21 @@ export interface Config { storageDir: string sourcesFile: string accessLogPath: string + embeddingModelPath: string tableName: string - ollamaHost: string - networkPolicy: NetworkPolicy - embedModel: string - llmModel: string + embeddingProvider: EmbeddingProvider + embeddingModel: string + transformersAllowRemoteModels: boolean redaction: RedactionConfig accessLog: boolean mcpMaxTopK: number topK: number chunkSize: number chunkOverlap: number + includeExtensions: string[] } -export type NetworkPolicy = "local-only" | "allow-private" | "allow-any" +export type EmbeddingProvider = "local-hash" | "transformers" export interface RedactionConfig { enabled: boolean @@ -39,11 +40,6 @@ export interface RedactionCount { count: number } -export interface HostClassification { - kind: "loopback" | "private" | "remote" | "invalid" - host: string -} - export interface SourceFile { absolutePath: string relativePath: string @@ -122,11 +118,12 @@ export interface DestroyIndexResult { export interface SecurityAuditReport { projectRoot: string zeroTelemetry: true - network: { - policy: NetworkPolicy - ollamaHost: string - host: string - classification: HostClassification["kind"] + providers: { + embedding: EmbeddingProvider + embeddingModel: string + embeddingModelPath: string + transformersAllowRemoteModels: boolean + llmGeneration: false } redaction: { enabled: boolean diff --git a/packages/mimir/src/version.ts b/packages/mimir/src/version.ts new file mode 100644 index 0000000..9216d62 --- /dev/null +++ b/packages/mimir/src/version.ts @@ -0,0 +1 @@ +export const VERSION = "0.4.0" diff --git a/packages/mimir/tsconfig.json b/packages/mimir/tsconfig.json new file mode 100644 index 0000000..6758f2d --- /dev/null +++ b/packages/mimir/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src/**/*.ts"], + "exclude": ["src/**/*.test.ts", "src/test-support/**/*.ts"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 36b0e5f..0b12988 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -7,7 +7,28 @@ settings: importers: .: + devDependencies: + '@biomejs/biome': + specifier: ^2.5.1 + version: 2.5.1 + '@commitlint/cli': + specifier: ^21.1.0 + version: 21.1.0(@types/node@24.13.2)(conventional-commits-parser@6.4.0)(typescript@5.9.3) + '@commitlint/config-conventional': + specifier: ^21.1.0 + version: 21.1.0 + yaml: + specifier: ^2.8.1 + version: 2.9.0 + + packages/mimir: dependencies: + '@huggingface/transformers': + specifier: ^4.2.0 + version: 4.2.0 + '@jcode.labs/mimir-tts': + specifier: workspace:* + version: link:../mimir-tts '@lancedb/lancedb': specifier: ^0.30.0 version: 0.30.0(apache-arrow@18.1.0) @@ -20,12 +41,12 @@ importers: fast-glob: specifier: ^3.3.3 version: 3.3.3 + fflate: + specifier: ^0.8.3 + version: 0.8.3 html-to-text: specifier: ^9.0.5 version: 9.0.5 - ollama: - specifier: ^0.6.3 - version: 0.6.3 picocolors: specifier: ^1.1.1 version: 1.1.1 @@ -39,15 +60,6 @@ importers: specifier: ^4.1.13 version: 4.4.3 devDependencies: - '@biomejs/biome': - specifier: ^2.5.1 - version: 2.5.1 - '@commitlint/cli': - specifier: ^21.1.0 - version: 21.1.0(@types/node@24.13.2)(conventional-commits-parser@6.4.0)(typescript@5.9.3) - '@commitlint/config-conventional': - specifier: ^21.1.0 - version: 21.1.0 '@types/html-to-text': specifier: ^9.0.4 version: 9.0.4 @@ -64,6 +76,25 @@ importers: specifier: ^4.0.15 version: 4.1.9(@types/node@24.13.2)(vite@8.1.0(@types/node@24.13.2)(jiti@2.6.1)(yaml@2.9.0)) + packages/mimir-tts: + dependencies: + '@huggingface/transformers': + specifier: ^4.2.0 + version: 4.2.0 + devDependencies: + '@types/node': + specifier: ^24.10.1 + version: 24.13.2 + publint: + specifier: ^0.3.21 + version: 0.3.21 + typescript: + specifier: ^5.9.3 + version: 5.9.3 + vitest: + specifier: ^4.0.15 + version: 4.1.9(@types/node@24.13.2)(vite@8.1.0(@types/node@24.13.2)(jiti@2.6.1)(yaml@2.9.0)) + packages: '@babel/code-frame@7.29.7': @@ -227,6 +258,169 @@ packages: peerDependencies: hono: ^4 + '@huggingface/jinja@0.5.9': + resolution: {integrity: sha512-uWTG+l3VJRsl7EXxYizuL3P+cCPoc3cRqbWWRcQN0FhejRfbdq0RNhCmbY/YDtnTcz9icdLYuLDjsnz4d8JMuw==} + engines: {node: '>=18'} + + '@huggingface/tokenizers@0.1.3': + resolution: {integrity: sha512-8rF/RRT10u+kn7YuUbUg0OF30K8rjTc78aHpxT+qJ1uWSqxT1MHi8+9ltwYfkFYJzT/oS+qw3JVfHtNMGAdqyA==} + + '@huggingface/transformers@4.2.0': + resolution: {integrity: sha512-8BRCoBMH0XsWaEIamuR0LrJGAfftgHAfb2Vrffy0VKlSAE/MnUJ5/h/zTfEP3fDIft+nk7TqB8xXEyABGitBjQ==} + + '@img/colour@1.1.0': + resolution: {integrity: sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==} + engines: {node: '>=18'} + + '@img/sharp-darwin-arm64@0.34.5': + resolution: {integrity: sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [darwin] + + '@img/sharp-darwin-x64@0.34.5': + resolution: {integrity: sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [darwin] + + '@img/sharp-libvips-darwin-arm64@1.2.4': + resolution: {integrity: sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==} + cpu: [arm64] + os: [darwin] + + '@img/sharp-libvips-darwin-x64@1.2.4': + resolution: {integrity: sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==} + cpu: [x64] + os: [darwin] + + '@img/sharp-libvips-linux-arm64@1.2.4': + resolution: {integrity: sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-arm@1.2.4': + resolution: {integrity: sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==} + cpu: [arm] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-ppc64@1.2.4': + resolution: {integrity: sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-riscv64@1.2.4': + resolution: {integrity: sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-s390x@1.2.4': + resolution: {integrity: sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-x64@1.2.4': + resolution: {integrity: sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linuxmusl-arm64@1.2.4': + resolution: {integrity: sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@img/sharp-libvips-linuxmusl-x64@1.2.4': + resolution: {integrity: sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==} + cpu: [x64] + os: [linux] + libc: [musl] + + '@img/sharp-linux-arm64@0.34.5': + resolution: {integrity: sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-arm@0.34.5': + resolution: {integrity: sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-ppc64@0.34.5': + resolution: {integrity: sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-riscv64@0.34.5': + resolution: {integrity: sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-s390x@0.34.5': + resolution: {integrity: sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-x64@0.34.5': + resolution: {integrity: sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@img/sharp-linuxmusl-arm64@0.34.5': + resolution: {integrity: sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@img/sharp-linuxmusl-x64@0.34.5': + resolution: {integrity: sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [linux] + libc: [musl] + + '@img/sharp-wasm32@0.34.5': + resolution: {integrity: sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [wasm32] + + '@img/sharp-win32-arm64@0.34.5': + resolution: {integrity: sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [win32] + + '@img/sharp-win32-ia32@0.34.5': + resolution: {integrity: sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [ia32] + os: [win32] + + '@img/sharp-win32-x64@0.34.5': + resolution: {integrity: sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [win32] + '@jridgewell/sourcemap-codec@1.5.5': resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==} @@ -315,6 +509,33 @@ packages: '@oxc-project/types@0.137.0': resolution: {integrity: sha512-WT+Gb24i8hmvo85AIv2oEYouEXkRlKAlT9WaCa3TfLgNCN+GhrJOGZuIlMouAh38Qe4QOx26eUOVsq70qXrywA==} + '@protobufjs/aspromise@1.1.2': + resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==} + + '@protobufjs/base64@1.1.2': + resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==} + + '@protobufjs/codegen@2.0.5': + resolution: {integrity: sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==} + + '@protobufjs/eventemitter@1.1.1': + resolution: {integrity: sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==} + + '@protobufjs/fetch@1.1.1': + resolution: {integrity: sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==} + + '@protobufjs/float@1.0.2': + resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==} + + '@protobufjs/path@1.1.2': + resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==} + + '@protobufjs/pool@1.1.0': + resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==} + + '@protobufjs/utf8@1.1.1': + resolution: {integrity: sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg==} + '@publint/pack@0.1.5': resolution: {integrity: sha512-edgyN2pP07uXiP4tJs0s8KVmU8M8i60YPbbI0/WDeok1mIJHRXz+CgD8I0nelwDkoCh3EWL/G5kGfbuHjsdbvw==} engines: {node: '>=18'} @@ -494,6 +715,10 @@ packages: resolution: {integrity: sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==} engines: {node: '>= 0.6'} + adm-zip@0.5.17: + resolution: {integrity: sha512-+Ut8d9LLqwEvHHJl1+PIHqoyDxFgVN847JTVM3Izi3xHDWPE4UtzzXysMZQs64DMcrJfBeS/uoEP4AD3HQHnQQ==} + engines: {node: '>=12.0'} + ajv-formats@3.0.1: resolution: {integrity: sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==} peerDependencies: @@ -543,6 +768,10 @@ packages: resolution: {integrity: sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==} engines: {node: '>=18'} + boolean@3.2.0: + resolution: {integrity: sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw==} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. + braces@3.0.3: resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} engines: {node: '>=8'} @@ -675,6 +904,14 @@ packages: resolution: {integrity: sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==} engines: {node: '>=0.10.0'} + define-data-property@1.1.4: + resolution: {integrity: sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==} + engines: {node: '>= 0.4'} + + define-properties@1.2.1: + resolution: {integrity: sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==} + engines: {node: '>= 0.4'} + depd@2.0.0: resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} engines: {node: '>= 0.8'} @@ -683,6 +920,9 @@ packages: resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} engines: {node: '>=8'} + detect-node@2.1.0: + resolution: {integrity: sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==} + dom-serializer@2.0.0: resolution: {integrity: sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==} @@ -743,6 +983,9 @@ packages: es-toolkit@1.49.0: resolution: {integrity: sha512-G5iZ6Pc/FNRY/soKZHC+TxGDD83rHUDXxzaWhGCX44vAv/tMs56WMusnm/KMNK+luUPsgA9U28cGr4RDlSzL2g==} + es6-error@4.1.1: + resolution: {integrity: sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==} + escalade@3.2.0: resolution: {integrity: sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==} engines: {node: '>=6'} @@ -750,6 +993,10 @@ packages: escape-html@1.0.3: resolution: {integrity: sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==} + escape-string-regexp@4.0.0: + resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} + engines: {node: '>=10'} + estree-walker@3.0.3: resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==} @@ -801,6 +1048,9 @@ packages: picomatch: optional: true + fflate@0.8.3: + resolution: {integrity: sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA==} + fill-range@7.1.1: resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} engines: {node: '>=8'} @@ -816,6 +1066,9 @@ packages: flatbuffers@24.12.23: resolution: {integrity: sha512-dLVCAISd5mhls514keQzmEG6QHmUUsNuWsb4tFafIUwvvgDjXhtfAYSKOzt5SWOy+qByV5pbsDZ+Vb7HUOBEdA==} + flatbuffers@25.9.23: + resolution: {integrity: sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==} + forwarded@0.2.0: resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} engines: {node: '>= 0.6'} @@ -858,18 +1111,32 @@ packages: resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==} engines: {node: '>= 6'} + global-agent@3.0.0: + resolution: {integrity: sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q==} + engines: {node: '>=10.0'} + global-directory@5.0.0: resolution: {integrity: sha512-1pgFdhK3J2LeM+dVf2Pd424yHx2ou338lC0ErNP2hPx4j8eW1Sp0XqSjNxtk6Tc4Kr5wlWtSvz8cn2yb7/SG/w==} engines: {node: '>=20'} + globalthis@1.0.4: + resolution: {integrity: sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ==} + engines: {node: '>= 0.4'} + gopd@1.2.0: resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} engines: {node: '>= 0.4'} + guid-typescript@1.0.9: + resolution: {integrity: sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==} + has-flag@4.0.0: resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} engines: {node: '>=8'} + has-property-descriptors@1.0.2: + resolution: {integrity: sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==} + has-symbols@1.1.0: resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==} engines: {node: '>= 0.4'} @@ -972,6 +1239,9 @@ packages: json-schema-typed@8.0.2: resolution: {integrity: sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==} + json-stringify-safe@5.0.1: + resolution: {integrity: sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==} + leac@0.6.0: resolution: {integrity: sha512-y+SqErxb8h7nE/fiEX07jsbuhrpO9lL8eca7/Y1nuWV2moNlXhyd59iDGcRf6moVyDMbmTNzL40SUyrFU/yDpg==} @@ -1055,9 +1325,16 @@ packages: lodash.camelcase@4.3.0: resolution: {integrity: sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==} + long@5.3.2: + resolution: {integrity: sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==} + magic-string@0.30.21: resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} + matcher@3.0.0: + resolution: {integrity: sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==} + engines: {node: '>=10'} + math-intrinsics@1.1.0: resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} engines: {node: '>= 0.4'} @@ -1114,13 +1391,14 @@ packages: resolution: {integrity: sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==} engines: {node: '>= 0.4'} + object-keys@1.1.1: + resolution: {integrity: sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==} + engines: {node: '>= 0.4'} + obug@2.1.3: resolution: {integrity: sha512-9miFgM2OFba7hB+pRgvtV84pYTBaoTHohvmIgiRt6dRIzbwEOIaNaP+dIlGs2fNFoB0SeISs0Jz5WFVRid6Xyg==} engines: {node: '>=12.20.0'} - ollama@0.6.3: - resolution: {integrity: sha512-KEWEhIqE5wtfzEIZbDCLH51VFZ6Z3ZSa6sIOg/E/tBV8S51flyqBOXi+bRxlOYKDf8i327zG9eSTb8IJxvm3Zg==} - on-finished@2.4.1: resolution: {integrity: sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==} engines: {node: '>= 0.8'} @@ -1128,6 +1406,19 @@ packages: once@1.4.0: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + onnxruntime-common@1.24.0-dev.20251116-b39e144322: + resolution: {integrity: sha512-BOoomdHYmNRL5r4iQ4bMvsl2t0/hzVQ3OM3PHD0gxeXu1PmggqBv3puZicEUVOA3AtHHYmqZtjMj9FOfGrATTw==} + + onnxruntime-common@1.24.3: + resolution: {integrity: sha512-GeuPZO6U/LBJXvwdaqHbuUmoXiEdeCjWi/EG7Y1HNnDwJYuk6WUbNXpF6luSUY8yASul3cmUlLGrCCL1ZgVXqA==} + + onnxruntime-node@1.24.3: + resolution: {integrity: sha512-JH7+czbc8ALA819vlTgcV+Q214/+VjGeBHDjX81+ZCD0PCVCIFGFNtT0V4sXG/1JXypKPgScQcB3ij/hk3YnTg==} + os: [win32, darwin, linux] + + onnxruntime-web@1.26.0-dev.20260416-b7804b056c: + resolution: {integrity: sha512-MD6Ss4GSpQBo6zqoJzyT9LRbKYs7x/JVN23FT24EcEvlqF4VuzPOeH6X38orZPKHQDbprn7K+SBpu0/mj2CQiw==} + package-manager-detector@1.6.0: resolution: {integrity: sha512-61A5ThoTiDG/C8s8UMZwSorAGwMJ0ERVGj2OjoW5pAalsNOg15+iQiPzrLJ4jhZ1HJzmC2PIHT2oEiH3R5fzNA==} @@ -1174,10 +1465,17 @@ packages: resolution: {integrity: sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==} engines: {node: '>=16.20.0'} + platform@1.3.6: + resolution: {integrity: sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==} + postcss@8.5.15: resolution: {integrity: sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==} engines: {node: ^10 || ^12 || >=14} + protobufjs@7.6.4: + resolution: {integrity: sha512-RJJPTTpvFfHcWLkIa2JFWK4XvtSzS0yEWDmunqHXli1h3JlkbcQZXDZdcWxv+JK3Xsl5/UFDPZ0iGm7DAengYw==} + engines: {node: '>=12.0.0'} + proxy-addr@2.0.7: resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} engines: {node: '>= 0.10'} @@ -1221,6 +1519,10 @@ packages: resolution: {integrity: sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==} engines: {iojs: '>=1.0.0', node: '>=0.10.0'} + roarr@2.15.4: + resolution: {integrity: sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==} + engines: {node: '>=8.0'} + rolldown@1.1.3: resolution: {integrity: sha512-1F1eEtUBtFvcGm1HQ9TiUIUHPQG7mSAODrhIzjxoUEFuo8OcbrGLiVLkevNgj84TE4lnHvnumwFjhJO5Eu135g==} engines: {node: ^20.19.0 || >=22.12.0} @@ -1243,6 +1545,9 @@ packages: selderee@0.11.0: resolution: {integrity: sha512-5TF+l7p4+OsnP8BCCvSyZiSPc4x4//p5uPwK8TCnVPJYRmU2aYKMpOXvw8zM5a5JvuuCGN1jmsMwuU2W02ukfA==} + semver-compare@1.0.0: + resolution: {integrity: sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==} + semver@7.8.5: resolution: {integrity: sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==} engines: {node: '>=10'} @@ -1252,6 +1557,10 @@ packages: resolution: {integrity: sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==} engines: {node: '>= 18'} + serialize-error@7.0.1: + resolution: {integrity: sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==} + engines: {node: '>=10'} + serve-static@2.2.1: resolution: {integrity: sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==} engines: {node: '>= 18'} @@ -1259,6 +1568,10 @@ packages: setprototypeof@1.2.0: resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} + sharp@0.34.5: + resolution: {integrity: sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + shebang-command@2.0.0: resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} engines: {node: '>=8'} @@ -1290,6 +1603,9 @@ packages: resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} engines: {node: '>=0.10.0'} + sprintf-js@1.1.3: + resolution: {integrity: sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==} + stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} @@ -1342,6 +1658,10 @@ packages: tslib@2.8.1: resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + type-fest@0.13.1: + resolution: {integrity: sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==} + engines: {node: '>=10'} + type-is@2.1.0: resolution: {integrity: sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==} engines: {node: '>= 18'} @@ -1465,9 +1785,6 @@ packages: jsdom: optional: true - whatwg-fetch@3.6.20: - resolution: {integrity: sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg==} - which@2.0.2: resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} engines: {node: '>= 8'} @@ -1696,6 +2013,114 @@ snapshots: dependencies: hono: 4.12.27 + '@huggingface/jinja@0.5.9': {} + + '@huggingface/tokenizers@0.1.3': {} + + '@huggingface/transformers@4.2.0': + dependencies: + '@huggingface/jinja': 0.5.9 + '@huggingface/tokenizers': 0.1.3 + onnxruntime-node: 1.24.3 + onnxruntime-web: 1.26.0-dev.20260416-b7804b056c + sharp: 0.34.5 + + '@img/colour@1.1.0': {} + + '@img/sharp-darwin-arm64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-darwin-arm64': 1.2.4 + optional: true + + '@img/sharp-darwin-x64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-darwin-x64': 1.2.4 + optional: true + + '@img/sharp-libvips-darwin-arm64@1.2.4': + optional: true + + '@img/sharp-libvips-darwin-x64@1.2.4': + optional: true + + '@img/sharp-libvips-linux-arm64@1.2.4': + optional: true + + '@img/sharp-libvips-linux-arm@1.2.4': + optional: true + + '@img/sharp-libvips-linux-ppc64@1.2.4': + optional: true + + '@img/sharp-libvips-linux-riscv64@1.2.4': + optional: true + + '@img/sharp-libvips-linux-s390x@1.2.4': + optional: true + + '@img/sharp-libvips-linux-x64@1.2.4': + optional: true + + '@img/sharp-libvips-linuxmusl-arm64@1.2.4': + optional: true + + '@img/sharp-libvips-linuxmusl-x64@1.2.4': + optional: true + + '@img/sharp-linux-arm64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-arm64': 1.2.4 + optional: true + + '@img/sharp-linux-arm@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-arm': 1.2.4 + optional: true + + '@img/sharp-linux-ppc64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-ppc64': 1.2.4 + optional: true + + '@img/sharp-linux-riscv64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-riscv64': 1.2.4 + optional: true + + '@img/sharp-linux-s390x@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-s390x': 1.2.4 + optional: true + + '@img/sharp-linux-x64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-x64': 1.2.4 + optional: true + + '@img/sharp-linuxmusl-arm64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-arm64': 1.2.4 + optional: true + + '@img/sharp-linuxmusl-x64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-x64': 1.2.4 + optional: true + + '@img/sharp-wasm32@0.34.5': + dependencies: + '@emnapi/runtime': 1.11.1 + optional: true + + '@img/sharp-win32-arm64@0.34.5': + optional: true + + '@img/sharp-win32-ia32@0.34.5': + optional: true + + '@img/sharp-win32-x64@0.34.5': + optional: true + '@jridgewell/sourcemap-codec@1.5.5': {} '@lancedb/lancedb-darwin-arm64@0.30.0': @@ -1775,6 +2200,26 @@ snapshots: '@oxc-project/types@0.137.0': {} + '@protobufjs/aspromise@1.1.2': {} + + '@protobufjs/base64@1.1.2': {} + + '@protobufjs/codegen@2.0.5': {} + + '@protobufjs/eventemitter@1.1.1': {} + + '@protobufjs/fetch@1.1.1': + dependencies: + '@protobufjs/aspromise': 1.1.2 + + '@protobufjs/float@1.0.2': {} + + '@protobufjs/path@1.1.2': {} + + '@protobufjs/pool@1.1.0': {} + + '@protobufjs/utf8@1.1.1': {} + '@publint/pack@0.1.5': dependencies: tinyexec: 1.2.4 @@ -1921,6 +2366,8 @@ snapshots: mime-types: 3.0.2 negotiator: 1.0.0 + adm-zip@0.5.17: {} + ajv-formats@3.0.1(ajv@8.20.0): optionalDependencies: ajv: 8.20.0 @@ -1976,6 +2423,8 @@ snapshots: transitivePeerDependencies: - supports-color + boolean@3.2.0: {} + braces@3.0.3: dependencies: fill-range: 7.1.1 @@ -2096,10 +2545,24 @@ snapshots: deepmerge@4.3.1: {} + define-data-property@1.1.4: + dependencies: + es-define-property: 1.0.1 + es-errors: 1.3.0 + gopd: 1.2.0 + + define-properties@1.2.1: + dependencies: + define-data-property: 1.1.4 + has-property-descriptors: 1.0.2 + object-keys: 1.1.1 + depd@2.0.0: {} detect-libc@2.1.2: {} + detect-node@2.1.0: {} + dom-serializer@2.0.0: dependencies: domelementtype: 2.3.0 @@ -2154,10 +2617,14 @@ snapshots: es-toolkit@1.49.0: {} + es6-error@4.1.1: {} + escalade@3.2.0: {} escape-html@1.0.3: {} + escape-string-regexp@4.0.0: {} + estree-walker@3.0.3: dependencies: '@types/estree': 1.0.9 @@ -2230,6 +2697,8 @@ snapshots: optionalDependencies: picomatch: 4.0.4 + fflate@0.8.3: {} + fill-range@7.1.1: dependencies: to-regex-range: 5.0.1 @@ -2251,6 +2720,8 @@ snapshots: flatbuffers@24.12.23: {} + flatbuffers@25.9.23: {} + forwarded@0.2.0: {} fresh@2.0.0: {} @@ -2294,14 +2765,34 @@ snapshots: dependencies: is-glob: 4.0.3 + global-agent@3.0.0: + dependencies: + boolean: 3.2.0 + es6-error: 4.1.1 + matcher: 3.0.0 + roarr: 2.15.4 + semver: 7.8.5 + serialize-error: 7.0.1 + global-directory@5.0.0: dependencies: ini: 6.0.0 + globalthis@1.0.4: + dependencies: + define-properties: 1.2.1 + gopd: 1.2.0 + gopd@1.2.0: {} + guid-typescript@1.0.9: {} + has-flag@4.0.0: {} + has-property-descriptors@1.0.2: + dependencies: + es-define-property: 1.0.1 + has-symbols@1.1.0: {} hasown@2.0.4: @@ -2386,6 +2877,8 @@ snapshots: json-schema-typed@8.0.2: {} + json-stringify-safe@5.0.1: {} + leac@0.6.0: {} lightningcss-android-arm64@1.32.0: @@ -2441,10 +2934,16 @@ snapshots: lodash.camelcase@4.3.0: {} + long@5.3.2: {} + magic-string@0.30.21: dependencies: '@jridgewell/sourcemap-codec': 1.5.5 + matcher@3.0.0: + dependencies: + escape-string-regexp: 4.0.0 + math-intrinsics@1.1.0: {} media-typer@1.1.0: {} @@ -2478,11 +2977,9 @@ snapshots: object-inspect@1.13.4: {} - obug@2.1.3: {} + object-keys@1.1.1: {} - ollama@0.6.3: - dependencies: - whatwg-fetch: 3.6.20 + obug@2.1.3: {} on-finished@2.4.1: dependencies: @@ -2492,6 +2989,25 @@ snapshots: dependencies: wrappy: 1.0.2 + onnxruntime-common@1.24.0-dev.20251116-b39e144322: {} + + onnxruntime-common@1.24.3: {} + + onnxruntime-node@1.24.3: + dependencies: + adm-zip: 0.5.17 + global-agent: 3.0.0 + onnxruntime-common: 1.24.3 + + onnxruntime-web@1.26.0-dev.20260416-b7804b056c: + dependencies: + flatbuffers: 25.9.23 + guid-typescript: 1.0.9 + long: 5.3.2 + onnxruntime-common: 1.24.0-dev.20251116-b39e144322 + platform: 1.3.6 + protobufjs: 7.6.4 + package-manager-detector@1.6.0: {} parent-module@1.0.1: @@ -2528,12 +3044,28 @@ snapshots: pkce-challenge@5.0.1: {} + platform@1.3.6: {} + postcss@8.5.15: dependencies: nanoid: 3.3.15 picocolors: 1.1.1 source-map-js: 1.2.1 + protobufjs@7.6.4: + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/base64': 1.1.2 + '@protobufjs/codegen': 2.0.5 + '@protobufjs/eventemitter': 1.1.1 + '@protobufjs/fetch': 1.1.1 + '@protobufjs/float': 1.0.2 + '@protobufjs/path': 1.1.2 + '@protobufjs/pool': 1.1.0 + '@protobufjs/utf8': 1.1.1 + '@types/node': 24.13.2 + long: 5.3.2 + proxy-addr@2.0.7: dependencies: forwarded: 0.2.0 @@ -2572,6 +3104,15 @@ snapshots: reusify@1.1.0: {} + roarr@2.15.4: + dependencies: + boolean: 3.2.0 + detect-node: 2.1.0 + globalthis: 1.0.4 + json-stringify-safe: 5.0.1 + semver-compare: 1.0.0 + sprintf-js: 1.1.3 + rolldown@1.1.3: dependencies: '@oxc-project/types': 0.137.0 @@ -2617,6 +3158,8 @@ snapshots: dependencies: parseley: 0.12.1 + semver-compare@1.0.0: {} + semver@7.8.5: {} send@1.2.1: @@ -2635,6 +3178,10 @@ snapshots: transitivePeerDependencies: - supports-color + serialize-error@7.0.1: + dependencies: + type-fest: 0.13.1 + serve-static@2.2.1: dependencies: encodeurl: 2.0.0 @@ -2646,6 +3193,37 @@ snapshots: setprototypeof@1.2.0: {} + sharp@0.34.5: + dependencies: + '@img/colour': 1.1.0 + detect-libc: 2.1.2 + semver: 7.8.5 + optionalDependencies: + '@img/sharp-darwin-arm64': 0.34.5 + '@img/sharp-darwin-x64': 0.34.5 + '@img/sharp-libvips-darwin-arm64': 1.2.4 + '@img/sharp-libvips-darwin-x64': 1.2.4 + '@img/sharp-libvips-linux-arm': 1.2.4 + '@img/sharp-libvips-linux-arm64': 1.2.4 + '@img/sharp-libvips-linux-ppc64': 1.2.4 + '@img/sharp-libvips-linux-riscv64': 1.2.4 + '@img/sharp-libvips-linux-s390x': 1.2.4 + '@img/sharp-libvips-linux-x64': 1.2.4 + '@img/sharp-libvips-linuxmusl-arm64': 1.2.4 + '@img/sharp-libvips-linuxmusl-x64': 1.2.4 + '@img/sharp-linux-arm': 0.34.5 + '@img/sharp-linux-arm64': 0.34.5 + '@img/sharp-linux-ppc64': 0.34.5 + '@img/sharp-linux-riscv64': 0.34.5 + '@img/sharp-linux-s390x': 0.34.5 + '@img/sharp-linux-x64': 0.34.5 + '@img/sharp-linuxmusl-arm64': 0.34.5 + '@img/sharp-linuxmusl-x64': 0.34.5 + '@img/sharp-wasm32': 0.34.5 + '@img/sharp-win32-arm64': 0.34.5 + '@img/sharp-win32-ia32': 0.34.5 + '@img/sharp-win32-x64': 0.34.5 + shebang-command@2.0.0: dependencies: shebang-regex: 3.0.0 @@ -2684,6 +3262,8 @@ snapshots: source-map-js@1.2.1: {} + sprintf-js@1.1.3: {} + stackback@0.0.2: {} statuses@2.0.2: {} @@ -2728,6 +3308,8 @@ snapshots: tslib@2.8.1: {} + type-fest@0.13.1: {} + type-is@2.1.0: dependencies: content-type: 2.0.0 @@ -2790,8 +3372,6 @@ snapshots: transitivePeerDependencies: - msw - whatwg-fetch@3.6.20: {} - which@2.0.2: dependencies: isexe: 2.0.0 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml new file mode 100644 index 0000000..f42ddbe --- /dev/null +++ b/pnpm-workspace.yaml @@ -0,0 +1,10 @@ +packages: + - "." + - "packages/*" + +strictDepBuilds: true + +allowBuilds: + onnxruntime-node: true + protobufjs: true + sharp: true diff --git a/scripts/release-artifacts.mjs b/scripts/release-artifacts.mjs index 737e853..b5794ad 100644 --- a/scripts/release-artifacts.mjs +++ b/scripts/release-artifacts.mjs @@ -7,19 +7,37 @@ import YAML from "yaml" const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..") const artifactsDir = path.join(repoRoot, "release-artifacts") -const packageJson = JSON.parse(await readFile(path.join(repoRoot, "package.json"), "utf8")) +const corePackageDir = "packages/mimir" +const packageDirs = ["packages/mimir-tts", corePackageDir] +const corePackageJson = await readPackageJson(corePackageDir) await rm(artifactsDir, { recursive: true, force: true }) await mkdir(artifactsDir, { recursive: true }) -const pack = run("npm", ["pack", "--json", "--pack-destination", artifactsDir]) -const packed = JSON.parse(pack.stdout) -const tarball = packed[0]?.filename -if (!tarball) { - throw new Error(`npm pack did not return a tarball filename: ${pack.stdout}`) +const packages = [] +for (const directory of packageDirs) { + const manifest = await readPackageJson(directory) + const pack = run( + "pnpm", + ["pack", "--json", "--pack-destination", artifactsDir], + packagePath(directory), + ) + const packed = JSON.parse(pack.stdout) + const tarball = tarballFilename(packed) + if (!tarball) { + throw new Error( + `pnpm pack did not return a tarball filename for ${manifest.name}: ${pack.stdout}`, + ) + } + packages.push({ + name: manifest.name, + version: manifest.version, + directory, + tarball, + }) } -const sbomFile = `${packageJson.name.replace("/", "-").replace("@", "")}-${packageJson.version}.sbom.cdx.json` +const sbomFile = `${packageNameForFile(corePackageJson.name)}-${corePackageJson.version}.sbom.cdx.json` await writeFile( path.join(artifactsDir, sbomFile), `${JSON.stringify(await buildCycloneDxSbom(), null, 2)}\n`, @@ -37,12 +55,12 @@ await writeFile( path.join(artifactsDir, "release-manifest.json"), `${JSON.stringify( { - package: packageJson.name, - version: packageJson.version, - tarball, + package: corePackageJson.name, + version: corePackageJson.version, + packages, sbom: sbomFile, checksums: "SHA256SUMS", - provenance: "npm publish --provenance is enforced by the protected GitHub Actions workflow.", + provenance: "pnpm publish --provenance is enforced by the protected GitHub Actions workflow.", }, null, 2, @@ -52,9 +70,27 @@ await writeFile( console.log(`Release artifacts written to ${path.relative(repoRoot, artifactsDir)}`) -function run(command, args) { +function packagePath(directory) { + return path.join(repoRoot, directory) +} + +async function readPackageJson(directory) { + return JSON.parse(await readFile(path.join(packagePath(directory), "package.json"), "utf8")) +} + +function tarballFilename(packed) { + const entry = Array.isArray(packed) ? packed[0] : packed + const filename = entry?.filename ?? entry?.path ?? entry?.name + return typeof filename === "string" ? path.basename(filename) : null +} + +function packageNameForFile(packageName) { + return packageName.replace("/", "-").replace("@", "") +} + +function run(command, args, cwd = repoRoot) { const result = spawnSync(command, args, { - cwd: repoRoot, + cwd, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], }) @@ -95,14 +131,14 @@ async function buildCycloneDxSbom() { metadata: { timestamp: new Date().toISOString(), tools: [ - { vendor: "JCode Labs", name: "Mimir release artifacts", version: packageJson.version }, + { vendor: "JCode Labs", name: "Mimir release artifacts", version: corePackageJson.version }, ], component: { type: "library", - name: packageJson.name, - version: packageJson.version, - licenses: [{ license: { id: packageJson.license } }], - purl: `pkg:npm/${packageJson.name}@${packageJson.version}`, + name: corePackageJson.name, + version: corePackageJson.version, + licenses: [{ license: { id: corePackageJson.license } }], + purl: `pkg:npm/${corePackageJson.name}@${corePackageJson.version}`, }, }, components, diff --git a/scripts/smoke.mjs b/scripts/smoke.mjs index 215cb14..118e833 100644 --- a/scripts/smoke.mjs +++ b/scripts/smoke.mjs @@ -1,18 +1,19 @@ import { spawn } from "node:child_process" -import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises" -import { createServer } from "node:http" +import { cp, mkdtemp, readFile, rm, writeFile } from "node:fs/promises" import { tmpdir } from "node:os" import path from "node:path" import { fileURLToPath } from "node:url" const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..") -const cliPath = path.join(repoRoot, "dist", "cli.js") +const corePackageRoot = path.join(repoRoot, "packages", "mimir") +const cliPath = path.join(corePackageRoot, "dist", "cli.js") const tempRoot = await mkdtemp(path.join(tmpdir(), "mimir-smoke-")) -const fakeOllama = await startFakeOllama() +const MCP_REQUEST_TIMEOUT_MS = 10_000 +const MCP_CLOSE_TIMEOUT_MS = 2_000 try { await runKb(["init"], tempRoot) - await configureProject(tempRoot, fakeOllama.url) + await configureProject(tempRoot) await writeFixtureDocuments(tempRoot) const ingest = await runKb(["ingest"], tempRoot) @@ -29,8 +30,8 @@ try { ) assertIncludes( ask.stdout, - "Fake Mimir answer citing [1].", - "ask should call the local LLM client", + "Mimir returns retrieval context only", + "ask should return retrieval context without calling an LLM", ) const audit = await runKb(["audit"], tempRoot) @@ -45,28 +46,105 @@ try { ) assertIncludes( security.stdout, - '"classification": "loopback"', - "security audit should classify local Ollama", + '"llmGeneration": false', + "security audit should report retrieval-only core behavior", + ) + + const audioDoctor = await runKb(["audio", "--doctor", "--json"], tempRoot) + assertIncludes( + audioDoctor.stdout, + '"pythonRequired": false', + "audio doctor should not require Python", + ) + assertIncludes( + audioDoctor.stdout, + '"outputFormat": "wav"', + "audio doctor should report WAV output", ) await runKb(["install-skill"], tempRoot) const skill = await readFile(path.join(tempRoot, ".mimir", "skills", "mimir", "SKILL.md"), "utf8") + const audioSkill = await readFile( + path.join(tempRoot, ".mimir", "skills", "mimir-audio-summary", "SKILL.md"), + "utf8", + ) assertIncludes(skill, "name: mimir", "install-skill should copy the bundled skill") + assertIncludes( + audioSkill, + "name: mimir-audio-summary", + "install-skill should copy the optional audio summary skill", + ) const gitignore = await readFile(path.join(tempRoot, ".gitignore"), "utf8") assertIncludes(gitignore, ".kb/", "init should ignore the Mimir config and index directory") assertIncludes(gitignore, ".mimir/", "install-skill should ignore generated agent kit files") await smokeMcp(tempRoot) + await smokeExampleWorkspace() const destroy = await runKb(["destroy-index", "--yes"], tempRoot) assertIncludes(destroy.stdout, "removed=true", "destroy-index should remove generated storage") console.log("Smoke test passed.") } finally { - await fakeOllama.close() await rm(tempRoot, { recursive: true, force: true }) } -async function configureProject(cwd, ollamaHost) { +async function smokeExampleWorkspace() { + const exampleSource = path.join(corePackageRoot, "examples", "sovereign-rag-demo") + const exampleTemp = await mkdtemp(path.join(tmpdir(), "mimir-example-")) + + try { + await cp(exampleSource, exampleTemp, { recursive: true }) + await configureProject(exampleTemp) + + const security = await runKb(["security-audit", "--strict"], exampleTemp) + assertIncludes( + security.stdout, + "llmGeneration=false", + "example security audit should keep LLM generation outside core", + ) + + const ingest = await runKb(["ingest"], exampleTemp) + assertIncludes(ingest.stdout, "errors=0", "example ingest should complete") + + const audit = await runKb(["audit"], exampleTemp) + assertIncludes(audit.stdout, "missingFromIndex=0", "example audit should find no missing files") + assertIncludes(audit.stdout, "staleInIndex=0", "example audit should find no stale files") + + const approvalSearch = await runKb( + ["search", "offline retrieval approval", "--top-k", "2"], + exampleTemp, + ) + assertIncludes( + approvalSearch.stdout, + "review-notes.evidence", + "example search should retrieve offline approval evidence", + ) + + const customExtensionSearch = await runKb( + ["search", "offline text-to-speech usage review", "--top-k", "10"], + exampleTemp, + ) + assertIncludes( + customExtensionSearch.stdout, + "review-notes.evidence", + "example search should index the custom .evidence extension", + ) + + const retrievalOnlyAsk = await runKb( + ["ask", "What evidence supports offline operation?", "--top-k", "2"], + exampleTemp, + ) + assertIncludes( + retrievalOnlyAsk.stdout, + "Mimir returns retrieval context only", + "example ask should return cited retrieval context", + ) + } finally { + await rm(exampleTemp, { recursive: true, force: true }) + } +} + +async function configureProject(cwd) { const configPath = path.join(cwd, ".kb", "config.json") const config = JSON.parse(await readFile(configPath, "utf8")) await writeFile( @@ -74,7 +152,7 @@ async function configureProject(cwd, ollamaHost) { `${JSON.stringify( { ...config, - ollamaHost, + embeddingProvider: "local-hash", chunkSize: 500, chunkOverlap: 50, topK: 2, @@ -209,11 +287,9 @@ function createJsonLineClient(child) { }) child.stderr.on("data", (chunk) => stderr.push(chunk)) child.on("close", () => { - for (const [id, entry] of pending) { + for (const entry of pending.values()) { clearTimeout(entry.timeout) - entry.reject( - new Error(`MCP process closed before response ${id}.\nstderr:\n${stderr.join("")}`), - ) + entry.reject(new Error(`MCP process closed before response.\nstderr:\n${stderr.join("")}`)) } pending.clear() }) @@ -230,7 +306,7 @@ function createJsonLineClient(child) { reject( new Error(`Timed out waiting for MCP response ${id}.\nstderr:\n${stderr.join("")}`), ) - }, 10_000) + }, MCP_REQUEST_TIMEOUT_MS) pending.set(id, { resolve, reject, timeout }) }) }, @@ -243,7 +319,7 @@ function createJsonLineClient(child) { const timeout = setTimeout(() => { child.kill("SIGKILL") resolve() - }, 2_000) + }, MCP_CLOSE_TIMEOUT_MS) child.on("close", () => { clearTimeout(timeout) resolve() @@ -261,72 +337,6 @@ function mcpText(response) { return content.map((item) => item.text ?? "").join("\n") } -async function startFakeOllama() { - const server = createServer(async (request, response) => { - const body = await readRequestJson(request) - - if (request.url === "/api/embed") { - const input = Array.isArray(body.input) ? body.input : [body.input] - writeJson(response, { embeddings: input.map(toEmbedding) }) - return - } - - if (request.url === "/api/chat") { - writeJson(response, { - model: body.model, - created_at: new Date(0).toISOString(), - message: { - role: "assistant", - content: "Fake Mimir answer citing [1].", - }, - done: true, - }) - return - } - - response.writeHead(404, { "content-type": "application/json" }) - response.end(JSON.stringify({ error: `Unhandled fake Ollama route: ${request.url}` })) - }) - - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)) - const address = server.address() - if (!address || typeof address === "string") { - throw new Error("Fake Ollama server did not bind to a TCP port.") - } - - return { - url: `http://127.0.0.1:${address.port}`, - close: () => new Promise((resolve) => server.close(resolve)), - } -} - -async function readRequestJson(request) { - const chunks = [] - for await (const chunk of request) { - chunks.push(chunk) - } - return JSON.parse(Buffer.concat(chunks).toString("utf8") || "{}") -} - -function writeJson(response, body) { - response.writeHead(200, { "content-type": "application/json" }) - response.end(JSON.stringify(body)) -} - -function toEmbedding(value) { - const text = String(value).toLowerCase() - return [ - countMatches(text, ["tax", "fiscal", "france", "french", "residency"]), - countMatches(text, ["thai", "thailand", "bangkok", "dtv"]), - countMatches(text, ["equipment", "subscription", "invoice"]), - Math.min(text.length / 1000, 1), - ] -} - -function countMatches(text, needles) { - return needles.reduce((count, needle) => count + (text.includes(needle) ? 1 : 0), 0) -} - function assertIncludes(actual, expected, message) { if (!actual.includes(expected)) { throw new Error(`${message}\nExpected to find: ${expected}\nActual:\n${actual}`) diff --git a/src/config.test.ts b/src/config.test.ts deleted file mode 100644 index 4ccd2dc..0000000 --- a/src/config.test.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" -import os from "node:os" -import path from "node:path" -import { afterEach, describe, expect, it } from "vitest" -import { findProjectRoot, loadConfig } from "./config.js" - -const tempDirs: string[] = [] - -afterEach(async () => { - for (const dir of tempDirs.splice(0)) { - await rm(dir, { recursive: true, force: true }) - } -}) - -describe("loadConfig", () => { - it("resolves project config upward and paths from the project root", async () => { - const root = await mkdtemp(path.join(os.tmpdir(), "jcode-kb-")) - tempDirs.push(root) - await writeFile(path.join(root, ".kb-config-placeholder"), "", "utf8") - await mkdir(path.join(root, ".kb"), { recursive: true }) - await writeFile( - path.join(root, ".kb/config.json"), - JSON.stringify({ rawDir: "docs", storageDir: ".kb/index" }), - "utf8", - ) - const nested = path.join(root, "packages/app") - await mkdir(nested, { recursive: true }) - - expect(findProjectRoot(nested)).toBe(root) - - const config = await loadConfig(nested) - expect(config.rawDir).toBe(path.join(root, "docs")) - expect(config.storageDir).toBe(path.join(root, ".kb/index")) - expect(config.accessLogPath).toBe(path.join(root, ".kb/access.log")) - expect(config.networkPolicy).toBe("local-only") - expect(config.redaction.enabled).toBe(true) - expect(config.accessLog).toBe(true) - expect(config.mcpMaxTopK).toBe(10) - }) -}) diff --git a/src/embeddings.ts b/src/embeddings.ts deleted file mode 100644 index 1b631d3..0000000 --- a/src/embeddings.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { Ollama } from "ollama" -import { assertNetworkPolicy } from "./network.js" -import type { Config } from "./types.js" - -export async function embedTexts(texts: string[], config: Config): Promise { - if (texts.length === 0) { - return [] - } - - assertNetworkPolicy(config) - const client = new Ollama({ host: config.ollamaHost }) - const response = await client.embed({ - model: config.embedModel, - input: texts, - }) - - if (!response.embeddings || response.embeddings.length !== texts.length) { - throw new Error( - `Expected ${texts.length} embeddings, received ${response.embeddings?.length ?? 0}.`, - ) - } - - return response.embeddings -} - -export async function embedText(text: string, config: Config): Promise { - const [embedding] = await embedTexts([text], config) - if (!embedding) { - throw new Error("No embedding returned for query.") - } - return embedding -} diff --git a/src/network.test.ts b/src/network.test.ts deleted file mode 100644 index 858ecce..0000000 --- a/src/network.test.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { describe, expect, it } from "vitest" -import { assertNetworkPolicy, classifyHost } from "./network.js" -import type { Config } from "./types.js" - -describe("network policy", () => { - it("classifies loopback and private hosts", () => { - expect(classifyHost("http://localhost:11434").kind).toBe("loopback") - expect(classifyHost("http://127.0.0.1:11434").kind).toBe("loopback") - expect(classifyHost("http://192.168.1.10:11434").kind).toBe("private") - expect(classifyHost("https://example.com").kind).toBe("remote") - }) - - it("blocks remote Ollama hosts by default", () => { - expect(() => assertNetworkPolicy(testConfig({ ollamaHost: "https://example.com" }))).toThrow( - /Refusing to send document text/, - ) - }) - - it("allows private hosts only when explicitly configured", () => { - expect(() => - assertNetworkPolicy( - testConfig({ ollamaHost: "http://192.168.1.10:11434", networkPolicy: "allow-private" }), - ), - ).not.toThrow() - }) -}) - -function testConfig(overrides: Partial = {}): Config { - return { - projectRoot: "/tmp/project", - rawDir: "/tmp/project/private", - storageDir: "/tmp/project/.kb/storage", - sourcesFile: "/tmp/project/.kb/sources.txt", - accessLogPath: "/tmp/project/.kb/access.log", - tableName: "chunks", - ollamaHost: "http://localhost:11434", - networkPolicy: "local-only", - embedModel: "nomic-embed-text", - llmModel: "gemma4:latest", - redaction: { enabled: true, builtIn: true, patterns: [] }, - accessLog: true, - mcpMaxTopK: 10, - topK: 5, - chunkSize: 1200, - chunkOverlap: 150, - ...overrides, - } -} diff --git a/src/network.ts b/src/network.ts deleted file mode 100644 index bf885cc..0000000 --- a/src/network.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { isIP } from "node:net" -import type { Config, HostClassification } from "./types.js" - -export function assertNetworkPolicy(config: Config): void { - const classification = classifyHost(config.ollamaHost) - - if (config.networkPolicy === "allow-any") { - return - } - - if (config.networkPolicy === "local-only" && classification.kind !== "loopback") { - throw new Error( - `Refusing to send document text to non-local Ollama host "${config.ollamaHost}". Set networkPolicy to "allow-private" or "allow-any" only if this is intentional.`, - ) - } - - if ( - config.networkPolicy === "allow-private" && - classification.kind !== "loopback" && - classification.kind !== "private" - ) { - throw new Error( - `Refusing to send document text to remote Ollama host "${config.ollamaHost}". Set networkPolicy to "allow-any" only if this is intentional.`, - ) - } -} - -export function classifyHost(input: string): HostClassification { - let url: URL - try { - url = new URL(input) - } catch { - return { kind: "invalid", host: input } - } - - const host = url.hostname.replace(/^\[(.*)\]$/, "$1").toLowerCase() - if (isLoopbackHost(host)) { - return { kind: "loopback", host } - } - - if (isPrivateHost(host)) { - return { kind: "private", host } - } - - return { kind: "remote", host } -} - -function isLoopbackHost(host: string): boolean { - if (host === "localhost" || host === "::1") { - return true - } - - if (isIP(host) === 4) { - return host.startsWith("127.") - } - - return false -} - -function isPrivateHost(host: string): boolean { - if (host === "host.docker.internal" || host.endsWith(".local")) { - return true - } - - if (isIP(host) === 4) { - const parts = host.split(".").map((part) => Number.parseInt(part, 10)) - const [first = 0, second = 0] = parts - return ( - first === 10 || - (first === 172 && second >= 16 && second <= 31) || - (first === 192 && second === 168) || - (first === 169 && second === 254) - ) - } - - if (isIP(host) === 6) { - return host.startsWith("fc") || host.startsWith("fd") || host.startsWith("fe80") - } - - return false -} diff --git a/src/parsing.ts b/src/parsing.ts deleted file mode 100644 index c2e00ab..0000000 --- a/src/parsing.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { readFile } from "node:fs/promises" -import { htmlToText } from "html-to-text" -import { extractText, getDocumentProxy } from "unpdf" -import YAML from "yaml" -import type { ParsedDocument, SourceFile } from "./types.js" - -export async function parseFile(file: SourceFile): Promise { - let text: string - - switch (file.extension) { - case ".pdf": - text = await parsePdf(file.absolutePath) - break - case ".html": - case ".htm": - text = htmlToText(await readFile(file.absolutePath, "utf8"), { - wordwrap: false, - selectors: [ - { selector: "a", options: { ignoreHref: true } }, - { selector: "img", format: "skip" }, - ], - }) - break - case ".json": - text = JSON.stringify(JSON.parse(await readFile(file.absolutePath, "utf8")), null, 2) - break - case ".yaml": - case ".yml": - text = YAML.stringify(YAML.parse(await readFile(file.absolutePath, "utf8"))) - break - default: - text = await readFile(file.absolutePath, "utf8") - } - - return { file, text: normalizeText(text) } -} - -async function parsePdf(filePath: string): Promise { - const buffer = await readFile(filePath) - const pdf = await getDocumentProxy(new Uint8Array(buffer)) - const result = await extractText(pdf, { mergePages: true }) - return result.text -} - -function normalizeText(input: string): string { - return input - .replace(/\r\n/g, "\n") - .replace(/[ \t]+\n/g, "\n") - .replace(/\n{4,}/g, "\n\n\n") - .trim() -} diff --git a/src/version.ts b/src/version.ts deleted file mode 100644 index d91ac72..0000000 --- a/src/version.ts +++ /dev/null @@ -1 +0,0 @@ -export const VERSION = "0.3.0" diff --git a/tsconfig.json b/tsconfig.base.json similarity index 66% rename from tsconfig.json rename to tsconfig.base.json index 7afafc0..f0de0ca 100644 --- a/tsconfig.json +++ b/tsconfig.base.json @@ -7,13 +7,11 @@ "declaration": true, "declarationMap": true, "sourceMap": true, - "outDir": "dist", - "rootDir": "src", "resolveJsonModule": true, "skipLibCheck": true, "noUncheckedIndexedAccess": true, - "exactOptionalPropertyTypes": true - }, - "include": ["src/**/*.ts"], - "exclude": ["src/**/*.test.ts"] + "exactOptionalPropertyTypes": true, + "noUnusedLocals": true, + "noUnusedParameters": true + } }