From 8de0a925c7708a119cea8ddff85bd53350ee87de Mon Sep 17 00:00:00 2001 From: Nuno Aguiar Date: Thu, 30 Apr 2026 03:32:00 +0100 Subject: [PATCH 1/7] Use Lucene/ElasticSearch opacks for wiki indexing backends --- mcps/mcp-wiki.yaml | 9 ++- mini-a-wiki.js | 152 +++++++++++++++++++++++++++++++++++++++++++-- mini-a.js | 9 ++- 3 files changed, 162 insertions(+), 8 deletions(-) diff --git a/mcps/mcp-wiki.yaml b/mcps/mcp-wiki.yaml index 30c00d8..c2dccfe 100644 --- a/mcps/mcp-wiki.yaml +++ b/mcps/mcp-wiki.yaml @@ -245,7 +245,7 @@ jobs: loadLib("mini-a-wiki.js") args.wikibackend = String(args.wikibackend).toLowerCase().trim() - if (["fs", "s3"].indexOf(args.wikibackend) < 0) args.wikibackend = "fs" + if (["fs", "s3", "es", "s3fs"].indexOf(args.wikibackend) < 0) args.wikibackend = "fs" args.wikiaccess = String(args.wikiaccess).toLowerCase().trim() if (["ro", "rw"].indexOf(args.wikiaccess) < 0) args.wikiaccess = "ro" @@ -255,7 +255,7 @@ jobs: backend: args.wikibackend } - if (args.wikibackend === "s3") { + if (args.wikibackend === "s3" || args.wikibackend === "s3fs") { cfg.bucket = args.wikibucket cfg.prefix = args.wikiprefix cfg.url = args.wikiurl @@ -264,6 +264,11 @@ jobs: cfg.region = args.wikiregion cfg.useVersion1 = args.wikiuseversion1 cfg.ignoreCertCheck = args.wikiignorecertcheck + } else if (args.wikibackend === "es") { + cfg.esurl = args.wikiurl + cfg.esindex = isString(args.wikiprefix) && args.wikiprefix.trim().length > 0 ? args.wikiprefix.trim() : "mini_a_wiki" + cfg.esuser = args.wikiaccesskey + cfg.espass = args.wikisecret } else { cfg.root = isString(args.wikiroot) && args.wikiroot.trim().length > 0 ? args.wikiroot.trim() : "." } diff --git a/mini-a-wiki.js b/mini-a-wiki.js index 85c74e7..f4aa20a 100644 --- a/mini-a-wiki.js +++ b/mini-a-wiki.js @@ -9,16 +9,92 @@ var MiniAWikiManager = function(config, loggerFn) { this.configure(config) } + +MiniAWikiManager.prototype._indexMeta = function() { + return { + file: ".mini-a-wiki-lucene.json", + hiddenNames: [".mini-a-wiki-lucene.json", ".mini-a-wiki-lucene.lock"] + } +} + +MiniAWikiManager.prototype._isHiddenPath = function(path) { + var p = isString(path) ? String(path).trim() : "" + if (p.length === 0) return false + var bn = p.split("/").pop() + var meta = this._indexMeta() + return meta.hiddenNames.indexOf(p) >= 0 || meta.hiddenNames.indexOf(bn) >= 0 +} + +MiniAWikiManager.prototype._safeListPages = function(prefix) { + var self = this + return this._backend.list(prefix).filter(function(p) { + return isString(p) && p.endsWith('.md') && !self._isHiddenPath(p) + }) +} + +MiniAWikiManager.prototype._rebuildSearchIndex = function() { + if (this._access !== 'rw') return + try { + var pages = this._safeListPages("") + var docs = [] + for (var i=0;i= 0 ? backendRaw : "fs" this._config = cfg - this._backend = this._backendType === "s3" ? this._makeS3Backend(cfg) : this._makeFsBackend(cfg) + this._backend = this._backendType === "s3" ? this._makeS3Backend(cfg) : (this._backendType === "es" ? this._makeEsBackend(cfg) : (this._backendType === "s3fs" ? this._makeS3FsBackend(cfg) : this._makeFsBackend(cfg))) this._bootstrapWiki() } @@ -186,6 +262,7 @@ MiniAWikiManager.prototype.init = function() { var created = [] var skipped = [] try { + this._rebuildSearchIndex() if (!hasAgents) { var agentsContent = [ "---", @@ -514,6 +591,47 @@ MiniAWikiManager.prototype._makeS3Backend = function(cfg) { } } +MiniAWikiManager.prototype._makeEsBackend = function(cfg) { + loadLib(getOPackPath("ElasticSearch") + "/elasticsearch.js") + var esurl = isString(cfg.esurl) ? cfg.esurl : "http://127.0.0.1:9200" + var index = isString(cfg.esindex) && cfg.esindex.length > 0 ? cfg.esindex : "mini_a_wiki" + var es = new ElasticSearch(esurl, cfg.esuser, cfg.espass) + var chName = "__mini_a_wiki_es_" + sha1(index).substring(0, 8) + es.createCh(index, ["path"], chName) + return { + type: "es", + list: function(pfx) { + var prefix = isString(pfx) ? pfx : "" + return $ch(chName).getAll({ query: { prefix: { path: prefix } }, size: 10000 }).map(r => r.path).filter(isString) + }, + read: function(path) { + var r = $ch(chName).get({ path: path }) + return isMap(r) ? r.raw : __ + }, + write: function(path, content) { $ch(chName).set({ path: path }, { path: path, raw: content }) }, + exists: function(path) { return isMap($ch(chName).get({ path: path })) }, + delete: function(path) { $ch(chName).unset({ path: path }) }, + close: function() { try { $ch(chName).destroy() } catch(e) {} } + } +} + +MiniAWikiManager.prototype._makeS3FsBackend = function(cfg) { + var fsb = this._makeFsBackend(cfg) + var s3b = this._makeS3Backend(cfg) + try { + var pages = s3b.list("") + for (var i = 0; i < pages.length; i++) { + var raw = s3b.read(pages[i]) + if (isString(raw)) fsb.write(pages[i], raw) + } + } catch(e) { + this._logFn("warn", "Failed to bootstrap s3fs wiki: " + __miniAErrMsg(e)) + } finally { + try { s3b.close() } catch(ig) {} + } + return fsb +} + MiniAWikiManager.prototype.close = function() { if (isObject(this._backend) && isFunction(this._backend.close)) { this._backend.close() @@ -800,6 +918,7 @@ MiniAWikiManager.prototype.write = function(path, metaOrRaw, body, options) { try { this._backend.write(path, this._serializeFrontmatter(updatedMeta, reparsed.body)) + this._rebuildSearchIndex() return { ok: true, path: path } } catch(e) { return { ok: false, error: __miniAErrMsg(e) } @@ -834,6 +953,7 @@ MiniAWikiManager.prototype.write = function(path, metaOrRaw, body, options) { try { var content = this._serializeFrontmatter(meta, bodyText) this._backend.write(path, content) + this._rebuildSearchIndex() return { ok: true, path: path } } catch(e) { return { ok: false, error: __miniAErrMsg(e) } @@ -850,9 +970,11 @@ MiniAWikiManager.prototype.delete = function(path) { } if (path === "AGENTS.md") return { ok: false, error: "cannot delete AGENTS.md (protected)" } + if (this._isHiddenPath(path)) return { ok: false, error: "cannot delete hidden wiki index files" } try { this._backend.delete(path) + this._rebuildSearchIndex() return { ok: true, path: path } } catch(e) { return { ok: false, error: __miniAErrMsg(e) } @@ -882,8 +1004,30 @@ MiniAWikiManager.prototype.search = function(query, options) { } var pages = scopedPath.length > 0 ? [scopedPath] : this.list("") + pages = pages.filter(p => !this._isHiddenPath(p)) var results = [] + if (!opts.regex && scopedPath.length === 0 && this._ensureLucene()) { + try { + var chName = "__mini_a_wiki_searchdb" + $ch(chName).create("searchdb", { path: this._getLuceneIndexPath(), idField: "id", contentField: "content" }) + var luceneHits = $ch(chName).getAll({ query: q, limit: limit }) + $ch(chName).destroy() + if (isArray(luceneHits) && luceneHits.length > 0) { + return luceneHits.map(function(h) { + return { + path: h.id || (isMap(h.payload) ? h.payload.path : __), + title: isMap(h.payload) && isString(h.payload.title) ? h.payload.title : (h.id || ""), + line: isNumber(h.line) ? h.line : 1, + snippet: isString(h.content) ? h.content.substring(0, 180) : q + } + }).filter(r => isString(r.path) && r.path.length > 0) + } + } catch(le) { + this._logFn("warn", "Lucene search fallback to scan: " + __miniAErrMsg(le)) + } + } + for (var i = 0; i < pages.length && results.length < limit; i++) { var raw = this._backend.read(pages[i]) if (!isString(raw)) continue diff --git a/mini-a.js b/mini-a.js index 73daa4e..4365215 100644 --- a/mini-a.js +++ b/mini-a.js @@ -8254,7 +8254,7 @@ MiniA.prototype._initWiki = function(args) { access : args.wikiaccess, backend: args.wikibackend } - if (args.wikibackend === "s3") { + if (args.wikibackend === "s3" || args.wikibackend === "s3fs") { cfg.bucket = args.wikibucket cfg.prefix = args.wikiprefix cfg.url = args.wikiurl @@ -8263,6 +8263,11 @@ MiniA.prototype._initWiki = function(args) { cfg.region = args.wikiregion cfg.useVersion1 = args.wikiuseversion1 cfg.ignoreCertCheck = args.wikiignorecertcheck + } else if (args.wikibackend === "es") { + cfg.esurl = args.wikiurl + cfg.esindex = isString(args.wikiprefix) && args.wikiprefix.trim().length > 0 ? args.wikiprefix.trim() : "mini_a_wiki" + cfg.esuser = args.wikiaccesskey + cfg.espass = args.wikisecret } else { cfg.root = isString(args.wikiroot) && args.wikiroot.trim().length > 0 ? args.wikiroot.trim() : "." } @@ -16498,7 +16503,7 @@ MiniA.prototype._startInternal = function(args, sessionStartTime) { if (["ro", "rw"].indexOf(String(args.wikiaccess).toLowerCase().trim()) < 0) args.wikiaccess = "ro" else args.wikiaccess = String(args.wikiaccess).toLowerCase().trim() args.wikibackend = _$(args.wikibackend, "args.wikibackend").isString().default("fs") - if (["fs", "s3"].indexOf(String(args.wikibackend).toLowerCase().trim()) < 0) args.wikibackend = "fs" + if (["fs", "s3", "es", "s3fs"].indexOf(String(args.wikibackend).toLowerCase().trim()) < 0) args.wikibackend = "fs" else args.wikibackend = String(args.wikibackend).toLowerCase().trim() args.wikiroot = _$(args.wikiroot, "args.wikiroot").isString().default(__) args.wikibucket = _$(args.wikibucket, "args.wikibucket").isString().default(__) From 2ebf57488251f3fa9d1f5ac6d2a3d5d218cf0089 Mon Sep 17 00:00:00 2001 From: nmaguiar Date: Thu, 30 Apr 2026 05:44:11 +0100 Subject: [PATCH 2/7] feat(wiki): add Elasticsearch and OpenSearch backend support Add Elasticsearch/OpenSearch as a wiki storage backend alongside the existing fs, s3, and s3fs options. Includes configurable index name, base URL, and optional basic authentication. Updates all documentation, MCP/web configs, and tests. --- CHEATSHEET.md | 35 +++++++++++++++++------- README.md | 13 +++++++++ USAGE.md | 15 +++++++++++ docs/WHATS-NEW.md | 12 ++++----- mcps/mcp-wiki.yaml | 10 +++---- mini-a-web.yaml | 10 +++---- mini-a-wiki.js | 67 +++++++++++++++++++++++++++++++--------------- mini-a.yaml | 57 +++++++++++++++++++++++++++++++++++++++ tests/wiki.js | 13 +++++++++ tests/wiki.yaml | 5 ++++ 10 files changed, 190 insertions(+), 47 deletions(-) diff --git a/CHEATSHEET.md b/CHEATSHEET.md index ea3b509..7b9b336 100644 --- a/CHEATSHEET.md +++ b/CHEATSHEET.md @@ -820,18 +820,29 @@ When a brand-new wiki is opened with `wikiaccess=rw`, Mini-A bootstraps two star |-----------|------|---------|-------------| | `usewiki` | boolean | `false` | Enable the wiki knowledge base | | `wikiaccess` | string | `ro` | Access mode: `ro` (read-only) or `rw` (read-write) | -| `wikibackend` | string | `fs` | Backend: `fs` (filesystem) or `s3` | +| `wikibackend` | string | `fs` | Backend: `fs` (filesystem), `s3`, `s3fs`, or `es` (Elasticsearch/OpenSearch) | | `wikiroot` | string | `.` | Root directory for the `fs` backend | -| `wikibucket` | string | - | S3 bucket name (`s3` backend) | -| `wikiprefix` | string | - | S3 key prefix (`s3` backend) | -| `wikiurl` | string | - | S3-compatible endpoint URL (`s3` backend) | -| `wikiaccesskey` | string | - | S3 access key (`s3` backend) | -| `wikisecret` | string | - | S3 secret key (`s3` backend) | -| `wikiregion` | string | - | S3 region (`s3` backend) | -| `wikiuseversion1` | boolean | `false` | Use S3 path-style (v1) signing (`s3` backend) | -| `wikiignorecertcheck` | boolean | `false` | Skip TLS certificate validation (`s3` backend) | +| `wikibucket` | string | - | S3 bucket name (`s3`/`s3fs` backend) | +| `wikiprefix` | string | - | S3 key prefix (`s3`/`s3fs`) or Elasticsearch index name (`es`, defaults to `mini_a_wiki`) | +| `wikiurl` | string | - | S3-compatible endpoint URL (`s3`/`s3fs`) or Elasticsearch/OpenSearch base URL (`es`; this is the CLI-facing `esurl`) | +| `wikiaccesskey` | string | - | S3 access key (`s3`/`s3fs`) or Elasticsearch username (`es`) | +| `wikisecret` | string | - | S3 secret key (`s3`/`s3fs`) or Elasticsearch password (`es`) | +| `wikiregion` | string | - | S3 region (`s3`/`s3fs` backend) | +| `wikiuseversion1` | boolean | `false` | Use S3 path-style (v1) signing (`s3`/`s3fs` backend) | +| `wikiignorecertcheck` | boolean | `false` | Skip TLS certificate validation (`s3`/`s3fs` backend) | | `wikilintstaleddays` | number | `90` | Days before a page without an `updated` update is marked stale in lint | +Elasticsearch/OpenSearch backend mapping: + +| Mini-A parameter | Internal wiki config | Meaning | +|------------------|----------------------|---------| +| `wikiurl` | `esurl` | Elasticsearch/OpenSearch base URL | +| `wikiprefix` | `esindex` | Index name; defaults to `mini_a_wiki` | +| `wikiaccesskey` | `esuser` | Optional basic-auth username | +| `wikisecret` | `espass` | Optional basic-auth password | + +If you are looking for `esurl=`, use `wikiurl=` with `wikibackend=es`. + ### Wiki Actions (agent) The agent uses the `wiki` action: @@ -874,6 +885,12 @@ mini-a goal="analyze and wiki" \ wikibucket=my-wiki-bucket wikiprefix=knowledge/ \ wikiurl=https://s3.amazonaws.com wikiaccesskey=AKI... wikisecret=xxx wikiregion=us-east-1 +# Elasticsearch/OpenSearch-backed wiki +mini-a goal="search and update the team wiki" \ + usewiki=true wikiaccess=rw wikibackend=es \ + wikiurl=http://localhost:9200 wikiprefix=mini_a_wiki \ + wikiaccesskey=elastic wikisecret=xxx + # Wiki + memory for maximum knowledge retention mini-a goal="deep research with persistent knowledge" \ usewiki=true wikiaccess=rw wikiroot=/shared/wiki \ diff --git a/README.md b/README.md index b54380c..c41ed74 100644 --- a/README.md +++ b/README.md @@ -462,6 +462,19 @@ Mini-A ships with complementary components: | `memorymaxentries` | Total memory-entry cap across all sections | `500` | | `memorycompactevery` | Run compaction/summarization every N memory mutations | `8` | | `memorydedup` | Deduplicate near-identical memory entries before append | `true` | +| `usewiki` | Enable persistent Markdown wiki knowledge base (`wiki` action and `/wiki` console commands) | `false` | +| `wikiaccess` | Wiki access mode (`ro` or `rw`) | `ro` | +| `wikibackend` | Wiki backend: `fs`, `s3`, `s3fs`, or `es` (Elasticsearch/OpenSearch) | `fs` | +| `wikiroot` | Filesystem wiki root when `wikibackend=fs` | `.` | +| `wikibucket` | S3 bucket for `s3`/`s3fs` wiki backends | - | +| `wikiprefix` | S3 key prefix for `s3`/`s3fs`, or Elasticsearch index name for `es` | - | +| `wikiurl` | S3 endpoint URL, or Elasticsearch/OpenSearch base URL when `wikibackend=es` (internal `esurl`) | - | +| `wikiaccesskey` | S3 access key, or Elasticsearch username when `wikibackend=es` | - | +| `wikisecret` | S3 secret key, or Elasticsearch password when `wikibackend=es` | - | +| `wikiregion` | S3 region for `s3`/`s3fs` wiki backends | - | +| `wikiuseversion1` | Use S3 signature v1/path-style compatibility for wiki access | `false` | +| `wikiignorecertcheck` | Disable TLS certificate checks for wiki S3 access | `false` | +| `wikilintstaleddays` | Stale-page age threshold used by wiki lint | `90` | | `useascii` | Encourage ASCII sketch outputs in agent responses | `false` | | `usemaps` | Encourage Leaflet-based interactive map outputs for geographic data | `false` | | `usemath` | Encourage LaTeX-style math formulas (`$...$`, `$$...$$`) for KaTeX rendering in the web UI | `false` | diff --git a/USAGE.md b/USAGE.md index 0506c29..4b02565 100644 --- a/USAGE.md +++ b/USAGE.md @@ -868,6 +868,21 @@ The `start()` method accepts various configuration options: - **`memorypromote`** (string, default: `""`): Comma-separated list of memory sections to auto-promote from the session store to the global store at session end. Uses a refresh-or-append strategy: near-duplicate global entries have their `confirmedAt` and `confirmCount` updated rather than duplicated; entirely new entries are appended. `memoryuser=true` sets this to `facts,decisions,summaries`. Set to `""` to disable promotion. - **`memorystaledays`** (number, default: `0`): Number of days after which a global memory entry that has not been re-confirmed by any session is marked `stale=true`. The sweep runs automatically after each auto-promotion pass. Stale entries are not deleted immediately — they are evicted by compaction when a section overflows `memorymaxpersection`, giving recently confirmed entries priority. Set to `0` to disable staleness tracking. `memoryuser=true` sets this to `30`. - **`memoryinject`** (string, one of `"summary"` or `"full"`, default: `"summary"`): Controls how working memory is embedded in the step context. `summary` (default) injects only section entry counts (e.g. `{facts:12,decisions:3}`) and enables the `memory_search` action for on-demand retrieval — reducing per-step memory token cost by ~95%. `full` restores the previous behaviour of embedding all compact entries in every step prompt. +- **`usewiki`** (boolean, default: `false`): Enable the persistent Markdown wiki knowledge base. +- **`wikiaccess`** (string, default: `ro`): Wiki access mode, `ro` or `rw`. +- **`wikibackend`** (string, default: `fs`): Wiki backend, one of `fs`, `s3`, `s3fs`, or `es` (Elasticsearch/OpenSearch). +- **`wikiroot`** (string, default: `.`): Filesystem root for `wikibackend=fs`. +- **`wikibucket`** (string, optional): S3 bucket for `wikibackend=s3` or `wikibackend=s3fs`. +- **`wikiprefix`** (string, optional): S3 key prefix for `s3`/`s3fs`; Elasticsearch index name for `es` (defaults to `mini_a_wiki`). +- **`wikiurl`** (string, optional): S3-compatible endpoint URL for `s3`/`s3fs`; Elasticsearch/OpenSearch base URL for `es` (this is the CLI-facing equivalent of the internal `esurl`). +- **`wikiaccesskey`** (string, optional): S3 access key for `s3`/`s3fs`; Elasticsearch username for `es`. +- **`wikisecret`** (string, optional): S3 secret key for `s3`/`s3fs`; Elasticsearch password for `es`. +- **`wikiregion`** (string, optional): S3 region for `s3`/`s3fs`. +- **`wikiuseversion1`** (boolean, default: `false`): Use S3 path-style/signature-v1 compatibility for `s3`/`s3fs`. +- **`wikiignorecertcheck`** (boolean, default: `false`): Disable TLS certificate checks for the S3 endpoint. +- **`wikilintstaleddays`** (number, default: `90`): Age threshold used by wiki lint stale-page checks. + +For the Elasticsearch/OpenSearch wiki backend, there is no separate top-level `esurl=` runtime argument; use `wikiurl=` with `wikibackend=es`. - **`mode`** (string): Apply a preset from [`mini-a-modes.yaml`](mini-a-modes.yaml), `~/.openaf-mini-a_modes.yaml`, or `~/.openaf-mini-a/modes.yaml` to prefill a bundle of related flags - **`agent`** (string): Path to a markdown agent profile (or inline markdown text) with YAML frontmatter metadata. Supported keys include `model`, `capabilities` (`useshell`, `readwrite`, `useutils`, `usetools`), `tools` (MCP entries such as `type: ojob`, `type: stdio` + `cmd`, `type: remote`, or `type: sse`), `constraints` (appended to `rules`), `knowledge`, `youare`, and `mini-a` (map of direct Mini-A arg overrides). When the profile uses Markdown front matter, any text after the closing `---` is used as the default `goal=` input unless you pass `goal=` explicitly. (`agentfile` remains a backward-compatible alias.) diff --git a/docs/WHATS-NEW.md b/docs/WHATS-NEW.md index bfbb89f..eaef6e3 100644 --- a/docs/WHATS-NEW.md +++ b/docs/WHATS-NEW.md @@ -115,13 +115,13 @@ See [docs/DELEGATION.md](DELEGATION.md) for full documentation including example |-----------|---------|-------------| | `usewiki` | `false` | Enable wiki knowledge base | | `wikiaccess` | `ro` | `ro` or `rw` | -| `wikibackend` | `fs` | `fs` or `s3` | +| `wikibackend` | `fs` | `fs`, `s3`, `s3fs`, or `es` | | `wikiroot` | `.` | Root directory (FS backend) | -| `wikibucket` | — | S3 bucket | -| `wikiprefix` | — | S3 key prefix | -| `wikiurl` | — | S3 endpoint URL | -| `wikiaccesskey` | — | S3 access key | -| `wikisecret` | — | S3 secret key | +| `wikibucket` | — | S3 bucket (`s3`/`s3fs`) | +| `wikiprefix` | — | S3 key prefix (`s3`/`s3fs`) or Elasticsearch index (`es`) | +| `wikiurl` | — | S3 endpoint URL or Elasticsearch/OpenSearch base URL (`esurl` internally) | +| `wikiaccesskey` | — | S3 access key or Elasticsearch username | +| `wikisecret` | — | S3 secret key or Elasticsearch password | | `wikiregion` | — | S3 region | | `wikiuseversion1` | `false` | S3 path-style signing | | `wikiignorecertcheck` | `false` | Skip TLS cert check | diff --git a/mcps/mcp-wiki.yaml b/mcps/mcp-wiki.yaml index c2dccfe..f5ec6e9 100644 --- a/mcps/mcp-wiki.yaml +++ b/mcps/mcp-wiki.yaml @@ -7,7 +7,7 @@ help: example : "8888" mandatory: false - name : wikibackend - desc : Wiki backend type (fs or s3) + desc : Wiki backend type (fs, s3, s3fs or es) example : "fs" mandatory: false - name : wikiaccess @@ -23,18 +23,18 @@ help: example : "my-wiki-bucket" mandatory: false - name : wikiprefix - desc : S3 key prefix for s3 backend + desc : S3 key prefix for s3/s3fs backend, or Elasticsearch index for es backend example : "wiki/" mandatory: false - name : wikiurl - desc : S3 endpoint/base URL + desc : S3 endpoint/base URL for s3/s3fs, or Elasticsearch/OpenSearch base URL for es (internal esurl) example : "https://s3.amazonaws.com" mandatory: false - name : wikiaccesskey - desc : S3 access key + desc : S3 access key, or Elasticsearch username for es backend mandatory: false - name : wikisecret - desc : S3 secret key + desc : S3 secret key, or Elasticsearch password for es backend mandatory: false - name : wikiregion desc : S3 region diff --git a/mini-a-web.yaml b/mini-a-web.yaml index 5addcfc..eccee34 100644 --- a/mini-a-web.yaml +++ b/mini-a-web.yaml @@ -648,7 +648,7 @@ help: example : "rw" mandatory: false - name : wikibackend - desc : Wiki backend (fs or s3) + desc : Wiki backend (fs, s3, s3fs or es) example : "fs" mandatory: false - name : wikiroot @@ -660,19 +660,19 @@ help: example : "mini-a-wiki" mandatory: false - name : wikiprefix - desc : Prefix path for the S3 wiki backend + desc : Prefix path for S3/s3fs wiki backends, or index name for the es backend example : "wiki/" mandatory: false - name : wikiurl - desc : S3 endpoint URL for the wiki backend + desc : S3 endpoint URL for s3/s3fs, or Elasticsearch/OpenSearch base URL for es (internal esurl) example : "https://s3.amazonaws.com" mandatory: false - name : wikiaccesskey - desc : S3 access key for the wiki backend + desc : S3 access key for s3/s3fs, or Elasticsearch username for es example : "AKIA..." mandatory: false - name : wikisecret - desc : S3 secret key for the wiki backend + desc : S3 secret key for s3/s3fs, or Elasticsearch password for es example : "secret" mandatory: false - name : wikiregion diff --git a/mini-a-wiki.js b/mini-a-wiki.js index f4aa20a..f124aa0 100644 --- a/mini-a-wiki.js +++ b/mini-a-wiki.js @@ -57,6 +57,7 @@ MiniAWikiManager.prototype._getLuceneIndexPath = function() { MiniAWikiManager.prototype._ensureLucene = function() { if (this._luceneReady === true) return true try { + includeOPack("lucene") var p = getOPackPath("lucene") if (!isString(p) || p.length === 0) return false loadLib(p + "/lucene.js") @@ -411,26 +412,36 @@ MiniAWikiManager.prototype.init = function() { } var __miniAWikiFsList = function(dir, normalizedPrefix, sep) { - if (!isString(dir) || dir.length === 0) return [] - if (!io.fileExists(dir) || io.fileInfo(dir).isDirectory !== true) return [] + if (isUnDef(dir)) return [] + dir = String(dir) + if (dir.length === 0) return [] + if (!io.fileExists(dir) || io.fileInfo(dir).isDirectory != true) return [] var dirPrefix = dir.endsWith(sep) ? dir : dir + sep var raw = listFilesRecursive(dir) - if (!isArray(raw)) raw = [] - - var selected = $from(raw) - .equals("isFile", true) - .ends("canonicalPath", ".md") - .match("canonicalPath", "^" + dirPrefix.replace(/([.*+?^${}()|[\]\\])/g, "\\$1")) - .select(function(entry) { - return normalizedPrefix + String(entry.canonicalPath).substring(dirPrefix.length).replace(/\\/g, "/") - }) - - var results = isArray(selected) ? selected : (isDef(selected) && isDef(selected.length) && !isString(selected) ? af.fromJavaArray(selected) : []) + var entries = [] + if (isArray(raw)) { + entries = raw + } else if (isMap(raw) && isArray(raw.files)) { + entries = raw.files + } else if (isDef(raw) && isFunction(raw.forEach)) { + raw.forEach(function(entry) { entries.push(entry) }) + } var dedup = [] var seen = {} - results.forEach(function(relPath) { + entries.forEach(function(entry) { + if (!isMap(entry) || entry.isFile != true) return + var entryPath = isString(entry.canonicalPath) ? entry.canonicalPath : "" + if (entryPath.length === 0 && isString(entry.filepath)) entryPath = entry.filepath + if (entryPath.length === 0 && isString(entry.path) && isString(entry.filename)) entryPath = entry.path + sep + entry.filename + if (entryPath.length === 0) return + + try { entryPath = new java.io.File(entryPath).getCanonicalPath() } catch(e) {} + if (!entryPath.endsWith(".md")) return + if (!entryPath.startsWith(dirPrefix)) return + + var relPath = normalizedPrefix + String(entryPath).substring(dirPrefix.length).replace(/\\/g, "/") if (!isString(relPath) || relPath.length === 0 || seen[relPath] === true) return seen[relPath] = true dedup.push(relPath) @@ -439,6 +450,13 @@ var __miniAWikiFsList = function(dir, normalizedPrefix, sep) { return dedup.sort() } +var __miniAWikiEsRowsToPaths = function(rows) { + if (!isArray(rows)) return [] + return rows.map(function(r) { + return isMap(r) && isString(r.path) ? r.path : __ + }).filter(isString) +} + var __miniAWikiNormalizePath = function(path, options) { var opts = isMap(options) ? options : {} if (!isString(path)) throw "path is required" @@ -476,7 +494,7 @@ MiniAWikiManager.prototype._makeFsBackend = function(cfg) { var sep = String(java.io.File.separator) var rawRoot = isDef(cfg.root) ? String(cfg.root).trim() : "" var root = rawRoot.length > 0 ? rawRoot : "." - var canonicalRoot = new java.io.File(root).getCanonicalPath() + var canonicalRoot = String(new java.io.File(root).getCanonicalPath()) var canonicalRootPrefix = canonicalRoot.endsWith(sep) ? canonicalRoot : canonicalRoot + sep var normalizePrefix = function(value) { var prefix = isDef(value) ? String(value).trim().replace(/\\/g, "/") : "" @@ -488,7 +506,7 @@ MiniAWikiManager.prototype._makeFsBackend = function(cfg) { return prefix } var resolvePath = function(relPath, allowMissingLeaf) { - var rel = isDef(relPath) ? __miniAWikiNormalizePath(relPath, { + var rel = (isDef(relPath) && String(relPath).length > 0) ? __miniAWikiNormalizePath(relPath, { allowDirectory : allowMissingLeaf !== true, requireMarkdown : allowMissingLeaf !== true }) : "" @@ -496,10 +514,10 @@ MiniAWikiManager.prototype._makeFsBackend = function(cfg) { var canonical if (allowMissingLeaf === true && !candidate.exists()) { var parent = candidate.getParentFile() - var parentCanonical = isDef(parent) ? parent.getCanonicalPath() : canonicalRoot + var parentCanonical = isDef(parent) ? String(parent.getCanonicalPath()) : canonicalRoot canonical = parentCanonical + sep + candidate.getName() } else { - canonical = candidate.getCanonicalPath() + canonical = String(candidate.getCanonicalPath()) } if (canonical !== canonicalRoot && !canonical.startsWith(canonicalRootPrefix)) { throw "path escapes wikiroot" @@ -517,7 +535,10 @@ MiniAWikiManager.prototype._makeFsBackend = function(cfg) { } catch(e) { return [] } }, read: function(path) { - try { return io.readFileString(resolvePath(path, false)) } catch(e) { return __ } + try { + var content = io.readFileString(resolvePath(path, false)) + return isDef(content) ? String(content) : __ + } catch(e) { return __ } }, write: function(path, content) { var full = resolvePath(path, true) @@ -592,7 +613,8 @@ MiniAWikiManager.prototype._makeS3Backend = function(cfg) { } MiniAWikiManager.prototype._makeEsBackend = function(cfg) { - loadLib(getOPackPath("ElasticSearch") + "/elasticsearch.js") + includeOPack("ElasticSearch") + loadLib("/elasticsearch.js") var esurl = isString(cfg.esurl) ? cfg.esurl : "http://127.0.0.1:9200" var index = isString(cfg.esindex) && cfg.esindex.length > 0 ? cfg.esindex : "mini_a_wiki" var es = new ElasticSearch(esurl, cfg.esuser, cfg.espass) @@ -602,7 +624,7 @@ MiniAWikiManager.prototype._makeEsBackend = function(cfg) { type: "es", list: function(pfx) { var prefix = isString(pfx) ? pfx : "" - return $ch(chName).getAll({ query: { prefix: { path: prefix } }, size: 10000 }).map(r => r.path).filter(isString) + return __miniAWikiEsRowsToPaths($ch(chName).getAll({ query: { prefix: { path: prefix } }, size: 10000 })) }, read: function(path) { var r = $ch(chName).get({ path: path }) @@ -1014,7 +1036,7 @@ MiniAWikiManager.prototype.search = function(query, options) { var luceneHits = $ch(chName).getAll({ query: q, limit: limit }) $ch(chName).destroy() if (isArray(luceneHits) && luceneHits.length > 0) { - return luceneHits.map(function(h) { + var validHits = luceneHits.map(function(h) { return { path: h.id || (isMap(h.payload) ? h.payload.path : __), title: isMap(h.payload) && isString(h.payload.title) ? h.payload.title : (h.id || ""), @@ -1022,6 +1044,7 @@ MiniAWikiManager.prototype.search = function(query, options) { snippet: isString(h.content) ? h.content.substring(0, 180) : q } }).filter(r => isString(r.path) && r.path.length > 0) + if (validHits.length > 0) return validHits } } catch(le) { this._logFn("warn", "Lucene search fallback to scan: " + __miniAErrMsg(le)) diff --git a/mini-a.yaml b/mini-a.yaml index 39f22ef..bcbaff8 100644 --- a/mini-a.yaml +++ b/mini-a.yaml @@ -599,6 +599,63 @@ help: example : "false" mandatory: false options : ["true", "false"] + - name : usewiki + desc : Enable the wiki knowledge base + example : "true" + mandatory: false + options : ["true", "false"] + - name : wikiaccess + desc : Wiki access mode (ro or rw) + example : "rw" + mandatory: false + options : ["ro", "rw"] + - name : wikibackend + desc : Wiki backend (fs, s3, s3fs or es) + example : "fs" + mandatory: false + options : ["fs", "s3", "s3fs", "es"] + - name : wikiroot + desc : Root directory for the filesystem wiki backend + example : "/shared/wiki" + mandatory: false + - name : wikibucket + desc : Bucket name for S3/s3fs wiki backends + example : "mini-a-wiki" + mandatory: false + - name : wikiprefix + desc : Prefix path for S3/s3fs wiki backends, or index name for the es backend + example : "wiki/" + mandatory: false + - name : wikiurl + desc : S3 endpoint URL for s3/s3fs, or Elasticsearch/OpenSearch base URL for es (internal esurl) + example : "https://s3.amazonaws.com" + mandatory: false + - name : wikiaccesskey + desc : S3 access key for s3/s3fs, or Elasticsearch username for es + example : "AKIA..." + mandatory: false + - name : wikisecret + desc : S3 secret key for s3/s3fs, or Elasticsearch password for es + example : "secret" + mandatory: false + - name : wikiregion + desc : S3 region for s3/s3fs wiki backends + example : "eu-west-1" + mandatory: false + - name : wikiuseversion1 + desc : Use S3 signature v1/path-style compatibility for wiki access + example : "false" + mandatory: false + options : ["true", "false"] + - name : wikiignorecertcheck + desc : Disable TLS certificate checks for wiki S3 access + example : "false" + mandatory: false + options : ["true", "false"] + - name : wikilintstaleddays + desc : Default stale-days threshold for wiki lint + example : "90" + mandatory: false - name : saveplannotes desc : Save execution notes back to the plan file after execution example : "true" diff --git a/tests/wiki.js b/tests/wiki.js index b3d0d5a..381ec2d 100644 --- a/tests/wiki.js +++ b/tests/wiki.js @@ -213,6 +213,19 @@ } } + exports.testEsRowsToPathsSkipsUndefinedRows = function() { + var paths = __miniAWikiEsRowsToPaths([ + __, + { path: "index.md" }, + {}, + { path: 42 }, + { path: "docs/page.md" } + ]) + ow.test.assert(paths.length, 2, "should keep only rows with string paths") + ow.test.assert(paths[0], "index.md", "should keep first valid path") + ow.test.assert(paths[1], "docs/page.md", "should keep second valid path") + } + exports.testFsBackendReadWrite = function() { var dir = createTestDir() try { diff --git a/tests/wiki.yaml b/tests/wiki.yaml index 6e6ff71..dcd6ace 100644 --- a/tests/wiki.yaml +++ b/tests/wiki.yaml @@ -65,6 +65,11 @@ jobs: to : oJob Test exec: args.func = args.tests.testFsBackendListHandlesIterableListFilesRecursiveShape +- name: MiniA Wiki Tests::EsRowsToPathsSkipsUndefinedRows + from: MiniA Wiki Tests::Init + to : oJob Test + exec: args.func = args.tests.testEsRowsToPathsSkipsUndefinedRows + - name: MiniA Wiki Tests::FsBackendReadWrite from: MiniA Wiki Tests::Init to : oJob Test From da5264c6d8a8f00a62bfaeebe0875a41fc42d8a1 Mon Sep 17 00:00:00 2001 From: Nuno Aguiar Date: Wed, 6 May 2026 05:31:31 +0100 Subject: [PATCH 3/7] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- mcps/mcp-wiki.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/mcps/mcp-wiki.yaml b/mcps/mcp-wiki.yaml index f5ec6e9..ed26286 100644 --- a/mcps/mcp-wiki.yaml +++ b/mcps/mcp-wiki.yaml @@ -264,6 +264,7 @@ jobs: cfg.region = args.wikiregion cfg.useVersion1 = args.wikiuseversion1 cfg.ignoreCertCheck = args.wikiignorecertcheck + if (args.wikibackend === "s3fs") cfg.root = isString(args.wikiroot) && args.wikiroot.trim().length > 0 ? args.wikiroot.trim() : "." } else if (args.wikibackend === "es") { cfg.esurl = args.wikiurl cfg.esindex = isString(args.wikiprefix) && args.wikiprefix.trim().length > 0 ? args.wikiprefix.trim() : "mini_a_wiki" From 09a8a622a1f2982a224c31d35ec7ce6179851aa6 Mon Sep 17 00:00:00 2001 From: Nuno Aguiar Date: Wed, 6 May 2026 05:32:11 +0100 Subject: [PATCH 4/7] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- mini-a-wiki.js | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/mini-a-wiki.js b/mini-a-wiki.js index f124aa0..a1ff9fb 100644 --- a/mini-a-wiki.js +++ b/mini-a-wiki.js @@ -75,12 +75,19 @@ MiniAWikiManager.prototype._rebuildLuceneIndex = function(docs) { try { var idxPath = this._getLuceneIndexPath() var chName = "__mini_a_wiki_searchdb" - $ch(chName).destroy() - $ch(chName).create("searchdb", { path: idxPath, idField: "id", contentField: "content" }) - ;(isArray(docs) ? docs : []).forEach(function(d) { - $ch(chName).set({ id: d.path }, { content: d.raw, payload: { path: d.path, title: d.title } }) - }) - $ch(chName).destroy() + try { + $ch(chName).destroy() + } catch(ignore) {} + try { + $ch(chName).create("searchdb", { path: idxPath, idField: "id", contentField: "content" }) + ;(isArray(docs) ? docs : []).forEach(function(d) { + $ch(chName).set({ id: d.path }, { content: d.raw, payload: { path: d.path, title: d.title } }) + }) + } finally { + try { + $ch(chName).destroy() + } catch(ignore2) {} + } } catch(e) { this._logFn("warn", "Failed to rebuild Lucene index: " + __miniAErrMsg(e)) } From 4e3c621d502748024421c3afc64f43b1763d4d3c Mon Sep 17 00:00:00 2001 From: Nuno Aguiar Date: Wed, 6 May 2026 05:32:41 +0100 Subject: [PATCH 5/7] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- mini-a-wiki.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/mini-a-wiki.js b/mini-a-wiki.js index a1ff9fb..61304f9 100644 --- a/mini-a-wiki.js +++ b/mini-a-wiki.js @@ -1039,9 +1039,13 @@ MiniAWikiManager.prototype.search = function(query, options) { if (!opts.regex && scopedPath.length === 0 && this._ensureLucene()) { try { var chName = "__mini_a_wiki_searchdb" - $ch(chName).create("searchdb", { path: this._getLuceneIndexPath(), idField: "id", contentField: "content" }) - var luceneHits = $ch(chName).getAll({ query: q, limit: limit }) - $ch(chName).destroy() + var luceneHits + try { + $ch(chName).create("searchdb", { path: this._getLuceneIndexPath(), idField: "id", contentField: "content" }) + luceneHits = $ch(chName).getAll({ query: q, limit: limit }) + } finally { + $ch(chName).destroy() + } if (isArray(luceneHits) && luceneHits.length > 0) { var validHits = luceneHits.map(function(h) { return { From 595fe389d4ca6320fbc9fbcfa94d9eb3a93ceda9 Mon Sep 17 00:00:00 2001 From: Nuno Aguiar Date: Wed, 6 May 2026 05:33:47 +0100 Subject: [PATCH 6/7] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- mini-a-wiki.js | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/mini-a-wiki.js b/mini-a-wiki.js index 61304f9..873bcba 100644 --- a/mini-a-wiki.js +++ b/mini-a-wiki.js @@ -646,18 +646,33 @@ MiniAWikiManager.prototype._makeEsBackend = function(cfg) { MiniAWikiManager.prototype._makeS3FsBackend = function(cfg) { var fsb = this._makeFsBackend(cfg) - var s3b = this._makeS3Backend(cfg) - try { - var pages = s3b.list("") - for (var i = 0; i < pages.length; i++) { - var raw = s3b.read(pages[i]) - if (isString(raw)) fsb.write(pages[i], raw) + var access = isString(cfg.wikiaccess) ? cfg.wikiaccess.toLowerCase() : "rw" + + if (access !== "ro") { + var s3b = this._makeS3Backend(cfg) + try { + var pages = s3b.list("") + for (var i = 0; i < pages.length; i++) { + var raw = s3b.read(pages[i]) + if (!isString(raw)) continue + + var shouldWrite = true + try { + if (isFunction(fsb.exists) && fsb.exists(pages[i])) { + var current = isFunction(fsb.read) ? fsb.read(pages[i]) : __ + shouldWrite = raw !== current + } + } catch(ig) {} + + if (shouldWrite) fsb.write(pages[i], raw) + } + } catch(e) { + this._logFn("warn", "Failed to bootstrap s3fs wiki: " + __miniAErrMsg(e)) + } finally { + try { s3b.close() } catch(ig) {} } - } catch(e) { - this._logFn("warn", "Failed to bootstrap s3fs wiki: " + __miniAErrMsg(e)) - } finally { - try { s3b.close() } catch(ig) {} } + return fsb } From a58de13a6d3bc041c4d7806f1e86f29b416b00a9 Mon Sep 17 00:00:00 2001 From: Nuno Aguiar Date: Wed, 6 May 2026 05:37:15 +0100 Subject: [PATCH 7/7] typo --- mini-a-wiki.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mini-a-wiki.js b/mini-a-wiki.js index 873bcba..c4c8b8a 100644 --- a/mini-a-wiki.js +++ b/mini-a-wiki.js @@ -58,9 +58,7 @@ MiniAWikiManager.prototype._ensureLucene = function() { if (this._luceneReady === true) return true try { includeOPack("lucene") - var p = getOPackPath("lucene") - if (!isString(p) || p.length === 0) return false - loadLib(p + "/lucene.js") + loadLib("lucene.js") this._luceneReady = true return true } catch(e) { @@ -621,7 +619,7 @@ MiniAWikiManager.prototype._makeS3Backend = function(cfg) { MiniAWikiManager.prototype._makeEsBackend = function(cfg) { includeOPack("ElasticSearch") - loadLib("/elasticsearch.js") + loadLib("elasticsearch.js") var esurl = isString(cfg.esurl) ? cfg.esurl : "http://127.0.0.1:9200" var index = isString(cfg.esindex) && cfg.esindex.length > 0 ? cfg.esindex : "mini_a_wiki" var es = new ElasticSearch(esurl, cfg.esuser, cfg.espass)