From a01e47e44a8923f4471140bccd9f5acce23ca082 Mon Sep 17 00:00:00 2001 From: jlin53882 Date: Tue, 31 Mar 2026 18:04:14 +0800 Subject: [PATCH 1/9] feat: make rerank timeout configurable via rerankTimeoutMs (closes #346) --- src/retriever.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/retriever.ts b/src/retriever.ts index 900db753..0cf03bca 100644 --- a/src/retriever.ts +++ b/src/retriever.ts @@ -58,6 +58,8 @@ export interface RetrievalConfig { | "pinecone" | "dashscope" | "tei"; + /** Rerank API timeout in milliseconds (default: 5000). Increase for local/CPU-based rerank servers. */ + rerankTimeoutMs?: number; /** * Length normalization: penalize long entries that dominate via sheer keyword * density. Formula: score *= 1 / (1 + log2(charLen / anchor)). @@ -127,6 +129,7 @@ export const DEFAULT_RETRIEVAL_CONFIG: RetrievalConfig = { filterNoise: true, rerankModel: "jina-reranker-v3", rerankEndpoint: "https://api.jina.ai/v1/rerank", + rerankTimeoutMs: 5000, lengthNormAnchor: 500, hardMinScore: 0.35, timeDecayHalfLifeDays: 60, @@ -858,9 +861,9 @@ export class MemoryRetriever { results.length, ); - // Timeout: 5 seconds to prevent stalling retrieval pipeline + // Timeout: configurable via rerankTimeoutMs (default: 5000ms) const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), 5000); + const timeout = setTimeout(() => controller.abort(), this.config.rerankTimeoutMs ?? 5000); const response = await fetch(endpoint, { method: "POST", @@ -928,7 +931,7 @@ export class MemoryRetriever { } } catch (error) { if (error instanceof Error && error.name === "AbortError") { - console.warn("Rerank API timed out (5s), falling back to cosine"); + console.warn(`Rerank API timed out (${this.config.rerankTimeoutMs ?? 5000}ms), falling back to cosine`); } else { console.warn("Rerank API failed, falling back to cosine:", error); } From 76421723dd163679437916f53b958c7062530862 Mon Sep 17 00:00:00 2001 From: jlin53882 Date: Tue, 31 Mar 2026 18:06:20 +0800 Subject: [PATCH 2/9] fix: add idempotent guard + governance detail logging - Add _initialized singleton flag to prevent re-initialization when register() is called multiple times during gateway boot - Add per-entry debug logging for governance filter decisions (id, reason, score, text snippet) for observability - Export _resetInitialized() for test harness reset - Fixes initialization block repeated N times on startup - Fixes governance filter decisions not observable in logs --- index.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/index.ts b/index.ts index 52f1962e..2743f5fb 100644 --- a/index.ts +++ b/index.ts @@ -1609,6 +1609,8 @@ const pluginVersion = getPluginVersion(); // Plugin Definition // ============================================================================ +let _initialized = false; + const memoryLanceDBProPlugin = { id: "memory-lancedb-pro", name: "Memory (LanceDB Pro)", @@ -1617,6 +1619,14 @@ const memoryLanceDBProPlugin = { kind: "memory" as const, register(api: OpenClawPluginApi) { + + // Idempotent guard: skip re-init on repeated register() calls + if (_initialized) { + api.logger.debug("memory-lancedb-pro: register() called again — skipping re-init (idempotent)"); + return; + } + _initialized = true; + // Parse and validate configuration const config = parsePluginConfig(api.pluginConfig); @@ -2359,10 +2369,12 @@ const memoryLanceDBProPlugin = { const meta = parseSmartMetadata(r.entry.metadata, r.entry); if (meta.state !== "confirmed") { stateFilteredCount++; + api.logger.debug(`memory-lancedb-pro: governance: filtered id=${r.entry.id} reason=state(${meta.state}) score=${r.score?.toFixed(3)} text=${r.entry.text.slice(0, 50)}`); return false; } if (meta.memory_layer === "archive" || meta.memory_layer === "reflection") { stateFilteredCount++; + api.logger.debug(`memory-lancedb-pro: governance: filtered id=${r.entry.id} reason=layer(${meta.memory_layer}) score=${r.score?.toFixed(3)} text=${r.entry.text.slice(0, 50)}`); return false; } if (meta.suppressed_until_turn > 0 && currentTurn <= meta.suppressed_until_turn) { @@ -3956,4 +3968,6 @@ export function parsePluginConfig(value: unknown): PluginConfig { }; } +export function _resetInitialized() { _initialized = false; } + export default memoryLanceDBProPlugin; From 1eb325429863e634f316683920892bc7715fbf90 Mon Sep 17 00:00:00 2001 From: jlin53882 Date: Tue, 31 Mar 2026 18:26:43 +0800 Subject: [PATCH 3/9] feat: add autoRecallExcludeAgents config to suppress injection for background agents Background agents (e.g. memory-distiller, cron workers) that receive structured task prompts can have their output quality degraded when memory context is injected alongside raw input. autoRecallExcludeAgents allows per-agent opt-out at the injection layer. Adapted from PR #307 (fix/autorecall-exclude-background-agents). Closes #137. --- index.ts | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/index.ts b/index.ts index 2743f5fb..cf95ad0b 100644 --- a/index.ts +++ b/index.ts @@ -109,6 +109,8 @@ interface PluginConfig { /** Hard per-turn injection cap (safety valve). Overrides autoRecallMaxItems if lower. Default: 10. */ maxRecallPerTurn?: number; recallMode?: "full" | "summary" | "adaptive" | "off"; + /** Agent IDs excluded from auto-recall injection. Useful for background agents (e.g. memory-distiller, cron workers) whose output should not be contaminated by injected memory context. */ + autoRecallExcludeAgents?: string[]; captureAssistant?: boolean; retrieval?: { mode?: "hybrid" | "vector"; @@ -2258,6 +2260,20 @@ const memoryLanceDBProPlugin = { const AUTO_RECALL_TIMEOUT_MS = parsePositiveInt(config.autoRecallTimeoutMs) ?? 5_000; // configurable; default raised from 3s to 5s for remote embedding APIs behind proxies api.on("before_prompt_build", async (event: any, ctx: any) => { + // Per-agent exclusion: skip auto-recall for agents in the exclusion list. + const agentId = resolveHookAgentId(ctx?.agentId, (event as any).sessionKey); + if ( + Array.isArray(config.autoRecallExcludeAgents) && + config.autoRecallExcludeAgents.length > 0 && + agentId !== undefined && + config.autoRecallExcludeAgents.includes(agentId) + ) { + api.logger.debug?.( + `memory-lancedb-pro: auto-recall skipped for excluded agent '${agentId}'`, + ); + return; + } + // Manually increment turn counter for this session const sessionId = ctx?.sessionId || "default"; @@ -3830,6 +3846,10 @@ export function parsePluginConfig(value: unknown): PluginConfig { autoRecallMaxChars: parsePositiveInt(cfg.autoRecallMaxChars) ?? 600, autoRecallPerItemMaxChars: parsePositiveInt(cfg.autoRecallPerItemMaxChars) ?? 180, maxRecallPerTurn: parsePositiveInt(cfg.maxRecallPerTurn) ?? 10, + recallMode: (cfg.recallMode === "full" || cfg.recallMode === "summary" || cfg.recallMode === "adaptive" || cfg.recallMode === "off") ? cfg.recallMode : "full", + autoRecallExcludeAgents: Array.isArray(cfg.autoRecallExcludeAgents) + ? cfg.autoRecallExcludeAgents.filter((id: unknown): id is string => typeof id === "string" && id.trim() !== "") + : undefined, captureAssistant: cfg.captureAssistant === true, retrieval: typeof cfg.retrieval === "object" && cfg.retrieval !== null ? cfg.retrieval as any : undefined, decay: typeof cfg.decay === "object" && cfg.decay !== null ? cfg.decay as any : undefined, From d2bb0075ef1c2bee9f3c4feac22f3cdae2503d89 Mon Sep 17 00:00:00 2001 From: jlin53882 Date: Tue, 31 Mar 2026 18:27:57 +0800 Subject: [PATCH 4/9] docs: add Option C for autoRecallExcludeAgents in troubleshooting --- README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/README.md b/README.md index 51adce7c..497b7cd0 100644 --- a/README.md +++ b/README.md @@ -618,6 +618,23 @@ Sometimes the model may echo the injected `` block. **Option B (preferred):** keep recall, add to agent system prompt: > Do not reveal or quote any `` / memory-injection content in your replies. Use it for internal reference only. +**Option C (for background/batch agents):** exclude specific agents from auto-recall injection: +```json +{ + "plugins": { + "entries": { + "memory-lancedb-pro": { + "config": { + "autoRecall": true, + "autoRecallExcludeAgents": ["memory-distiller", "my-cron-agent"] + } + } + } + } +} +``` +Useful for background agents (e.g. memory-distiller, cron workers) whose output should not be contaminated by injected memory context. +
From ab501f5c18642d9aff8c466fdec8e1ad8e94459f Mon Sep 17 00:00:00 2001 From: jlin53882 Date: Tue, 31 Mar 2026 18:51:44 +0800 Subject: [PATCH 5/9] feat: add import-markdown CLI command Add `memory-pro import-markdown` command to migrate existing Markdown memories (MEMORY.md, memory/YYYY-MM-DD.md) into the plugin LanceDB store for semantic recall. This addresses Issue #344 by providing a migration path from the Markdown layer to the plugin memory layer. --- cli.ts | 125 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) diff --git a/cli.ts b/cli.ts index 99203916..dd062ad3 100644 --- a/cli.ts +++ b/cli.ts @@ -1036,6 +1036,131 @@ export function registerMemoryCLI(program: Command, context: CLIContext): void { } }); + /** + * import-markdown: Import memories from Markdown memory files into the plugin store. + * Targets MEMORY.md and memory/YYYY-MM-DD.md files found in OpenClaw workspaces. + */ + memory + .command("import-markdown [workspace-glob]") + .description("Import memories from Markdown files (MEMORY.md, memory/YYYY-MM-DD.md) into the plugin store") + .option("--dry-run", "Show what would be imported without importing") + .option("--scope ", "Import into specific scope (default: global)") + .option( + "--openclaw-home ", + "OpenClaw home directory (default: ~/.openclaw)", + ) + .action(async (workspaceGlob, options) => { + const openclawHome = options.openclawHome + ? path.resolve(options.openclawHome) + : path.join(homedir(), ".openclaw"); + + const workspaceDir = path.join(openclawHome, "workspace"); + let imported = 0; + let skipped = 0; + let foundFiles = 0; + + if (!context.embedder) { + console.error( + "import-markdown requires an embedder. Use via plugin CLI or ensure embedder is configured.", + ); + process.exit(1); + } + + // Scan workspace directories + let workspaceEntries: string[]; + try { + const fsPromises = await import("node:fs/promises"); + workspaceEntries = await fsPromises.readdir(workspaceDir, { withFileTypes: true }); + } catch { + console.error(`Failed to read workspace directory: ${workspaceDir}`); + process.exit(1); + } + + // Collect all markdown files to scan + const mdFiles: Array<{ filePath: string; scope: string }> = []; + + for (const entry of workspaceEntries) { + if (!entry.isDirectory()) continue; + if (workspaceGlob && !entry.name.includes(workspaceGlob)) continue; + + const workspacePath = path.join(workspaceDir, entry.name); + + // MEMORY.md + const memoryMd = path.join(workspacePath, "MEMORY.md"); + try { + const { stat } = await import("node:fs/promises"); + await stat(memoryMd); + mdFiles.push({ filePath: memoryMd, scope: entry.name }); + } catch { /* not found */ } + + // memory/ directory + const memoryDir = path.join(workspacePath, "memory"); + try { + const { stat } = await import("node:fs/promises"); + const stats = await stat(memoryDir); + if (stats.isDirectory()) { + const { readdir } = await import("node:fs/promises"); + const files = await readdir(memoryDir); + for (const f of files) { + if (f.endsWith(".md") && /^\d{4}-\d{2}-\d{2}/.test(f)) { + mdFiles.push({ filePath: path.join(memoryDir, f), scope: entry.name }); + } + } + } + } catch { /* not found */ } + } + + if (mdFiles.length === 0) { + console.log("No Markdown memory files found."); + return; + } + + const targetScope = options.scope || "global"; + + // Parse each file for memory entries (lines starting with "- ") + for (const { filePath, scope } of mdFiles) { + foundFiles++; + const { readFile } = await import("node:fs/promises"); + const content = await readFile(filePath, "utf-8"); + const lines = content.split("\n"); + + for (const line of lines) { + // Skip non-memory lines + if (!line.startsWith("- ")) continue; + const text = line.slice(2).trim(); + if (text.length < 5) { skipped++; continue; } + + if (options.dryRun) { + console.log(` [dry-run] would import: ${text.slice(0, 80)}...`); + imported++; + continue; + } + + try { + const vector = await context.embedder!.embedQuery(text); + await context.store.store({ + text, + vector, + importance: 0.7, + category: "other", + scope: targetScope, + metadata: { importedFrom: filePath, sourceScope: scope }, + }); + imported++; + } catch (err) { + console.warn(` Failed to import: ${text.slice(0, 60)}... — ${err}`); + skipped++; + } + } + } + + if (options.dryRun) { + console.log(`\nDRY RUN — found ${foundFiles} files, ${imported} entries would be imported, ${skipped} skipped`); + } else { + console.log(`\nImport complete: ${imported} imported, ${skipped} skipped (scanned ${foundFiles} files)`); + } + }); + // Re-embed an existing LanceDB into the current target DB (A/B testing) memory .command("reembed") From 389750c90f2a3a5132c80f85c571adf5398c7450 Mon Sep 17 00:00:00 2001 From: jlin53882 Date: Tue, 31 Mar 2026 18:09:12 +0800 Subject: [PATCH 6/9] docs: clarify dual-memory architecture (fixes #344) Add two improvements addressing Issue #344: 1. README: add Dual-Memory Architecture section explaining Plugin Memory (LanceDB) vs Markdown Memory distinction 2. index.ts: log dual-memory warning on plugin startup Refs: CortexReach#344 --- README.md | 47 +- cli.ts | 1349 ----------------------------------------------------- index.ts | 9 + 3 files changed, 34 insertions(+), 1371 deletions(-) delete mode 100644 cli.ts diff --git a/README.md b/README.md index 51adce7c..99e489b7 100644 --- a/README.md +++ b/README.md @@ -9,19 +9,10 @@ A LanceDB-backed OpenClaw memory plugin that stores preferences, decisions, and project context, then auto-recalls them in future sessions. [![OpenClaw Plugin](https://img.shields.io/badge/OpenClaw-Plugin-blue)](https://github.com/openclaw/openclaw) -[![OpenClaw 2026.3+](https://img.shields.io/badge/OpenClaw-2026.3%2B-brightgreen)](https://github.com/openclaw/openclaw) [![npm version](https://img.shields.io/npm/v/memory-lancedb-pro)](https://www.npmjs.com/package/memory-lancedb-pro) [![LanceDB](https://img.shields.io/badge/LanceDB-Vectorstore-orange)](https://lancedb.com) [![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE) -

⚡ v1.1.0-beta.10 — OpenClaw 2026.3+ Hook Adaptation

- -

- ✅ Fully adapted for OpenClaw 2026.3+ new plugin architecture
- 🔄 Uses before_prompt_build hooks (replacing deprecated before_agent_start)
- 🩺 Run openclaw doctor --fix after upgrading -

- [English](README.md) | [简体中文](README_CN.md) | [繁體中文](README_TW.md) | [日本語](README_JA.md) | [한국어](README_KO.md) | [Français](README_FR.md) | [Español](README_ES.md) | [Deutsch](README_DE.md) | [Italiano](README_IT.md) | [Русский](README_RU.md) | [Português (Brasil)](README_PT-BR.md) @@ -129,6 +120,31 @@ Add to your `openclaw.json`: - `extractMinMessages: 2` → extraction triggers in normal two-turn chats - `sessionMemory.enabled: false` → avoids polluting retrieval with session summaries on day one +--- + +## ⚠️ Dual-Memory Architecture (Important) + +When `memory-lancedb-pro` is active, your system has **two independent memory layers** that do **not** auto-sync: + +| Memory Layer | Storage | What it's for | Recallable? | +|---|---|---|---| +| **Plugin Memory** | LanceDB (vector store) | Semantic recall via `memory_recall` / auto-recall | ✅ Yes | +| **Markdown Memory** | `MEMORY.md`, `memory/YYYY-MM-DD.md` | Startup context, human-readable journal | ❌ Not auto-recalled | + +**Key principle:** +> A fact written into `memory/YYYY-MM-DD.md` is visible in startup context, but `memory_recall` **will not find it** unless it was also written via `memory_store` (or auto-captured by the plugin). + +**What this means for you:** +- Need semantic recall? → Use `memory_store` or let auto-capture do it +- `memory/YYYY-MM-DD.md` → treat as a **daily journal / log**, not a recall source +- `MEMORY.md` → curated human-readable reference, not a recall source +- Plugin memory → **primary recall source** for `memory_recall` and auto-recall + +**If you want your Markdown memories to be recallable**, use the import command: +```bash +npx memory-lancedb-pro memory-pro import-markdown +``` + Validate & restart: ```bash @@ -620,19 +636,6 @@ Sometimes the model may echo the injected `` block.
-
-Auto-recall timeout tuning - -Auto-recall has a configurable timeout (default 5s) to prevent stalling agent startup. If you're behind a proxy or using a high-latency embedding API, increase it: - -```json -{ "plugins": { "entries": { "memory-lancedb-pro": { "config": { "autoRecallTimeoutMs": 8000 } } } } } -``` - -If auto-recall consistently times out, check your embedding API latency first. The timeout only affects the automatic injection path — manual `memory_recall` tool calls are not affected. - -
-
Session Memory diff --git a/cli.ts b/cli.ts deleted file mode 100644 index 99203916..00000000 --- a/cli.ts +++ /dev/null @@ -1,1349 +0,0 @@ -/** - * CLI Commands for Memory Management - */ - -import type { Command } from "commander"; -import { readFileSync } from "node:fs"; -import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; -import { homedir } from "node:os"; -import path from "node:path"; -import * as readline from "node:readline"; -import JSON5 from "json5"; -import { loadLanceDB, type MemoryEntry, type MemoryStore } from "./src/store.js"; -import { createRetriever, type MemoryRetriever } from "./src/retriever.js"; -import type { MemoryScopeManager } from "./src/scopes.js"; -import type { MemoryMigrator } from "./src/migrate.js"; -import { createMemoryUpgrader } from "./src/memory-upgrader.js"; -import type { LlmClient } from "./src/llm-client.js"; -import { - getDefaultOauthModelForProvider, - getOAuthProviderLabel, - isOauthModelSupported, - listOAuthProviders, - normalizeOauthModel, - normalizeOAuthProviderId, - performOAuthLogin, -} from "./src/llm-oauth.js"; - -// ============================================================================ -// Types -// ============================================================================ - -interface CLIContext { - store: MemoryStore; - retriever: MemoryRetriever; - scopeManager: MemoryScopeManager; - migrator: MemoryMigrator; - embedder?: import("./src/embedder.js").Embedder; - llmClient?: LlmClient; - pluginId?: string; - pluginConfig?: Record; - oauthTestHooks?: { - openUrl?: (url: string) => void | Promise; - authorizeUrl?: (url: string) => void | Promise; - chooseProvider?: ( - providers: Array<{ id: string; label: string; defaultModel: string }>, - currentProviderId: string, - ) => string | Promise; - }; -} - -// ============================================================================ -// Utility Functions -// ============================================================================ - -function getPluginVersion(): string { - try { - const pkgUrl = new URL("./package.json", import.meta.url); - const pkg = JSON.parse(readFileSync(pkgUrl, "utf8")) as { version?: string }; - return pkg.version || "unknown"; - } catch { - return "unknown"; - } -} - -function clampInt(value: number, min: number, max: number): number { - const n = Number.isFinite(value) ? value : min; - return Math.max(min, Math.min(max, Math.trunc(n))); -} - -function resolveOpenClawConfigPath(explicit?: string): string { - const openclawHome = resolveOpenClawHome(); - if (explicit && explicit.trim()) { - return path.resolve(explicit.trim()); - } - - const fromEnv = process.env.OPENCLAW_CONFIG_PATH?.trim(); - if (fromEnv) { - return path.resolve(fromEnv); - } - - return path.join(openclawHome, "openclaw.json"); -} - -function resolveOpenClawHome(): string { - return process.env.OPENCLAW_HOME?.trim() - ? path.resolve(process.env.OPENCLAW_HOME.trim()) - : path.join(homedir(), ".openclaw"); -} - -function resolveDefaultOauthPath(): string { - return path.join(resolveOpenClawHome(), ".memory-lancedb-pro", "oauth.json"); -} - -function resolveLoginOauthPath(rawPath: unknown): string { - const trimmed = typeof rawPath === "string" ? rawPath.trim() : ""; - const candidate = trimmed || resolveDefaultOauthPath(); - return path.resolve(candidate); -} - -function resolveConfiguredOauthPath(configPath: string, rawPath: unknown): string { - const trimmed = typeof rawPath === "string" ? rawPath.trim() : ""; - if (!trimmed) { - return resolveDefaultOauthPath(); - } - if (path.isAbsolute(trimmed)) { - return trimmed; - } - return path.resolve(path.dirname(configPath), trimmed); -} - -type RestorableApiKeyLlmConfig = { - auth?: "api-key"; - apiKey?: string; - model?: string; - baseURL?: string; - timeoutMs?: number; -}; - -type OAuthLlmBackup = { - version: 1; - hadLlmConfig: boolean; - llm: RestorableApiKeyLlmConfig; -}; - -function isPlainObject(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} - -function isOauthLlmConfig(value: unknown): boolean { - return isPlainObject(value) && value.auth === "oauth"; -} - -function extractRestorableApiKeyLlmConfig(value: unknown): RestorableApiKeyLlmConfig { - if (!isPlainObject(value)) { - return {}; - } - - const result: RestorableApiKeyLlmConfig = {}; - if (value.auth === "api-key") { - result.auth = "api-key"; - } - if (typeof value.apiKey === "string") { - result.apiKey = value.apiKey; - } - if (typeof value.model === "string") { - result.model = value.model; - } - if (typeof value.baseURL === "string") { - result.baseURL = value.baseURL; - } - if (typeof value.timeoutMs === "number" && Number.isFinite(value.timeoutMs) && value.timeoutMs > 0) { - result.timeoutMs = Math.trunc(value.timeoutMs); - } - return result; -} - -function extractOauthSafeLlmConfig(value: unknown): RestorableApiKeyLlmConfig { - if (!isPlainObject(value)) { - return {}; - } - - const result: RestorableApiKeyLlmConfig = {}; - if (typeof value.baseURL === "string") { - result.baseURL = value.baseURL; - } - if (typeof value.timeoutMs === "number" && Number.isFinite(value.timeoutMs) && value.timeoutMs > 0) { - result.timeoutMs = Math.trunc(value.timeoutMs); - } - return result; -} - -function hasRestorableApiKeyLlmConfig(value: RestorableApiKeyLlmConfig): boolean { - return Object.keys(value).length > 0; -} - -function buildLogoutFallbackLlmConfig(value: unknown): RestorableApiKeyLlmConfig { - if (isOauthLlmConfig(value)) { - return extractOauthSafeLlmConfig(value); - } - return extractRestorableApiKeyLlmConfig(value); -} - -function getOauthBackupPath(oauthPath: string): string { - const parsed = path.parse(oauthPath); - const fileName = parsed.ext - ? `${parsed.name}.llm-backup${parsed.ext}` - : `${parsed.base}.llm-backup.json`; - return path.join(parsed.dir, fileName); -} - -async function saveOauthLlmBackup(oauthPath: string, llm: unknown, hadLlmConfig: boolean): Promise { - const backupPath = getOauthBackupPath(oauthPath); - const payload: OAuthLlmBackup = { - version: 1, - hadLlmConfig, - llm: extractRestorableApiKeyLlmConfig(llm), - }; - await mkdir(path.dirname(backupPath), { recursive: true }); - await writeFile(backupPath, JSON.stringify(payload, null, 2) + "\n", "utf8"); -} - -async function loadOauthLlmBackup(oauthPath: string): Promise { - const backupPath = getOauthBackupPath(oauthPath); - try { - const raw = await readFile(backupPath, "utf8"); - const parsed = JSON.parse(raw); - if (!isPlainObject(parsed) || parsed.version !== 1 || typeof parsed.hadLlmConfig !== "boolean") { - return null; - } - return { - version: 1, - hadLlmConfig: parsed.hadLlmConfig, - llm: extractRestorableApiKeyLlmConfig(parsed.llm), - }; - } catch { - return null; - } -} - -const OAUTH_PROVIDER_CHOICES = listOAuthProviders() - .map((provider) => `${provider.id} (${provider.label})`) - .join(", "); - -function pickOauthProvider(currentProvider: string | undefined, overrideProvider: string | undefined): { - providerId: string; - source: "override" | "config" | "default"; -} { - if (overrideProvider && overrideProvider.trim()) { - return { providerId: normalizeOAuthProviderId(overrideProvider), source: "override" }; - } - - if (currentProvider && currentProvider.trim()) { - try { - return { providerId: normalizeOAuthProviderId(currentProvider), source: "config" }; - } catch { - // Fall back to the default provider when the saved config is stale or invalid. - } - } - - return { providerId: normalizeOAuthProviderId(), source: "default" }; -} - -async function promptOauthProviderSelection( - currentProviderId: string, - testHook?: CLIContext["oauthTestHooks"]["chooseProvider"], -): Promise<{ providerId: string; source: "prompt" | "default" }> { - const providers = listOAuthProviders(); - if (providers.length === 0) { - throw new Error("No OAuth providers are available."); - } - - if (testHook) { - const selected = await testHook(providers, currentProviderId); - return { providerId: normalizeOAuthProviderId(selected), source: "prompt" }; - } - - if (!process.stdin.isTTY || !process.stdout.isTTY) { - return { providerId: currentProviderId, source: "default" }; - } - - let selectedIndex = providers.findIndex((provider) => provider.id === currentProviderId); - if (selectedIndex < 0) selectedIndex = 0; - - readline.emitKeypressEvents(process.stdin); - const canSetRawMode = typeof process.stdin.setRawMode === "function"; - const previousRawMode = canSetRawMode ? !!process.stdin.isRaw : false; - const menuLines = 2 + providers.length; - let hasRendered = false; - - const render = () => { - if (hasRendered) { - readline.moveCursor(process.stdout, 0, -menuLines); - readline.cursorTo(process.stdout, 0); - readline.clearScreenDown(process.stdout); - } else { - process.stdout.write("\n"); - hasRendered = true; - } - - process.stdout.write("Select OAuth provider\n"); - process.stdout.write("Use arrow keys and Enter.\n"); - providers.forEach((provider, index) => { - const marker = index === selectedIndex ? ">" : " "; - process.stdout.write( - `${marker} ${provider.label} (${provider.id}) [default model: ${provider.defaultModel}]\n`, - ); - }); - }; - - return await new Promise((resolve, reject) => { - const cleanup = () => { - process.stdin.off("keypress", onKeypress); - if (canSetRawMode) { - process.stdin.setRawMode(previousRawMode); - } - process.stdin.pause(); - process.stdout.write("\n"); - }; - - const onKeypress = (_str: string, key: { name?: string; ctrl?: boolean }) => { - if (key.ctrl && key.name === "c") { - cleanup(); - reject(new Error("OAuth login cancelled while selecting a provider.")); - return; - } - - if (key.name === "escape") { - cleanup(); - reject(new Error("OAuth login cancelled while selecting a provider.")); - return; - } - - if (key.name === "up" || key.name === "left") { - selectedIndex = (selectedIndex - 1 + providers.length) % providers.length; - render(); - return; - } - - if (key.name === "down" || key.name === "right") { - selectedIndex = (selectedIndex + 1) % providers.length; - render(); - return; - } - - if (key.name === "return" || key.name === "enter") { - const provider = providers[selectedIndex]; - cleanup(); - resolve({ providerId: provider.id, source: "prompt" }); - } - }; - - render(); - process.stdin.on("keypress", onKeypress); - process.stdin.resume(); - if (canSetRawMode) { - process.stdin.setRawMode(true); - } - }); -} - -async function resolveOauthProviderSelection( - currentProvider: string | undefined, - overrideProvider: string | undefined, - chooseProviderHook?: CLIContext["oauthTestHooks"]["chooseProvider"], -): Promise<{ providerId: string; source: "override" | "config" | "default" | "prompt" }> { - if (overrideProvider && overrideProvider.trim()) { - return pickOauthProvider(currentProvider, overrideProvider); - } - - const initial = pickOauthProvider(currentProvider, undefined); - return await promptOauthProviderSelection(initial.providerId, chooseProviderHook); -} - -function pickOauthModel( - providerId: string, - currentModel: string | undefined, - overrideModel: string | undefined, -): { model: string; source: "override" | "config" | "default" } { - if (overrideModel && overrideModel.trim()) { - if (!isOauthModelSupported(providerId, overrideModel)) { - throw new Error( - `Model "${overrideModel}" is not supported for OAuth provider ${providerId}. Use a compatible model such as ${getDefaultOauthModelForProvider(providerId)}.`, - ); - } - return { model: overrideModel.trim(), source: "override" }; - } - - if (isOauthModelSupported(providerId, currentModel)) { - return { model: currentModel!.trim(), source: "config" }; - } - - return { model: getDefaultOauthModelForProvider(providerId), source: "default" }; -} - -async function loadOpenClawConfig(configPath: string): Promise> { - const raw = await readFile(configPath, "utf8"); - const parsed = JSON5.parse(raw); - if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { - throw new Error(`Invalid OpenClaw config at ${configPath}: expected object`); - } - return parsed as Record; -} - -function ensurePluginConfigRoot(config: Record, pluginId: string): Record { - config.plugins ||= {}; - config.plugins.entries ||= {}; - config.plugins.entries[pluginId] ||= { enabled: true, config: {} }; - const entry = config.plugins.entries[pluginId]; - entry.enabled = true; - entry.config ||= {}; - return entry.config as Record; -} - -async function saveOpenClawConfig(configPath: string, config: Record): Promise { - await mkdir(path.dirname(configPath), { recursive: true }); - await writeFile(configPath, JSON.stringify(config, null, 2) + "\n", "utf8"); -} - -function formatMemory(memory: any, index?: number): string { - const prefix = index !== undefined ? `${index + 1}. ` : ""; - const id = memory?.id ? String(memory.id) : "unknown"; - const date = new Date(memory.timestamp || memory.createdAt || Date.now()).toISOString().split('T')[0]; - const fullText = String(memory.text || ""); - const text = fullText.slice(0, 100) + (fullText.length > 100 ? "..." : ""); - return `${prefix}[${id}] [${memory.category}:${memory.scope}] ${text} (${date})`; -} - -function formatJson(obj: any): string { - return JSON.stringify(obj, null, 2); -} - -async function sleep(ms: number): Promise { - await new Promise(resolve => setTimeout(resolve, ms)); -} - -// ============================================================================ -// CLI Command Implementations -// ============================================================================ - -export function registerMemoryCLI(program: Command, context: CLIContext): void { - const getSearchRetriever = (): MemoryRetriever => { - if (!context.embedder) { - return context.retriever; - } - return createRetriever(context.store, context.embedder, context.retriever.getConfig()); - }; - - const runSearch = async ( - query: string, - limit: number, - scopeFilter?: string[], - category?: string, - ) => { - let results = await getSearchRetriever().retrieve({ - query, - limit, - scopeFilter, - category, - source: "cli", - }); - - if (results.length === 0 && context.embedder) { - await sleep(75); - results = await getSearchRetriever().retrieve({ - query, - limit, - scopeFilter, - category, - source: "cli", - }); - } - - return results; - }; - - const memory = program - .command("memory-pro") - .description("Enhanced memory management commands (LanceDB Pro)"); - - // Version - memory - .command("version") - .description("Print plugin version") - .action(() => { - console.log(getPluginVersion()); - }); - - const auth = memory - .command("auth") - .description("Manage OAuth authentication for smart-extraction LLM access"); - - auth - .command("login") - .description("Authenticate with ChatGPT/Codex in a browser, save the plugin OAuth file, and switch this plugin to llm.auth=oauth") - .option("--config ", "OpenClaw config file to update") - .option("--provider ", `OAuth provider to use (${OAUTH_PROVIDER_CHOICES})`) - .option("--model ", "Override the model saved into llm.model") - .option("--oauth-path ", "OAuth file path (default: ~/.openclaw/.memory-lancedb-pro/oauth.json)") - .option("--timeout ", "OAuth callback timeout in seconds", "120") - .option("--no-browser", "Do not auto-open the browser; print the authorization URL only") - .action(async (options) => { - try { - const pluginId = context.pluginId || "memory-lancedb-pro"; - const currentLlm = context.pluginConfig?.llm; - const currentProvider = currentLlm && typeof currentLlm === "object" && typeof (currentLlm as any).oauthProvider === "string" - ? String((currentLlm as any).oauthProvider) - : undefined; - const selectedProvider = await resolveOauthProviderSelection( - currentProvider, - options.provider, - context.oauthTestHooks?.chooseProvider, - ); - const currentModel = currentLlm && typeof currentLlm === "object" && typeof (currentLlm as any).model === "string" - ? String((currentLlm as any).model) - : undefined; - const selectedModel = pickOauthModel(selectedProvider.providerId, currentModel, options.model); - const oauthModel = normalizeOauthModel(selectedModel.model); - const configPath = resolveOpenClawConfigPath(options.config); - const oauthPath = resolveLoginOauthPath(options.oauthPath); - const timeoutMs = clampInt((parseInt(options.timeout, 10) || 120) * 1000, 15_000, 900_000); - - if (selectedModel.source === "default" && currentModel && currentModel.trim()) { - console.log( - `Configured llm.model "${currentModel}" is not supported by provider ${selectedProvider.providerId}. Falling back to ${getDefaultOauthModelForProvider(selectedProvider.providerId)}.`, - ); - } - - console.log(`Config file: ${configPath}`); - console.log(`Provider: ${getOAuthProviderLabel(selectedProvider.providerId)} (${selectedProvider.providerId}, ${selectedProvider.source})`); - console.log(`OAuth file: ${oauthPath}`); - console.log(`Model: ${oauthModel} (${selectedModel.source})`); - - const { session } = await performOAuthLogin({ - authPath: oauthPath, - timeoutMs, - noBrowser: options.browser === false, - model: selectedModel.model, - providerId: selectedProvider.providerId, - onOpenUrl: context.oauthTestHooks?.openUrl, - onAuthorizeUrl: async (url) => { - console.log(`Authorization URL: ${url}`); - await context.oauthTestHooks?.authorizeUrl?.(url); - }, - }); - - const openclawConfig = await loadOpenClawConfig(configPath); - const pluginConfig = ensurePluginConfigRoot(openclawConfig, pluginId); - const hadLlmConfig = isPlainObject(pluginConfig.llm); - const existingLlm = hadLlmConfig ? { ...(pluginConfig.llm as Record) } : {}; - const wasOauthMode = isOauthLlmConfig(existingLlm); - - if (!wasOauthMode) { - await saveOauthLlmBackup(oauthPath, pluginConfig.llm, hadLlmConfig); - } - - const nextLlm = wasOauthMode ? { ...existingLlm } : extractOauthSafeLlmConfig(existingLlm); - delete nextLlm.apiKey; - if (!wasOauthMode) { - delete nextLlm.baseURL; - } - pluginConfig.llm = { - ...nextLlm, - auth: "oauth", - oauthProvider: selectedProvider.providerId, - model: oauthModel, - oauthPath, - }; - await saveOpenClawConfig(configPath, openclawConfig); - - console.log(`OAuth login completed for account ${session.accountId}.`); - console.log( - `Updated ${pluginId} config: llm.auth=oauth, llm.oauthProvider=${selectedProvider.providerId}, llm.oauthPath=${oauthPath}, llm.model=${oauthModel}`, - ); - } catch (error) { - console.error("OAuth login failed:", error); - process.exit(1); - } - }); - - auth - .command("status") - .description("Show the current OAuth configuration for this plugin") - .option("--config ", "OpenClaw config file to inspect") - .action(async (options) => { - try { - const pluginId = context.pluginId || "memory-lancedb-pro"; - const configPath = resolveOpenClawConfigPath(options.config); - const openclawConfig = await loadOpenClawConfig(configPath); - const pluginConfig = ensurePluginConfigRoot(openclawConfig, pluginId); - const llm = typeof pluginConfig.llm === "object" && pluginConfig.llm ? pluginConfig.llm as Record : {}; - const oauthProviderRaw = typeof llm.oauthProvider === "string" && llm.oauthProvider.trim() - ? llm.oauthProvider.trim() - : normalizeOAuthProviderId(); - let oauthProviderDisplay = `${oauthProviderRaw} (unknown)`; - try { - oauthProviderDisplay = `${normalizeOAuthProviderId(oauthProviderRaw)} (${getOAuthProviderLabel(oauthProviderRaw)})`; - } catch { - // Leave the raw provider id visible for debugging stale or unsupported configs. - } - const oauthPath = resolveConfiguredOauthPath(configPath, llm.oauthPath); - - let tokenInfo = "missing"; - try { - const session = await readFile(oauthPath, "utf8"); - tokenInfo = session.trim() ? "present" : "empty"; - } catch { - tokenInfo = "missing"; - } - - console.log(`Config file: ${configPath}`); - console.log(`Plugin: ${pluginId}`); - console.log(`llm.auth: ${typeof llm.auth === "string" ? llm.auth : "api-key"}`); - console.log(`llm.oauthProvider: ${oauthProviderDisplay}`); - console.log(`llm.model: ${typeof llm.model === "string" ? llm.model : "openai/gpt-oss-120b"}`); - console.log(`llm.oauthPath: ${oauthPath}`); - console.log(`oauth file: ${tokenInfo}`); - } catch (error) { - console.error("OAuth status failed:", error); - process.exit(1); - } - }); - - auth - .command("logout") - .description("Delete the plugin OAuth file and switch this plugin back to llm.auth=api-key") - .option("--config ", "OpenClaw config file to update") - .option("--oauth-path ", "OAuth file path to remove") - .action(async (options) => { - try { - const pluginId = context.pluginId || "memory-lancedb-pro"; - const configPath = resolveOpenClawConfigPath(options.config); - const openclawConfig = await loadOpenClawConfig(configPath); - const pluginConfig = ensurePluginConfigRoot(openclawConfig, pluginId); - const llm = typeof pluginConfig.llm === "object" && pluginConfig.llm ? pluginConfig.llm as Record : {}; - const oauthPath = - options.oauthPath && String(options.oauthPath).trim() - ? resolveLoginOauthPath(options.oauthPath) - : resolveConfiguredOauthPath(configPath, llm.oauthPath); - const backupPath = getOauthBackupPath(oauthPath); - const backup = await loadOauthLlmBackup(oauthPath); - - await rm(oauthPath, { force: true }); - await rm(backupPath, { force: true }); - - if (backup) { - if (backup.hadLlmConfig) { - pluginConfig.llm = { ...backup.llm }; - } else { - delete pluginConfig.llm; - } - } else { - const fallbackLlm = buildLogoutFallbackLlmConfig(llm); - if (hasRestorableApiKeyLlmConfig(fallbackLlm)) { - pluginConfig.llm = fallbackLlm; - } else { - delete pluginConfig.llm; - } - } - await saveOpenClawConfig(configPath, openclawConfig); - - console.log(`Deleted OAuth file: ${oauthPath}`); - console.log(`Updated ${pluginId} config: llm.auth=api-key`); - } catch (error) { - console.error("OAuth logout failed:", error); - process.exit(1); - } - }); - - // List memories - memory - .command("list") - .description("List memories with optional filtering") - .option("--scope ", "Filter by scope") - .option("--category ", "Filter by category") - .option("--limit ", "Maximum number of results", "20") - .option("--offset ", "Number of results to skip", "0") - .option("--json", "Output as JSON") - .action(async (options) => { - try { - const limit = parseInt(options.limit) || 20; - const offset = parseInt(options.offset) || 0; - - let scopeFilter: string[] | undefined; - if (options.scope) { - scopeFilter = [options.scope]; - } - - const memories = await context.store.list( - scopeFilter, - options.category, - limit, - offset - ); - - if (options.json) { - console.log(formatJson(memories)); - } else { - if (memories.length === 0) { - console.log("No memories found."); - } else { - console.log(`Found ${memories.length} memories:\n`); - memories.forEach((memory, i) => { - console.log(formatMemory(memory, offset + i)); - }); - } - } - } catch (error) { - console.error("Failed to list memories:", error); - process.exit(1); - } - }); - - // Search memories - memory - .command("search ") - .description("Search memories using hybrid retrieval") - .option("--scope ", "Search within specific scope") - .option("--category ", "Filter by category") - .option("--limit ", "Maximum number of results", "10") - .option("--json", "Output as JSON") - .action(async (query, options) => { - try { - const limit = parseInt(options.limit) || 10; - - let scopeFilter: string[] | undefined; - if (options.scope) { - scopeFilter = [options.scope]; - } - - const results = await runSearch(query, limit, scopeFilter, options.category); - - if (options.json) { - console.log(formatJson(results)); - } else { - if (results.length === 0) { - console.log("No relevant memories found."); - } else { - console.log(`Found ${results.length} memories:\n`); - results.forEach((result, i) => { - const sources = []; - if (result.sources.vector) sources.push("vector"); - if (result.sources.bm25) sources.push("BM25"); - if (result.sources.reranked) sources.push("reranked"); - - console.log( - `${i + 1}. [${result.entry.id}] [${result.entry.category}:${result.entry.scope}] ${result.entry.text} ` + - `(${(result.score * 100).toFixed(0)}%, ${sources.join('+')})` - ); - }); - } - } - } catch (error) { - console.error("Search failed:", error); - process.exit(1); - } - }); - - // Memory statistics - memory - .command("stats") - .description("Show memory statistics") - .option("--scope ", "Stats for specific scope") - .option("--json", "Output as JSON") - .action(async (options) => { - try { - let scopeFilter: string[] | undefined; - if (options.scope) { - scopeFilter = [options.scope]; - } - - const stats = await context.store.stats(scopeFilter); - const scopeStats = context.scopeManager.getStats(); - const retrievalConfig = context.retriever.getConfig(); - - const summary = { - memory: stats, - scopes: scopeStats, - retrieval: { - mode: retrievalConfig.mode, - hasFtsSupport: context.store.hasFtsSupport, - }, - }; - - if (options.json) { - console.log(formatJson(summary)); - } else { - console.log(`Memory Statistics:`); - console.log(`• Total memories: ${stats.totalCount}`); - console.log(`• Available scopes: ${scopeStats.totalScopes}`); - console.log(`• Retrieval mode: ${retrievalConfig.mode}`); - console.log(`• FTS support: ${context.store.hasFtsSupport ? 'Yes' : 'No'}`); - console.log(); - - console.log("Memories by scope:"); - Object.entries(stats.scopeCounts).forEach(([scope, count]) => { - console.log(` • ${scope}: ${count}`); - }); - console.log(); - - console.log("Memories by category:"); - Object.entries(stats.categoryCounts).forEach(([category, count]) => { - console.log(` • ${category}: ${count}`); - }); - } - } catch (error) { - console.error("Failed to get statistics:", error); - process.exit(1); - } - }); - - // Delete memory - memory - .command("delete ") - .description("Delete a specific memory by ID") - .option("--scope ", "Scope to delete from (for access control)") - .action(async (id, options) => { - try { - let scopeFilter: string[] | undefined; - if (options.scope) { - scopeFilter = [options.scope]; - } - - const deleted = await context.store.delete(id, scopeFilter); - - if (deleted) { - console.log(`Memory ${id} deleted successfully.`); - } else { - console.log(`Memory ${id} not found or access denied.`); - process.exit(1); - } - } catch (error) { - console.error("Failed to delete memory:", error); - process.exit(1); - } - }); - - // Bulk delete - memory - .command("delete-bulk") - .description("Bulk delete memories with filters") - .option("--scope ", "Scopes to delete from (required)") - .option("--before ", "Delete memories before this date (YYYY-MM-DD)") - .option("--dry-run", "Show what would be deleted without actually deleting") - .action(async (options) => { - try { - if (!options.scope || options.scope.length === 0) { - console.error("At least one scope must be specified for safety."); - process.exit(1); - } - - let beforeTimestamp: number | undefined; - if (options.before) { - const date = new Date(options.before); - if (isNaN(date.getTime())) { - console.error("Invalid date format. Use YYYY-MM-DD."); - process.exit(1); - } - beforeTimestamp = date.getTime(); - } - - if (options.dryRun) { - console.log("DRY RUN - No memories will be deleted"); - console.log(`Filters: scopes=${options.scope.join(',')}, before=${options.before || 'none'}`); - - // Show what would be deleted - const stats = await context.store.stats(options.scope); - console.log(`Would delete from ${stats.totalCount} memories in matching scopes.`); - } else { - const deletedCount = await context.store.bulkDelete(options.scope, beforeTimestamp); - console.log(`Deleted ${deletedCount} memories.`); - } - } catch (error) { - console.error("Bulk delete failed:", error); - process.exit(1); - } - }); - - // Export memories - memory - .command("export") - .description("Export memories to JSON") - .option("--scope ", "Export specific scope") - .option("--category ", "Export specific category") - .option("--output ", "Output file (default: stdout)") - .action(async (options) => { - try { - let scopeFilter: string[] | undefined; - if (options.scope) { - scopeFilter = [options.scope]; - } - - const memories = await context.store.list( - scopeFilter, - options.category, - 1000 // Large limit for export - ); - - const exportData = { - version: "1.0", - exportedAt: new Date().toISOString(), - count: memories.length, - filters: { - scope: options.scope, - category: options.category, - }, - memories: memories.map(m => ({ - ...m, - vector: undefined, // Exclude vectors to reduce size - })), - }; - - const output = formatJson(exportData); - - if (options.output) { - const fs = await import("node:fs/promises"); - await fs.writeFile(options.output, output); - console.log(`Exported ${memories.length} memories to ${options.output}`); - } else { - console.log(output); - } - } catch (error) { - console.error("Export failed:", error); - process.exit(1); - } - }); - - // Import memories - memory - .command("import ") - .description("Import memories from JSON file") - .option("--scope ", "Import into specific scope") - .option("--dry-run", "Show what would be imported without actually importing") - .action(async (file, options) => { - try { - const fs = await import("node:fs/promises"); - const content = await fs.readFile(file, "utf-8"); - const data = JSON.parse(content); - - if (!data.memories || !Array.isArray(data.memories)) { - throw new Error("Invalid import file format"); - } - - if (options.dryRun) { - console.log("DRY RUN - No memories will be imported"); - console.log(`Would import ${data.memories.length} memories`); - if (options.scope) { - console.log(`Target scope: ${options.scope}`); - } - return; - } - - console.log(`Importing ${data.memories.length} memories...`); - - let imported = 0; - let skipped = 0; - - if (!context.embedder) { - console.error("Import requires an embedder (not available in basic CLI mode)."); - console.error("Use the plugin's memory_store tool or pass embedder to createMemoryCLI."); - return; - } - - const targetScope = options.scope || context.scopeManager.getDefaultScope(); - - for (const memory of data.memories) { - try { - const text = memory.text; - if (!text || typeof text !== "string" || text.length < 2) { - skipped++; - continue; - } - - const categoryRaw = memory.category; - const category: MemoryEntry["category"] = - categoryRaw === "preference" || - categoryRaw === "fact" || - categoryRaw === "decision" || - categoryRaw === "entity" || - categoryRaw === "other" - ? categoryRaw - : "other"; - - const importanceRaw = Number(memory.importance); - const importance = Number.isFinite(importanceRaw) - ? Math.max(0, Math.min(1, importanceRaw)) - : 0.7; - - const timestampRaw = Number(memory.timestamp); - const timestamp = Number.isFinite(timestampRaw) ? timestampRaw : Date.now(); - - const metadataRaw = memory.metadata; - const metadata = - typeof metadataRaw === "string" - ? metadataRaw - : metadataRaw != null - ? JSON.stringify(metadataRaw) - : "{}"; - - const idRaw = memory.id; - const id = typeof idRaw === "string" && idRaw.length > 0 ? idRaw : undefined; - - // Idempotency: if the import file includes an id and we already have it, skip. - if (id && (await context.store.hasId(id))) { - skipped++; - continue; - } - - // Back-compat dedupe: if no id provided, do a best-effort similarity check. - if (!id) { - const existing = await context.retriever.retrieve({ - query: text, - limit: 1, - scopeFilter: [targetScope], - }); - if (existing.length > 0 && existing[0].score > 0.95) { - skipped++; - continue; - } - } - - const vector = await context.embedder.embedPassage(text); - - if (id) { - await context.store.importEntry({ - id, - text, - vector, - category, - scope: targetScope, - importance, - timestamp, - metadata, - }); - } else { - await context.store.store({ - text, - vector, - importance, - category, - scope: targetScope, - metadata, - }); - } - - imported++; - } catch (error) { - console.warn(`Failed to import memory: ${error}`); - skipped++; - } - } - - console.log(`Import completed: ${imported} imported, ${skipped} skipped`); - } catch (error) { - console.error("Import failed:", error); - process.exit(1); - } - }); - - // Re-embed an existing LanceDB into the current target DB (A/B testing) - memory - .command("reembed") - .description("Re-embed memories from a source LanceDB database into the current target database") - .requiredOption("--source-db ", "Source LanceDB database directory") - .option("--batch-size ", "Batch size for embedding calls", "32") - .option("--limit ", "Limit number of rows to process (for testing)") - .option("--dry-run", "Show what would be re-embedded without writing") - .option("--skip-existing", "Skip entries whose id already exists in the target DB") - .option("--force", "Allow using the same source-db as the target dbPath (DANGEROUS)") - .action(async (options) => { - try { - if (!context.embedder) { - console.error("Re-embed requires an embedder (not available in basic CLI mode)."); - return; - } - - const fs = await import("node:fs/promises"); - - const sourceDbPath = options.sourceDb as string; - const batchSize = clampInt(parseInt(options.batchSize, 10) || 32, 1, 128); - const limit = options.limit ? clampInt(parseInt(options.limit, 10) || 0, 1, 1000000) : undefined; - const dryRun = options.dryRun === true; - const skipExisting = options.skipExisting === true; - const force = options.force === true; - - // Safety: prevent accidental in-place re-embedding - let sourceReal = sourceDbPath; - let targetReal = context.store.dbPath; - try { - sourceReal = await fs.realpath(sourceDbPath); - } catch { } - try { - targetReal = await fs.realpath(context.store.dbPath); - } catch { } - - if (!force && sourceReal === targetReal) { - console.error("Refusing to re-embed in-place: source-db equals target dbPath. Use a new dbPath or pass --force."); - process.exit(1); - } - - const lancedb = await loadLanceDB(); - const db = await lancedb.connect(sourceDbPath); - const table = await db.openTable("memories"); - - let query = table - .query() - .select(["id", "text", "category", "scope", "importance", "timestamp", "metadata"]); - - if (limit) query = query.limit(limit); - - const rows = (await query.toArray()) - .filter((r: any) => r && typeof r.text === "string" && r.text.trim().length > 0) - .filter((r: any) => r.id && r.id !== "__schema__"); - - if (rows.length === 0) { - console.log("No source memories found."); - return; - } - - console.log( - `Re-embedding ${rows.length} memories from ${sourceDbPath} → ${context.store.dbPath} (batchSize=${batchSize})` - ); - - if (dryRun) { - console.log("DRY RUN - No memories will be written"); - console.log(`First example: ${rows[0].id?.slice?.(0, 8)} ${String(rows[0].text).slice(0, 80)}`); - return; - } - - let processed = 0; - let imported = 0; - let skipped = 0; - - for (let i = 0; i < rows.length; i += batchSize) { - const batch = rows.slice(i, i + batchSize); - const texts = batch.map((r: any) => String(r.text)); - const vectors = await context.embedder.embedBatchPassage(texts); - - for (let j = 0; j < batch.length; j++) { - processed++; - const row = batch[j]; - const vector = vectors[j]; - - if (!vector || vector.length === 0) { - skipped++; - continue; - } - - const id = String(row.id); - if (skipExisting) { - const exists = await context.store.hasId(id); - if (exists) { - skipped++; - continue; - } - } - - const entry: MemoryEntry = { - id, - text: String(row.text), - vector, - category: (row.category as any) || "other", - scope: (row.scope as string | undefined) || "global", - importance: (row.importance != null) ? Number(row.importance) : 0.7, - timestamp: (row.timestamp != null) ? Number(row.timestamp) : Date.now(), - metadata: typeof row.metadata === "string" ? row.metadata : "{}", - }; - - await context.store.importEntry(entry); - imported++; - } - - if (processed % 100 === 0 || processed === rows.length) { - console.log(`Progress: ${processed}/${rows.length} processed, ${imported} imported, ${skipped} skipped`); - } - } - - console.log(`Re-embed completed: ${imported} imported, ${skipped} skipped (processed=${processed}).`); - } catch (error) { - console.error("Re-embed failed:", error); - process.exit(1); - } - }); - - // Upgrade legacy memories to new smart memory format - memory - .command("upgrade") - .description("Upgrade legacy memories to new 6-category L0/L1/L2 smart memory format") - .option("--dry-run", "Show upgrade statistics without modifying data") - .option("--batch-size ", "Number of memories per batch", "10") - .option("--no-llm", "Skip LLM calls; use simple text truncation for L0/L1") - .option("--limit ", "Maximum number of memories to upgrade") - .option("--scope ", "Only upgrade memories in this scope") - .action(async (options) => { - try { - const upgrader = createMemoryUpgrader( - context.store, - options.llm === false ? null : (context.llmClient ?? null), - { log: console.log }, - ); - - // Show current status first - const scopeFilter = options.scope ? [options.scope] : undefined; - const counts = await upgrader.countLegacy(scopeFilter); - - console.log(`Memory Upgrade Status:`); - console.log(`• Total memories: ${counts.total}`); - console.log(`• Legacy (needs upgrade): ${counts.legacy}`); - console.log(`• Already new format: ${counts.total - counts.legacy}`); - if (Object.keys(counts.byCategory).length > 0) { - console.log(`• Legacy by category:`); - Object.entries(counts.byCategory).forEach(([cat, n]) => { - console.log(` ${cat}: ${n}`); - }); - } - - if (counts.legacy === 0) { - console.log(`\nAll memories are already in the new format. No upgrade needed.`); - return; - } - - if (options.dryRun) { - console.log(`\n[DRY-RUN] Would upgrade ${counts.legacy} memories.`); - return; - } - - console.log(`\nStarting upgrade...`); - const result = await upgrader.upgrade({ - dryRun: false, - batchSize: parseInt(options.batchSize) || 10, - noLlm: options.llm === false, - limit: options.limit ? parseInt(options.limit) : undefined, - scopeFilter, - }); - - console.log(`\nUpgrade Results:`); - console.log(`• Upgraded: ${result.upgraded}`); - console.log(`• Already new format: ${result.skipped}`); - if (result.errors.length > 0) { - console.log(`• Errors: ${result.errors.length}`); - result.errors.slice(0, 5).forEach(err => console.log(` - ${err}`)); - if (result.errors.length > 5) { - console.log(` ... and ${result.errors.length - 5} more`); - } - } - } catch (error) { - console.error("Upgrade failed:", error); - process.exit(1); - } - }); - - // Migration commands - const migrate = memory - .command("migrate") - .description("Migration utilities"); - - migrate - .command("check") - .description("Check if migration is needed from legacy memory-lancedb") - .option("--source ", "Specific source database path") - .action(async (options) => { - try { - const check = await context.migrator.checkMigrationNeeded(options.source); - - console.log("Migration Check Results:"); - console.log(`• Legacy database found: ${check.sourceFound ? 'Yes' : 'No'}`); - if (check.sourceDbPath) { - console.log(`• Source path: ${check.sourceDbPath}`); - } - if (check.entryCount !== undefined) { - console.log(`• Entries to migrate: ${check.entryCount}`); - } - console.log(`• Migration needed: ${check.needed ? 'Yes' : 'No'}`); - } catch (error) { - console.error("Migration check failed:", error); - process.exit(1); - } - }); - - migrate - .command("run") - .description("Run migration from legacy memory-lancedb") - .option("--source ", "Specific source database path") - .option("--default-scope ", "Default scope for migrated data", "global") - .option("--dry-run", "Show what would be migrated without actually migrating") - .option("--skip-existing", "Skip entries that already exist") - .action(async (options) => { - try { - const result = await context.migrator.migrate({ - sourceDbPath: options.source, - defaultScope: options.defaultScope, - dryRun: options.dryRun, - skipExisting: options.skipExisting, - }); - - console.log("Migration Results:"); - console.log(`• Status: ${result.success ? 'Success' : 'Failed'}`); - console.log(`• Migrated: ${result.migratedCount}`); - console.log(`• Skipped: ${result.skippedCount}`); - if (result.errors.length > 0) { - console.log(`• Errors: ${result.errors.length}`); - result.errors.forEach(error => console.log(` - ${error}`)); - } - console.log(`• Summary: ${result.summary}`); - - if (!result.success) { - process.exit(1); - } - } catch (error) { - console.error("Migration failed:", error); - process.exit(1); - } - }); - - migrate - .command("verify") - .description("Verify migration results") - .option("--source ", "Specific source database path") - .action(async (options) => { - try { - const result = await context.migrator.verifyMigration(options.source); - - console.log("Migration Verification:"); - console.log(`• Valid: ${result.valid ? 'Yes' : 'No'}`); - console.log(`• Source count: ${result.sourceCount}`); - console.log(`• Target count: ${result.targetCount}`); - - if (result.issues.length > 0) { - console.log("• Issues:"); - result.issues.forEach(issue => console.log(` - ${issue}`)); - } - - if (!result.valid) { - process.exit(1); - } - } catch (error) { - console.error("Verification failed:", error); - process.exit(1); - } - }); - - // reindex-fts: Rebuild FTS index - program - .command("reindex-fts") - .description("Rebuild the BM25 full-text search index") - .action(async () => { - try { - const status = context.store.getFtsStatus(); - console.log(`FTS status before: available=${status.available}, lastError=${status.lastError || "none"}`); - const result = await context.store.rebuildFtsIndex(); - if (result.success) { - console.log("✅ FTS index rebuilt successfully"); - } else { - console.error("❌ FTS rebuild failed:", result.error); - process.exit(1); - } - } catch (error) { - console.error("FTS rebuild error:", error); - process.exit(1); - } - }); -} - -// ============================================================================ -// Factory Function -// ============================================================================ - -export function createMemoryCLI(context: CLIContext) { - return ({ program }: { program: Command }) => registerMemoryCLI(program, context); -} diff --git a/index.ts b/index.ts index 52f1962e..32ff66c6 100644 --- a/index.ts +++ b/index.ts @@ -1993,6 +1993,15 @@ const memoryLanceDBProPlugin = { ); logReg(`memory-lancedb-pro: diagnostic build tag loaded (${DIAG_BUILD_TAG})`); + // Dual-memory model warning: help users understand the two-layer architecture + // Runs synchronously and logs warnings; does NOT block gateway startup. + api.logger.info( + `[memory-lancedb-pro] memory_recall queries the plugin store (LanceDB), not MEMORY.md.\n` + + ` - Plugin memory (LanceDB) = primary recall source for semantic search\n` + + ` - MEMORY.md / memory/YYYY-MM-DD.md = startup context / journal only\n` + + ` - Use memory_store or auto-capture for recallable memories.\n` + ); + api.on("message_received", (event: any, ctx: any) => { const conversationKey = buildAutoCaptureConversationKeyFromIngress( ctx.channelId, From cc669b1028eb8431263e4863f8eaeded4e9889cf Mon Sep 17 00:00:00 2001 From: jlin53882 Date: Tue, 31 Mar 2026 19:08:47 +0800 Subject: [PATCH 7/9] fix: address review feedback - schema completeness + timeout cleanup in finally Reviewer: rwmjhb (CortexReach) 1. Schema completeness: add rerankTimeoutMs to openclaw.plugin.json retrieval schema and index.ts PluginConfig.retrieval 2. Timeout cleanup: move clearTimeout into finally block so timer is always cleared even on fast failure paths 3. Also update warning message to show actual configured timeout value Closes #346 --- index.ts | 2 ++ openclaw.plugin.json | 12 ++++++++++++ src/retriever.ts | 19 +++++++++++-------- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/index.ts b/index.ts index 52f1962e..6614c699 100644 --- a/index.ts +++ b/index.ts @@ -120,6 +120,8 @@ interface PluginConfig { rerankApiKey?: string; rerankModel?: string; rerankEndpoint?: string; + /** Rerank API timeout in milliseconds (default: 5000). Increase for local/CPU-based rerank servers. */ + rerankTimeoutMs?: number; rerankProvider?: | "jina" | "siliconflow" diff --git a/openclaw.plugin.json b/openclaw.plugin.json index a2cfb1f5..976b8a19 100644 --- a/openclaw.plugin.json +++ b/openclaw.plugin.json @@ -319,6 +319,12 @@ "default": "https://api.jina.ai/v1/rerank", "description": "Reranker API endpoint URL. Compatible with Jina-compatible endpoints and dedicated adapters such as TEI, SiliconFlow, Voyage, Pinecone, and DashScope." }, + "rerankTimeoutMs": { + "type": "integer", + "minimum": 1, + "default": 5000, + "description": "Rerank API timeout in milliseconds (default: 5000). Increase for local/CPU-based rerank servers." + }, "rerankProvider": { "type": "string", "enum": [ @@ -1078,6 +1084,12 @@ "help": "Custom reranker API endpoint URL", "advanced": true }, + "retrieval.rerankTimeoutMs": { + "label": "Rerank Timeout (ms)", + "placeholder": "5000", + "help": "Rerank API timeout in milliseconds. Increase for local/CPU-based rerank servers.", + "advanced": true + }, "retrieval.rerankProvider": { "label": "Reranker Provider", "help": "Provider format: jina (default), siliconflow, voyage, pinecone, dashscope, or tei", diff --git a/src/retriever.ts b/src/retriever.ts index 0cf03bca..bf3aeee2 100644 --- a/src/retriever.ts +++ b/src/retriever.ts @@ -865,14 +865,17 @@ export class MemoryRetriever { const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), this.config.rerankTimeoutMs ?? 5000); - const response = await fetch(endpoint, { - method: "POST", - headers, - body: JSON.stringify(body), - signal: controller.signal, - }); - - clearTimeout(timeout); + let response: Response; + try { + response = await fetch(endpoint, { + method: "POST", + headers, + body: JSON.stringify(body), + signal: controller.signal, + }); + } finally { + clearTimeout(timeout); + } if (response.ok) { const data: unknown = await response.json(); From bf5a14eccca8fe4bbe6a1e5f3bcd03bfadf70977 Mon Sep 17 00:00:00 2001 From: jlin53882 Date: Tue, 31 Mar 2026 19:17:59 +0800 Subject: [PATCH 8/9] restore: restore accidentally deleted files from upstream/master MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Files were deleted from origin/master during earlier rebase/cherry-pick operations. These deletions were unintentional — the PRs were meant to add features, not delete existing functionality. Restored from upstream/master: - src/: admission-control, admission-stats, auto-capture-cleanup, batch-dedup, clawteam-scope, identity-addressing, intent-analyzer, llm-oauth, memory-compactor, preference-slots, retrieval-stats, retrieval-trace, session-compressor, workspace-boundary - scripts/: governance-maintenance.mjs, migrate-governance-metadata.mjs - test/: 25 test files restored - README translations: DE, ES, FR, IT, JA, KO, PT-BR, RU, TW --- README_DE.md | 773 +++++++++++++++++++++ README_ES.md | 773 +++++++++++++++++++++ README_FR.md | 773 +++++++++++++++++++++ README_IT.md | 773 +++++++++++++++++++++ README_JA.md | 773 +++++++++++++++++++++ README_KO.md | 773 +++++++++++++++++++++ README_PT-BR.md | 773 +++++++++++++++++++++ README_RU.md | 773 +++++++++++++++++++++ README_TW.md | 773 +++++++++++++++++++++ scripts/governance-maintenance.mjs | 130 ++++ scripts/migrate-governance-metadata.mjs | 110 +++ src/admission-control.ts | 748 ++++++++++++++++++++ src/admission-stats.ts | 332 +++++++++ src/auto-capture-cleanup.ts | 94 +++ src/batch-dedup.ts | 146 ++++ src/clawteam-scope.ts | 63 ++ src/identity-addressing.ts | 201 ++++++ src/intent-analyzer.ts | 259 +++++++ src/llm-oauth.ts | 675 ++++++++++++++++++ src/memory-compactor.ts | 403 +++++++++++ src/preference-slots.ts | 76 ++ src/retrieval-stats.ts | 152 ++++ src/retrieval-trace.ts | 173 +++++ src/session-compressor.ts | 331 +++++++++ src/workspace-boundary.ts | 154 ++++ test/batch-dedup.test.mjs | 196 ++++++ test/cjk-recursion-regression.test.mjs | 338 +++++++++ test/clawteam-scope.test.mjs | 128 ++++ test/cli-oauth-login.test.mjs | 577 +++++++++++++++ test/cross-process-lock.test.mjs | 119 ++++ test/embedder-ollama-abort.test.mjs | 99 +++ test/governance-metadata.test.mjs | 72 ++ test/intent-analyzer.test.mjs | 209 ++++++ test/llm-api-key-client.test.mjs | 68 ++ test/llm-oauth-client.test.mjs | 307 ++++++++ test/memory-compactor.test.mjs | 292 ++++++++ test/memory-governance-tools.test.mjs | 168 +++++ test/memory-upgrader-diagnostics.test.mjs | 71 ++ test/preference-slots.test.mjs | 280 ++++++++ test/reflection-bypass-hook.test.mjs | 194 ++++++ test/resolve-env-vars-array.test.mjs | 181 +++++ test/retrieval-trace.test.mjs | 309 ++++++++ test/retriever-tag-query.test.mjs | 290 ++++++++ test/scope-access-undefined.test.mjs | 191 +++++ test/session-compressor.test.mjs | 342 +++++++++ test/session-summary-before-reset.test.mjs | 166 +++++ test/smart-extractor-scope-filter.test.mjs | 98 +++ test/store-empty-scope-filter.test.mjs | 45 ++ test/strip-envelope-metadata.test.mjs | 206 ++++++ test/workflow-fork-guards.test.mjs | 26 + 50 files changed, 15976 insertions(+) create mode 100644 README_DE.md create mode 100644 README_ES.md create mode 100644 README_FR.md create mode 100644 README_IT.md create mode 100644 README_JA.md create mode 100644 README_KO.md create mode 100644 README_PT-BR.md create mode 100644 README_RU.md create mode 100644 README_TW.md create mode 100644 scripts/governance-maintenance.mjs create mode 100644 scripts/migrate-governance-metadata.mjs create mode 100644 src/admission-control.ts create mode 100644 src/admission-stats.ts create mode 100644 src/auto-capture-cleanup.ts create mode 100644 src/batch-dedup.ts create mode 100644 src/clawteam-scope.ts create mode 100644 src/identity-addressing.ts create mode 100644 src/intent-analyzer.ts create mode 100644 src/llm-oauth.ts create mode 100644 src/memory-compactor.ts create mode 100644 src/preference-slots.ts create mode 100644 src/retrieval-stats.ts create mode 100644 src/retrieval-trace.ts create mode 100644 src/session-compressor.ts create mode 100644 src/workspace-boundary.ts create mode 100644 test/batch-dedup.test.mjs create mode 100644 test/cjk-recursion-regression.test.mjs create mode 100644 test/clawteam-scope.test.mjs create mode 100644 test/cli-oauth-login.test.mjs create mode 100644 test/cross-process-lock.test.mjs create mode 100644 test/embedder-ollama-abort.test.mjs create mode 100644 test/governance-metadata.test.mjs create mode 100644 test/intent-analyzer.test.mjs create mode 100644 test/llm-api-key-client.test.mjs create mode 100644 test/llm-oauth-client.test.mjs create mode 100644 test/memory-compactor.test.mjs create mode 100644 test/memory-governance-tools.test.mjs create mode 100644 test/memory-upgrader-diagnostics.test.mjs create mode 100644 test/preference-slots.test.mjs create mode 100644 test/reflection-bypass-hook.test.mjs create mode 100644 test/resolve-env-vars-array.test.mjs create mode 100644 test/retrieval-trace.test.mjs create mode 100644 test/retriever-tag-query.test.mjs create mode 100644 test/scope-access-undefined.test.mjs create mode 100644 test/session-compressor.test.mjs create mode 100644 test/session-summary-before-reset.test.mjs create mode 100644 test/smart-extractor-scope-filter.test.mjs create mode 100644 test/store-empty-scope-filter.test.mjs create mode 100644 test/strip-envelope-metadata.test.mjs create mode 100644 test/workflow-fork-guards.test.mjs diff --git a/README_DE.md b/README_DE.md new file mode 100644 index 00000000..104c259b --- /dev/null +++ b/README_DE.md @@ -0,0 +1,773 @@ +
+ +# 🧠 memory-lancedb-pro · 🦞OpenClaw Plugin + +**KI-Gedächtnisassistent für [OpenClaw](https://github.com/openclaw/openclaw)-Agenten** + +*Geben Sie Ihrem KI-Agenten ein Gehirn, das sich wirklich erinnert — über Sitzungen, Agenten und Zeit hinweg.* + +Ein LanceDB-basiertes OpenClaw-Langzeitgedächtnis-Plugin, das Präferenzen, Entscheidungen und Projektkontext speichert und in zukünftigen Sitzungen automatisch abruft. + +[![OpenClaw Plugin](https://img.shields.io/badge/OpenClaw-Plugin-blue)](https://github.com/openclaw/openclaw) +[![npm version](https://img.shields.io/npm/v/memory-lancedb-pro)](https://www.npmjs.com/package/memory-lancedb-pro) +[![LanceDB](https://img.shields.io/badge/LanceDB-Vectorstore-orange)](https://lancedb.com) +[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE) + +[English](README.md) | [简体中文](README_CN.md) | [繁體中文](README_TW.md) | [日本語](README_JA.md) | [한국어](README_KO.md) | [Français](README_FR.md) | [Español](README_ES.md) | [Deutsch](README_DE.md) | [Italiano](README_IT.md) | [Русский](README_RU.md) | [Português (Brasil)](README_PT-BR.md) + +
+ +--- + +## Warum memory-lancedb-pro? + +Die meisten KI-Agenten leiden unter Amnesie. Sie vergessen alles, sobald Sie einen neuen Chat starten. + +**memory-lancedb-pro** ist ein produktionsreifes Langzeitgedächtnis-Plugin für OpenClaw, das Ihren Agenten in einen echten **KI-Gedächtnisassistenten** verwandelt — es erfasst automatisch, was wichtig ist, lässt Rauschen natürlich verblassen und ruft die richtige Erinnerung zum richtigen Zeitpunkt ab. Kein manuelles Taggen, keine Konfigurationsprobleme. + +### Ihr KI-Gedächtnisassistent in Aktion + +**Ohne Gedächtnis — jede Sitzung beginnt bei null:** + +> **Sie:** „Verwende Tabs für die Einrückung, füge immer Fehlerbehandlung hinzu." +> *(nächste Sitzung)* +> **Sie:** „Ich habe es dir schon gesagt — Tabs, nicht Leerzeichen!" 😤 +> *(nächste Sitzung)* +> **Sie:** „…ernsthaft, Tabs. Und Fehlerbehandlung. Schon wieder." + +**Mit memory-lancedb-pro — Ihr Agent lernt und erinnert sich:** + +> **Sie:** „Verwende Tabs für die Einrückung, füge immer Fehlerbehandlung hinzu." +> *(nächste Sitzung — Agent ruft automatisch Ihre Präferenzen ab)* +> **Agent:** *(wendet still Tabs + Fehlerbehandlung an)* ✅ +> **Sie:** „Warum haben wir letzten Monat PostgreSQL statt MongoDB gewählt?" +> **Agent:** „Basierend auf unserer Diskussion am 12. Februar waren die Hauptgründe…" ✅ + +Das ist der Unterschied, den ein **KI-Gedächtnisassistent** macht — er lernt Ihren Stil, erinnert sich an vergangene Entscheidungen und liefert personalisierte Antworten, ohne dass Sie sich wiederholen müssen. + +### Was kann es noch? + +| | Was Sie bekommen | +|---|---| +| **Auto-Capture** | Ihr Agent lernt aus jeder Unterhaltung — kein manuelles `memory_store` nötig | +| **Intelligente Extraktion** | LLM-gestützte 6-Kategorien-Klassifikation: Profile, Präferenzen, Entitäten, Ereignisse, Fälle, Muster | +| **Intelligentes Vergessen** | Weibull-Zerfallsmodell — wichtige Erinnerungen bleiben, Rauschen verblasst natürlich | +| **Hybride Suche** | Vektor + BM25 Volltextsuche, fusioniert mit Cross-Encoder-Reranking | +| **Kontextinjektion** | Relevante Erinnerungen tauchen automatisch vor jeder Antwort auf | +| **Multi-Scope-Isolation** | Gedächtnisgrenzen pro Agent, pro Benutzer, pro Projekt | +| **Jeder Anbieter** | OpenAI, Jina, Gemini, Ollama oder jede OpenAI-kompatible API | +| **Vollständiges Toolkit** | CLI, Backup, Migration, Upgrade, Export/Import — produktionsbereit | + +--- + +## Schnellstart + +### Option A: Ein-Klick-Installationsskript (empfohlen) + +Das community-gepflegte **[Setup-Skript](https://github.com/CortexReach/toolbox/tree/main/memory-lancedb-pro-setup)** erledigt Installation, Upgrade und Reparatur in einem Befehl: + +```bash +curl -fsSL https://raw.githubusercontent.com/CortexReach/toolbox/main/memory-lancedb-pro-setup/setup-memory.sh -o setup-memory.sh +bash setup-memory.sh +``` + +> Siehe [Ökosystem](#ökosystem) unten für die vollständige Liste der abgedeckten Szenarien und andere Community-Tools. + +### Option B: Manuelle Installation + +**Über OpenClaw CLI (empfohlen):** +```bash +openclaw plugins install memory-lancedb-pro@beta +``` + +**Oder über npm:** +```bash +npm i memory-lancedb-pro@beta +``` +> Bei npm-Installation müssen Sie auch das Plugin-Installationsverzeichnis als **absoluten** Pfad in `plugins.load.paths` Ihrer `openclaw.json` hinzufügen. Dies ist das häufigste Einrichtungsproblem. + +Fügen Sie zu Ihrer `openclaw.json` hinzu: + +```json +{ + "plugins": { + "slots": { "memory": "memory-lancedb-pro" }, + "entries": { + "memory-lancedb-pro": { + "enabled": true, + "config": { + "embedding": { + "provider": "openai-compatible", + "apiKey": "${OPENAI_API_KEY}", + "model": "text-embedding-3-small" + }, + "autoCapture": true, + "autoRecall": true, + "smartExtraction": true, + "extractMinMessages": 2, + "extractMaxChars": 8000, + "sessionMemory": { "enabled": false } + } + } + } + } +} +``` + +**Warum diese Standardwerte?** +- `autoCapture` + `smartExtraction` → Ihr Agent lernt automatisch aus jeder Unterhaltung +- `autoRecall` → relevante Erinnerungen werden vor jeder Antwort injiziert +- `extractMinMessages: 2` → Extraktion wird bei normalen Zwei-Runden-Chats ausgelöst +- `sessionMemory.enabled: false` → vermeidet Verschmutzung der Suche durch Sitzungszusammenfassungen am Anfang + +Validieren und neu starten: + +```bash +openclaw config validate +openclaw gateway restart +openclaw logs --follow --plain | grep "memory-lancedb-pro" +``` + +Sie sollten sehen: +- `memory-lancedb-pro: smart extraction enabled` +- `memory-lancedb-pro@...: plugin registered` + +Fertig! Ihr Agent verfügt jetzt über Langzeitgedächtnis. + +
+Weitere Installationswege (bestehende Benutzer, Upgrades) + +**Bereits OpenClaw-Benutzer?** + +1. Fügen Sie das Plugin mit einem **absoluten** `plugins.load.paths`-Eintrag hinzu +2. Binden Sie den Memory-Slot: `plugins.slots.memory = "memory-lancedb-pro"` +3. Überprüfen: `openclaw plugins info memory-lancedb-pro && openclaw memory-pro stats` + +**Upgrade von vor v1.1.0?** + +```bash +# 1) Backup +openclaw memory-pro export --scope global --output memories-backup.json +# 2) Testlauf +openclaw memory-pro upgrade --dry-run +# 3) Upgrade ausführen +openclaw memory-pro upgrade +# 4) Überprüfen +openclaw memory-pro stats +``` + +Siehe `CHANGELOG-v1.1.0.md` für Verhaltensänderungen und Upgrade-Begründung. + +
+ +
+Telegram-Bot-Schnellimport (zum Aufklappen klicken) + +Wenn Sie die Telegram-Integration von OpenClaw verwenden, ist es am einfachsten, einen Importbefehl direkt an den Hauptbot zu senden, anstatt die Konfiguration manuell zu bearbeiten. + +Senden Sie diese Nachricht: + +```text +Help me connect this memory plugin with the most user-friendly configuration: https://github.com/CortexReach/memory-lancedb-pro + +Requirements: +1. Set it as the only active memory plugin +2. Use Jina for embedding +3. Use Jina for reranker +4. Use gpt-4o-mini for the smart-extraction LLM +5. Enable autoCapture, autoRecall, smartExtraction +6. extractMinMessages=2 +7. sessionMemory.enabled=false +8. captureAssistant=false +9. retrieval mode=hybrid, vectorWeight=0.7, bm25Weight=0.3 +10. rerank=cross-encoder, candidatePoolSize=12, minScore=0.6, hardMinScore=0.62 +11. Generate the final openclaw.json config directly, not just an explanation +``` + +
+ +--- + +## Ökosystem + +memory-lancedb-pro ist das Kern-Plugin. Die Community hat Tools darum herum gebaut, um Einrichtung und tägliche Nutzung noch reibungsloser zu machen: + +### Setup-Skript — Ein-Klick-Installation, Upgrade und Reparatur + +> **[CortexReach/toolbox/memory-lancedb-pro-setup](https://github.com/CortexReach/toolbox/tree/main/memory-lancedb-pro-setup)** + +Nicht nur ein einfacher Installer — das Skript behandelt intelligent eine Vielzahl realer Szenarien: + +| Ihre Situation | Was das Skript macht | +|---|---| +| Nie installiert | Frischer Download → Abhängigkeiten installieren → Konfiguration wählen → in openclaw.json schreiben → Neustart | +| Per `git clone` installiert, auf altem Commit hängen geblieben | Automatisches `git fetch` + `checkout` auf neueste Version → Abhängigkeiten neu installieren → Verifizieren | +| Konfiguration hat ungültige Felder | Automatische Erkennung per Schema-Filter, nicht unterstützte Felder entfernen | +| Per `npm` installiert | Überspringt Git-Update, erinnert Sie daran, `npm update` selbst auszuführen | +| `openclaw` CLI durch ungültige Konfiguration defekt | Fallback: Workspace-Pfad direkt aus der `openclaw.json`-Datei lesen | +| `extensions/` statt `plugins/` | Automatische Erkennung des Plugin-Standorts aus Konfiguration oder Dateisystem | +| Bereits aktuell | Nur Gesundheitschecks ausführen, keine Änderungen | + +```bash +bash setup-memory.sh # Installieren oder upgraden +bash setup-memory.sh --dry-run # Nur Vorschau +bash setup-memory.sh --beta # Pre-Release-Versionen einschließen +bash setup-memory.sh --uninstall # Konfiguration zurücksetzen und Plugin entfernen +``` + +Eingebaute Anbieter-Presets: **Jina / DashScope / SiliconFlow / OpenAI / Ollama**, oder bringen Sie Ihre eigene OpenAI-kompatible API mit. Für die vollständige Nutzung (einschließlich `--ref`, `--selfcheck-only` und mehr) siehe das [Setup-Skript README](https://github.com/CortexReach/toolbox/tree/main/memory-lancedb-pro-setup). + +### Claude Code / OpenClaw Skill — KI-geführte Konfiguration + +> **[CortexReach/memory-lancedb-pro-skill](https://github.com/CortexReach/memory-lancedb-pro-skill)** + +Installieren Sie diesen Skill und Ihr KI-Agent (Claude Code oder OpenClaw) erhält tiefgreifendes Wissen über alle Funktionen von memory-lancedb-pro. Sagen Sie einfach **„hilf mir die beste Konfiguration zu aktivieren"** und erhalten Sie: + +- **Geführter 7-Schritte-Konfigurationsworkflow** mit 4 Bereitstellungsplänen: + - Full Power (Jina + OpenAI) / Budget (kostenloser SiliconFlow Reranker) / Simple (nur OpenAI) / Vollständig lokal (Ollama, null API-Kosten) +- **Alle 9 MCP-Tools** korrekt verwendet: `memory_recall`, `memory_store`, `memory_forget`, `memory_update`, `memory_stats`, `memory_list`, `self_improvement_log`, `self_improvement_extract_skill`, `self_improvement_review` *(vollständiges Toolset erfordert `enableManagementTools: true` — die Standard-Schnellstart-Konfiguration stellt nur die 4 Kern-Tools bereit)* +- **Vermeidung häufiger Fallstricke**: Workspace-Plugin-Aktivierung, `autoRecall` standardmäßig false, jiti-Cache, Umgebungsvariablen, Scope-Isolation und mehr + +**Installation für Claude Code:** +```bash +git clone https://github.com/CortexReach/memory-lancedb-pro-skill.git ~/.claude/skills/memory-lancedb-pro +``` + +**Installation für OpenClaw:** +```bash +git clone https://github.com/CortexReach/memory-lancedb-pro-skill.git ~/.openclaw/workspace/skills/memory-lancedb-pro-skill +``` + +--- + +## Video-Tutorial + +> Vollständige Anleitung: Installation, Konfiguration und Funktionsweise der hybriden Suche. + +[![YouTube Video](https://img.shields.io/badge/YouTube-Watch%20Now-red?style=for-the-badge&logo=youtube)](https://youtu.be/MtukF1C8epQ) +**https://youtu.be/MtukF1C8epQ** + +[![Bilibili Video](https://img.shields.io/badge/Bilibili-Watch%20Now-00A1D6?style=for-the-badge&logo=bilibili&logoColor=white)](https://www.bilibili.com/video/BV1zUf2BGEgn/) +**https://www.bilibili.com/video/BV1zUf2BGEgn/** + +--- + +## Architektur + +``` +┌─────────────────────────────────────────────────────────┐ +│ index.ts (Einstiegspunkt) │ +│ Plugin-Registrierung · Config-Parsing · Lifecycle-Hooks│ +└────────┬──────────┬──────────┬──────────┬───────────────┘ + │ │ │ │ + ┌────▼───┐ ┌────▼───┐ ┌───▼────┐ ┌──▼──────────┐ + │ store │ │embedder│ │retriever│ │ scopes │ + │ .ts │ │ .ts │ │ .ts │ │ .ts │ + └────────┘ └────────┘ └────────┘ └─────────────┘ + │ │ + ┌────▼───┐ ┌─────▼──────────┐ + │migrate │ │noise-filter.ts │ + │ .ts │ │adaptive- │ + └────────┘ │retrieval.ts │ + └────────────────┘ + ┌─────────────┐ ┌──────────┐ + │ tools.ts │ │ cli.ts │ + │ (Agent-API) │ │ (CLI) │ + └─────────────┘ └──────────┘ +``` + +> Für eine detaillierte Analyse der vollständigen Architektur siehe [docs/memory_architecture_analysis.md](docs/memory_architecture_analysis.md). + +
+Dateireferenz (zum Aufklappen klicken) + +| Datei | Zweck | +| --- | --- | +| `index.ts` | Plugin-Einstiegspunkt. Registriert sich bei der OpenClaw Plugin API, parst Konfiguration, bindet Lifecycle-Hooks ein | +| `openclaw.plugin.json` | Plugin-Metadaten + vollständige JSON-Schema-Konfigurationsdeklaration | +| `cli.ts` | CLI-Befehle: `memory-pro list/search/stats/delete/delete-bulk/export/import/reembed/upgrade/migrate` | +| `src/store.ts` | LanceDB-Speicherschicht. Tabellenerstellung / FTS-Indexierung / Vektorsuche / BM25-Suche / CRUD | +| `src/embedder.ts` | Embedding-Abstraktion. Kompatibel mit jedem OpenAI-kompatiblen API-Anbieter | +| `src/retriever.ts` | Hybride Suchmaschine. Vektor + BM25 → Hybride Fusion → Rerank → Lifecycle-Zerfall → Filter | +| `src/scopes.ts` | Multi-Scope-Zugriffskontrolle | +| `src/tools.ts` | Agent-Tool-Definitionen: `memory_recall`, `memory_store`, `memory_forget`, `memory_update` + Verwaltungstools | +| `src/noise-filter.ts` | Filtert Agent-Ablehnungen, Meta-Fragen, Begrüßungen und minderwertige Inhalte | +| `src/adaptive-retrieval.ts` | Bestimmt, ob eine Abfrage Gedächtnisabruf benötigt | +| `src/migrate.ts` | Migration vom eingebauten `memory-lancedb` zu Pro | +| `src/smart-extractor.ts` | LLM-gestützte 6-Kategorien-Extraktion mit L0/L1/L2 Schichtspeicherung und zweistufiger Deduplizierung | +| `src/decay-engine.ts` | Weibull Stretched-Exponential-Zerfallsmodell | +| `src/tier-manager.ts` | Dreistufige Beförderung/Herabstufung: Peripheral ↔ Working ↔ Core | + +
+ +--- + +## Kernfunktionen + +### Hybride Suche + +``` +Query → embedQuery() ─┐ + ├─→ Hybride Fusion → Rerank → Lifecycle-Zerfall-Boost → Längennorm → Filter +Query → BM25 FTS ─────┘ +``` + +- **Vektorsuche** — semantische Ähnlichkeit über LanceDB ANN (Kosinus-Distanz) +- **BM25 Volltextsuche** — exakte Schlüsselwortübereinstimmung über LanceDB FTS-Index +- **Hybride Fusion** — Vektorscore als Basis, BM25-Treffer erhalten gewichteten Boost (kein Standard-RRF — optimiert für reale Abrufqualität) +- **Konfigurierbare Gewichte** — `vectorWeight`, `bm25Weight`, `minScore` + +### Cross-Encoder Reranking + +- Eingebaute Adapter für **Jina**, **SiliconFlow**, **Voyage AI** und **Pinecone** +- Kompatibel mit jedem Jina-kompatiblen Endpunkt (z.B. Hugging Face TEI, DashScope) +- Hybrid-Scoring: 60% Cross-Encoder + 40% ursprünglicher fusionierter Score +- Graceful Degradation: Rückfall auf Kosinus-Ähnlichkeit bei API-Ausfall + +### Mehrstufige Scoring-Pipeline + +| Stufe | Effekt | +| --- | --- | +| **Hybride Fusion** | Kombiniert semantische und exakte Suche | +| **Cross-Encoder Rerank** | Fördert semantisch präzise Treffer | +| **Lifecycle-Zerfall-Boost** | Weibull-Aktualität + Zugriffshäufigkeit + Wichtigkeit × Konfidenz | +| **Längennormalisierung** | Verhindert, dass lange Einträge dominieren (Anker: 500 Zeichen) | +| **Harter Mindestscore** | Entfernt irrelevante Ergebnisse (Standard: 0.35) | +| **MMR-Diversität** | Kosinus-Ähnlichkeit > 0.85 → herabgestuft | + +### Intelligente Gedächtnisextraktion (v1.1.0) + +- **LLM-gestützte 6-Kategorien-Extraktion**: Profil, Präferenzen, Entitäten, Ereignisse, Fälle, Muster +- **L0/L1/L2 Schichtspeicherung**: L0 (Einzeiler-Index) → L1 (strukturierte Zusammenfassung) → L2 (vollständige Erzählung) +- **Zweistufige Deduplizierung**: Vektor-Ähnlichkeits-Vorfilter (≥0.7) → LLM semantische Entscheidung (CREATE/MERGE/SKIP) +- **Kategoriebasierte Zusammenführung**: `profile` wird immer zusammengeführt, `events`/`cases` sind nur anfügbar + +### Gedächtnis-Lebenszyklusverwaltung (v1.1.0) + +- **Weibull-Zerfallsmotor**: Gesamtscore = Aktualität + Häufigkeit + intrinsischer Wert +- **Dreistufige Beförderung**: `Peripheral ↔ Working ↔ Core` mit konfigurierbaren Schwellenwerten +- **Zugriffsverstärkung**: Häufig abgerufene Erinnerungen zerfallen langsamer (Spaced-Repetition-Stil) +- **Wichtigkeitsmodulierte Halbwertszeit**: Wichtige Erinnerungen zerfallen langsamer + +### Multi-Scope-Isolation + +- Eingebaute Scopes: `global`, `agent:`, `custom:`, `project:`, `user:` +- Zugriffskontrolle auf Agentenebene über `scopes.agentAccess` +- Standard: Jeder Agent greift auf `global` + seinen eigenen `agent:`-Scope zu + +### Auto-Capture und Auto-Recall + +- **Auto-Capture** (`agent_end`): Extrahiert Präferenzen/Fakten/Entscheidungen/Entitäten aus Gesprächen, dedupliziert, speichert bis zu 3 pro Runde +- **Auto-Recall** (`before_agent_start`): Injiziert ``-Kontext (bis zu 3 Einträge) + +### Rauschfilterung und adaptive Suche + +- Filtert minderwertige Inhalte: Agent-Ablehnungen, Meta-Fragen, Begrüßungen +- Überspringt Suche bei: Begrüßungen, Slash-Befehlen, einfachen Bestätigungen, Emoji +- Erzwingt Suche bei Gedächtnis-Schlüsselwörtern („erinnere dich", „vorher", „letztes Mal") +- CJK-bewusste Schwellenwerte (Chinesisch: 6 Zeichen vs Englisch: 15 Zeichen) + +--- + +
+Vergleich mit dem eingebauten memory-lancedb (zum Aufklappen klicken) + +| Funktion | Eingebautes `memory-lancedb` | **memory-lancedb-pro** | +| --- | :---: | :---: | +| Vektorsuche | Ja | Ja | +| BM25 Volltextsuche | - | Ja | +| Hybride Fusion (Vektor + BM25) | - | Ja | +| Cross-Encoder Rerank (Multi-Anbieter) | - | Ja | +| Aktualitäts-Boost und Zeitzerfall | - | Ja | +| Längennormalisierung | - | Ja | +| MMR-Diversität | - | Ja | +| Multi-Scope-Isolation | - | Ja | +| Rauschfilterung | - | Ja | +| Adaptive Suche | - | Ja | +| Verwaltungs-CLI | - | Ja | +| Sitzungsgedächtnis | - | Ja | +| Aufgabenbezogene Embeddings | - | Ja | +| **LLM Intelligente Extraktion (6 Kategorien)** | - | Ja (v1.1.0) | +| **Weibull-Zerfall + Stufenbeförderung** | - | Ja (v1.1.0) | +| Beliebiges OpenAI-kompatibles Embedding | Eingeschränkt | Ja | + +
+ +--- + +## Konfiguration + +
+Vollständiges Konfigurationsbeispiel + +```json +{ + "embedding": { + "apiKey": "${JINA_API_KEY}", + "model": "jina-embeddings-v5-text-small", + "baseURL": "https://api.jina.ai/v1", + "dimensions": 1024, + "taskQuery": "retrieval.query", + "taskPassage": "retrieval.passage", + "normalized": true + }, + "dbPath": "~/.openclaw/memory/lancedb-pro", + "autoCapture": true, + "autoRecall": true, + "retrieval": { + "mode": "hybrid", + "vectorWeight": 0.7, + "bm25Weight": 0.3, + "minScore": 0.3, + "rerank": "cross-encoder", + "rerankApiKey": "${JINA_API_KEY}", + "rerankModel": "jina-reranker-v3", + "rerankEndpoint": "https://api.jina.ai/v1/rerank", + "rerankProvider": "jina", + "candidatePoolSize": 20, + "recencyHalfLifeDays": 14, + "recencyWeight": 0.1, + "filterNoise": true, + "lengthNormAnchor": 500, + "hardMinScore": 0.35, + "timeDecayHalfLifeDays": 60, + "reinforcementFactor": 0.5, + "maxHalfLifeMultiplier": 3 + }, + "enableManagementTools": false, + "scopes": { + "default": "global", + "definitions": { + "global": { "description": "Shared knowledge" }, + "agent:discord-bot": { "description": "Discord bot private" } + }, + "agentAccess": { + "discord-bot": ["global", "agent:discord-bot"] + } + }, + "sessionMemory": { + "enabled": false, + "messageCount": 15 + }, + "smartExtraction": true, + "llm": { + "apiKey": "${OPENAI_API_KEY}", + "model": "gpt-4o-mini", + "baseURL": "https://api.openai.com/v1" + }, + "extractMinMessages": 2, + "extractMaxChars": 8000 +} +``` + +
+ +
+Embedding-Anbieter + +Funktioniert mit **jeder OpenAI-kompatiblen Embedding-API**: + +| Anbieter | Modell | Basis-URL | Dimensionen | +| --- | --- | --- | --- | +| **Jina** (empfohlen) | `jina-embeddings-v5-text-small` | `https://api.jina.ai/v1` | 1024 | +| **OpenAI** | `text-embedding-3-small` | `https://api.openai.com/v1` | 1536 | +| **Voyage** | `voyage-4-lite` / `voyage-4` | `https://api.voyageai.com/v1` | 1024 / 1024 | +| **Google Gemini** | `gemini-embedding-001` | `https://generativelanguage.googleapis.com/v1beta/openai/` | 3072 | +| **Ollama** (lokal) | `nomic-embed-text` | `http://localhost:11434/v1` | anbieterspezifisch | + +
+ +
+Rerank-Anbieter + +Cross-Encoder Reranking unterstützt mehrere Anbieter über `rerankProvider`: + +| Anbieter | `rerankProvider` | Beispielmodell | +| --- | --- | --- | +| **Jina** (Standard) | `jina` | `jina-reranker-v3` | +| **SiliconFlow** (kostenlose Stufe verfügbar) | `siliconflow` | `BAAI/bge-reranker-v2-m3` | +| **Voyage AI** | `voyage` | `rerank-2.5` | +| **Pinecone** | `pinecone` | `bge-reranker-v2-m3` | + +Jeder Jina-kompatible Rerank-Endpunkt funktioniert ebenfalls — setzen Sie `rerankProvider: "jina"` und verweisen Sie `rerankEndpoint` auf Ihren Dienst (z.B. Hugging Face TEI, DashScope `qwen3-rerank`). + +
+ +
+Intelligente Extraktion (LLM) — v1.1.0 + +Wenn `smartExtraction` aktiviert ist (Standard: `true`), verwendet das Plugin ein LLM, um Erinnerungen intelligent zu extrahieren und zu klassifizieren, anstatt regex-basierte Auslöser zu verwenden. + +| Feld | Typ | Standard | Beschreibung | +|-------|------|---------|-------------| +| `smartExtraction` | boolean | `true` | LLM-gestützte 6-Kategorien-Extraktion aktivieren/deaktivieren | +| `llm.auth` | string | `api-key` | `api-key` verwendet `llm.apiKey` / `embedding.apiKey`; `oauth` verwendet standardmäßig eine plugin-spezifische OAuth-Token-Datei | +| `llm.apiKey` | string | *(Rückfall auf `embedding.apiKey`)* | API-Schlüssel für den LLM-Anbieter | +| `llm.model` | string | `openai/gpt-oss-120b` | LLM-Modellname | +| `llm.baseURL` | string | *(Rückfall auf `embedding.baseURL`)* | LLM-API-Endpunkt | +| `llm.oauthProvider` | string | `openai-codex` | OAuth-Anbieter-ID bei `llm.auth` = `oauth` | +| `llm.oauthPath` | string | `~/.openclaw/.memory-lancedb-pro/oauth.json` | OAuth-Token-Datei bei `llm.auth` = `oauth` | +| `llm.timeoutMs` | number | `30000` | LLM-Anfrage-Timeout in Millisekunden | +| `extractMinMessages` | number | `2` | Mindestanzahl an Nachrichten bevor Extraktion ausgelöst wird | +| `extractMaxChars` | number | `8000` | Maximale Zeichenanzahl, die an das LLM gesendet wird | + + +OAuth `llm`-Konfiguration (vorhandenen Codex / ChatGPT Login-Cache für LLM-Aufrufe verwenden): +```json +{ + "llm": { + "auth": "oauth", + "oauthProvider": "openai-codex", + "model": "gpt-5.4", + "oauthPath": "${HOME}/.openclaw/.memory-lancedb-pro/oauth.json", + "timeoutMs": 30000 + } +} +``` + +Hinweise zu `llm.auth: "oauth"`: + +- `llm.oauthProvider` ist derzeit `openai-codex`. +- OAuth-Tokens werden standardmäßig unter `~/.openclaw/.memory-lancedb-pro/oauth.json` gespeichert. +- Sie können `llm.oauthPath` setzen, wenn Sie die Datei an einem anderen Ort speichern möchten. +- `auth login` erstellt eine Sicherung der vorherigen api-key `llm`-Konfiguration neben der OAuth-Datei, und `auth logout` stellt diese Sicherung bei Verfügbarkeit wieder her. +- Der Wechsel von `api-key` zu `oauth` überträgt `llm.baseURL` nicht automatisch. Setzen Sie es im OAuth-Modus nur manuell, wenn Sie absichtlich ein benutzerdefiniertes ChatGPT/Codex-kompatibles Backend verwenden möchten. + +
+ +
+Lebenszyklus-Konfiguration (Zerfall + Stufen) + +| Feld | Standard | Beschreibung | +|-------|---------|-------------| +| `decay.recencyHalfLifeDays` | `30` | Basis-Halbwertszeit für Weibull-Aktualitätszerfall | +| `decay.frequencyWeight` | `0.3` | Gewichtung der Zugriffshäufigkeit im Gesamtscore | +| `decay.intrinsicWeight` | `0.3` | Gewichtung von `Wichtigkeit × Konfidenz` | +| `decay.betaCore` | `0.8` | Weibull-Beta für `core`-Erinnerungen | +| `decay.betaWorking` | `1.0` | Weibull-Beta für `working`-Erinnerungen | +| `decay.betaPeripheral` | `1.3` | Weibull-Beta für `peripheral`-Erinnerungen | +| `tier.coreAccessThreshold` | `10` | Mindestanzahl Abrufe vor Beförderung zu `core` | +| `tier.peripheralAgeDays` | `60` | Altersschwelle für die Herabstufung veralteter Erinnerungen | + +
+ +
+Zugriffsverstärkung + +Häufig abgerufene Erinnerungen zerfallen langsamer (Spaced-Repetition-Stil). + +Konfigurationsschlüssel (unter `retrieval`): +- `reinforcementFactor` (0-2, Standard: `0.5`) — auf `0` setzen zum Deaktivieren +- `maxHalfLifeMultiplier` (1-10, Standard: `3`) — harte Obergrenze für die effektive Halbwertszeit + +
+ +--- + +## CLI-Befehle + +```bash +openclaw memory-pro list [--scope global] [--category fact] [--limit 20] [--json] +openclaw memory-pro search "query" [--scope global] [--limit 10] [--json] +openclaw memory-pro stats [--scope global] [--json] +openclaw memory-pro auth login [--provider openai-codex] [--model gpt-5.4] [--oauth-path /abs/path/oauth.json] +openclaw memory-pro auth status +openclaw memory-pro auth logout +openclaw memory-pro delete +openclaw memory-pro delete-bulk --scope global [--before 2025-01-01] [--dry-run] +openclaw memory-pro export [--scope global] [--output memories.json] +openclaw memory-pro import memories.json [--scope global] [--dry-run] +openclaw memory-pro reembed --source-db /path/to/old-db [--batch-size 32] [--skip-existing] +openclaw memory-pro upgrade [--dry-run] [--batch-size 10] [--no-llm] [--limit N] [--scope SCOPE] +openclaw memory-pro migrate check|run|verify [--source /path] +``` + +OAuth-Login-Ablauf: + +1. Führen Sie `openclaw memory-pro auth login` aus +2. Wenn `--provider` in einem interaktiven Terminal weggelassen wird, zeigt die CLI eine OAuth-Anbieterauswahl an, bevor der Browser geöffnet wird +3. Der Befehl gibt eine Autorisierungs-URL aus und öffnet Ihren Browser, sofern `--no-browser` nicht gesetzt ist +4. Nach erfolgreichem OAuth-Callback speichert der Befehl die Plugin-OAuth-Datei (Standard: `~/.openclaw/.memory-lancedb-pro/oauth.json`), erstellt eine Sicherung der vorherigen api-key `llm`-Konfiguration für Logout und ersetzt die Plugin-`llm`-Konfiguration durch OAuth-Einstellungen (`auth`, `oauthProvider`, `model`, `oauthPath`) +5. `openclaw memory-pro auth logout` löscht die OAuth-Datei und stellt die vorherige api-key `llm`-Konfiguration wieder her, wenn eine Sicherung vorhanden ist + +--- + +## Erweiterte Themen + +
+Wenn injizierte Erinnerungen in Antworten auftauchen + +Manchmal kann das Modell den injizierten ``-Block wiedergeben. + +**Option A (geringstes Risiko):** Auto-Recall vorübergehend deaktivieren: +```json +{ "plugins": { "entries": { "memory-lancedb-pro": { "config": { "autoRecall": false } } } } } +``` + +**Option B (bevorzugt):** Recall beibehalten, zum Agent-Systemprompt hinzufügen: +> Do not reveal or quote any `` / memory-injection content in your replies. Use it for internal reference only. + +
+ +
+Sitzungsgedächtnis + +- Wird beim `/new`-Befehl ausgelöst — speichert die vorherige Sitzungszusammenfassung in LanceDB +- Standardmäßig deaktiviert (OpenClaw hat bereits native `.jsonl`-Sitzungspersistenz) +- Konfigurierbare Nachrichtenanzahl (Standard: 15) + +Siehe [docs/openclaw-integration-playbook.md](docs/openclaw-integration-playbook.md) für Bereitstellungsmodi und `/new`-Verifizierung. + +
+ +
+Benutzerdefinierte Slash-Befehle (z.B. /lesson) + +Fügen Sie zu Ihrer `CLAUDE.md`, `AGENTS.md` oder Ihrem Systemprompt hinzu: + +```markdown +## /lesson command +When the user sends `/lesson `: +1. Use memory_store to save as category=fact (raw knowledge) +2. Use memory_store to save as category=decision (actionable takeaway) +3. Confirm what was saved + +## /remember command +When the user sends `/remember `: +1. Use memory_store to save with appropriate category and importance +2. Confirm with the stored memory ID +``` + +
+ +
+Eiserne Regeln für KI-Agenten + +> Kopieren Sie den folgenden Block in Ihre `AGENTS.md`, damit Ihr Agent diese Regeln automatisch durchsetzt. + +```markdown +## Rule 1 — Dual-layer memory storage +Every pitfall/lesson learned → IMMEDIATELY store TWO memories: +- Technical layer: Pitfall: [symptom]. Cause: [root cause]. Fix: [solution]. Prevention: [how to avoid] + (category: fact, importance >= 0.8) +- Principle layer: Decision principle ([tag]): [behavioral rule]. Trigger: [when]. Action: [what to do] + (category: decision, importance >= 0.85) + +## Rule 2 — LanceDB hygiene +Entries must be short and atomic (< 500 chars). No raw conversation summaries or duplicates. + +## Rule 3 — Recall before retry +On ANY tool failure, ALWAYS memory_recall with relevant keywords BEFORE retrying. + +## Rule 4 — Confirm target codebase +Confirm you are editing memory-lancedb-pro vs built-in memory-lancedb before changes. + +## Rule 5 — Clear jiti cache after plugin code changes +After modifying .ts files under plugins/, MUST run rm -rf /tmp/jiti/ BEFORE openclaw gateway restart. +``` + +
+ +
+Datenbankschema + +LanceDB-Tabelle `memories`: + +| Feld | Typ | Beschreibung | +| --- | --- | --- | +| `id` | string (UUID) | Primärschlüssel | +| `text` | string | Gedächtnistext (FTS-indiziert) | +| `vector` | float[] | Embedding-Vektor | +| `category` | string | Speicherkategorie: `preference` / `fact` / `decision` / `entity` / `reflection` / `other` | +| `scope` | string | Scope-Bezeichner (z.B. `global`, `agent:main`) | +| `importance` | float | Wichtigkeitsscore 0-1 | +| `timestamp` | int64 | Erstellungszeitstempel (ms) | +| `metadata` | string (JSON) | Erweiterte Metadaten | + +Häufige `metadata`-Schlüssel in v1.1.0: `l0_abstract`, `l1_overview`, `l2_content`, `memory_category`, `tier`, `access_count`, `confidence`, `last_accessed_at` + +> **Hinweis zu Kategorien:** Das Top-Level-Feld `category` verwendet 6 Speicherkategorien. Die 6-Kategorien-semantischen Labels der intelligenten Extraktion (`profile` / `preferences` / `entities` / `events` / `cases` / `patterns`) werden in `metadata.memory_category` gespeichert. + +
+ +
+Fehlerbehebung + +### "Cannot mix BigInt and other types" (LanceDB / Apache Arrow) + +Bei LanceDB 0.26+ können einige numerische Spalten als `BigInt` zurückgegeben werden. Aktualisieren Sie auf **memory-lancedb-pro >= 1.0.14** — dieses Plugin konvertiert Werte nun mit `Number(...)` vor arithmetischen Operationen. + +
+ +--- + +## Dokumentation + +| Dokument | Beschreibung | +| --- | --- | +| [OpenClaw Integrations-Playbook](docs/openclaw-integration-playbook.md) | Bereitstellungsmodi, Verifizierung, Regressionsmatrix | +| [Gedächtnisarchitektur-Analyse](docs/memory_architecture_analysis.md) | Vollständige Architektur-Tiefenanalyse | +| [CHANGELOG v1.1.0](docs/CHANGELOG-v1.1.0.md) | Verhaltensänderungen v1.1.0 und Upgrade-Begründung | +| [Langkontext-Chunking](docs/long-context-chunking.md) | Chunking-Strategie für lange Dokumente | + +--- + +## Beta: Smart Memory v1.1.0 + +> Status: Beta — verfügbar über `npm i memory-lancedb-pro@beta`. Stabile Benutzer auf `latest` sind nicht betroffen. + +| Funktion | Beschreibung | +|---------|-------------| +| **Intelligente Extraktion** | LLM-gestützte 6-Kategorien-Extraktion mit L0/L1/L2 Metadaten. Rückfall auf Regex wenn deaktiviert. | +| **Lebenszyklus-Scoring** | Weibull-Zerfall in die Suche integriert — häufige und wichtige Erinnerungen ranken höher. | +| **Stufenverwaltung** | Dreistufiges System (Core → Working → Peripheral) mit automatischer Beförderung/Herabstufung. | + +Feedback: [GitHub Issues](https://github.com/CortexReach/memory-lancedb-pro/issues) · Zurücksetzen: `npm i memory-lancedb-pro@latest` + +--- + +## Abhängigkeiten + +| Paket | Zweck | +| --- | --- | +| `@lancedb/lancedb` ≥0.26.2 | Vektordatenbank (ANN + FTS) | +| `openai` ≥6.21.0 | OpenAI-kompatibler Embedding-API-Client | +| `@sinclair/typebox` 0.34.48 | JSON-Schema-Typdefinitionen | + +--- + +## Contributors + +

+@win4r +@kctony +@Akatsuki-Ryu +@JasonSuz +@Minidoracat +@furedericca-lab +@joe2643 +@AliceLJY +@chenjiyong +

+ +Full list: [Contributors](https://github.com/CortexReach/memory-lancedb-pro/graphs/contributors) + +## Star History + + + + + + Star History Chart + + + +## Lizenz + +MIT + +--- + +## Mein WeChat QR-Code + + diff --git a/README_ES.md b/README_ES.md new file mode 100644 index 00000000..fe4d173c --- /dev/null +++ b/README_ES.md @@ -0,0 +1,773 @@ +
+ +# 🧠 memory-lancedb-pro · 🦞OpenClaw Plugin + +**Asistente de Memoria IA para Agentes [OpenClaw](https://github.com/openclaw/openclaw)** + +*Dale a tu agente de IA un cerebro que realmente recuerda — entre sesiones, entre agentes, a lo largo del tiempo.* + +Un plugin de memoria para OpenClaw respaldado por LanceDB que almacena preferencias, decisiones y contexto de proyectos, y los recupera automáticamente en sesiones futuras. + +[![OpenClaw Plugin](https://img.shields.io/badge/OpenClaw-Plugin-blue)](https://github.com/openclaw/openclaw) +[![npm version](https://img.shields.io/npm/v/memory-lancedb-pro)](https://www.npmjs.com/package/memory-lancedb-pro) +[![LanceDB](https://img.shields.io/badge/LanceDB-Vectorstore-orange)](https://lancedb.com) +[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE) + +[English](README.md) | [简体中文](README_CN.md) | [繁體中文](README_TW.md) | [日本語](README_JA.md) | [한국어](README_KO.md) | [Français](README_FR.md) | [Español](README_ES.md) | [Deutsch](README_DE.md) | [Italiano](README_IT.md) | [Русский](README_RU.md) | [Português (Brasil)](README_PT-BR.md) + +
+ +--- + +## ¿Por qué memory-lancedb-pro? + +La mayoría de los agentes de IA tienen amnesia. Olvidan todo en el momento en que inicias un nuevo chat. + +**memory-lancedb-pro** es un plugin de memoria a largo plazo de nivel productivo para OpenClaw que convierte a tu agente en un **Asistente de Memoria IA** — captura automáticamente lo que importa, deja que el ruido se desvanezca naturalmente y recupera el recuerdo correcto en el momento adecuado. Sin etiquetado manual, sin complicaciones de configuración. + +### Tu Asistente de Memoria IA en acción + +**Sin memoria — cada sesión comienza desde cero:** + +> **Tú:** "Usa tabulaciones para la indentación, siempre agrega manejo de errores." +> *(siguiente sesión)* +> **Tú:** "¡Ya te lo dije — tabulaciones, no espacios!" 😤 +> *(siguiente sesión)* +> **Tú:** "...en serio, tabulaciones. Y manejo de errores. Otra vez." + +**Con memory-lancedb-pro — tu agente aprende y recuerda:** + +> **Tú:** "Usa tabulaciones para la indentación, siempre agrega manejo de errores." +> *(siguiente sesión — el agente recupera automáticamente tus preferencias)* +> **Agente:** *(aplica silenciosamente tabulaciones + manejo de errores)* ✅ +> **Tú:** "¿Por qué elegimos PostgreSQL en lugar de MongoDB el mes pasado?" +> **Agente:** "Basándome en nuestra discusión del 12 de febrero, las razones principales fueron..." ✅ + +Esa es la diferencia que hace un **Asistente de Memoria IA** — aprende tu estilo, recuerda decisiones pasadas y entrega respuestas personalizadas sin que tengas que repetirte. + +### ¿Qué más puede hacer? + +| | Lo que obtienes | +|---|---| +| **Auto-Capture** | Tu agente aprende de cada conversación — sin necesidad de `memory_store` manual | +| **Smart Extraction** | Clasificación de 6 categorías impulsada por LLM: perfiles, preferencias, entidades, eventos, casos, patrones | +| **Olvido Inteligente** | Modelo de decaimiento Weibull — los recuerdos importantes permanecen, el ruido se desvanece naturalmente | +| **Recuperación Híbrida** | Búsqueda vectorial + BM25 de texto completo, fusionada con reranking por cross-encoder | +| **Inyección de Contexto** | Los recuerdos relevantes aparecen automáticamente antes de cada respuesta | +| **Aislamiento Multi-Scope** | Límites de memoria por agente, por usuario, por proyecto | +| **Cualquier Proveedor** | OpenAI, Jina, Gemini, Ollama, o cualquier API compatible con OpenAI | +| **Kit Completo de Herramientas** | CLI, respaldo, migración, actualización, exportar/importar — listo para producción | + +--- + +## Inicio Rápido + +### Opción A: Script de instalación con un clic (Recomendado) + +El **[script de instalación](https://github.com/CortexReach/toolbox/tree/main/memory-lancedb-pro-setup)** mantenido por la comunidad gestiona la instalación, actualización y reparación en un solo comando: + +```bash +curl -fsSL https://raw.githubusercontent.com/CortexReach/toolbox/main/memory-lancedb-pro-setup/setup-memory.sh -o setup-memory.sh +bash setup-memory.sh +``` + +> Consulta [Ecosistema](#ecosistema) más abajo para ver la lista completa de escenarios que cubre el script y otras herramientas de la comunidad. + +### Opción B: Instalación Manual + +**Mediante la CLI de OpenClaw (recomendado):** +```bash +openclaw plugins install memory-lancedb-pro@beta +``` + +**O mediante npm:** +```bash +npm i memory-lancedb-pro@beta +``` +> Si usas npm, también necesitarás agregar el directorio de instalación del plugin como una ruta **absoluta** en `plugins.load.paths` en tu `openclaw.json`. Este es el problema de configuración más común. + +Agrega a tu `openclaw.json`: + +```json +{ + "plugins": { + "slots": { "memory": "memory-lancedb-pro" }, + "entries": { + "memory-lancedb-pro": { + "enabled": true, + "config": { + "embedding": { + "provider": "openai-compatible", + "apiKey": "${OPENAI_API_KEY}", + "model": "text-embedding-3-small" + }, + "autoCapture": true, + "autoRecall": true, + "smartExtraction": true, + "extractMinMessages": 2, + "extractMaxChars": 8000, + "sessionMemory": { "enabled": false } + } + } + } + } +} +``` + +**¿Por qué estos valores predeterminados?** +- `autoCapture` + `smartExtraction` → tu agente aprende de cada conversación automáticamente +- `autoRecall` → los recuerdos relevantes se inyectan antes de cada respuesta +- `extractMinMessages: 2` → la extracción se activa en chats normales de dos turnos +- `sessionMemory.enabled: false` → evita contaminar la recuperación con resúmenes de sesión desde el primer día + +Valida y reinicia: + +```bash +openclaw config validate +openclaw gateway restart +openclaw logs --follow --plain | grep "memory-lancedb-pro" +``` + +Deberías ver: +- `memory-lancedb-pro: smart extraction enabled` +- `memory-lancedb-pro@...: plugin registered` + +¡Listo! Tu agente ahora tiene memoria a largo plazo. + +
+Más rutas de instalación (usuarios existentes, actualizaciones) + +**¿Ya usas OpenClaw?** + +1. Agrega el plugin con una entrada **absoluta** en `plugins.load.paths` +2. Vincula el slot de memoria: `plugins.slots.memory = "memory-lancedb-pro"` +3. Verifica: `openclaw plugins info memory-lancedb-pro && openclaw memory-pro stats` + +**¿Actualizando desde una versión anterior a v1.1.0?** + +```bash +# 1) Respaldo +openclaw memory-pro export --scope global --output memories-backup.json +# 2) Ejecución de prueba +openclaw memory-pro upgrade --dry-run +# 3) Ejecutar actualización +openclaw memory-pro upgrade +# 4) Verificar +openclaw memory-pro stats +``` + +Consulta `CHANGELOG-v1.1.0.md` para los cambios de comportamiento y la justificación de la actualización. + +
+ +
+Importación rápida para Bot de Telegram (clic para expandir) + +Si usas la integración de Telegram de OpenClaw, la forma más fácil es enviar un comando de importación directamente al Bot principal en lugar de editar la configuración manualmente. + +Envía este mensaje (en inglés, ya que es un prompt para el bot): + +```text +Help me connect this memory plugin with the most user-friendly configuration: https://github.com/CortexReach/memory-lancedb-pro + +Requirements: +1. Set it as the only active memory plugin +2. Use Jina for embedding +3. Use Jina for reranker +4. Use gpt-4o-mini for the smart-extraction LLM +5. Enable autoCapture, autoRecall, smartExtraction +6. extractMinMessages=2 +7. sessionMemory.enabled=false +8. captureAssistant=false +9. retrieval mode=hybrid, vectorWeight=0.7, bm25Weight=0.3 +10. rerank=cross-encoder, candidatePoolSize=12, minScore=0.6, hardMinScore=0.62 +11. Generate the final openclaw.json config directly, not just an explanation +``` + +
+ +--- + +## Ecosistema + +memory-lancedb-pro es el plugin principal. La comunidad ha desarrollado herramientas a su alrededor para hacer que la configuración y el uso diario sean aún más sencillos: + +### Script de Instalación — Instala, actualiza y repara con un solo clic + +> **[CortexReach/toolbox/memory-lancedb-pro-setup](https://github.com/CortexReach/toolbox/tree/main/memory-lancedb-pro-setup)** + +Mucho más que un simple instalador — el script gestiona de forma inteligente una amplia variedad de escenarios reales: + +| Tu situación | Lo que hace el script | +|---|---| +| Nunca instalado | Descarga nueva → instala dependencias → elige configuración → escribe en openclaw.json → reinicia | +| Instalado vía `git clone`, atascado en un commit antiguo | `git fetch` + `checkout` automático a la última versión → reinstala dependencias → verifica | +| La configuración tiene campos inválidos | Auto-detección mediante filtro de esquema, elimina campos no soportados | +| Instalado vía `npm` | Omite la actualización de git, te recuerda ejecutar `npm update` por tu cuenta | +| CLI de `openclaw` rota por configuración inválida | Alternativa: lee la ruta del workspace directamente del archivo `openclaw.json` | +| `extensions/` en lugar de `plugins/` | Auto-detección de la ubicación del plugin desde la configuración o el sistema de archivos | +| Ya está actualizado | Solo ejecuta verificaciones de salud, sin cambios | + +```bash +bash setup-memory.sh # Instalar o actualizar +bash setup-memory.sh --dry-run # Solo previsualización +bash setup-memory.sh --beta # Incluir versiones preliminares +bash setup-memory.sh --uninstall # Revertir configuración y eliminar plugin +``` + +Configuraciones preestablecidas de proveedores: **Jina / DashScope / SiliconFlow / OpenAI / Ollama**, o usa tu propia API compatible con OpenAI. Para la referencia completa (incluyendo `--ref`, `--selfcheck-only` y más), consulta el [README del script de instalación](https://github.com/CortexReach/toolbox/tree/main/memory-lancedb-pro-setup). + +### Skill para Claude Code / OpenClaw — Configuración Guiada por IA + +> **[CortexReach/memory-lancedb-pro-skill](https://github.com/CortexReach/memory-lancedb-pro-skill)** + +Instala este skill y tu agente de IA (Claude Code u OpenClaw) obtiene un conocimiento profundo de cada característica de memory-lancedb-pro. Solo di **"ayúdame a habilitar la mejor configuración"** y obtén: + +- **Flujo de configuración guiado en 7 pasos** con 4 planes de despliegue: + - Potencia Total (Jina + OpenAI) / Económico (reranker gratuito de SiliconFlow) / Simple (solo OpenAI) / Totalmente Local (Ollama, sin costo de API) +- **Las 9 herramientas MCP** usadas correctamente: `memory_recall`, `memory_store`, `memory_forget`, `memory_update`, `memory_stats`, `memory_list`, `self_improvement_log`, `self_improvement_extract_skill`, `self_improvement_review` *(el conjunto completo de herramientas requiere `enableManagementTools: true` — la configuración de Inicio Rápido predeterminada expone las 4 herramientas principales)* +- **Prevención de errores comunes**: habilitación del plugin en el workspace, `autoRecall` desactivado por defecto, caché de jiti, variables de entorno, aislamiento de scope, y más + +**Instalar para Claude Code:** +```bash +git clone https://github.com/CortexReach/memory-lancedb-pro-skill.git ~/.claude/skills/memory-lancedb-pro +``` + +**Instalar para OpenClaw:** +```bash +git clone https://github.com/CortexReach/memory-lancedb-pro-skill.git ~/.openclaw/workspace/skills/memory-lancedb-pro-skill +``` + +--- + +## Tutorial en Video + +> Recorrido completo: instalación, configuración y funcionamiento interno de la recuperación híbrida. + +[![YouTube Video](https://img.shields.io/badge/YouTube-Watch%20Now-red?style=for-the-badge&logo=youtube)](https://youtu.be/MtukF1C8epQ) +**https://youtu.be/MtukF1C8epQ** + +[![Bilibili Video](https://img.shields.io/badge/Bilibili-Watch%20Now-00A1D6?style=for-the-badge&logo=bilibili&logoColor=white)](https://www.bilibili.com/video/BV1zUf2BGEgn/) +**https://www.bilibili.com/video/BV1zUf2BGEgn/** + +--- + +## Arquitectura + +``` +┌─────────────────────────────────────────────────────────┐ +│ index.ts (Entry Point) │ +│ Plugin Registration · Config Parsing · Lifecycle Hooks │ +└────────┬──────────┬──────────┬──────────┬───────────────┘ + │ │ │ │ + ┌────▼───┐ ┌────▼───┐ ┌───▼────┐ ┌──▼──────────┐ + │ store │ │embedder│ │retriever│ │ scopes │ + │ .ts │ │ .ts │ │ .ts │ │ .ts │ + └────────┘ └────────┘ └────────┘ └─────────────┘ + │ │ + ┌────▼───┐ ┌─────▼──────────┐ + │migrate │ │noise-filter.ts │ + │ .ts │ │adaptive- │ + └────────┘ │retrieval.ts │ + └────────────────┘ + ┌─────────────┐ ┌──────────┐ + │ tools.ts │ │ cli.ts │ + │ (Agent API) │ │ (CLI) │ + └─────────────┘ └──────────┘ +``` + +> Para un análisis detallado de la arquitectura completa, consulta [docs/memory_architecture_analysis.md](docs/memory_architecture_analysis.md). + +
+Referencia de Archivos (clic para expandir) + +| Archivo | Propósito | +| --- | --- | +| `index.ts` | Punto de entrada del plugin. Se registra con la API de Plugins de OpenClaw, analiza la configuración, monta hooks de ciclo de vida | +| `openclaw.plugin.json` | Metadatos del plugin + declaración completa de configuración con JSON Schema | +| `cli.ts` | Comandos CLI: `memory-pro list/search/stats/delete/delete-bulk/export/import/reembed/upgrade/migrate` | +| `src/store.ts` | Capa de almacenamiento LanceDB. Creación de tablas / Indexación FTS / Búsqueda vectorial / Búsqueda BM25 / CRUD | +| `src/embedder.ts` | Abstracción de embeddings. Compatible con cualquier proveedor de API compatible con OpenAI | +| `src/retriever.ts` | Motor de recuperación híbrida. Vector + BM25 → Fusión Híbrida → Rerank → Decaimiento de Ciclo de Vida → Filtro | +| `src/scopes.ts` | Control de acceso multi-scope | +| `src/tools.ts` | Definiciones de herramientas del agente: `memory_recall`, `memory_store`, `memory_forget`, `memory_update` + herramientas de gestión | +| `src/noise-filter.ts` | Filtra rechazos del agente, meta-preguntas, saludos y contenido de baja calidad | +| `src/adaptive-retrieval.ts` | Determina si una consulta necesita recuperación de memoria | +| `src/migrate.ts` | Migración desde `memory-lancedb` integrado a Pro | +| `src/smart-extractor.ts` | Extracción de 6 categorías impulsada por LLM con almacenamiento en capas L0/L1/L2 y deduplicación en dos etapas | +| `src/decay-engine.ts` | Modelo de decaimiento exponencial estirado de Weibull | +| `src/tier-manager.ts` | Promoción/degradación en tres niveles: Peripheral ↔ Working ↔ Core | + +
+ +--- + +## Características Principales + +### Recuperación Híbrida + +``` +Query → embedQuery() ─┐ + ├─→ Hybrid Fusion → Rerank → Lifecycle Decay Boost → Length Norm → Filter +Query → BM25 FTS ─────┘ +``` + +- **Búsqueda Vectorial** — similitud semántica mediante LanceDB ANN (distancia coseno) +- **Búsqueda de Texto Completo BM25** — coincidencia exacta de palabras clave mediante índice FTS de LanceDB +- **Fusión Híbrida** — puntuación vectorial como base, los resultados de BM25 reciben un impulso ponderado (no es RRF estándar — ajustado para calidad de recuperación en el mundo real) +- **Pesos Configurables** — `vectorWeight`, `bm25Weight`, `minScore` + +### Reranking con Cross-Encoder + +- Adaptadores integrados para **Jina**, **SiliconFlow**, **Voyage AI** y **Pinecone** +- Compatible con cualquier endpoint compatible con Jina (por ejemplo, Hugging Face TEI, DashScope) +- Puntuación híbrida: 60% cross-encoder + 40% puntuación fusionada original +- Degradación elegante: recurre a similitud coseno en caso de fallo de la API + +### Pipeline de Puntuación Multi-Etapa + +| Etapa | Efecto | +| --- | --- | +| **Fusión Híbrida** | Combina recuperación semántica y de coincidencia exacta | +| **Rerank con Cross-Encoder** | Promueve resultados semánticamente precisos | +| **Impulso por Decaimiento de Ciclo de Vida** | Frescura Weibull + frecuencia de acceso + importancia × confianza | +| **Normalización de Longitud** | Evita que entradas largas dominen (ancla: 500 caracteres) | +| **Puntuación Mínima Estricta** | Elimina resultados irrelevantes (predeterminado: 0.35) | +| **Diversidad MMR** | Similitud coseno > 0.85 → degradado | + +### Extracción Inteligente de Memoria (v1.1.0) + +- **Extracción de 6 Categorías con LLM**: perfil, preferencias, entidades, eventos, casos, patrones +- **Almacenamiento en Capas L0/L1/L2**: L0 (índice de una oración) → L1 (resumen estructurado) → L2 (narrativa completa) +- **Deduplicación en Dos Etapas**: pre-filtro de similitud vectorial (≥0.7) → decisión semántica por LLM (CREATE/MERGE/SKIP) +- **Fusión por Categoría**: `profile` siempre se fusiona, `events`/`cases` son solo de adición + +### Gestión del Ciclo de Vida de la Memoria (v1.1.0) + +- **Motor de Decaimiento Weibull**: puntuación compuesta = recencia + frecuencia + valor intrínseco +- **Promoción en Tres Niveles**: `Peripheral ↔ Working ↔ Core` con umbrales configurables +- **Refuerzo por Acceso**: los recuerdos frecuentemente recuperados decaen más lentamente (estilo repetición espaciada) +- **Vida Media Modulada por Importancia**: los recuerdos importantes decaen más lentamente + +### Aislamiento Multi-Scope + +- Scopes integrados: `global`, `agent:`, `custom:`, `project:`, `user:` +- Control de acceso a nivel de agente mediante `scopes.agentAccess` +- Predeterminado: cada agente accede a `global` + su propio scope `agent:` + +### Auto-Capture y Auto-Recall + +- **Auto-Capture** (`agent_end`): extrae preferencia/hecho/decisión/entidad de las conversaciones, deduplica, almacena hasta 3 por turno +- **Auto-Recall** (`before_agent_start`): inyecta contexto `` (hasta 3 entradas) + +### Filtrado de Ruido y Recuperación Adaptativa + +- Filtra contenido de baja calidad: rechazos del agente, meta-preguntas, saludos +- Omite la recuperación para saludos, comandos slash, confirmaciones simples, emojis +- Fuerza la recuperación para palabras clave de memoria ("recuerda", "anteriormente", "la última vez") +- Umbrales adaptados a CJK (chino: 6 caracteres vs inglés: 15 caracteres) + +--- + +
+Comparación con memory-lancedb integrado (clic para expandir) + +| Característica | `memory-lancedb` integrado | **memory-lancedb-pro** | +| --- | :---: | :---: | +| Búsqueda vectorial | Sí | Sí | +| Búsqueda de texto completo BM25 | - | Sí | +| Fusión híbrida (Vector + BM25) | - | Sí | +| Rerank con cross-encoder (multi-proveedor) | - | Sí | +| Impulso por recencia y decaimiento temporal | - | Sí | +| Normalización de longitud | - | Sí | +| Diversidad MMR | - | Sí | +| Aislamiento multi-scope | - | Sí | +| Filtrado de ruido | - | Sí | +| Recuperación adaptativa | - | Sí | +| CLI de gestión | - | Sí | +| Memoria de sesión | - | Sí | +| Embeddings adaptados a la tarea | - | Sí | +| **Extracción Inteligente con LLM (6 categorías)** | - | Sí (v1.1.0) | +| **Decaimiento Weibull + Promoción por Niveles** | - | Sí (v1.1.0) | +| Cualquier embedding compatible con OpenAI | Limitado | Sí | + +
+ +--- + +## Configuración + +
+Ejemplo de Configuración Completa + +```json +{ + "embedding": { + "apiKey": "${JINA_API_KEY}", + "model": "jina-embeddings-v5-text-small", + "baseURL": "https://api.jina.ai/v1", + "dimensions": 1024, + "taskQuery": "retrieval.query", + "taskPassage": "retrieval.passage", + "normalized": true + }, + "dbPath": "~/.openclaw/memory/lancedb-pro", + "autoCapture": true, + "autoRecall": true, + "retrieval": { + "mode": "hybrid", + "vectorWeight": 0.7, + "bm25Weight": 0.3, + "minScore": 0.3, + "rerank": "cross-encoder", + "rerankApiKey": "${JINA_API_KEY}", + "rerankModel": "jina-reranker-v3", + "rerankEndpoint": "https://api.jina.ai/v1/rerank", + "rerankProvider": "jina", + "candidatePoolSize": 20, + "recencyHalfLifeDays": 14, + "recencyWeight": 0.1, + "filterNoise": true, + "lengthNormAnchor": 500, + "hardMinScore": 0.35, + "timeDecayHalfLifeDays": 60, + "reinforcementFactor": 0.5, + "maxHalfLifeMultiplier": 3 + }, + "enableManagementTools": false, + "scopes": { + "default": "global", + "definitions": { + "global": { "description": "Shared knowledge" }, + "agent:discord-bot": { "description": "Discord bot private" } + }, + "agentAccess": { + "discord-bot": ["global", "agent:discord-bot"] + } + }, + "sessionMemory": { + "enabled": false, + "messageCount": 15 + }, + "smartExtraction": true, + "llm": { + "apiKey": "${OPENAI_API_KEY}", + "model": "gpt-4o-mini", + "baseURL": "https://api.openai.com/v1" + }, + "extractMinMessages": 2, + "extractMaxChars": 8000 +} +``` + +
+ +
+Proveedores de Embedding + +Funciona con **cualquier API de embedding compatible con OpenAI**: + +| Proveedor | Modelo | URL Base | Dimensiones | +| --- | --- | --- | --- | +| **Jina** (recomendado) | `jina-embeddings-v5-text-small` | `https://api.jina.ai/v1` | 1024 | +| **OpenAI** | `text-embedding-3-small` | `https://api.openai.com/v1` | 1536 | +| **Voyage** | `voyage-4-lite` / `voyage-4` | `https://api.voyageai.com/v1` | 1024 / 1024 | +| **Google Gemini** | `gemini-embedding-001` | `https://generativelanguage.googleapis.com/v1beta/openai/` | 3072 | +| **Ollama** (local) | `nomic-embed-text` | `http://localhost:11434/v1` | específico del proveedor | + +
+ +
+Proveedores de Rerank + +El reranking con cross-encoder admite múltiples proveedores mediante `rerankProvider`: + +| Proveedor | `rerankProvider` | Modelo de Ejemplo | +| --- | --- | --- | +| **Jina** (predeterminado) | `jina` | `jina-reranker-v3` | +| **SiliconFlow** (nivel gratuito disponible) | `siliconflow` | `BAAI/bge-reranker-v2-m3` | +| **Voyage AI** | `voyage` | `rerank-2.5` | +| **Pinecone** | `pinecone` | `bge-reranker-v2-m3` | + +Cualquier endpoint de rerank compatible con Jina también funciona — configura `rerankProvider: "jina"` y apunta `rerankEndpoint` a tu servicio (por ejemplo, Hugging Face TEI, DashScope `qwen3-rerank`). + +
+ +
+Smart Extraction (LLM) — v1.1.0 + +Cuando `smartExtraction` está habilitado (predeterminado: `true`), el plugin utiliza un LLM para extraer y clasificar recuerdos de forma inteligente en lugar de disparadores basados en regex. + +| Campo | Tipo | Predeterminado | Descripción | +|-------|------|----------------|-------------| +| `smartExtraction` | boolean | `true` | Habilitar/deshabilitar la extracción de 6 categorías impulsada por LLM | +| `llm.auth` | string | `api-key` | `api-key` usa `llm.apiKey` / `embedding.apiKey`; `oauth` usa un archivo de token OAuth con alcance de plugin por defecto | +| `llm.apiKey` | string | *(recurre a `embedding.apiKey`)* | Clave API para el proveedor de LLM | +| `llm.model` | string | `openai/gpt-oss-120b` | Nombre del modelo LLM | +| `llm.baseURL` | string | *(recurre a `embedding.baseURL`)* | Endpoint de la API del LLM | +| `llm.oauthProvider` | string | `openai-codex` | ID del proveedor OAuth usado cuando `llm.auth` es `oauth` | +| `llm.oauthPath` | string | `~/.openclaw/.memory-lancedb-pro/oauth.json` | Archivo de token OAuth usado cuando `llm.auth` es `oauth` | +| `llm.timeoutMs` | number | `30000` | Tiempo de espera de solicitud LLM en milisegundos | +| `extractMinMessages` | number | `2` | Mensajes mínimos antes de que se active la extracción | +| `extractMaxChars` | number | `8000` | Máximo de caracteres enviados al LLM | + + +Configuración de `llm` con OAuth (usa la caché de inicio de sesión existente de Codex / ChatGPT para llamadas al LLM): +```json +{ + "llm": { + "auth": "oauth", + "oauthProvider": "openai-codex", + "model": "gpt-5.4", + "oauthPath": "${HOME}/.openclaw/.memory-lancedb-pro/oauth.json", + "timeoutMs": 30000 + } +} +``` + +Notas para `llm.auth: "oauth"`: + +- `llm.oauthProvider` es actualmente `openai-codex`. +- Los tokens OAuth se almacenan por defecto en `~/.openclaw/.memory-lancedb-pro/oauth.json`. +- Puedes configurar `llm.oauthPath` si deseas almacenar ese archivo en otra ubicación. +- `auth login` guarda una copia de la configuración anterior de `llm` con api-key junto al archivo OAuth, y `auth logout` restaura esa copia cuando está disponible. +- Cambiar de `api-key` a `oauth` no transfiere automáticamente `llm.baseURL`. Configúralo manualmente en modo OAuth solo cuando intencionalmente quieras un backend personalizado compatible con ChatGPT/Codex. + +
+ +
+Configuración del Ciclo de Vida (Decaimiento + Nivel) + +| Campo | Predeterminado | Descripción | +|-------|----------------|-------------| +| `decay.recencyHalfLifeDays` | `30` | Vida media base para el decaimiento de recencia Weibull | +| `decay.frequencyWeight` | `0.3` | Peso de la frecuencia de acceso en la puntuación compuesta | +| `decay.intrinsicWeight` | `0.3` | Peso de `importancia × confianza` | +| `decay.betaCore` | `0.8` | Beta de Weibull para memorias `core` | +| `decay.betaWorking` | `1.0` | Beta de Weibull para memorias `working` | +| `decay.betaPeripheral` | `1.3` | Beta de Weibull para memorias `peripheral` | +| `tier.coreAccessThreshold` | `10` | Mínimo de recuperaciones antes de promover a `core` | +| `tier.peripheralAgeDays` | `60` | Umbral de antigüedad para degradar memorias inactivas | + +
+ +
+Refuerzo por Acceso + +Los recuerdos frecuentemente recuperados decaen más lentamente (estilo repetición espaciada). + +Claves de configuración (bajo `retrieval`): +- `reinforcementFactor` (0-2, predeterminado: `0.5`) — establece `0` para deshabilitar +- `maxHalfLifeMultiplier` (1-10, predeterminado: `3`) — límite máximo de vida media efectiva + +
+ +--- + +## Comandos CLI + +```bash +openclaw memory-pro list [--scope global] [--category fact] [--limit 20] [--json] +openclaw memory-pro search "query" [--scope global] [--limit 10] [--json] +openclaw memory-pro stats [--scope global] [--json] +openclaw memory-pro auth login [--provider openai-codex] [--model gpt-5.4] [--oauth-path /abs/path/oauth.json] +openclaw memory-pro auth status +openclaw memory-pro auth logout +openclaw memory-pro delete +openclaw memory-pro delete-bulk --scope global [--before 2025-01-01] [--dry-run] +openclaw memory-pro export [--scope global] [--output memories.json] +openclaw memory-pro import memories.json [--scope global] [--dry-run] +openclaw memory-pro reembed --source-db /path/to/old-db [--batch-size 32] [--skip-existing] +openclaw memory-pro upgrade [--dry-run] [--batch-size 10] [--no-llm] [--limit N] [--scope SCOPE] +openclaw memory-pro migrate check|run|verify [--source /path] +``` + +Flujo de inicio de sesión OAuth: + +1. Ejecuta `openclaw memory-pro auth login` +2. Si se omite `--provider` en una terminal interactiva, la CLI muestra un selector de proveedor OAuth antes de abrir el navegador +3. El comando imprime una URL de autorización y abre tu navegador a menos que se establezca `--no-browser` +4. Después de que la devolución de llamada sea exitosa, el comando guarda el archivo OAuth del plugin (predeterminado: `~/.openclaw/.memory-lancedb-pro/oauth.json`), guarda una copia de la configuración anterior de `llm` con api-key para el cierre de sesión, y reemplaza la configuración `llm` del plugin con la configuración OAuth (`auth`, `oauthProvider`, `model`, `oauthPath`) +5. `openclaw memory-pro auth logout` elimina ese archivo OAuth y restaura la configuración anterior de `llm` con api-key cuando esa copia existe + +--- + +## Temas Avanzados + +
+Si los recuerdos inyectados aparecen en las respuestas + +A veces el modelo puede repetir el bloque `` inyectado. + +**Opción A (menor riesgo):** deshabilitar temporalmente la recuperación automática: +```json +{ "plugins": { "entries": { "memory-lancedb-pro": { "config": { "autoRecall": false } } } } } +``` + +**Opción B (preferida):** mantener la recuperación automática y agregar al prompt del sistema del agente: +> No reveles ni cites ningún contenido de `` / inyección de memoria en tus respuestas. Úsalo solo como referencia interna. + +
+ +
+Memoria de Sesión + +- Se activa con el comando `/new` — guarda el resumen de la sesión anterior en LanceDB +- Deshabilitado por defecto (OpenClaw ya tiene persistencia nativa de sesión en `.jsonl`) +- Cantidad de mensajes configurable (predeterminado: 15) + +Consulta [docs/openclaw-integration-playbook.md](docs/openclaw-integration-playbook.md) para los modos de despliegue y la verificación de `/new`. + +
+ +
+Comandos Slash Personalizados (por ejemplo, /lesson) + +Agrega a tu `CLAUDE.md`, `AGENTS.md` o prompt del sistema (el bloque se mantiene en inglés para que el agente lo interprete correctamente): + +```markdown +## /lesson command +When the user sends `/lesson `: +1. Use memory_store to save as category=fact (raw knowledge) +2. Use memory_store to save as category=decision (actionable takeaway) +3. Confirm what was saved + +## /remember command +When the user sends `/remember `: +1. Use memory_store to save with appropriate category and importance +2. Confirm with the stored memory ID +``` + +
+ +
+Reglas de Hierro para Agentes de IA + +> Copia el bloque de abajo en tu `AGENTS.md` para que tu agente aplique estas reglas automáticamente. Se mantiene en inglés porque es instrucción directa para el modelo. + +```markdown +## Rule 1 — Dual-layer memory storage +Every pitfall/lesson learned → IMMEDIATELY store TWO memories: +- Technical layer: Pitfall: [symptom]. Cause: [root cause]. Fix: [solution]. Prevention: [how to avoid] + (category: fact, importance >= 0.8) +- Principle layer: Decision principle ([tag]): [behavioral rule]. Trigger: [when]. Action: [what to do] + (category: decision, importance >= 0.85) + +## Rule 2 — LanceDB hygiene +Entries must be short and atomic (< 500 chars). No raw conversation summaries or duplicates. + +## Rule 3 — Recall before retry +On ANY tool failure, ALWAYS memory_recall with relevant keywords BEFORE retrying. + +## Rule 4 — Confirm target codebase +Confirm you are editing memory-lancedb-pro vs built-in memory-lancedb before changes. + +## Rule 5 — Clear jiti cache after plugin code changes +After modifying .ts files under plugins/, MUST run rm -rf /tmp/jiti/ BEFORE openclaw gateway restart. +``` + +
+ +
+Esquema de la Base de Datos + +Tabla LanceDB `memories`: + +| Campo | Tipo | Descripción | +| --- | --- | --- | +| `id` | string (UUID) | Clave primaria | +| `text` | string | Texto del recuerdo (indexado con FTS) | +| `vector` | float[] | Vector de embedding | +| `category` | string | Categoría de almacenamiento: `preference` / `fact` / `decision` / `entity` / `reflection` / `other` | +| `scope` | string | Identificador de scope (por ejemplo, `global`, `agent:main`) | +| `importance` | float | Puntuación de importancia 0-1 | +| `timestamp` | int64 | Marca de tiempo de creación (ms) | +| `metadata` | string (JSON) | Metadatos extendidos | + +Claves comunes de `metadata` en v1.1.0: `l0_abstract`, `l1_overview`, `l2_content`, `memory_category`, `tier`, `access_count`, `confidence`, `last_accessed_at` + +> **Nota sobre categorías:** El campo de nivel superior `category` usa 6 categorías de almacenamiento. Las 6 etiquetas semánticas de categoría de Smart Extraction (`profile` / `preferences` / `entities` / `events` / `cases` / `patterns`) se almacenan en `metadata.memory_category`. + +
+ +
+Solución de Problemas + +### "Cannot mix BigInt and other types" (LanceDB / Apache Arrow) + +En LanceDB 0.26+, algunas columnas numéricas pueden devolverse como `BigInt`. Actualiza a **memory-lancedb-pro >= 1.0.14** — este plugin ahora convierte los valores usando `Number(...)` antes de realizar operaciones aritméticas. + +
+ +--- + +## Documentación + +| Documento | Descripción | +| --- | --- | +| [Manual de Integración con OpenClaw](docs/openclaw-integration-playbook.md) | Modos de despliegue, verificación, matriz de regresión | +| [Análisis de la Arquitectura de Memoria](docs/memory_architecture_analysis.md) | Análisis detallado de la arquitectura completa | +| [CHANGELOG v1.1.0](docs/CHANGELOG-v1.1.0.md) | Cambios de comportamiento en v1.1.0 y justificación de la actualización | +| [Fragmentación de Contexto Largo](docs/long-context-chunking.md) | Estrategia de fragmentación para documentos largos | + +--- + +## Beta: Smart Memory v1.1.0 + +> Estado: Beta — disponible mediante `npm i memory-lancedb-pro@beta`. Los usuarios estables en `latest` no se ven afectados. + +| Característica | Descripción | +|----------------|-------------| +| **Smart Extraction** | Extracción de 6 categorías impulsada por LLM con metadatos L0/L1/L2. Recurre a regex cuando está deshabilitado. | +| **Puntuación de Ciclo de Vida** | Decaimiento Weibull integrado en la recuperación — los recuerdos de alta frecuencia y alta importancia se clasifican mejor. | +| **Gestión de Niveles** | Sistema de tres niveles (Core → Working → Peripheral) con promoción/degradación automática. | + +Comentarios: [GitHub Issues](https://github.com/CortexReach/memory-lancedb-pro/issues) · Revertir: `npm i memory-lancedb-pro@latest` + +--- + +## Dependencias + +| Paquete | Propósito | +| --- | --- | +| `@lancedb/lancedb` ≥0.26.2 | Base de datos vectorial (ANN + FTS) | +| `openai` ≥6.21.0 | Cliente de API de Embedding compatible con OpenAI | +| `@sinclair/typebox` 0.34.48 | Definiciones de tipos con JSON Schema | + +--- + +## Contributors + +

+@win4r +@kctony +@Akatsuki-Ryu +@JasonSuz +@Minidoracat +@furedericca-lab +@joe2643 +@AliceLJY +@chenjiyong +

+ +Full list: [Contributors](https://github.com/CortexReach/memory-lancedb-pro/graphs/contributors) + +## Star History + + + + + + Star History Chart + + + +## Licencia + +MIT + +--- + +## Mi Código QR de WeChat + + diff --git a/README_FR.md b/README_FR.md new file mode 100644 index 00000000..19f5fc45 --- /dev/null +++ b/README_FR.md @@ -0,0 +1,773 @@ +
+ +# 🧠 memory-lancedb-pro · 🦞OpenClaw Plugin + +**Assistant Mémoire IA pour les Agents [OpenClaw](https://github.com/openclaw/openclaw)** + +*Donnez à votre agent IA un cerveau qui se souvient vraiment — entre les sessions, entre les agents, dans le temps.* + +Un plugin de mémoire long terme pour OpenClaw basé sur LanceDB qui stocke les préférences, les décisions et le contexte du projet, puis les rappelle automatiquement dans les sessions futures. + +[![OpenClaw Plugin](https://img.shields.io/badge/OpenClaw-Plugin-blue)](https://github.com/openclaw/openclaw) +[![npm version](https://img.shields.io/npm/v/memory-lancedb-pro)](https://www.npmjs.com/package/memory-lancedb-pro) +[![LanceDB](https://img.shields.io/badge/LanceDB-Vectorstore-orange)](https://lancedb.com) +[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE) + +[English](README.md) | [简体中文](README_CN.md) | [繁體中文](README_TW.md) | [日本語](README_JA.md) | [한국어](README_KO.md) | [Français](README_FR.md) | [Español](README_ES.md) | [Deutsch](README_DE.md) | [Italiano](README_IT.md) | [Русский](README_RU.md) | [Português (Brasil)](README_PT-BR.md) + +
+ +--- + +## Pourquoi memory-lancedb-pro ? + +La plupart des agents IA souffrent d'amnésie. Ils oublient tout dès que vous démarrez une nouvelle conversation. + +**memory-lancedb-pro** est un plugin de mémoire long terme de niveau production pour OpenClaw qui transforme votre agent en un véritable **Assistant Mémoire IA** — il capture automatiquement ce qui compte, laisse le bruit s'estomper naturellement et retrouve le bon souvenir au bon moment. Pas d'étiquetage manuel, pas de configuration compliquée. + +### Votre Assistant Mémoire IA en action + +**Sans mémoire — chaque session repart de zéro :** + +> **Vous :** « Utilise des tabulations pour l'indentation, ajoute toujours la gestion d'erreurs. » +> *(session suivante)* +> **Vous :** « Je t'ai déjà dit — des tabulations, pas des espaces ! » 😤 +> *(session suivante)* +> **Vous :** « …sérieusement, des tabulations. Et la gestion d'erreurs. Encore. » + +**Avec memory-lancedb-pro — votre agent apprend et se souvient :** + +> **Vous :** « Utilise des tabulations pour l'indentation, ajoute toujours la gestion d'erreurs. » +> *(session suivante — l'agent rappelle automatiquement vos préférences)* +> **Agent :** *(applique silencieusement tabulations + gestion d'erreurs)* ✅ +> **Vous :** « Pourquoi avons-nous choisi PostgreSQL plutôt que MongoDB le mois dernier ? » +> **Agent :** « Selon notre discussion du 12 février, les raisons principales étaient… » ✅ + +Voilà la différence que fait un **Assistant Mémoire IA** — il apprend votre style, rappelle les décisions passées et fournit des réponses personnalisées sans que vous ayez à vous répéter. + +### Que peut-il faire d'autre ? + +| | Ce que vous obtenez | +|---|---| +| **Capture automatique** | Votre agent apprend de chaque conversation — pas besoin de `memory_store` manuel | +| **Extraction intelligente** | Classification LLM en 6 catégories : profils, préférences, entités, événements, cas, patterns | +| **Oubli intelligent** | Modèle de décroissance Weibull — les souvenirs importants restent, le bruit s'estompe | +| **Recherche hybride** | Recherche vectorielle + BM25 plein texte, fusionnée avec un reranking cross-encoder | +| **Injection de contexte** | Les souvenirs pertinents remontent automatiquement avant chaque réponse | +| **Isolation multi-scope** | Limites mémoire par agent, par utilisateur, par projet | +| **Tout fournisseur** | OpenAI, Jina, Gemini, Ollama ou toute API compatible OpenAI | +| **Boîte à outils complète** | CLI, sauvegarde, migration, mise à niveau, export/import — prêt pour la production | + +--- + +## Démarrage rapide + +### Option A : Script d'installation en un clic (recommandé) + +Le **[script d'installation](https://github.com/CortexReach/toolbox/tree/main/memory-lancedb-pro-setup)** maintenu par la communauté gère l'installation, la mise à niveau et la réparation en une seule commande : + +```bash +curl -fsSL https://raw.githubusercontent.com/CortexReach/toolbox/main/memory-lancedb-pro-setup/setup-memory.sh -o setup-memory.sh +bash setup-memory.sh +``` + +> Consultez [Écosystème](#écosystème) ci-dessous pour la liste complète des scénarios couverts et les autres outils communautaires. + +### Option B : Installation manuelle + +**Via OpenClaw CLI (recommandé) :** +```bash +openclaw plugins install memory-lancedb-pro@beta +``` + +**Ou via npm :** +```bash +npm i memory-lancedb-pro@beta +``` +> Si vous utilisez npm, vous devrez également ajouter le répertoire d'installation du plugin comme chemin **absolu** dans `plugins.load.paths` de votre `openclaw.json`. C'est le problème de configuration le plus courant. + +Ajoutez à votre `openclaw.json` : + +```json +{ + "plugins": { + "slots": { "memory": "memory-lancedb-pro" }, + "entries": { + "memory-lancedb-pro": { + "enabled": true, + "config": { + "embedding": { + "provider": "openai-compatible", + "apiKey": "${OPENAI_API_KEY}", + "model": "text-embedding-3-small" + }, + "autoCapture": true, + "autoRecall": true, + "smartExtraction": true, + "extractMinMessages": 2, + "extractMaxChars": 8000, + "sessionMemory": { "enabled": false } + } + } + } + } +} +``` + +**Pourquoi ces valeurs par défaut ?** +- `autoCapture` + `smartExtraction` → votre agent apprend automatiquement de chaque conversation +- `autoRecall` → les souvenirs pertinents sont injectés avant chaque réponse +- `extractMinMessages: 2` → l'extraction se déclenche dans les conversations normales à deux tours +- `sessionMemory.enabled: false` → évite de polluer la recherche avec des résumés de session au début + +Validez et redémarrez : + +```bash +openclaw config validate +openclaw gateway restart +openclaw logs --follow --plain | grep "memory-lancedb-pro" +``` + +Vous devriez voir : +- `memory-lancedb-pro: smart extraction enabled` +- `memory-lancedb-pro@...: plugin registered` + +Terminé ! Votre agent dispose maintenant d'une mémoire long terme. + +
+Plus de chemins d'installation (utilisateurs existants, mises à niveau) + +**Déjà utilisateur d'OpenClaw ?** + +1. Ajoutez le plugin avec un chemin **absolu** dans `plugins.load.paths` +2. Liez le slot mémoire : `plugins.slots.memory = "memory-lancedb-pro"` +3. Vérifiez : `openclaw plugins info memory-lancedb-pro && openclaw memory-pro stats` + +**Mise à niveau depuis une version antérieure à v1.1.0 ?** + +```bash +# 1) Sauvegarde +openclaw memory-pro export --scope global --output memories-backup.json +# 2) Simulation +openclaw memory-pro upgrade --dry-run +# 3) Exécution de la mise à niveau +openclaw memory-pro upgrade +# 4) Vérification +openclaw memory-pro stats +``` + +Consultez `CHANGELOG-v1.1.0.md` pour les changements de comportement et la justification de la mise à niveau. + +
+ +
+Import rapide Telegram Bot (cliquez pour développer) + +Si vous utilisez l'intégration Telegram d'OpenClaw, le plus simple est d'envoyer une commande d'import directement au Bot principal au lieu de modifier manuellement la configuration. + +Envoyez ce message : + +```text +Help me connect this memory plugin with the most user-friendly configuration: https://github.com/CortexReach/memory-lancedb-pro + +Requirements: +1. Set it as the only active memory plugin +2. Use Jina for embedding +3. Use Jina for reranker +4. Use gpt-4o-mini for the smart-extraction LLM +5. Enable autoCapture, autoRecall, smartExtraction +6. extractMinMessages=2 +7. sessionMemory.enabled=false +8. captureAssistant=false +9. retrieval mode=hybrid, vectorWeight=0.7, bm25Weight=0.3 +10. rerank=cross-encoder, candidatePoolSize=12, minScore=0.6, hardMinScore=0.62 +11. Generate the final openclaw.json config directly, not just an explanation +``` + +
+ +--- + +## Écosystème + +memory-lancedb-pro est le plugin principal. La communauté a construit des outils autour pour faciliter l'installation et l'utilisation quotidienne : + +### Script d'installation — Installation, mise à niveau et réparation en un clic + +> **[CortexReach/toolbox/memory-lancedb-pro-setup](https://github.com/CortexReach/toolbox/tree/main/memory-lancedb-pro-setup)** + +Pas un simple installateur — le script gère intelligemment de nombreux scénarios réels : + +| Votre situation | Ce que fait le script | +|---|---| +| Jamais installé | Téléchargement → installation des dépendances → choix de la config → écriture dans openclaw.json → redémarrage | +| Installé via `git clone`, bloqué sur un ancien commit | `git fetch` + `checkout` automatique vers la dernière version → réinstallation des dépendances → vérification | +| La config contient des champs invalides | Détection automatique via filtre de schéma, suppression des champs non supportés | +| Installé via `npm` | Saute la mise à jour git, rappelle d'exécuter `npm update` soi-même | +| CLI `openclaw` cassé à cause d'une config invalide | Solution de repli : lecture directe du chemin workspace depuis le fichier `openclaw.json` | +| `extensions/` au lieu de `plugins/` | Détection automatique de l'emplacement du plugin depuis la config ou le système de fichiers | +| Déjà à jour | Exécution des vérifications de santé uniquement, aucune modification | + +```bash +bash setup-memory.sh # Installer ou mettre à niveau +bash setup-memory.sh --dry-run # Aperçu uniquement +bash setup-memory.sh --beta # Inclure les versions pré-release +bash setup-memory.sh --uninstall # Restaurer la config et supprimer le plugin +``` + +Presets de fournisseurs intégrés : **Jina / DashScope / SiliconFlow / OpenAI / Ollama**, ou apportez votre propre API compatible OpenAI. Pour l'utilisation complète (incluant `--ref`, `--selfcheck-only`, etc.), consultez le [README du script d'installation](https://github.com/CortexReach/toolbox/tree/main/memory-lancedb-pro-setup). + +### Claude Code / OpenClaw Skill — Configuration guidée par IA + +> **[CortexReach/memory-lancedb-pro-skill](https://github.com/CortexReach/memory-lancedb-pro-skill)** + +Installez ce Skill et votre agent IA (Claude Code ou OpenClaw) acquiert une connaissance approfondie de toutes les fonctionnalités de memory-lancedb-pro. Dites simplement **« aide-moi à activer la meilleure config »** et obtenez : + +- **Workflow de configuration guidé en 7 étapes** avec 4 plans de déploiement : + - Full Power (Jina + OpenAI) / Budget (reranker SiliconFlow gratuit) / Simple (OpenAI uniquement) / Entièrement local (Ollama, zéro coût API) +- **Les 9 outils MCP** utilisés correctement : `memory_recall`, `memory_store`, `memory_forget`, `memory_update`, `memory_stats`, `memory_list`, `self_improvement_log`, `self_improvement_extract_skill`, `self_improvement_review` *(l'ensemble complet nécessite `enableManagementTools: true` — la config Quick Start par défaut expose les 4 outils principaux)* +- **Évitement des pièges courants** : activation du plugin workspace, `autoRecall` par défaut à false, cache jiti, variables d'environnement, isolation des scopes, etc. + +**Installation pour Claude Code :** +```bash +git clone https://github.com/CortexReach/memory-lancedb-pro-skill.git ~/.claude/skills/memory-lancedb-pro +``` + +**Installation pour OpenClaw :** +```bash +git clone https://github.com/CortexReach/memory-lancedb-pro-skill.git ~/.openclaw/workspace/skills/memory-lancedb-pro-skill +``` + +--- + +## Tutoriel vidéo + +> Présentation complète : installation, configuration et fonctionnement interne de la recherche hybride. + +[![YouTube Video](https://img.shields.io/badge/YouTube-Watch%20Now-red?style=for-the-badge&logo=youtube)](https://youtu.be/MtukF1C8epQ) +**https://youtu.be/MtukF1C8epQ** + +[![Bilibili Video](https://img.shields.io/badge/Bilibili-Watch%20Now-00A1D6?style=for-the-badge&logo=bilibili&logoColor=white)](https://www.bilibili.com/video/BV1zUf2BGEgn/) +**https://www.bilibili.com/video/BV1zUf2BGEgn/** + +--- + +## Architecture + +``` +┌─────────────────────────────────────────────────────────┐ +│ index.ts (Point d'entrée) │ +│ Enregistrement du plugin · Parsing config · Hooks │ +└────────┬──────────┬──────────┬──────────┬───────────────┘ + │ │ │ │ + ┌────▼───┐ ┌────▼───┐ ┌───▼────┐ ┌──▼──────────┐ + │ store │ │embedder│ │retriever│ │ scopes │ + │ .ts │ │ .ts │ │ .ts │ │ .ts │ + └────────┘ └────────┘ └────────┘ └─────────────┘ + │ │ + ┌────▼───┐ ┌─────▼──────────┐ + │migrate │ │noise-filter.ts │ + │ .ts │ │adaptive- │ + └────────┘ │retrieval.ts │ + └────────────────┘ + ┌─────────────┐ ┌──────────┐ + │ tools.ts │ │ cli.ts │ + │ (API Agent) │ │ (CLI) │ + └─────────────┘ └──────────┘ +``` + +> Pour une analyse approfondie de l'architecture complète, consultez [docs/memory_architecture_analysis.md](docs/memory_architecture_analysis.md). + +
+Référence des fichiers (cliquez pour développer) + +| Fichier | Rôle | +| --- | --- | +| `index.ts` | Point d'entrée du plugin. S'enregistre auprès de l'API Plugin OpenClaw, parse la config, monte les hooks de cycle de vie | +| `openclaw.plugin.json` | Métadonnées du plugin + déclaration complète du JSON Schema de config | +| `cli.ts` | Commandes CLI : `memory-pro list/search/stats/delete/delete-bulk/export/import/reembed/upgrade/migrate` | +| `src/store.ts` | Couche de stockage LanceDB. Création de tables / Indexation FTS / Recherche vectorielle / Recherche BM25 / CRUD | +| `src/embedder.ts` | Abstraction d'embedding. Compatible avec tout fournisseur API compatible OpenAI | +| `src/retriever.ts` | Moteur de recherche hybride. Vectoriel + BM25 → Fusion hybride → Rerank → Décroissance cycle de vie → Filtre | +| `src/scopes.ts` | Contrôle d'accès multi-scope | +| `src/tools.ts` | Définitions des outils agent : `memory_recall`, `memory_store`, `memory_forget`, `memory_update` + outils de gestion | +| `src/noise-filter.ts` | Filtre les refus d'agent, les méta-questions, les salutations et le contenu de faible qualité | +| `src/adaptive-retrieval.ts` | Détermine si une requête nécessite une recherche en mémoire | +| `src/migrate.ts` | Migration depuis `memory-lancedb` intégré vers Pro | +| `src/smart-extractor.ts` | Extraction LLM en 6 catégories avec stockage L0/L1/L2 et déduplication en deux étapes | +| `src/decay-engine.ts` | Modèle de décroissance exponentielle étirée Weibull | +| `src/tier-manager.ts` | Promotion/rétrogradation à trois niveaux : Périphérique ↔ Travail ↔ Noyau | + +
+ +--- + +## Fonctionnalités principales + +### Recherche hybride + +``` +Requête → embedQuery() ─┐ + ├─→ Fusion hybride → Rerank → Boost décroissance → Normalisation longueur → Filtre +Requête → BM25 FTS ─────┘ +``` + +- **Recherche vectorielle** — similarité sémantique via LanceDB ANN (distance cosinus) +- **Recherche plein texte BM25** — correspondance exacte de mots-clés via l'index FTS de LanceDB +- **Fusion hybride** — score vectoriel comme base, les résultats BM25 reçoivent un boost pondéré (pas du RRF standard — optimisé pour la qualité de rappel réelle) +- **Poids configurables** — `vectorWeight`, `bm25Weight`, `minScore` + +### Reranking Cross-Encoder + +- Adaptateurs intégrés pour **Jina**, **SiliconFlow**, **Voyage AI** et **Pinecone** +- Compatible avec tout endpoint compatible Jina (ex. Hugging Face TEI, DashScope) +- Scoring hybride : 60% cross-encoder + 40% score fusionné original +- Dégradation gracieuse : repli sur la similarité cosinus en cas d'échec API + +### Pipeline de scoring multi-étapes + +| Étape | Effet | +| --- | --- | +| **Fusion hybride** | Combine rappel sémantique et correspondance exacte | +| **Rerank Cross-Encoder** | Promeut les résultats sémantiquement précis | +| **Boost décroissance cycle de vie** | Fraîcheur Weibull + fréquence d'accès + importance × confiance | +| **Normalisation de longueur** | Empêche les entrées longues de dominer (ancre : 500 caractères) | +| **Score minimum dur** | Supprime les résultats non pertinents (par défaut : 0.35) | +| **Diversité MMR** | Similarité cosinus > 0.85 → rétrogradé | + +### Extraction mémoire intelligente (v1.1.0) + +- **Extraction LLM en 6 catégories** : profil, préférences, entités, événements, cas, patterns +- **Stockage par couches L0/L1/L2** : L0 (index en une phrase) → L1 (résumé structuré) → L2 (récit complet) +- **Déduplication en deux étapes** : pré-filtre de similarité vectorielle (≥0.7) → décision sémantique LLM (CREATE/MERGE/SKIP) +- **Fusion sensible aux catégories** : `profile` fusionne toujours, `events`/`cases` en ajout uniquement + +### Gestion du cycle de vie mémoire (v1.1.0) + +- **Moteur de décroissance Weibull** : score composite = fraîcheur + fréquence + valeur intrinsèque +- **Promotion à trois niveaux** : `Périphérique ↔ Travail ↔ Noyau` avec seuils configurables +- **Renforcement par accès** : les souvenirs fréquemment rappelés décroissent plus lentement (style répétition espacée) +- **Demi-vie modulée par l'importance** : les souvenirs importants décroissent plus lentement + +### Isolation multi-scope + +- Scopes intégrés : `global`, `agent:`, `custom:`, `project:`, `user:` +- Contrôle d'accès au niveau agent via `scopes.agentAccess` +- Par défaut : chaque agent accède à `global` + son propre scope `agent:` + +### Capture automatique et rappel automatique + +- **Capture auto** (`agent_end`) : extrait préférences/faits/décisions/entités des conversations, déduplique, stocke jusqu'à 3 par tour +- **Rappel auto** (`before_agent_start`) : injecte le contexte `` (jusqu'à 3 entrées) + +### Filtrage du bruit et recherche adaptative + +- Filtre le contenu de faible qualité : refus d'agent, méta-questions, salutations +- Ignore la recherche pour : salutations, commandes slash, confirmations simples, emoji +- Force la recherche pour les mots-clés mémoire (« souviens-toi », « précédemment », « la dernière fois ») +- Seuils CJK (chinois : 6 caractères vs anglais : 15 caractères) + +--- + +
+Comparaison avec memory-lancedb intégré (cliquez pour développer) + +| Fonctionnalité | `memory-lancedb` intégré | **memory-lancedb-pro** | +| --- | :---: | :---: | +| Recherche vectorielle | Oui | Oui | +| Recherche plein texte BM25 | - | Oui | +| Fusion hybride (Vectoriel + BM25) | - | Oui | +| Rerank cross-encoder (multi-fournisseur) | - | Oui | +| Boost de fraîcheur et décroissance temporelle | - | Oui | +| Normalisation de longueur | - | Oui | +| Diversité MMR | - | Oui | +| Isolation multi-scope | - | Oui | +| Filtrage du bruit | - | Oui | +| Recherche adaptative | - | Oui | +| CLI de gestion | - | Oui | +| Mémoire de session | - | Oui | +| Embeddings sensibles aux tâches | - | Oui | +| **Extraction intelligente LLM (6 catégories)** | - | Oui (v1.1.0) | +| **Décroissance Weibull + Promotion par niveaux** | - | Oui (v1.1.0) | +| Tout embedding compatible OpenAI | Limité | Oui | + +
+ +--- + +## Configuration + +
+Exemple de configuration complète + +```json +{ + "embedding": { + "apiKey": "${JINA_API_KEY}", + "model": "jina-embeddings-v5-text-small", + "baseURL": "https://api.jina.ai/v1", + "dimensions": 1024, + "taskQuery": "retrieval.query", + "taskPassage": "retrieval.passage", + "normalized": true + }, + "dbPath": "~/.openclaw/memory/lancedb-pro", + "autoCapture": true, + "autoRecall": true, + "retrieval": { + "mode": "hybrid", + "vectorWeight": 0.7, + "bm25Weight": 0.3, + "minScore": 0.3, + "rerank": "cross-encoder", + "rerankApiKey": "${JINA_API_KEY}", + "rerankModel": "jina-reranker-v3", + "rerankEndpoint": "https://api.jina.ai/v1/rerank", + "rerankProvider": "jina", + "candidatePoolSize": 20, + "recencyHalfLifeDays": 14, + "recencyWeight": 0.1, + "filterNoise": true, + "lengthNormAnchor": 500, + "hardMinScore": 0.35, + "timeDecayHalfLifeDays": 60, + "reinforcementFactor": 0.5, + "maxHalfLifeMultiplier": 3 + }, + "enableManagementTools": false, + "scopes": { + "default": "global", + "definitions": { + "global": { "description": "Shared knowledge" }, + "agent:discord-bot": { "description": "Discord bot private" } + }, + "agentAccess": { + "discord-bot": ["global", "agent:discord-bot"] + } + }, + "sessionMemory": { + "enabled": false, + "messageCount": 15 + }, + "smartExtraction": true, + "llm": { + "apiKey": "${OPENAI_API_KEY}", + "model": "gpt-4o-mini", + "baseURL": "https://api.openai.com/v1" + }, + "extractMinMessages": 2, + "extractMaxChars": 8000 +} +``` + +
+ +
+Fournisseurs d'embedding + +Fonctionne avec **toute API d'embedding compatible OpenAI** : + +| Fournisseur | Modèle | Base URL | Dimensions | +| --- | --- | --- | --- | +| **Jina** (recommandé) | `jina-embeddings-v5-text-small` | `https://api.jina.ai/v1` | 1024 | +| **OpenAI** | `text-embedding-3-small` | `https://api.openai.com/v1` | 1536 | +| **Voyage** | `voyage-4-lite` / `voyage-4` | `https://api.voyageai.com/v1` | 1024 / 1024 | +| **Google Gemini** | `gemini-embedding-001` | `https://generativelanguage.googleapis.com/v1beta/openai/` | 3072 | +| **Ollama** (local) | `nomic-embed-text` | `http://localhost:11434/v1` | selon le modèle | + +
+ +
+Fournisseurs de reranking + +Le reranking cross-encoder supporte plusieurs fournisseurs via `rerankProvider` : + +| Fournisseur | `rerankProvider` | Modèle exemple | +| --- | --- | --- | +| **Jina** (par défaut) | `jina` | `jina-reranker-v3` | +| **SiliconFlow** (niveau gratuit disponible) | `siliconflow` | `BAAI/bge-reranker-v2-m3` | +| **Voyage AI** | `voyage` | `rerank-2.5` | +| **Pinecone** | `pinecone` | `bge-reranker-v2-m3` | + +Tout endpoint de reranking compatible Jina fonctionne également — définissez `rerankProvider: "jina"` et pointez `rerankEndpoint` vers votre service (ex. Hugging Face TEI, DashScope `qwen3-rerank`). + +
+ +
+Extraction intelligente (LLM) — v1.1.0 + +Quand `smartExtraction` est activé (par défaut : `true`), le plugin utilise un LLM pour extraire et classifier intelligemment les souvenirs au lieu de déclencheurs basés sur des regex. + +| Champ | Type | Défaut | Description | +|-------|------|---------|-------------| +| `smartExtraction` | boolean | `true` | Activer/désactiver l'extraction LLM en 6 catégories | +| `llm.auth` | string | `api-key` | `api-key` utilise `llm.apiKey` / `embedding.apiKey` ; `oauth` utilise un fichier token OAuth au niveau plugin | +| `llm.apiKey` | string | *(repli sur `embedding.apiKey`)* | Clé API pour le fournisseur LLM | +| `llm.model` | string | `openai/gpt-oss-120b` | Nom du modèle LLM | +| `llm.baseURL` | string | *(repli sur `embedding.baseURL`)* | Point de terminaison API LLM | +| `llm.oauthProvider` | string | `openai-codex` | ID du fournisseur OAuth utilisé quand `llm.auth` est `oauth` | +| `llm.oauthPath` | string | `~/.openclaw/.memory-lancedb-pro/oauth.json` | Fichier token OAuth utilisé quand `llm.auth` est `oauth` | +| `llm.timeoutMs` | number | `30000` | Timeout des requêtes LLM en millisecondes | +| `extractMinMessages` | number | `2` | Nombre minimum de messages avant le déclenchement de l'extraction | +| `extractMaxChars` | number | `8000` | Nombre maximum de caractères envoyés au LLM | + + +OAuth `llm` config (utiliser le cache de connexion Codex / ChatGPT existant pour les appels LLM) : +```json +{ + "llm": { + "auth": "oauth", + "oauthProvider": "openai-codex", + "model": "gpt-5.4", + "oauthPath": "${HOME}/.openclaw/.memory-lancedb-pro/oauth.json", + "timeoutMs": 30000 + } +} +``` + +Notes pour `llm.auth: "oauth"` : + +- `llm.oauthProvider` est actuellement `openai-codex`. +- Les tokens OAuth sont stockés par défaut dans `~/.openclaw/.memory-lancedb-pro/oauth.json`. +- Vous pouvez définir `llm.oauthPath` si vous souhaitez stocker ce fichier ailleurs. +- `auth login` sauvegarde la configuration `llm` api-key précédente à côté du fichier OAuth, et `auth logout` restaure cette sauvegarde lorsqu'elle est disponible. +- Passer de `api-key` à `oauth` ne transfère pas automatiquement `llm.baseURL`. Définissez-le manuellement en mode OAuth uniquement si vous souhaitez intentionnellement un backend personnalisé compatible ChatGPT/Codex. + +
+ +
+Configuration du cycle de vie (Décroissance + Niveaux) + +| Champ | Défaut | Description | +|-------|---------|-------------| +| `decay.recencyHalfLifeDays` | `30` | Demi-vie de base pour la décroissance Weibull | +| `decay.frequencyWeight` | `0.3` | Poids de la fréquence d'accès dans le score composite | +| `decay.intrinsicWeight` | `0.3` | Poids de `importance × confiance` | +| `decay.betaCore` | `0.8` | Beta Weibull pour les souvenirs `noyau` | +| `decay.betaWorking` | `1.0` | Beta Weibull pour les souvenirs `travail` | +| `decay.betaPeripheral` | `1.3` | Beta Weibull pour les souvenirs `périphériques` | +| `tier.coreAccessThreshold` | `10` | Nombre minimum de rappels avant promotion en `noyau` | +| `tier.peripheralAgeDays` | `60` | Seuil d'âge pour la rétrogradation des souvenirs obsolètes | + +
+ +
+Renforcement par accès + +Les souvenirs fréquemment rappelés décroissent plus lentement (style répétition espacée). + +Clés de config (sous `retrieval`) : +- `reinforcementFactor` (0-2, défaut : `0.5`) — mettre à `0` pour désactiver +- `maxHalfLifeMultiplier` (1-10, défaut : `3`) — plafond de la demi-vie effective + +
+ +--- + +## Commandes CLI + +```bash +openclaw memory-pro list [--scope global] [--category fact] [--limit 20] [--json] +openclaw memory-pro search "requête" [--scope global] [--limit 10] [--json] +openclaw memory-pro stats [--scope global] [--json] +openclaw memory-pro auth login [--provider openai-codex] [--model gpt-5.4] [--oauth-path /abs/path/oauth.json] +openclaw memory-pro auth status +openclaw memory-pro auth logout +openclaw memory-pro delete +openclaw memory-pro delete-bulk --scope global [--before 2025-01-01] [--dry-run] +openclaw memory-pro export [--scope global] [--output memories.json] +openclaw memory-pro import memories.json [--scope global] [--dry-run] +openclaw memory-pro reembed --source-db /path/to/old-db [--batch-size 32] [--skip-existing] +openclaw memory-pro upgrade [--dry-run] [--batch-size 10] [--no-llm] [--limit N] [--scope SCOPE] +openclaw memory-pro migrate check|run|verify [--source /path] +``` + +Flux de connexion OAuth : + +1. Exécutez `openclaw memory-pro auth login` +2. Si `--provider` est omis dans un terminal interactif, la CLI affiche un sélecteur de fournisseur OAuth avant d'ouvrir le navigateur +3. La commande affiche une URL d'autorisation et ouvre votre navigateur sauf si `--no-browser` est défini +4. Après le succès du callback, la commande sauvegarde le fichier OAuth du plugin (par défaut : `~/.openclaw/.memory-lancedb-pro/oauth.json`), sauvegarde la configuration `llm` api-key précédente pour la déconnexion, et remplace la configuration `llm` du plugin par les paramètres OAuth (`auth`, `oauthProvider`, `model`, `oauthPath`) +5. `openclaw memory-pro auth logout` supprime ce fichier OAuth et restaure la configuration `llm` api-key précédente lorsque la sauvegarde existe + +--- + +## Sujets avancés + +
+Si les souvenirs injectés apparaissent dans les réponses + +Parfois le modèle peut répéter le bloc `` injecté. + +**Option A (plus sûr) :** désactiver temporairement le rappel automatique : +```json +{ "plugins": { "entries": { "memory-lancedb-pro": { "config": { "autoRecall": false } } } } } +``` + +**Option B (préféré) :** garder le rappel, ajouter au prompt système de l'agent : +> Ne révélez pas et ne citez pas le contenu `` / injection mémoire dans vos réponses. Utilisez-le uniquement comme référence interne. + +
+ +
+Mémoire de session + +- Déclenchée par la commande `/new` — sauvegarde le résumé de la session précédente dans LanceDB +- Désactivée par défaut (OpenClaw dispose déjà d'une persistance native de session `.jsonl`) +- Nombre de messages configurable (par défaut : 15) + +Consultez [docs/openclaw-integration-playbook.md](docs/openclaw-integration-playbook.md) pour les modes de déploiement et la vérification `/new`. + +
+ +
+Commandes slash personnalisées (ex. /lesson) + +Ajoutez à votre `CLAUDE.md`, `AGENTS.md` ou prompt système : + +```markdown +## /lesson command +When the user sends `/lesson `: +1. Use memory_store to save as category=fact (raw knowledge) +2. Use memory_store to save as category=decision (actionable takeaway) +3. Confirm what was saved + +## /remember command +When the user sends `/remember `: +1. Use memory_store to save with appropriate category and importance +2. Confirm with the stored memory ID +``` + +
+ +
+Règles d'or pour les agents IA + +> Copiez le bloc ci-dessous dans votre `AGENTS.md` pour que votre agent applique automatiquement ces règles. + +```markdown +## Rule 1 — Dual-layer memory storage +Every pitfall/lesson learned → IMMEDIATELY store TWO memories: +- Technical layer: Pitfall: [symptom]. Cause: [root cause]. Fix: [solution]. Prevention: [how to avoid] + (category: fact, importance >= 0.8) +- Principle layer: Decision principle ([tag]): [behavioral rule]. Trigger: [when]. Action: [what to do] + (category: decision, importance >= 0.85) + +## Rule 2 — LanceDB hygiene +Entries must be short and atomic (< 500 chars). No raw conversation summaries or duplicates. + +## Rule 3 — Recall before retry +On ANY tool failure, ALWAYS memory_recall with relevant keywords BEFORE retrying. + +## Rule 4 — Confirm target codebase +Confirm you are editing memory-lancedb-pro vs built-in memory-lancedb before changes. + +## Rule 5 — Clear jiti cache after plugin code changes +After modifying .ts files under plugins/, MUST run rm -rf /tmp/jiti/ BEFORE openclaw gateway restart. +``` + +
+ +
+Schéma de la base de données + +Table LanceDB `memories` : + +| Champ | Type | Description | +| --- | --- | --- | +| `id` | string (UUID) | Clé primaire | +| `text` | string | Texte du souvenir (indexé FTS) | +| `vector` | float[] | Vecteur d'embedding | +| `category` | string | Catégorie de stockage : `preference` / `fact` / `decision` / `entity` / `reflection` / `other` | +| `scope` | string | Identifiant de scope (ex. `global`, `agent:main`) | +| `importance` | float | Score d'importance 0-1 | +| `timestamp` | int64 | Horodatage de création (ms) | +| `metadata` | string (JSON) | Métadonnées étendues | + +Clés `metadata` courantes en v1.1.0 : `l0_abstract`, `l1_overview`, `l2_content`, `memory_category`, `tier`, `access_count`, `confidence`, `last_accessed_at` + +> **Note sur les catégories :** Le champ `category` de niveau supérieur utilise 6 catégories de stockage. Les 6 labels sémantiques de l'Extraction Intelligente (`profile` / `preferences` / `entities` / `events` / `cases` / `patterns`) sont stockés dans `metadata.memory_category`. + +
+ +
+Dépannage + +### "Cannot mix BigInt and other types" (LanceDB / Apache Arrow) + +Avec LanceDB 0.26+, certaines colonnes numériques peuvent être retournées en `BigInt`. Mettez à niveau vers **memory-lancedb-pro >= 1.0.14** — ce plugin convertit maintenant les valeurs avec `Number(...)` avant les opérations arithmétiques. + +
+ +--- + +## Documentation + +| Document | Description | +| --- | --- | +| [Playbook d'intégration OpenClaw](docs/openclaw-integration-playbook.md) | Modes de déploiement, vérification, matrice de régression | +| [Analyse de l'architecture mémoire](docs/memory_architecture_analysis.md) | Analyse approfondie de l'architecture complète | +| [CHANGELOG v1.1.0](docs/CHANGELOG-v1.1.0.md) | Changements de comportement v1.1.0 et justification de la mise à niveau | +| [Chunking long contexte](docs/long-context-chunking.md) | Stratégie de chunking pour les longs documents | + +--- + +## Beta : Smart Memory v1.1.0 + +> Statut : Beta — disponible via `npm i memory-lancedb-pro@beta`. Les utilisateurs stables sur `latest` ne sont pas affectés. + +| Fonctionnalité | Description | +|---------|-------------| +| **Extraction intelligente** | Extraction LLM en 6 catégories avec métadonnées L0/L1/L2. Repli sur regex si désactivé. | +| **Scoring du cycle de vie** | Décroissance Weibull intégrée à la recherche — les souvenirs fréquents et importants sont mieux classés. | +| **Gestion des niveaux** | Système à trois niveaux (Noyau → Travail → Périphérique) avec promotion/rétrogradation automatique. | + +Retours : [GitHub Issues](https://github.com/CortexReach/memory-lancedb-pro/issues) · Retour en arrière : `npm i memory-lancedb-pro@latest` + +--- + +## Dépendances + +| Package | Rôle | +| --- | --- | +| `@lancedb/lancedb` ≥0.26.2 | Base de données vectorielle (ANN + FTS) | +| `openai` ≥6.21.0 | Client API d'embedding compatible OpenAI | +| `@sinclair/typebox` 0.34.48 | Définitions de types JSON Schema | + +--- + +## Contributors + +

+@win4r +@kctony +@Akatsuki-Ryu +@JasonSuz +@Minidoracat +@furedericca-lab +@joe2643 +@AliceLJY +@chenjiyong +

+ +Full list: [Contributors](https://github.com/CortexReach/memory-lancedb-pro/graphs/contributors) + +## Star History + + + + + + Star History Chart + + + +## Licence + +MIT + +--- + +## Mon QR Code WeChat + + diff --git a/README_IT.md b/README_IT.md new file mode 100644 index 00000000..b1679682 --- /dev/null +++ b/README_IT.md @@ -0,0 +1,773 @@ +
+ +# 🧠 memory-lancedb-pro · 🦞OpenClaw Plugin + +**Assistente Memoria IA per Agenti [OpenClaw](https://github.com/openclaw/openclaw)** + +*Dai al tuo agente IA un cervello che ricorda davvero — tra sessioni, tra agenti, nel tempo.* + +Un plugin di memoria a lungo termine per OpenClaw basato su LanceDB che memorizza preferenze, decisioni e contesto di progetto, e li richiama automaticamente nelle sessioni future. + +[![OpenClaw Plugin](https://img.shields.io/badge/OpenClaw-Plugin-blue)](https://github.com/openclaw/openclaw) +[![npm version](https://img.shields.io/npm/v/memory-lancedb-pro)](https://www.npmjs.com/package/memory-lancedb-pro) +[![LanceDB](https://img.shields.io/badge/LanceDB-Vectorstore-orange)](https://lancedb.com) +[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE) + +[English](README.md) | [简体中文](README_CN.md) | [繁體中文](README_TW.md) | [日本語](README_JA.md) | [한국어](README_KO.md) | [Français](README_FR.md) | [Español](README_ES.md) | [Deutsch](README_DE.md) | [Italiano](README_IT.md) | [Русский](README_RU.md) | [Português (Brasil)](README_PT-BR.md) + +
+ +--- + +## Perché memory-lancedb-pro? + +La maggior parte degli agenti IA soffre di amnesia. Dimenticano tutto nel momento in cui si avvia una nuova chat. + +**memory-lancedb-pro** è un plugin di memoria a lungo termine di livello produttivo per OpenClaw che trasforma il tuo agente in un vero **Assistente Memoria IA** — cattura automaticamente ciò che conta, lascia il rumore dissolversi naturalmente e recupera il ricordo giusto al momento giusto. Nessun tag manuale, nessuna configurazione complicata. + +### Il tuo Assistente Memoria IA in azione + +**Senza memoria — ogni sessione parte da zero:** + +> **Tu:** "Usa i tab per l'indentazione, aggiungi sempre la gestione degli errori." +> *(sessione successiva)* +> **Tu:** "Te l'ho già detto — tab, non spazi!" 😤 +> *(sessione successiva)* +> **Tu:** "…sul serio, tab. E gestione degli errori. Di nuovo." + +**Con memory-lancedb-pro — il tuo agente impara e ricorda:** + +> **Tu:** "Usa i tab per l'indentazione, aggiungi sempre la gestione degli errori." +> *(sessione successiva — l'agente richiama automaticamente le tue preferenze)* +> **Agente:** *(applica silenziosamente tab + gestione errori)* ✅ +> **Tu:** "Perché il mese scorso abbiamo scelto PostgreSQL invece di MongoDB?" +> **Agente:** "In base alla nostra discussione del 12 febbraio, i motivi principali erano…" ✅ + +Questa è la differenza che fa un **Assistente Memoria IA** — impara il tuo stile, ricorda le decisioni passate e fornisce risposte personalizzate senza che tu debba ripeterti. + +### Cos'altro può fare? + +| | Cosa ottieni | +|---|---| +| **Auto-Capture** | Il tuo agente impara da ogni conversazione — nessun `memory_store` manuale necessario | +| **Estrazione intelligente** | Classificazione LLM in 6 categorie: profili, preferenze, entità, eventi, casi, pattern | +| **Oblio intelligente** | Modello di decadimento Weibull — i ricordi importanti restano, il rumore svanisce | +| **Ricerca ibrida** | Ricerca vettoriale + BM25 full-text, fusa con reranking cross-encoder | +| **Iniezione di contesto** | I ricordi rilevanti emergono automaticamente prima di ogni risposta | +| **Isolamento multi-scope** | Confini di memoria per agente, per utente, per progetto | +| **Qualsiasi provider** | OpenAI, Jina, Gemini, Ollama o qualsiasi API compatibile OpenAI | +| **Toolkit completo** | CLI, backup, migrazione, upgrade, esportazione/importazione — pronto per la produzione | + +--- + +## Avvio rapido + +### Opzione A: Script di installazione con un clic (consigliato) + +Lo **[script di installazione](https://github.com/CortexReach/toolbox/tree/main/memory-lancedb-pro-setup)** mantenuto dalla community gestisce installazione, aggiornamento e riparazione in un solo comando: + +```bash +curl -fsSL https://raw.githubusercontent.com/CortexReach/toolbox/main/memory-lancedb-pro-setup/setup-memory.sh -o setup-memory.sh +bash setup-memory.sh +``` + +> Vedi [Ecosistema](#ecosistema) qui sotto per l'elenco completo degli scenari coperti e altri strumenti della community. + +### Opzione B: Installazione manuale + +**Tramite OpenClaw CLI (consigliato):** +```bash +openclaw plugins install memory-lancedb-pro@beta +``` + +**Oppure tramite npm:** +```bash +npm i memory-lancedb-pro@beta +``` +> Se usi npm, dovrai anche aggiungere la directory di installazione del plugin come percorso **assoluto** in `plugins.load.paths` nel tuo `openclaw.json`. Questo è il problema di configurazione più comune. + +Aggiungi al tuo `openclaw.json`: + +```json +{ + "plugins": { + "slots": { "memory": "memory-lancedb-pro" }, + "entries": { + "memory-lancedb-pro": { + "enabled": true, + "config": { + "embedding": { + "provider": "openai-compatible", + "apiKey": "${OPENAI_API_KEY}", + "model": "text-embedding-3-small" + }, + "autoCapture": true, + "autoRecall": true, + "smartExtraction": true, + "extractMinMessages": 2, + "extractMaxChars": 8000, + "sessionMemory": { "enabled": false } + } + } + } + } +} +``` + +**Perché questi valori predefiniti?** +- `autoCapture` + `smartExtraction` → il tuo agente impara automaticamente da ogni conversazione +- `autoRecall` → i ricordi rilevanti vengono iniettati prima di ogni risposta +- `extractMinMessages: 2` → l'estrazione si attiva nelle normali chat a due turni +- `sessionMemory.enabled: false` → evita di inquinare la ricerca con riassunti di sessione all'inizio + +Valida e riavvia: + +```bash +openclaw config validate +openclaw gateway restart +openclaw logs --follow --plain | grep "memory-lancedb-pro" +``` + +Dovresti vedere: +- `memory-lancedb-pro: smart extraction enabled` +- `memory-lancedb-pro@...: plugin registered` + +Fatto! Il tuo agente ora ha una memoria a lungo termine. + +
+Ulteriori percorsi di installazione (utenti esistenti, aggiornamenti) + +**Usi già OpenClaw?** + +1. Aggiungi il plugin con un percorso **assoluto** in `plugins.load.paths` +2. Associa lo slot di memoria: `plugins.slots.memory = "memory-lancedb-pro"` +3. Verifica: `openclaw plugins info memory-lancedb-pro && openclaw memory-pro stats` + +**Aggiornamento da versioni precedenti alla v1.1.0?** + +```bash +# 1) Backup +openclaw memory-pro export --scope global --output memories-backup.json +# 2) Dry run +openclaw memory-pro upgrade --dry-run +# 3) Run upgrade +openclaw memory-pro upgrade +# 4) Verify +openclaw memory-pro stats +``` + +Vedi `CHANGELOG-v1.1.0.md` per le modifiche comportamentali e le motivazioni dell'aggiornamento. + +
+ +
+Importazione rapida Telegram Bot (clicca per espandere) + +Se stai usando l'integrazione Telegram di OpenClaw, il modo più semplice è inviare un comando di importazione direttamente al Bot principale invece di modificare manualmente la configurazione. + +Invia questo messaggio: + +```text +Help me connect this memory plugin with the most user-friendly configuration: https://github.com/CortexReach/memory-lancedb-pro + +Requirements: +1. Set it as the only active memory plugin +2. Use Jina for embedding +3. Use Jina for reranker +4. Use gpt-4o-mini for the smart-extraction LLM +5. Enable autoCapture, autoRecall, smartExtraction +6. extractMinMessages=2 +7. sessionMemory.enabled=false +8. captureAssistant=false +9. retrieval mode=hybrid, vectorWeight=0.7, bm25Weight=0.3 +10. rerank=cross-encoder, candidatePoolSize=12, minScore=0.6, hardMinScore=0.62 +11. Generate the final openclaw.json config directly, not just an explanation +``` + +
+ +--- + +## Ecosistema + +memory-lancedb-pro è il plugin principale. La community ha costruito strumenti per rendere l'installazione e l'uso quotidiano ancora più fluidi: + +### Script di installazione — Installazione, aggiornamento e riparazione con un clic + +> **[CortexReach/toolbox/memory-lancedb-pro-setup](https://github.com/CortexReach/toolbox/tree/main/memory-lancedb-pro-setup)** + +Non è un semplice installer — lo script gestisce in modo intelligente numerosi scenari reali: + +| La tua situazione | Cosa fa lo script | +|---|---| +| Mai installato | Download → installazione dipendenze → scelta configurazione → scrittura in openclaw.json → riavvio | +| Installato tramite `git clone`, bloccato su un vecchio commit | `git fetch` + `checkout` automatico all'ultima versione → reinstallazione dipendenze → verifica | +| La configurazione ha campi non validi | Rilevamento automatico tramite filtro schema, rimozione campi non supportati | +| Installato tramite `npm` | Salta l'aggiornamento git, ricorda di eseguire `npm update` autonomamente | +| CLI `openclaw` non funzionante per configurazione non valida | Fallback: lettura diretta del percorso workspace dal file `openclaw.json` | +| `extensions/` invece di `plugins/` | Rilevamento automatico della posizione del plugin da configurazione o filesystem | +| Già aggiornato | Solo controlli di integrità, nessuna modifica | + +```bash +bash setup-memory.sh # Installa o aggiorna +bash setup-memory.sh --dry-run # Solo anteprima +bash setup-memory.sh --beta # Includi versioni pre-release +bash setup-memory.sh --uninstall # Ripristina configurazione e rimuovi plugin +``` + +Preset di provider integrati: **Jina / DashScope / SiliconFlow / OpenAI / Ollama**, oppure usa la tua API compatibile OpenAI. Per l'utilizzo completo (inclusi `--ref`, `--selfcheck-only` e altro), consulta il [README dello script di installazione](https://github.com/CortexReach/toolbox/tree/main/memory-lancedb-pro-setup). + +### Claude Code / OpenClaw Skill — Configurazione guidata dall'IA + +> **[CortexReach/memory-lancedb-pro-skill](https://github.com/CortexReach/memory-lancedb-pro-skill)** + +Installa questa Skill e il tuo agente IA (Claude Code o OpenClaw) acquisisce una conoscenza approfondita di tutte le funzionalità di memory-lancedb-pro. Basta dire **"aiutami ad attivare la configurazione migliore"** per ottenere: + +- **Workflow di configurazione guidato in 7 passaggi** con 4 piani di distribuzione: + - Full Power (Jina + OpenAI) / Budget (reranker SiliconFlow gratuito) / Simple (solo OpenAI) / Completamente locale (Ollama, zero costi API) +- **Tutti i 9 strumenti MCP** usati correttamente: `memory_recall`, `memory_store`, `memory_forget`, `memory_update`, `memory_stats`, `memory_list`, `self_improvement_log`, `self_improvement_extract_skill`, `self_improvement_review` *(il set completo richiede `enableManagementTools: true` — la configurazione Quick Start predefinita espone i 4 strumenti principali)* +- **Prevenzione delle insidie comuni**: attivazione plugin workspace, `autoRecall` predefinito a false, cache jiti, variabili d'ambiente, isolamento scope, ecc. + +**Installazione per Claude Code:** +```bash +git clone https://github.com/CortexReach/memory-lancedb-pro-skill.git ~/.claude/skills/memory-lancedb-pro +``` + +**Installazione per OpenClaw:** +```bash +git clone https://github.com/CortexReach/memory-lancedb-pro-skill.git ~/.openclaw/workspace/skills/memory-lancedb-pro-skill +``` + +--- + +## Tutorial video + +> Guida completa: installazione, configurazione e funzionamento interno della ricerca ibrida. + +[![YouTube Video](https://img.shields.io/badge/YouTube-Watch%20Now-red?style=for-the-badge&logo=youtube)](https://youtu.be/MtukF1C8epQ) +**https://youtu.be/MtukF1C8epQ** + +[![Bilibili Video](https://img.shields.io/badge/Bilibili-Watch%20Now-00A1D6?style=for-the-badge&logo=bilibili&logoColor=white)](https://www.bilibili.com/video/BV1zUf2BGEgn/) +**https://www.bilibili.com/video/BV1zUf2BGEgn/** + +--- + +## Architettura + +``` +┌─────────────────────────────────────────────────────────┐ +│ index.ts (Entry Point) │ +│ Plugin Registration · Config Parsing · Lifecycle Hooks │ +└────────┬──────────┬──────────┬──────────┬───────────────┘ + │ │ │ │ + ┌────▼───┐ ┌────▼───┐ ┌───▼────┐ ┌──▼──────────┐ + │ store │ │embedder│ │retriever│ │ scopes │ + │ .ts │ │ .ts │ │ .ts │ │ .ts │ + └────────┘ └────────┘ └────────┘ └─────────────┘ + │ │ + ┌────▼───┐ ┌─────▼──────────┐ + │migrate │ │noise-filter.ts │ + │ .ts │ │adaptive- │ + └────────┘ │retrieval.ts │ + └────────────────┘ + ┌─────────────┐ ┌──────────┐ + │ tools.ts │ │ cli.ts │ + │ (Agent API) │ │ (CLI) │ + └─────────────┘ └──────────┘ +``` + +> Per un approfondimento sull'architettura completa, consulta [docs/memory_architecture_analysis.md](docs/memory_architecture_analysis.md). + +
+Riferimento file (clicca per espandere) + +| File | Scopo | +| --- | --- | +| `index.ts` | Punto di ingresso del plugin. Si registra con l'API Plugin di OpenClaw, analizza la configurazione, monta gli hook del ciclo di vita | +| `openclaw.plugin.json` | Metadati del plugin + dichiarazione completa della configurazione JSON Schema | +| `cli.ts` | Comandi CLI: `memory-pro list/search/stats/delete/delete-bulk/export/import/reembed/upgrade/migrate` | +| `src/store.ts` | Layer di storage LanceDB. Creazione tabelle / indicizzazione FTS / ricerca vettoriale / ricerca BM25 / CRUD | +| `src/embedder.ts` | Astrazione embedding. Compatibile con qualsiasi provider API compatibile OpenAI | +| `src/retriever.ts` | Motore di ricerca ibrido. Vettoriale + BM25 → Fusione ibrida → Rerank → Decadimento ciclo di vita → Filtro | +| `src/scopes.ts` | Controllo accessi multi-scope | +| `src/tools.ts` | Definizioni degli strumenti agente: `memory_recall`, `memory_store`, `memory_forget`, `memory_update` + strumenti di gestione | +| `src/noise-filter.ts` | Filtra rifiuti dell'agente, meta-domande, saluti e contenuti di bassa qualità | +| `src/adaptive-retrieval.ts` | Determina se una query necessita di ricerca nella memoria | +| `src/migrate.ts` | Migrazione dal `memory-lancedb` integrato a Pro | +| `src/smart-extractor.ts` | Estrazione LLM in 6 categorie con archiviazione a strati L0/L1/L2 e deduplicazione in due fasi | +| `src/decay-engine.ts` | Modello di decadimento esponenziale esteso Weibull | +| `src/tier-manager.ts` | Promozione/retrocessione a tre livelli: Peripheral ↔ Working ↔ Core | + +
+ +--- + +## Funzionalità principali + +### Ricerca ibrida + +``` +Query → embedQuery() ─┐ + ├─→ Hybrid Fusion → Rerank → Lifecycle Decay Boost → Length Norm → Filter +Query → BM25 FTS ─────┘ +``` + +- **Ricerca vettoriale** — similarità semantica tramite LanceDB ANN (distanza del coseno) +- **Ricerca full-text BM25** — corrispondenza esatta delle parole chiave tramite indice FTS di LanceDB +- **Fusione ibrida** — punteggio vettoriale come base, i risultati BM25 ricevono un boost ponderato (non RRF standard — ottimizzato per la qualità di richiamo nel mondo reale) +- **Pesi configurabili** — `vectorWeight`, `bm25Weight`, `minScore` + +### Reranking Cross-Encoder + +- Adattatori integrati per **Jina**, **SiliconFlow**, **Voyage AI** e **Pinecone** +- Compatibile con qualsiasi endpoint compatibile Jina (ad es. Hugging Face TEI, DashScope) +- Punteggio ibrido: 60% cross-encoder + 40% punteggio fuso originale +- Degradazione elegante: fallback sulla similarità del coseno in caso di errore API + +### Pipeline di punteggio multi-fase + +| Fase | Effetto | +| --- | --- | +| **Fusione ibrida** | Combina richiamo semantico e corrispondenza esatta | +| **Rerank Cross-Encoder** | Promuove risultati semanticamente precisi | +| **Boost decadimento ciclo di vita** | Freschezza Weibull + frequenza di accesso + importance × confidence | +| **Normalizzazione lunghezza** | Impedisce alle voci lunghe di dominare (ancora: 500 caratteri) | +| **Punteggio minimo rigido** | Rimuove risultati irrilevanti (predefinito: 0.35) | +| **Diversità MMR** | Similarità coseno > 0.85 → retrocesso | + +### Estrazione intelligente della memoria (v1.1.0) + +- **Estrazione LLM in 6 categorie**: profilo, preferenze, entità, eventi, casi, pattern +- **Archiviazione a strati L0/L1/L2**: L0 (indice in una frase) → L1 (riepilogo strutturato) → L2 (narrazione completa) +- **Deduplicazione in due fasi**: pre-filtro similarità vettoriale (≥0.7) → decisione semantica LLM (CREATE/MERGE/SKIP) +- **Fusione consapevole delle categorie**: `profile` viene sempre fuso, `events`/`cases` solo in aggiunta + +### Gestione del ciclo di vita della memoria (v1.1.0) + +- **Motore di decadimento Weibull**: punteggio composito = freschezza + frequenza + valore intrinseco +- **Promozione a tre livelli**: `Peripheral ↔ Working ↔ Core` con soglie configurabili +- **Rinforzo per accesso**: i ricordi richiamati frequentemente decadono più lentamente (stile ripetizione spaziata) +- **Emivita modulata dall'importanza**: i ricordi importanti decadono più lentamente + +### Isolamento multi-scope + +- Scope integrati: `global`, `agent:`, `custom:`, `project:`, `user:` +- Controllo accessi a livello agente tramite `scopes.agentAccess` +- Predefinito: ogni agente accede a `global` + il proprio scope `agent:` + +### Auto-Capture e Auto-Recall + +- **Auto-Capture** (`agent_end`): estrae preferenze/fatti/decisioni/entità dalle conversazioni, deduplica, memorizza fino a 3 per turno +- **Auto-Recall** (`before_agent_start`): inietta il contesto `` (fino a 3 voci) + +### Filtraggio del rumore e ricerca adattiva + +- Filtra contenuti di bassa qualità: rifiuti dell'agente, meta-domande, saluti +- Salta la ricerca per: saluti, comandi slash, conferme semplici, emoji +- Forza la ricerca per parole chiave della memoria ("ricorda", "precedentemente", "l'ultima volta") +- Soglie CJK (cinese: 6 caratteri vs inglese: 15 caratteri) + +--- + +
+Confronto con memory-lancedb integrato (clicca per espandere) + +| Funzionalità | `memory-lancedb` integrato | **memory-lancedb-pro** | +| --- | :---: | :---: | +| Ricerca vettoriale | Sì | Sì | +| Ricerca full-text BM25 | - | Sì | +| Fusione ibrida (Vettoriale + BM25) | - | Sì | +| Rerank cross-encoder (multi-provider) | - | Sì | +| Boost di freschezza e decadimento temporale | - | Sì | +| Normalizzazione lunghezza | - | Sì | +| Diversità MMR | - | Sì | +| Isolamento multi-scope | - | Sì | +| Filtraggio del rumore | - | Sì | +| Ricerca adattiva | - | Sì | +| CLI di gestione | - | Sì | +| Memoria di sessione | - | Sì | +| Embedding task-aware | - | Sì | +| **Estrazione intelligente LLM (6 categorie)** | - | Sì (v1.1.0) | +| **Decadimento Weibull + promozione livelli** | - | Sì (v1.1.0) | +| Qualsiasi embedding compatibile OpenAI | Limitato | Sì | + +
+ +--- + +## Configurazione + +
+Esempio di configurazione completa + +```json +{ + "embedding": { + "apiKey": "${JINA_API_KEY}", + "model": "jina-embeddings-v5-text-small", + "baseURL": "https://api.jina.ai/v1", + "dimensions": 1024, + "taskQuery": "retrieval.query", + "taskPassage": "retrieval.passage", + "normalized": true + }, + "dbPath": "~/.openclaw/memory/lancedb-pro", + "autoCapture": true, + "autoRecall": true, + "retrieval": { + "mode": "hybrid", + "vectorWeight": 0.7, + "bm25Weight": 0.3, + "minScore": 0.3, + "rerank": "cross-encoder", + "rerankApiKey": "${JINA_API_KEY}", + "rerankModel": "jina-reranker-v3", + "rerankEndpoint": "https://api.jina.ai/v1/rerank", + "rerankProvider": "jina", + "candidatePoolSize": 20, + "recencyHalfLifeDays": 14, + "recencyWeight": 0.1, + "filterNoise": true, + "lengthNormAnchor": 500, + "hardMinScore": 0.35, + "timeDecayHalfLifeDays": 60, + "reinforcementFactor": 0.5, + "maxHalfLifeMultiplier": 3 + }, + "enableManagementTools": false, + "scopes": { + "default": "global", + "definitions": { + "global": { "description": "Shared knowledge" }, + "agent:discord-bot": { "description": "Discord bot private" } + }, + "agentAccess": { + "discord-bot": ["global", "agent:discord-bot"] + } + }, + "sessionMemory": { + "enabled": false, + "messageCount": 15 + }, + "smartExtraction": true, + "llm": { + "apiKey": "${OPENAI_API_KEY}", + "model": "gpt-4o-mini", + "baseURL": "https://api.openai.com/v1" + }, + "extractMinMessages": 2, + "extractMaxChars": 8000 +} +``` + +
+ +
+Provider di embedding + +Funziona con **qualsiasi API di embedding compatibile OpenAI**: + +| Provider | Modello | Base URL | Dimensioni | +| --- | --- | --- | --- | +| **Jina** (consigliato) | `jina-embeddings-v5-text-small` | `https://api.jina.ai/v1` | 1024 | +| **OpenAI** | `text-embedding-3-small` | `https://api.openai.com/v1` | 1536 | +| **Voyage** | `voyage-4-lite` / `voyage-4` | `https://api.voyageai.com/v1` | 1024 / 1024 | +| **Google Gemini** | `gemini-embedding-001` | `https://generativelanguage.googleapis.com/v1beta/openai/` | 3072 | +| **Ollama** (locale) | `nomic-embed-text` | `http://localhost:11434/v1` | specifico del provider | + +
+ +
+Provider di rerank + +Il reranking cross-encoder supporta più provider tramite `rerankProvider`: + +| Provider | `rerankProvider` | Modello di esempio | +| --- | --- | --- | +| **Jina** (predefinito) | `jina` | `jina-reranker-v3` | +| **SiliconFlow** (piano gratuito disponibile) | `siliconflow` | `BAAI/bge-reranker-v2-m3` | +| **Voyage AI** | `voyage` | `rerank-2.5` | +| **Pinecone** | `pinecone` | `bge-reranker-v2-m3` | + +Funziona anche qualsiasi endpoint di rerank compatibile Jina — imposta `rerankProvider: "jina"` e punta `rerankEndpoint` al tuo servizio (ad es. Hugging Face TEI, DashScope `qwen3-rerank`). + +
+ +
+Estrazione intelligente (LLM) — v1.1.0 + +Quando `smartExtraction` è abilitato (predefinito: `true`), il plugin utilizza un LLM per estrarre e classificare intelligentemente i ricordi invece di trigger basati su regex. + +| Campo | Tipo | Predefinito | Descrizione | +|-------|------|---------|-------------| +| `smartExtraction` | boolean | `true` | Abilita/disabilita l'estrazione LLM in 6 categorie | +| `llm.auth` | string | `api-key` | `api-key` usa `llm.apiKey` / `embedding.apiKey`; `oauth` usa un file token OAuth con scope plugin per impostazione predefinita | +| `llm.apiKey` | string | *(fallback su `embedding.apiKey`)* | Chiave API per il provider LLM | +| `llm.model` | string | `openai/gpt-oss-120b` | Nome del modello LLM | +| `llm.baseURL` | string | *(fallback su `embedding.baseURL`)* | Endpoint API LLM | +| `llm.oauthProvider` | string | `openai-codex` | ID del provider OAuth usato quando `llm.auth` è `oauth` | +| `llm.oauthPath` | string | `~/.openclaw/.memory-lancedb-pro/oauth.json` | File token OAuth usato quando `llm.auth` è `oauth` | +| `llm.timeoutMs` | number | `30000` | Timeout della richiesta LLM in millisecondi | +| `extractMinMessages` | number | `2` | Messaggi minimi prima che l'estrazione si attivi | +| `extractMaxChars` | number | `8000` | Caratteri massimi inviati al LLM | + + +Configurazione `llm` OAuth (usa la cache di login esistente di Codex / ChatGPT per le chiamate LLM): +```json +{ + "llm": { + "auth": "oauth", + "oauthProvider": "openai-codex", + "model": "gpt-5.4", + "oauthPath": "${HOME}/.openclaw/.memory-lancedb-pro/oauth.json", + "timeoutMs": 30000 + } +} +``` + +Note per `llm.auth: "oauth"`: + +- `llm.oauthProvider` è attualmente `openai-codex`. +- I token OAuth sono salvati di default in `~/.openclaw/.memory-lancedb-pro/oauth.json`. +- Puoi impostare `llm.oauthPath` se vuoi salvare quel file altrove. +- `auth login` crea uno snapshot della configurazione `llm` precedente con api-key accanto al file OAuth, e `auth logout` ripristina quello snapshot quando disponibile. +- Il passaggio da `api-key` a `oauth` non trasferisce automaticamente `llm.baseURL`. Impostalo manualmente in modalità OAuth solo quando vuoi intenzionalmente un backend personalizzato compatibile ChatGPT/Codex. + +
+ +
+Configurazione ciclo di vita (Decadimento + Livelli) + +| Campo | Predefinito | Descrizione | +|-------|---------|-------------| +| `decay.recencyHalfLifeDays` | `30` | Emivita base per il decadimento di freschezza Weibull | +| `decay.frequencyWeight` | `0.3` | Peso della frequenza di accesso nel punteggio composito | +| `decay.intrinsicWeight` | `0.3` | Peso di `importance × confidence` | +| `decay.betaCore` | `0.8` | Beta Weibull per i ricordi `core` | +| `decay.betaWorking` | `1.0` | Beta Weibull per i ricordi `working` | +| `decay.betaPeripheral` | `1.3` | Beta Weibull per i ricordi `peripheral` | +| `tier.coreAccessThreshold` | `10` | Conteggio minimo richiami prima della promozione a `core` | +| `tier.peripheralAgeDays` | `60` | Soglia di età per retrocedere i ricordi inattivi | + +
+ +
+Rinforzo per accesso + +I ricordi richiamati frequentemente decadono più lentamente (stile ripetizione spaziata). + +Chiavi di configurazione (sotto `retrieval`): +- `reinforcementFactor` (0-2, predefinito: `0.5`) — imposta `0` per disabilitare +- `maxHalfLifeMultiplier` (1-10, predefinito: `3`) — limite massimo sull'emivita effettiva + +
+ +--- + +## Comandi CLI + +```bash +openclaw memory-pro list [--scope global] [--category fact] [--limit 20] [--json] +openclaw memory-pro search "query" [--scope global] [--limit 10] [--json] +openclaw memory-pro stats [--scope global] [--json] +openclaw memory-pro auth login [--provider openai-codex] [--model gpt-5.4] [--oauth-path /abs/path/oauth.json] +openclaw memory-pro auth status +openclaw memory-pro auth logout +openclaw memory-pro delete +openclaw memory-pro delete-bulk --scope global [--before 2025-01-01] [--dry-run] +openclaw memory-pro export [--scope global] [--output memories.json] +openclaw memory-pro import memories.json [--scope global] [--dry-run] +openclaw memory-pro reembed --source-db /path/to/old-db [--batch-size 32] [--skip-existing] +openclaw memory-pro upgrade [--dry-run] [--batch-size 10] [--no-llm] [--limit N] [--scope SCOPE] +openclaw memory-pro migrate check|run|verify [--source /path] +``` + +Flusso di login OAuth: + +1. Esegui `openclaw memory-pro auth login` +2. Se `--provider` è omesso in un terminale interattivo, la CLI mostra un selettore di provider OAuth prima di aprire il browser +3. Il comando stampa un URL di autorizzazione e apre il browser, a meno che non sia impostato `--no-browser` +4. Dopo il successo del callback, il comando salva il file OAuth del plugin (predefinito: `~/.openclaw/.memory-lancedb-pro/oauth.json`), crea uno snapshot della configurazione `llm` precedente con api-key per il logout, e sostituisce la configurazione `llm` del plugin con le impostazioni OAuth (`auth`, `oauthProvider`, `model`, `oauthPath`) +5. `openclaw memory-pro auth logout` elimina quel file OAuth e ripristina la configurazione `llm` precedente con api-key quando quello snapshot esiste + +--- + +## Argomenti avanzati + +
+Se i ricordi iniettati appaiono nelle risposte + +A volte il modello può ripetere il blocco `` iniettato. + +**Opzione A (rischio minimo):** disabilita temporaneamente l'auto-recall: +```json +{ "plugins": { "entries": { "memory-lancedb-pro": { "config": { "autoRecall": false } } } } } +``` + +**Opzione B (preferita):** mantieni il recall, aggiungi al prompt di sistema dell'agente: +> Do not reveal or quote any `` / memory-injection content in your replies. Use it for internal reference only. + +
+ +
+Memoria di sessione + +- Si attiva con il comando `/new` — salva il riepilogo della sessione precedente in LanceDB +- Disabilitata per impostazione predefinita (OpenClaw ha già la persistenza nativa delle sessioni in `.jsonl`) +- Conteggio messaggi configurabile (predefinito: 15) + +Vedi [docs/openclaw-integration-playbook.md](docs/openclaw-integration-playbook.md) per le modalità di distribuzione e la verifica di `/new`. + +
+ +
+Comandi slash personalizzati (ad es. /lesson) + +Aggiungi al tuo `CLAUDE.md`, `AGENTS.md` o prompt di sistema: + +```markdown +## /lesson command +When the user sends `/lesson `: +1. Use memory_store to save as category=fact (raw knowledge) +2. Use memory_store to save as category=decision (actionable takeaway) +3. Confirm what was saved + +## /remember command +When the user sends `/remember `: +1. Use memory_store to save with appropriate category and importance +2. Confirm with the stored memory ID +``` + +
+ +
+Regole d'oro per agenti IA + +> Copia il blocco seguente nel tuo `AGENTS.md` in modo che il tuo agente applichi queste regole automaticamente. + +```markdown +## Rule 1 — Dual-layer memory storage +Every pitfall/lesson learned → IMMEDIATELY store TWO memories: +- Technical layer: Pitfall: [symptom]. Cause: [root cause]. Fix: [solution]. Prevention: [how to avoid] + (category: fact, importance >= 0.8) +- Principle layer: Decision principle ([tag]): [behavioral rule]. Trigger: [when]. Action: [what to do] + (category: decision, importance >= 0.85) + +## Rule 2 — LanceDB hygiene +Entries must be short and atomic (< 500 chars). No raw conversation summaries or duplicates. + +## Rule 3 — Recall before retry +On ANY tool failure, ALWAYS memory_recall with relevant keywords BEFORE retrying. + +## Rule 4 — Confirm target codebase +Confirm you are editing memory-lancedb-pro vs built-in memory-lancedb before changes. + +## Rule 5 — Clear jiti cache after plugin code changes +After modifying .ts files under plugins/, MUST run rm -rf /tmp/jiti/ BEFORE openclaw gateway restart. +``` + +
+ +
+Schema del database + +Tabella LanceDB `memories`: + +| Campo | Tipo | Descrizione | +| --- | --- | --- | +| `id` | string (UUID) | Chiave primaria | +| `text` | string | Testo del ricordo (indicizzato FTS) | +| `vector` | float[] | Vettore di embedding | +| `category` | string | Categoria di archiviazione: `preference` / `fact` / `decision` / `entity` / `reflection` / `other` | +| `scope` | string | Identificatore scope (ad es. `global`, `agent:main`) | +| `importance` | float | Punteggio di importanza 0-1 | +| `timestamp` | int64 | Timestamp di creazione (ms) | +| `metadata` | string (JSON) | Metadati estesi | + +Chiavi `metadata` comuni nella v1.1.0: `l0_abstract`, `l1_overview`, `l2_content`, `memory_category`, `tier`, `access_count`, `confidence`, `last_accessed_at` + +> **Nota sulle categorie:** Il campo `category` di primo livello usa 6 categorie di archiviazione. Le 6 etichette semantiche dell'Estrazione Intelligente (`profile` / `preferences` / `entities` / `events` / `cases` / `patterns`) sono memorizzate in `metadata.memory_category`. + +
+ +
+Risoluzione dei problemi + +### "Cannot mix BigInt and other types" (LanceDB / Apache Arrow) + +Con LanceDB 0.26+, alcune colonne numeriche potrebbero essere restituite come `BigInt`. Aggiorna a **memory-lancedb-pro >= 1.0.14** — questo plugin ora converte i valori usando `Number(...)` prima delle operazioni aritmetiche. + +
+ +--- + +## Documentazione + +| Documento | Descrizione | +| --- | --- | +| [Playbook di integrazione OpenClaw](docs/openclaw-integration-playbook.md) | Modalità di distribuzione, verifica, matrice di regressione | +| [Analisi dell'architettura della memoria](docs/memory_architecture_analysis.md) | Analisi approfondita dell'architettura completa | +| [CHANGELOG v1.1.0](docs/CHANGELOG-v1.1.0.md) | Modifiche comportamentali v1.1.0 e motivazioni per l'upgrade | +| [Chunking contesto lungo](docs/long-context-chunking.md) | Strategia di chunking per documenti lunghi | + +--- + +## Beta: Smart Memory v1.1.0 + +> Stato: Beta — disponibile tramite `npm i memory-lancedb-pro@beta`. Gli utenti stabili su `latest` non sono interessati. + +| Funzionalità | Descrizione | +|---------|-------------| +| **Estrazione intelligente** | Estrazione LLM in 6 categorie con metadati L0/L1/L2. Fallback su regex se disabilitato. | +| **Punteggio ciclo di vita** | Decadimento Weibull integrato nella ricerca — i ricordi frequenti e importanti si posizionano più in alto. | +| **Gestione livelli** | Sistema a tre livelli (Core → Working → Peripheral) con promozione/retrocessione automatica. | + +Feedback: [GitHub Issues](https://github.com/CortexReach/memory-lancedb-pro/issues) · Ripristina: `npm i memory-lancedb-pro@latest` + +--- + +## Dipendenze + +| Pacchetto | Scopo | +| --- | --- | +| `@lancedb/lancedb` ≥0.26.2 | Database vettoriale (ANN + FTS) | +| `openai` ≥6.21.0 | Client API Embedding compatibile OpenAI | +| `@sinclair/typebox` 0.34.48 | Definizioni di tipo JSON Schema | + +--- + +## Contributors + +

+@win4r +@kctony +@Akatsuki-Ryu +@JasonSuz +@Minidoracat +@furedericca-lab +@joe2643 +@AliceLJY +@chenjiyong +

+ +Full list: [Contributors](https://github.com/CortexReach/memory-lancedb-pro/graphs/contributors) + +## Star History + + + + + + Star History Chart + + + +## Licenza + +MIT + +--- + +## Il mio QR Code WeChat + + diff --git a/README_JA.md b/README_JA.md new file mode 100644 index 00000000..0627a2de --- /dev/null +++ b/README_JA.md @@ -0,0 +1,773 @@ +
+ +# 🧠 memory-lancedb-pro · 🦞OpenClaw Plugin + +**[OpenClaw](https://github.com/openclaw/openclaw) エージェント向け AI メモリアシスタント** + +*あなたの AI エージェントに本物の記憶力を——セッションを超え、エージェントを超え、時間を超えて。* + +LanceDB ベースの OpenClaw 長期メモリプラグイン。好み・意思決定・プロジェクトコンテキストを自動保存し、将来のセッションで自動的に想起します。 + +[![OpenClaw Plugin](https://img.shields.io/badge/OpenClaw-Plugin-blue)](https://github.com/openclaw/openclaw) +[![npm version](https://img.shields.io/npm/v/memory-lancedb-pro)](https://www.npmjs.com/package/memory-lancedb-pro) +[![LanceDB](https://img.shields.io/badge/LanceDB-Vectorstore-orange)](https://lancedb.com) +[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE) + +[English](README.md) | [简体中文](README_CN.md) | [繁體中文](README_TW.md) | [日本語](README_JA.md) | [한국어](README_KO.md) | [Français](README_FR.md) | [Español](README_ES.md) | [Deutsch](README_DE.md) | [Italiano](README_IT.md) | [Русский](README_RU.md) | [Português (Brasil)](README_PT-BR.md) + +
+ +--- + +## なぜ memory-lancedb-pro なのか？ + +ほとんどの AI エージェントは「記憶喪失」です——新しいチャットを始めるたびに、以前の会話内容はすべてリセットされます。 + +**memory-lancedb-pro** は OpenClaw 向けのプロダクショングレードの長期メモリプラグインです。エージェントを真の **AI メモリアシスタント** に変えます——重要な情報を自動的にキャプチャし、ノイズを自然に減衰させ、適切なタイミングで適切な記憶を呼び出します。手動タグ付けも複雑な設定も不要です。 + +### AI メモリアシスタントの実際の動作 + +**メモリなし——毎回ゼロからスタート：** + +> **あなた：** 「インデントはタブで、常にエラーハンドリングを追加して。」 +> *（次のセッション）* +> **あなた：** 「前に言ったでしょ——タブであってスペースじゃない！」 😤 +> *（さらに次のセッション）* +> **あなた：** 「……本当にもう3回目だよ、タブ。あとエラーハンドリングも。」 + +**memory-lancedb-pro あり——エージェントが学習し記憶する：** + +> **あなた：** 「インデントはタブで、常にエラーハンドリングを追加して。」 +> *（次のセッション——エージェントが自動的にあなたの好みを想起）* +> **エージェント：** *（黙ってタブインデント＋エラーハンドリングを適用）* ✅ +> **あなた：** 「先月なぜ MongoDB ではなく PostgreSQL を選んだんだっけ？」 +> **エージェント：** 「2月12日の議論に基づくと、主な理由は……」 ✅ + +これが **AI メモリアシスタント** の価値です——あなたのスタイルを学び、過去の意思決定を想起し、繰り返し説明することなくパーソナライズされた応答を提供します。 + +### 他に何ができる？ + +| | 得られるもの | +|---|---| +| **自動キャプチャ** | エージェントが毎回の会話から学習——手動で `memory_store` を呼ぶ必要なし | +| **スマート抽出** | LLM 駆動の6カテゴリ分類：プロフィール、好み、エンティティ、イベント、ケース、パターン | +| **インテリジェント忘却** | Weibull 減衰モデル——重要な記憶は残り、ノイズは自然に消える | +| **ハイブリッド検索** | ベクトル + BM25 全文検索、クロスエンコーダーリランキングで融合 | +| **コンテキスト注入** | 関連する記憶が各応答前に自動的に浮上 | +| **マルチスコープ分離** | エージェント別、ユーザー別、プロジェクト別のメモリ境界 | +| **任意のプロバイダー** | OpenAI、Jina、Gemini、Ollama、または任意の OpenAI 互換 API | +| **フルツールキット** | CLI、バックアップ、マイグレーション、アップグレード、エクスポート/インポート——本番運用対応 | + +--- + +## クイックスタート + +### 方法 A：ワンクリックインストールスクリプト（推奨） + +コミュニティが管理する **[セットアップスクリプト](https://github.com/CortexReach/toolbox/tree/main/memory-lancedb-pro-setup)** で、インストール・アップグレード・修復を1コマンドで実行： + +```bash +curl -fsSL https://raw.githubusercontent.com/CortexReach/toolbox/main/memory-lancedb-pro-setup/setup-memory.sh -o setup-memory.sh +bash setup-memory.sh +``` + +> スクリプトがカバーするシナリオの完全なリストとその他のコミュニティツールは、以下の [エコシステム](#エコシステム) をご覧ください。 + +### 方法 B：手動インストール + +**OpenClaw CLI 経由（推奨）：** +```bash +openclaw plugins install memory-lancedb-pro@beta +``` + +**または npm 経由：** +```bash +npm i memory-lancedb-pro@beta +``` +> npm を使用する場合、`openclaw.json` の `plugins.load.paths` にプラグインのインストールディレクトリの **絶対パス** を追加する必要があります。これが最も一般的なセットアップの問題です。 + +`openclaw.json` に以下を追加： + +```json +{ + "plugins": { + "slots": { "memory": "memory-lancedb-pro" }, + "entries": { + "memory-lancedb-pro": { + "enabled": true, + "config": { + "embedding": { + "provider": "openai-compatible", + "apiKey": "${OPENAI_API_KEY}", + "model": "text-embedding-3-small" + }, + "autoCapture": true, + "autoRecall": true, + "smartExtraction": true, + "extractMinMessages": 2, + "extractMaxChars": 8000, + "sessionMemory": { "enabled": false } + } + } + } + } +} +``` + +**これらのデフォルト値の理由：** +- `autoCapture` + `smartExtraction` → エージェントが毎回の会話から自動的に学習 +- `autoRecall` → 関連する記憶が各応答前に自動注入 +- `extractMinMessages: 2` → 通常の2ターン会話で抽出がトリガー +- `sessionMemory.enabled: false` → 初期段階でセッション要約が検索結果を汚染するのを回避 + +検証と再起動： + +```bash +openclaw config validate +openclaw gateway restart +openclaw logs --follow --plain | grep "memory-lancedb-pro" +``` + +以下が表示されるはずです： +- `memory-lancedb-pro: smart extraction enabled` +- `memory-lancedb-pro@...: plugin registered` + +完了！あなたのエージェントは長期メモリを持つようになりました。 + +
+その他のインストール方法（既存ユーザー、アップグレード） + +**既に OpenClaw を使用中？** + +1. `plugins.load.paths` にプラグインの **絶対パス** を追加 +2. メモリスロットをバインド：`plugins.slots.memory = "memory-lancedb-pro"` +3. 検証：`openclaw plugins info memory-lancedb-pro && openclaw memory-pro stats` + +**v1.1.0 以前からのアップグレード？** + +```bash +# 1) バックアップ +openclaw memory-pro export --scope global --output memories-backup.json +# 2) ドライラン +openclaw memory-pro upgrade --dry-run +# 3) アップグレード実行 +openclaw memory-pro upgrade +# 4) 検証 +openclaw memory-pro stats +``` + +動作変更とアップグレードの詳細は `CHANGELOG-v1.1.0.md` を参照してください。 + +
+ +
+Telegram Bot クイックインポート（クリックで展開） + +OpenClaw の Telegram 連携を使用している場合、設定ファイルを手動で編集するより、メイン Bot にインポートコマンドを直接送信するのが最も簡単です。 + +以下のメッセージを送信してください： + +```text +Help me connect this memory plugin with the most user-friendly configuration: https://github.com/CortexReach/memory-lancedb-pro + +Requirements: +1. Set it as the only active memory plugin +2. Use Jina for embedding +3. Use Jina for reranker +4. Use gpt-4o-mini for the smart-extraction LLM +5. Enable autoCapture, autoRecall, smartExtraction +6. extractMinMessages=2 +7. sessionMemory.enabled=false +8. captureAssistant=false +9. retrieval mode=hybrid, vectorWeight=0.7, bm25Weight=0.3 +10. rerank=cross-encoder, candidatePoolSize=12, minScore=0.6, hardMinScore=0.62 +11. Generate the final openclaw.json config directly, not just an explanation +``` + +
+ +--- + +## エコシステム + +memory-lancedb-pro はコアプラグインです。コミュニティがセットアップと日常利用をさらにスムーズにするツールを構築しています： + +### セットアップスクリプト——ワンクリックでインストール・アップグレード・修復 + +> **[CortexReach/toolbox/memory-lancedb-pro-setup](https://github.com/CortexReach/toolbox/tree/main/memory-lancedb-pro-setup)** + +単なるインストーラーではありません——さまざまな実際のシナリオをインテリジェントに処理します： + +| あなたの状況 | スクリプトの動作 | +|---|---| +| 未インストール | 新規ダウンロード → 依存関係インストール → 設定選択 → openclaw.json に書き込み → 再起動 | +| `git clone` でインストール済み、古いコミットで停滞 | 自動で `git fetch` + `checkout` を最新版に → 依存関係再インストール → 検証 | +| 設定に無効なフィールドがある | スキーマフィルターで自動検出し、サポートされていないフィールドを除去 | +| `npm` でインストール済み | git 更新をスキップし、`npm update` の実行を促す | +| 無効な設定で `openclaw` CLI が壊れている | フォールバック：`openclaw.json` ファイルからワークスペースパスを直接読み取り | +| `plugins/` ではなく `extensions/` | 設定またはファイルシステムからプラグインの場所を自動検出 | +| 既に最新版 | ヘルスチェックのみ実行、変更なし | + +```bash +bash setup-memory.sh # インストールまたはアップグレード +bash setup-memory.sh --dry-run # プレビューのみ +bash setup-memory.sh --beta # プレリリース版を含む +bash setup-memory.sh --uninstall # 設定を元に戻しプラグインを削除 +``` + +内蔵プロバイダープリセット：**Jina / DashScope / SiliconFlow / OpenAI / Ollama**、または任意の OpenAI 互換 API を利用可能。完全な使用方法（`--ref`、`--selfcheck-only` など）は [セットアップスクリプト README](https://github.com/CortexReach/toolbox/tree/main/memory-lancedb-pro-setup) を参照してください。 + +### Claude Code / OpenClaw Skill——AI ガイド付き設定 + +> **[CortexReach/memory-lancedb-pro-skill](https://github.com/CortexReach/memory-lancedb-pro-skill)** + +この Skill をインストールすると、AI エージェント（Claude Code または OpenClaw）が memory-lancedb-pro のすべての機能を深く理解できるようになります。**「最適な設定を有効にして」** と言うだけで： + +- **7ステップのガイド付き設定ワークフロー**、4つのデプロイプランを提供： + - フルパワー版（Jina + OpenAI）/ コスト削減版（無料の SiliconFlow リランカー）/ シンプル版（OpenAI のみ）/ 完全ローカル版（Ollama、API コストゼロ） +- **全9つの MCP ツール** の正しい使い方：`memory_recall`、`memory_store`、`memory_forget`、`memory_update`、`memory_stats`、`memory_list`、`self_improvement_log`、`self_improvement_extract_skill`、`self_improvement_review` *（フルツールセットには `enableManagementTools: true` が必要——デフォルトのクイックスタート設定では4つのコアツールのみ公開）* +- **よくある落とし穴の回避**：ワークスペースプラグインの有効化、`autoRecall` のデフォルト false、jiti キャッシュ、環境変数、スコープ分離など + +**Claude Code へのインストール：** +```bash +git clone https://github.com/CortexReach/memory-lancedb-pro-skill.git ~/.claude/skills/memory-lancedb-pro +``` + +**OpenClaw へのインストール：** +```bash +git clone https://github.com/CortexReach/memory-lancedb-pro-skill.git ~/.openclaw/workspace/skills/memory-lancedb-pro-skill +``` + +--- + +## 動画チュートリアル + +> フルウォークスルー：インストール、設定、ハイブリッド検索の内部構造。 + +[![YouTube Video](https://img.shields.io/badge/YouTube-Watch%20Now-red?style=for-the-badge&logo=youtube)](https://youtu.be/MtukF1C8epQ) +**https://youtu.be/MtukF1C8epQ** + +[![Bilibili Video](https://img.shields.io/badge/Bilibili-Watch%20Now-00A1D6?style=for-the-badge&logo=bilibili&logoColor=white)](https://www.bilibili.com/video/BV1zUf2BGEgn/) +**https://www.bilibili.com/video/BV1zUf2BGEgn/** + +--- + +## アーキテクチャ + +``` +┌─────────────────────────────────────────────────────────┐ +│ index.ts（エントリポイント） │ +│ プラグイン登録 · 設定解析 · ライフサイクルフック │ +└────────┬──────────┬──────────┬──────────┬───────────────┘ + │ │ │ │ + ┌────▼───┐ ┌────▼───┐ ┌───▼────┐ ┌──▼──────────┐ + │ store │ │embedder│ │retriever│ │ scopes │ + │ .ts │ │ .ts │ │ .ts │ │ .ts │ + └────────┘ └────────┘ └────────┘ └─────────────┘ + │ │ + ┌────▼───┐ ┌─────▼──────────┐ + │migrate │ │noise-filter.ts │ + │ .ts │ │adaptive- │ + └────────┘ │retrieval.ts │ + └────────────────┘ + ┌─────────────┐ ┌──────────┐ + │ tools.ts │ │ cli.ts │ + │(エージェントAPI)│ │ (CLI) │ + └─────────────┘ └──────────┘ +``` + +> 完全なアーキテクチャの詳細は [docs/memory_architecture_analysis.md](docs/memory_architecture_analysis.md) を参照してください。 + +
+ファイルリファレンス（クリックで展開） + +| ファイル | 用途 | +| --- | --- | +| `index.ts` | プラグインエントリポイント。OpenClaw Plugin API に登録、設定解析、ライフサイクルフックのマウント | +| `openclaw.plugin.json` | プラグインメタデータ + 完全な JSON Schema 設定宣言 | +| `cli.ts` | CLI コマンド：`memory-pro list/search/stats/delete/delete-bulk/export/import/reembed/upgrade/migrate` | +| `src/store.ts` | LanceDB ストレージレイヤー。テーブル作成 / FTS インデックス / ベクトル検索 / BM25 検索 / CRUD | +| `src/embedder.ts` | Embedding 抽象レイヤー。任意の OpenAI 互換 API プロバイダーに対応 | +| `src/retriever.ts` | ハイブリッド検索エンジン。ベクトル + BM25 → ハイブリッド融合 → リランク → ライフサイクル減衰 → フィルタ | +| `src/scopes.ts` | マルチスコープアクセス制御 | +| `src/tools.ts` | エージェントツール定義：`memory_recall`、`memory_store`、`memory_forget`、`memory_update` + 管理ツール | +| `src/noise-filter.ts` | エージェントの拒否応答、メタ質問、挨拶などの低品質コンテンツをフィルタリング | +| `src/adaptive-retrieval.ts` | クエリがメモリ検索を必要とするかどうかを判定 | +| `src/migrate.ts` | 内蔵 `memory-lancedb` から Pro へのマイグレーション | +| `src/smart-extractor.ts` | LLM 駆動の6カテゴリ抽出、L0/L1/L2 階層ストレージと2段階重複排除対応 | +| `src/decay-engine.ts` | Weibull 伸長指数関数減衰モデル | +| `src/tier-manager.ts` | 3段階昇格/降格：周辺 ↔ ワーキング ↔ コア | + +
+ +--- + +## コア機能 + +### ハイブリッド検索 + +``` +クエリ → embedQuery() ─┐ + ├─→ ハイブリッド融合 → リランク → ライフサイクル減衰ブースト → 長さ正規化 → フィルタ +クエリ → BM25 全文 ─────┘ +``` + +- **ベクトル検索** — LanceDB ANN によるセマンティック類似度（コサイン距離） +- **BM25 全文検索** — LanceDB FTS インデックスによる正確なキーワードマッチング +- **ハイブリッド融合** — ベクトルスコアをベースに、BM25 ヒットに重み付きブーストを適用（標準 RRF ではなく、実際の再現率品質に最適化） +- **設定可能な重み** — `vectorWeight`、`bm25Weight`、`minScore` + +### クロスエンコーダーリランキング + +- **Jina**、**SiliconFlow**、**Voyage AI**、**Pinecone** の組み込みアダプター +- 任意の Jina 互換エンドポイント（例：Hugging Face TEI、DashScope）に対応 +- ハイブリッドスコアリング：60% クロスエンコーダー + 40% 元の融合スコア +- グレースフルデグラデーション：API 失敗時にコサイン類似度にフォールバック + +### マルチステージスコアリングパイプライン + +| ステージ | 効果 | +| --- | --- | +| **ハイブリッド融合** | セマンティック検索と完全一致検索を統合 | +| **クロスエンコーダーリランク** | セマンティックに正確なヒットを上位に昇格 | +| **ライフサイクル減衰ブースト** | Weibull 鮮度 + アクセス頻度 + 重要度 × 信頼度 | +| **長さ正規化** | 長いエントリが結果を支配するのを防止（アンカー：500文字） | +| **ハード最低スコア** | 無関係な結果を除去（デフォルト：0.35） | +| **MMR 多様性** | コサイン類似度 > 0.85 → 降格 | + +### スマートメモリ抽出（v1.1.0） + +- **LLM 駆動の6カテゴリ抽出**：プロフィール、好み、エンティティ、イベント、ケース、パターン +- **L0/L1/L2 階層ストレージ**：L0（一文の索引）→ L1（構造化サマリー）→ L2（完全な記述） +- **2段階重複排除**：ベクトル類似度プレフィルタ（≥0.7）→ LLM セマンティック判定（CREATE/MERGE/SKIP） +- **カテゴリ対応マージ**：`profile` は常にマージ、`events`/`cases` は追記のみ + +### メモリライフサイクル管理（v1.1.0） + +- **Weibull 減衰エンジン**：複合スコア = 鮮度 + 頻度 + 内在的価値 +- **3段階昇格**：`周辺 ↔ ワーキング ↔ コア`、閾値は設定可能 +- **アクセス強化**：頻繁に想起される記憶はより遅く減衰（間隔反復スタイル） +- **重要度変調半減期**：重要な記憶はより遅く減衰 + +### マルチスコープ分離 + +- 組み込みスコープ：`global`、`agent:`、`custom:`、`project:`、`user:` +- `scopes.agentAccess` によるエージェントレベルのアクセス制御 +- デフォルト：各エージェントが `global` + 自身の `agent:` スコープにアクセス + +### 自動キャプチャ＆自動想起 + +- **自動キャプチャ**（`agent_end`）：会話から好み/事実/決定/エンティティを抽出、重複排除後、1ターンあたり最大3件を保存 +- **自動想起**（`before_agent_start`）：`` コンテキストを注入（最大3件） + +### ノイズフィルタリング＆アダプティブ検索 + +- 低品質コンテンツをフィルタリング：エージェントの拒否応答、メタ質問、挨拶 +- 検索をスキップ：挨拶、スラッシュコマンド、簡単な確認、絵文字 +- 強制検索：メモリキーワード（「覚えている」「以前」「前回」） +- CJK 対応の閾値（中国語：6文字、英語：15文字） + +--- + +
+内蔵 memory-lancedb との比較（クリックで展開） + +| 機能 | 内蔵 `memory-lancedb` | **memory-lancedb-pro** | +| --- | :---: | :---: | +| ベクトル検索 | あり | あり | +| BM25 全文検索 | - | あり | +| ハイブリッド融合（ベクトル + BM25） | - | あり | +| クロスエンコーダーリランク（マルチプロバイダー） | - | あり | +| 鮮度ブースト＆時間減衰 | - | あり | +| 長さ正規化 | - | あり | +| MMR 多様性 | - | あり | +| マルチスコープ分離 | - | あり | +| ノイズフィルタリング | - | あり | +| アダプティブ検索 | - | あり | +| 管理 CLI | - | あり | +| セッションメモリ | - | あり | +| タスク対応 Embedding | - | あり | +| **LLM スマート抽出（6カテゴリ）** | - | あり（v1.1.0） | +| **Weibull 減衰 + 階層昇格** | - | あり（v1.1.0） | +| 任意の OpenAI 互換 Embedding | 限定的 | あり | + +
+ +--- + +## 設定 + +
+完全な設定例 + +```json +{ + "embedding": { + "apiKey": "${JINA_API_KEY}", + "model": "jina-embeddings-v5-text-small", + "baseURL": "https://api.jina.ai/v1", + "dimensions": 1024, + "taskQuery": "retrieval.query", + "taskPassage": "retrieval.passage", + "normalized": true + }, + "dbPath": "~/.openclaw/memory/lancedb-pro", + "autoCapture": true, + "autoRecall": true, + "retrieval": { + "mode": "hybrid", + "vectorWeight": 0.7, + "bm25Weight": 0.3, + "minScore": 0.3, + "rerank": "cross-encoder", + "rerankApiKey": "${JINA_API_KEY}", + "rerankModel": "jina-reranker-v3", + "rerankEndpoint": "https://api.jina.ai/v1/rerank", + "rerankProvider": "jina", + "candidatePoolSize": 20, + "recencyHalfLifeDays": 14, + "recencyWeight": 0.1, + "filterNoise": true, + "lengthNormAnchor": 500, + "hardMinScore": 0.35, + "timeDecayHalfLifeDays": 60, + "reinforcementFactor": 0.5, + "maxHalfLifeMultiplier": 3 + }, + "enableManagementTools": false, + "scopes": { + "default": "global", + "definitions": { + "global": { "description": "Shared knowledge" }, + "agent:discord-bot": { "description": "Discord bot private" } + }, + "agentAccess": { + "discord-bot": ["global", "agent:discord-bot"] + } + }, + "sessionMemory": { + "enabled": false, + "messageCount": 15 + }, + "smartExtraction": true, + "llm": { + "apiKey": "${OPENAI_API_KEY}", + "model": "gpt-4o-mini", + "baseURL": "https://api.openai.com/v1" + }, + "extractMinMessages": 2, + "extractMaxChars": 8000 +} +``` + +
+ +
+Embedding プロバイダー + +**任意の OpenAI 互換 Embedding API** で動作： + +| プロバイダー | モデル | Base URL | 次元数 | +| --- | --- | --- | --- | +| **Jina**（推奨） | `jina-embeddings-v5-text-small` | `https://api.jina.ai/v1` | 1024 | +| **OpenAI** | `text-embedding-3-small` | `https://api.openai.com/v1` | 1536 | +| **Voyage** | `voyage-4-lite` / `voyage-4` | `https://api.voyageai.com/v1` | 1024 / 1024 | +| **Google Gemini** | `gemini-embedding-001` | `https://generativelanguage.googleapis.com/v1beta/openai/` | 3072 | +| **Ollama**（ローカル） | `nomic-embed-text` | `http://localhost:11434/v1` | プロバイダー依存 | + +
+ +
+リランクプロバイダー + +クロスエンコーダーリランキングは `rerankProvider` で複数のプロバイダーをサポート： + +| プロバイダー | `rerankProvider` | モデル例 | +| --- | --- | --- | +| **Jina**（デフォルト） | `jina` | `jina-reranker-v3` | +| **SiliconFlow**（無料枠あり） | `siliconflow` | `BAAI/bge-reranker-v2-m3` | +| **Voyage AI** | `voyage` | `rerank-2.5` | +| **Pinecone** | `pinecone` | `bge-reranker-v2-m3` | + +任意の Jina 互換リランクエンドポイントも使用可能——`rerankProvider: "jina"` を設定し、`rerankEndpoint` をあなたのサービスに向けてください（例：Hugging Face TEI、DashScope `qwen3-rerank`）。 + +
+ +
+スマート抽出（LLM）— v1.1.0 + +`smartExtraction` が有効（デフォルト：`true`）の場合、プラグインは正規表現ベースのトリガーの代わりに LLM を使用してインテリジェントにメモリを抽出・分類します。 + +| フィールド | 型 | デフォルト | 説明 | +|-------|------|---------|-------------| +| `smartExtraction` | boolean | `true` | LLM 駆動の6カテゴリ抽出の有効化/無効化 | +| `llm.auth` | string | `api-key` | `api-key` は `llm.apiKey` / `embedding.apiKey` を使用；`oauth` はデフォルトでプラグインスコープの OAuth トークンファイルを使用 | +| `llm.apiKey` | string | *（`embedding.apiKey` にフォールバック）* | LLM プロバイダーの API キー | +| `llm.model` | string | `openai/gpt-oss-120b` | LLM モデル名 | +| `llm.baseURL` | string | *（`embedding.baseURL` にフォールバック）* | LLM API エンドポイント | +| `llm.oauthProvider` | string | `openai-codex` | `llm.auth` が `oauth` の場合に使用する OAuth プロバイダー ID | +| `llm.oauthPath` | string | `~/.openclaw/.memory-lancedb-pro/oauth.json` | `llm.auth` が `oauth` の場合に使用する OAuth トークンファイル | +| `llm.timeoutMs` | number | `30000` | LLM リクエストタイムアウト（ミリ秒） | +| `extractMinMessages` | number | `2` | 抽出がトリガーされる最小メッセージ数 | +| `extractMaxChars` | number | `8000` | LLM に送信される最大文字数 | + + +OAuth `llm` 設定（既存の Codex / ChatGPT ログインキャッシュを使用して LLM 呼び出しを行う）： +```json +{ + "llm": { + "auth": "oauth", + "oauthProvider": "openai-codex", + "model": "gpt-5.4", + "oauthPath": "${HOME}/.openclaw/.memory-lancedb-pro/oauth.json", + "timeoutMs": 30000 + } +} +``` + +`llm.auth: "oauth"` に関する注意点： + +- `llm.oauthProvider` は現在 `openai-codex` です。 +- OAuth トークンのデフォルト保存先は `~/.openclaw/.memory-lancedb-pro/oauth.json` です。 +- 別の場所に保存したい場合は `llm.oauthPath` を設定してください。 +- `auth login` は OAuth ファイルの隣に以前の api-key モードの `llm` 設定のスナップショットを保存し、`auth logout` は利用可能な場合にそのスナップショットを復元します。 +- `api-key` から `oauth` への切り替え時、`llm.baseURL` は自動的に引き継がれません。意図的にカスタム ChatGPT/Codex 互換バックエンドを使用する場合のみ、OAuth モードで手動設定してください。 + +
+ +
+ライフサイクル設定（減衰 + 階層） + +| フィールド | デフォルト | 説明 | +|-------|---------|-------------| +| `decay.recencyHalfLifeDays` | `30` | Weibull 鮮度減衰のベース半減期 | +| `decay.frequencyWeight` | `0.3` | 複合スコアにおけるアクセス頻度の重み | +| `decay.intrinsicWeight` | `0.3` | `重要度 × 信頼度` の重み | +| `decay.betaCore` | `0.8` | `コア` メモリの Weibull ベータ | +| `decay.betaWorking` | `1.0` | `ワーキング` メモリの Weibull ベータ | +| `decay.betaPeripheral` | `1.3` | `周辺` メモリの Weibull ベータ | +| `tier.coreAccessThreshold` | `10` | `コア` に昇格するために必要な最小想起回数 | +| `tier.peripheralAgeDays` | `60` | 古いメモリを降格するための日数閾値 | + +
+ +
+アクセス強化 + +頻繁に想起されるメモリはより遅く減衰します（間隔反復スタイル）。 + +設定キー（`retrieval` 内）： +- `reinforcementFactor`（0-2、デフォルト：`0.5`）— `0` に設定すると無効化 +- `maxHalfLifeMultiplier`（1-10、デフォルト：`3`）— 実効半減期のハードキャップ + +
+ +--- + +## CLI コマンド + +```bash +openclaw memory-pro list [--scope global] [--category fact] [--limit 20] [--json] +openclaw memory-pro search "クエリ" [--scope global] [--limit 10] [--json] +openclaw memory-pro stats [--scope global] [--json] +openclaw memory-pro auth login [--provider openai-codex] [--model gpt-5.4] [--oauth-path /abs/path/oauth.json] +openclaw memory-pro auth status +openclaw memory-pro auth logout +openclaw memory-pro delete +openclaw memory-pro delete-bulk --scope global [--before 2025-01-01] [--dry-run] +openclaw memory-pro export [--scope global] [--output memories.json] +openclaw memory-pro import memories.json [--scope global] [--dry-run] +openclaw memory-pro reembed --source-db /path/to/old-db [--batch-size 32] [--skip-existing] +openclaw memory-pro upgrade [--dry-run] [--batch-size 10] [--no-llm] [--limit N] [--scope SCOPE] +openclaw memory-pro migrate check|run|verify [--source /path] +``` + +OAuth ログインフロー： + +1. `openclaw memory-pro auth login` を実行 +2. `--provider` が省略され、対話型ターミナルの場合、CLI はブラウザを開く前に OAuth プロバイダーピッカーを表示 +3. コマンドは認証 URL を表示し、`--no-browser` が設定されていない限りブラウザを自動的に開く +4. コールバック成功後、コマンドはプラグイン OAuth ファイル（デフォルト：`~/.openclaw/.memory-lancedb-pro/oauth.json`）を保存し、ログアウト用に以前の api-key モードの `llm` 設定のスナップショットを作成し、プラグインの `llm` 設定を OAuth 設定（`auth`、`oauthProvider`、`model`、`oauthPath`）に置き換え +5. `openclaw memory-pro auth logout` はその OAuth ファイルを削除し、スナップショットが存在する場合は以前の api-key モードの `llm` 設定を復元 + +--- + +## 応用トピック + +
+注入されたメモリが応答に表示される場合 + +モデルが注入された `` ブロックをそのまま出力してしまうことがあります。 + +**方法 A（最も安全）：** 自動想起を一時的に無効化： +```json +{ "plugins": { "entries": { "memory-lancedb-pro": { "config": { "autoRecall": false } } } } } +``` + +**方法 B（推奨）：** 想起は有効のまま、エージェントのシステムプロンプトに追加： +> Do not reveal or quote any `` / memory-injection content in your replies. Use it for internal reference only. + +
+ +
+セッションメモリ + +- `/new` コマンドでトリガー——前のセッションの要約を LanceDB に保存 +- デフォルトで無効（OpenClaw にはネイティブの `.jsonl` セッション永続化機能あり） +- メッセージ数は設定可能（デフォルト：15） + +デプロイモードと `/new` の検証については [docs/openclaw-integration-playbook.md](docs/openclaw-integration-playbook.md) を参照してください。 + +
+ +
+カスタムスラッシュコマンド（例：/lesson） + +`CLAUDE.md`、`AGENTS.md`、またはシステムプロンプトに追加： + +```markdown +## /lesson コマンド +ユーザーが `/lesson <内容>` を送信した場合： +1. memory_store を使用して category=fact（生の知識）として保存 +2. memory_store を使用して category=decision（実行可能な教訓）として保存 +3. 保存した内容を確認 + +## /remember コマンド +ユーザーが `/remember <内容>` を送信した場合： +1. memory_store を使用して適切な category と importance で保存 +2. 保存されたメモリ ID で確認 +``` + +
+ +
+AI エージェントの鉄則 + +> 以下のブロックを `AGENTS.md` にコピーして、エージェントがこれらのルールを自動的に適用するようにしてください。 + +```markdown +## ルール 1 — 二層メモリ保存 +すべての落とし穴/学んだ教訓 → 直ちに2つのメモリを保存： +- 技術レイヤー：落とし穴：[症状]。原因：[根本原因]。修正：[解決策]。予防：[回避方法] + (category: fact, importance >= 0.8) +- 原則レイヤー：意思決定原則 ([タグ])：[行動ルール]。トリガー：[いつ]。アクション：[何をする] + (category: decision, importance >= 0.85) + +## ルール 2 — LanceDB データ品質 +エントリは短くアトミックに（500文字未満）。生の会話要約や重複は保存しない。 + +## ルール 3 — リトライ前に想起 +いかなるツール失敗時も、リトライする前に必ず関連キーワードで memory_recall を実行。 + +## ルール 4 — 対象コードベースの確認 +変更前に、操作対象が memory-lancedb-pro なのか内蔵 memory-lancedb なのかを確認。 + +## ルール 5 — プラグインコード変更後に jiti キャッシュをクリア +plugins/ 配下の .ts ファイルを変更した後、openclaw gateway restart の前に必ず rm -rf /tmp/jiti/ を実行。 +``` + +
+ +
+データベーススキーマ + +LanceDB テーブル `memories`： + +| フィールド | 型 | 説明 | +| --- | --- | --- | +| `id` | string (UUID) | 主キー | +| `text` | string | メモリテキスト（FTS インデックス付き） | +| `vector` | float[] | Embedding ベクトル | +| `category` | string | ストレージカテゴリ：`preference` / `fact` / `decision` / `entity` / `reflection` / `other` | +| `scope` | string | スコープ識別子（例：`global`、`agent:main`） | +| `importance` | float | 重要度スコア 0-1 | +| `timestamp` | int64 | 作成タイムスタンプ（ミリ秒） | +| `metadata` | string (JSON) | 拡張メタデータ | + +v1.1.0 の一般的な `metadata` キー：`l0_abstract`、`l1_overview`、`l2_content`、`memory_category`、`tier`、`access_count`、`confidence`、`last_accessed_at` + +> **カテゴリに関する注意：** トップレベルの `category` フィールドは6つのストレージカテゴリを使用します。スマート抽出の6カテゴリセマンティックラベル（`profile` / `preferences` / `entities` / `events` / `cases` / `patterns`）は `metadata.memory_category` に保存されます。 + +
+ +
+トラブルシューティング + +### "Cannot mix BigInt and other types"（LanceDB / Apache Arrow） + +LanceDB 0.26+ では、一部の数値カラムが `BigInt` として返されることがあります。**memory-lancedb-pro >= 1.0.14** にアップグレードしてください——プラグインは算術演算の前に `Number(...)` で値を変換するようになっています。 + +
+ +--- + +## ドキュメント + +| ドキュメント | 説明 | +| --- | --- | +| [OpenClaw 統合プレイブック](docs/openclaw-integration-playbook.md) | デプロイモード、検証、リグレッションマトリックス | +| [メモリアーキテクチャ分析](docs/memory_architecture_analysis.md) | 完全なアーキテクチャ詳細解説 | +| [CHANGELOG v1.1.0](docs/CHANGELOG-v1.1.0.md) | v1.1.0 の動作変更とアップグレード根拠 | +| [ロングコンテキストチャンキング](docs/long-context-chunking.md) | 長文ドキュメントのチャンキング戦略 | + +--- + +## Beta：スマートメモリ v1.1.0 + +> ステータス：Beta——`npm i memory-lancedb-pro@beta` でインストール可能。`latest` を使用している安定版ユーザーには影響しません。 + +| 機能 | 説明 | +|---------|-------------| +| **スマート抽出** | LLM 駆動の6カテゴリ抽出、L0/L1/L2 メタデータ対応。無効時は正規表現にフォールバック。 | +| **ライフサイクルスコアリング** | Weibull 減衰を検索に統合——高頻度・高重要度のメモリが上位にランク。 | +| **階層管理** | 3段階システム（コア → ワーキング → 周辺）、自動昇格/降格。 | + +フィードバック：[GitHub Issues](https://github.com/CortexReach/memory-lancedb-pro/issues) · 元に戻す：`npm i memory-lancedb-pro@latest` + +--- + +## 依存関係 + +| パッケージ | 用途 | +| --- | --- | +| `@lancedb/lancedb` ≥0.26.2 | ベクトルデータベース（ANN + FTS） | +| `openai` ≥6.21.0 | OpenAI 互換 Embedding API クライアント | +| `@sinclair/typebox` 0.34.48 | JSON Schema 型定義 | + +--- + +## コントリビューター + +

+@win4r +@kctony +@Akatsuki-Ryu +@JasonSuz +@Minidoracat +@furedericca-lab +@joe2643 +@AliceLJY +@chenjiyong +

+ +全リスト：[Contributors](https://github.com/CortexReach/memory-lancedb-pro/graphs/contributors) + +## Star 履歴 + + + + + + Star History Chart + + + +## ライセンス + +MIT + +--- + +## WeChat QR コード + + diff --git a/README_KO.md b/README_KO.md new file mode 100644 index 00000000..c8f165e6 --- /dev/null +++ b/README_KO.md @@ -0,0 +1,773 @@ +
+ +# 🧠 memory-lancedb-pro · 🦞OpenClaw Plugin + +**[OpenClaw](https://github.com/openclaw/openclaw) 에이전트를 위한 AI 메모리 어시스턴트** + +*AI 에이전트에게 진짜 기억하는 두뇌를 선물하세요 — 세션을 넘어, 에이전트를 넘어, 시간을 넘어.* + +LanceDB 기반 OpenClaw 메모리 플러그인으로, 사용자 선호도·의사결정·프로젝트 맥락을 저장하고 이후 세션에서 자동으로 불러옵니다. + +[![OpenClaw Plugin](https://img.shields.io/badge/OpenClaw-Plugin-blue)](https://github.com/openclaw/openclaw) +[![npm version](https://img.shields.io/npm/v/memory-lancedb-pro)](https://www.npmjs.com/package/memory-lancedb-pro) +[![LanceDB](https://img.shields.io/badge/LanceDB-Vectorstore-orange)](https://lancedb.com) +[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE) + +[English](README.md) | [简体中文](README_CN.md) | [繁體中文](README_TW.md) | [日本語](README_JA.md) | [한국어](README_KO.md) | [Français](README_FR.md) | [Español](README_ES.md) | [Deutsch](README_DE.md) | [Italiano](README_IT.md) | [Русский](README_RU.md) | [Português (Brasil)](README_PT-BR.md) + +
+ +--- + +## 왜 memory-lancedb-pro인가? + +대부분의 AI 에이전트는 건망증이 있습니다. 새 채팅을 시작하는 순간 모든 것을 잊어버립니다. + +**memory-lancedb-pro**는 OpenClaw를 위한 프로덕션 수준의 장기 기억 플러그인으로, 에이전트를 **AI 메모리 어시스턴트**로 바꿔줍니다 — 중요한 내용을 자동으로 캡처하고, 노이즈는 자연스럽게 희미해지게 하며, 적시에 적절한 기억을 검색합니다. 수동 태그 지정도, 복잡한 설정도 필요 없습니다. + +### AI 메모리 어시스턴트 실제 사용 모습 + +**기억 없이 — 매 세션이 처음부터 시작:** + +> **사용자:** "들여쓰기에 탭을 사용하고, 항상 에러 처리를 추가해." +> *(다음 세션)* +> **사용자:** "이미 말했잖아 — 스페이스 말고 탭이라고!" 😤 +> *(다음 세션)* +> **사용자:** "...진짜로, 탭이라고. 에러 처리도. 또." + +**memory-lancedb-pro와 함께 — 에이전트가 학습하고 기억합니다:** + +> **사용자:** "들여쓰기에 탭을 사용하고, 항상 에러 처리를 추가해." +> *(다음 세션 — 에이전트가 사용자 선호도를 자동으로 불러옴)* +> **에이전트:** *(자동으로 탭 + 에러 처리 적용)* ✅ +> **사용자:** "지난달에 왜 MongoDB 대신 PostgreSQL을 선택했지?" +> **에이전트:** "2월 12일 논의 내용에 따르면, 주요 이유는..." ✅ + +이것이 **AI 메모리 어시스턴트**가 만드는 차이입니다 — 사용자의 스타일을 학습하고, 과거 결정을 불러오며, 반복 없이 개인화된 응답을 제공합니다. + +### 그 외 무엇을 할 수 있나요? + +| | 제공 기능 | +|---|---| +| **Auto-Capture** | 에이전트가 모든 대화에서 학습 — 수동 `memory_store` 불필요 | +| **Smart Extraction** | LLM 기반 6개 카테고리 분류: profile, preferences, entities, events, cases, patterns | +| **Intelligent Forgetting** | Weibull 감쇠 모델 — 중요한 기억은 유지, 노이즈는 자연스럽게 사라짐 | +| **Hybrid Retrieval** | 벡터 + BM25 전문 검색, Cross-Encoder 리랭킹으로 융합 | +| **Context Injection** | 관련 기억이 매 응답 전에 자동으로 불러와짐 | +| **Multi-Scope Isolation** | 에이전트별, 사용자별, 프로젝트별 메모리 경계 | +| **Any Provider** | OpenAI, Jina, Gemini, Ollama 또는 OpenAI 호환 API 모두 지원 | +| **Full Toolkit** | CLI, 백업, 마이그레이션, 업그레이드, 내보내기/가져오기 — 프로덕션 환경에 적합 | + +--- + +## 빠른 시작 + +### 옵션 A: 원클릭 설치 스크립트 (권장) + +커뮤니티에서 관리하는 **[설치 스크립트](https://github.com/CortexReach/toolbox/tree/main/memory-lancedb-pro-setup)**가 설치, 업그레이드, 복구를 하나의 명령어로 처리합니다: + +```bash +curl -fsSL https://raw.githubusercontent.com/CortexReach/toolbox/main/memory-lancedb-pro-setup/setup-memory.sh -o setup-memory.sh +bash setup-memory.sh +``` + +> 스크립트가 다루는 전체 시나리오와 기타 커뮤니티 도구 목록은 아래 [에코시스템](#에코시스템)을 참조하세요. + +### 옵션 B: 수동 설치 + +**OpenClaw CLI를 통한 설치 (권장):** +```bash +openclaw plugins install memory-lancedb-pro@beta +``` + +**또는 npm을 통한 설치:** +```bash +npm i memory-lancedb-pro@beta +``` +> npm을 사용하는 경우, `openclaw.json`의 `plugins.load.paths`에 플러그인 설치 디렉터리의 **절대** 경로를 추가해야 합니다. 이것이 가장 흔한 설정 문제입니다. + +`openclaw.json`에 다음을 추가하세요: + +```json +{ + "plugins": { + "slots": { "memory": "memory-lancedb-pro" }, + "entries": { + "memory-lancedb-pro": { + "enabled": true, + "config": { + "embedding": { + "provider": "openai-compatible", + "apiKey": "${OPENAI_API_KEY}", + "model": "text-embedding-3-small" + }, + "autoCapture": true, + "autoRecall": true, + "smartExtraction": true, + "extractMinMessages": 2, + "extractMaxChars": 8000, + "sessionMemory": { "enabled": false } + } + } + } + } +} +``` + +**왜 이러한 기본값인가?** +- `autoCapture` + `smartExtraction` → 에이전트가 모든 대화에서 자동으로 학습 +- `autoRecall` → 매 응답 전에 관련 기억이 주입됨 +- `extractMinMessages: 2` → 일반적인 두 턴 대화에서 추출이 시작됨 +- `sessionMemory.enabled: false` → 초기에 세션 요약으로 검색이 오염되는 것을 방지 + +검증 및 재시작: + +```bash +openclaw config validate +openclaw gateway restart +openclaw logs --follow --plain | grep "memory-lancedb-pro" +``` + +다음이 표시되어야 합니다: +- `memory-lancedb-pro: smart extraction enabled` +- `memory-lancedb-pro@...: plugin registered` + +완료! 이제 에이전트가 장기 기억을 갖게 됩니다. + +
+추가 설치 경로 (기존 사용자, 업그레이드) + +**이미 OpenClaw를 사용 중인 경우:** + +1. **절대** 경로의 `plugins.load.paths` 항목으로 플러그인 추가 +2. 메모리 슬롯 바인딩: `plugins.slots.memory = "memory-lancedb-pro"` +3. 확인: `openclaw plugins info memory-lancedb-pro && openclaw memory-pro stats` + +**v1.1.0 이전 버전에서 업그레이드하는 경우:** + +```bash +# 1) 백업 +openclaw memory-pro export --scope global --output memories-backup.json +# 2) 시뮬레이션 실행 +openclaw memory-pro upgrade --dry-run +# 3) 업그레이드 실행 +openclaw memory-pro upgrade +# 4) 확인 +openclaw memory-pro stats +``` + +동작 변경사항과 업그레이드 근거는 `CHANGELOG-v1.1.0.md`를 참조하세요. + +
+ +
+Telegram 봇 빠른 가져오기 (클릭하여 펼치기) + +OpenClaw의 Telegram 연동을 사용하는 경우, 수동으로 설정을 편집하는 대신 메인 봇에 가져오기 명령어를 직접 보내는 것이 가장 쉬운 방법입니다. + +다음 메시지를 전송하세요 (봇에 그대로 복사하여 붙여넣기하는 영문 프롬프트입니다): + +```text +Help me connect this memory plugin with the most user-friendly configuration: https://github.com/CortexReach/memory-lancedb-pro + +Requirements: +1. Set it as the only active memory plugin +2. Use Jina for embedding +3. Use Jina for reranker +4. Use gpt-4o-mini for the smart-extraction LLM +5. Enable autoCapture, autoRecall, smartExtraction +6. extractMinMessages=2 +7. sessionMemory.enabled=false +8. captureAssistant=false +9. retrieval mode=hybrid, vectorWeight=0.7, bm25Weight=0.3 +10. rerank=cross-encoder, candidatePoolSize=12, minScore=0.6, hardMinScore=0.62 +11. Generate the final openclaw.json config directly, not just an explanation +``` + +
+ +--- + +## 에코시스템 + +memory-lancedb-pro는 핵심 플러그인입니다. 커뮤니티에서 설정과 일상적인 사용을 더욱 원활하게 만드는 도구들을 구축했습니다: + +### 설치 스크립트 — 원클릭 설치, 업그레이드 및 복구 + +> **[CortexReach/toolbox/memory-lancedb-pro-setup](https://github.com/CortexReach/toolbox/tree/main/memory-lancedb-pro-setup)** + +단순한 인스톨러가 아닙니다 — 스크립트가 다양한 실제 시나리오를 지능적으로 처리합니다: + +| 상황 | 스크립트의 동작 | +|---|---| +| 설치한 적 없음 | 새로 다운로드 → 의존성 설치 → 설정 선택 → openclaw.json에 기록 → 재시작 | +| `git clone`으로 설치, 이전 커밋에서 멈춤 | 자동 `git fetch` + `checkout`으로 최신 버전 이동 → 의존성 재설치 → 확인 | +| 설정에 유효하지 않은 필드 존재 | 스키마 필터를 통한 자동 감지, 지원되지 않는 필드 제거 | +| `npm`으로 설치 | git 업데이트 건너뜀, `npm update` 직접 실행 알림 | +| 유효하지 않은 설정으로 `openclaw` CLI 동작 불가 | 대체 방법: `openclaw.json` 파일에서 직접 워크스페이스 경로 읽기 | +| `plugins/` 대신 `extensions/` 사용 | 설정 또는 파일시스템에서 플러그인 위치 자동 감지 | +| 이미 최신 상태 | 상태 확인만 실행, 변경 없음 | + +```bash +bash setup-memory.sh # 설치 또는 업그레이드 +bash setup-memory.sh --dry-run # 미리보기만 +bash setup-memory.sh --beta # 사전 릴리스 버전 포함 +bash setup-memory.sh --uninstall # 설정 복원 및 플러그인 제거 +``` + +내장 프로바이더 프리셋: **Jina / DashScope / SiliconFlow / OpenAI / Ollama**, 또는 자체 OpenAI 호환 API를 사용할 수 있습니다. `--ref`, `--selfcheck-only` 등 전체 사용법은 [설치 스크립트 README](https://github.com/CortexReach/toolbox/tree/main/memory-lancedb-pro-setup)를 참조하세요. + +### Claude Code / OpenClaw Skill — AI 가이드 설정 + +> **[CortexReach/memory-lancedb-pro-skill](https://github.com/CortexReach/memory-lancedb-pro-skill)** + +이 Skill을 설치하면 AI 에이전트(Claude Code 또는 OpenClaw)가 memory-lancedb-pro의 모든 기능에 대한 깊은 지식을 갖게 됩니다. **"최적의 설정을 도와줘"**라고 말하면 다음을 제공합니다: + +- **가이드 7단계 설정 워크플로우**와 4가지 배포 계획: + - Full Power (Jina + OpenAI) / Budget (무료 SiliconFlow 리랭커) / Simple (OpenAI만) / Fully Local (Ollama, API 비용 제로) +- **모든 9개 MCP 도구**의 올바른 사용법: `memory_recall`, `memory_store`, `memory_forget`, `memory_update`, `memory_stats`, `memory_list`, `self_improvement_log`, `self_improvement_extract_skill`, `self_improvement_review` *(전체 도구 세트를 사용하려면 `enableManagementTools: true`가 필요합니다 — 기본 빠른 시작 설정은 4개 핵심 도구만 노출합니다)* +- **일반적인 함정 방지**: 워크스페이스 플러그인 활성화, `autoRecall` 기본값 false, jiti 캐시, 환경 변수, 스코프 격리 등 + +**Claude Code용 설치:** +```bash +git clone https://github.com/CortexReach/memory-lancedb-pro-skill.git ~/.claude/skills/memory-lancedb-pro +``` + +**OpenClaw용 설치:** +```bash +git clone https://github.com/CortexReach/memory-lancedb-pro-skill.git ~/.openclaw/workspace/skills/memory-lancedb-pro-skill +``` + +--- + +## 비디오 튜토리얼 + +> 전체 안내: 설치, 설정, 하이브리드 검색 내부 구조. + +[![YouTube Video](https://img.shields.io/badge/YouTube-Watch%20Now-red?style=for-the-badge&logo=youtube)](https://youtu.be/MtukF1C8epQ) +**https://youtu.be/MtukF1C8epQ** + +[![Bilibili Video](https://img.shields.io/badge/Bilibili-Watch%20Now-00A1D6?style=for-the-badge&logo=bilibili&logoColor=white)](https://www.bilibili.com/video/BV1zUf2BGEgn/) +**https://www.bilibili.com/video/BV1zUf2BGEgn/** + +--- + +## 아키텍처 + +``` +┌─────────────────────────────────────────────────────────┐ +│ index.ts (진입점) │ +│ 플러그인 등록 · 설정 파싱 · 라이프사이클 훅 │ +└────────┬──────────┬──────────┬──────────┬───────────────┘ + │ │ │ │ + ┌────▼───┐ ┌────▼───┐ ┌───▼────┐ ┌──▼──────────┐ + │ store │ │embedder│ │retriever│ │ scopes │ + │ .ts │ │ .ts │ │ .ts │ │ .ts │ + └────────┘ └────────┘ └────────┘ └─────────────┘ + │ │ + ┌────▼───┐ ┌─────▼──────────┐ + │migrate │ │noise-filter.ts │ + │ .ts │ │adaptive- │ + └────────┘ │retrieval.ts │ + └────────────────┘ + ┌─────────────┐ ┌──────────┐ + │ tools.ts │ │ cli.ts │ + │ (에이전트API)│ │ (CLI) │ + └─────────────┘ └──────────┘ +``` + +> 전체 아키텍처에 대한 심층 분석은 [docs/memory_architecture_analysis.md](docs/memory_architecture_analysis.md)를 참조하세요. + +
+파일 레퍼런스 (클릭하여 펼치기) + +| 파일 | 용도 | +| --- | --- | +| `index.ts` | 플러그인 진입점. OpenClaw Plugin API에 등록, 설정 파싱, 라이프사이클 훅 마운트 | +| `openclaw.plugin.json` | 플러그인 메타데이터 + 전체 JSON Schema 설정 선언 | +| `cli.ts` | CLI 명령어: `memory-pro list/search/stats/delete/delete-bulk/export/import/reembed/upgrade/migrate` | +| `src/store.ts` | LanceDB 스토리지 레이어. 테이블 생성 / FTS 인덱싱 / 벡터 검색 / BM25 검색 / CRUD | +| `src/embedder.ts` | 임베딩 추상화. OpenAI 호환 API 프로바이더 모두 지원 | +| `src/retriever.ts` | 하이브리드 검색 엔진. 벡터 + BM25 → 하이브리드 퓨전 → 리랭크 → 라이프사이클 감쇠 → 필터 | +| `src/scopes.ts` | 멀티 스코프 접근 제어 | +| `src/tools.ts` | 에이전트 도구 정의: `memory_recall`, `memory_store`, `memory_forget`, `memory_update` + 관리 도구 | +| `src/noise-filter.ts` | 에이전트 거절, 메타 질문, 인사, 저품질 콘텐츠 필터링 | +| `src/adaptive-retrieval.ts` | 쿼리에 메모리 검색이 필요한지 판단 | +| `src/migrate.ts` | 내장 `memory-lancedb`에서 Pro로의 마이그레이션 | +| `src/smart-extractor.ts` | LLM 기반 6개 카테고리 추출 + L0/L1/L2 계층 저장 + 2단계 중복 제거 | +| `src/decay-engine.ts` | Weibull 확장 지수 감쇠 모델 | +| `src/tier-manager.ts` | 3단계 승격/강등: Peripheral ↔ Working ↔ Core | + +
+ +--- + +## 핵심 기능 + +### 하이브리드 검색 + +``` +Query → embedQuery() ─┐ + ├─→ Hybrid Fusion → Rerank → Lifecycle Decay Boost → Length Norm → Filter +Query → BM25 FTS ─────┘ +``` + +- **벡터 검색** — LanceDB ANN을 통한 의미적 유사도 (코사인 거리) +- **BM25 전문 검색** — LanceDB FTS 인덱스를 통한 정확한 키워드 매칭 +- **하이브리드 퓨전** — 벡터 스코어를 기본으로, BM25 히트에 가중 부스트 적용 (표준 RRF가 아님 — 실제 검색 품질에 맞게 튜닝됨) +- **가중치 설정 가능** — `vectorWeight`, `bm25Weight`, `minScore` + +### Cross-Encoder 리랭킹 + +- **Jina**, **SiliconFlow**, **Voyage AI**, **Pinecone** 내장 어댑터 +- Jina 호환 엔드포인트와 호환 (예: Hugging Face TEI, DashScope) +- 하이브리드 스코어링: Cross-Encoder 60% + 원래 퓨전 스코어 40% +- 그레이스풀 디그레이데이션: API 실패 시 코사인 유사도로 폴백 + +### 다단계 스코어링 파이프라인 + +| 단계 | 효과 | +| --- | --- | +| **하이브리드 퓨전** | 의미적 검색과 정확한 매칭 결합 | +| **Cross-Encoder 리랭크** | 의미적으로 정확한 결과 승격 | +| **라이프사이클 감쇠 부스트** | Weibull 최신성 + 접근 빈도 + 중요도 × 신뢰도 | +| **길이 정규화** | 긴 항목이 결과를 지배하는 것을 방지 (앵커: 500자) | +| **최소 점수 하한** | 관련 없는 결과 제거 (기본값: 0.35) | +| **MMR 다양성** | 코사인 유사도 > 0.85 → 강등 | + +### Smart Memory Extraction (v1.1.0) + +- **LLM 기반 6개 카테고리 추출**: profile, preferences, entities, events, cases, patterns +- **L0/L1/L2 계층 저장**: L0 (한 줄 인덱스) → L1 (구조화된 요약) → L2 (전체 내러티브) +- **2단계 중복 제거**: 벡터 유사도 사전 필터 (≥0.7) → LLM 의미 판단 (CREATE/MERGE/SKIP) +- **카테고리 인식 병합**: `profile`은 항상 병합, `events`/`cases`는 추가 전용 + +### 메모리 라이프사이클 관리 (v1.1.0) + +- **Weibull 감쇠 엔진**: 복합 점수 = 최신성 + 빈도 + 내재적 가치 +- **3단계 승격**: `Peripheral ↔ Working ↔ Core`, 설정 가능한 임계값 +- **접근 강화**: 자주 불러오는 기억은 더 느리게 감쇠 (간격 반복 학습 방식) +- **중요도 조절 반감기**: 중요한 기억은 더 느리게 감쇠 + +### Multi-Scope 격리 + +- 내장 스코프: `global`, `agent:`, `custom:`, `project:`, `user:` +- `scopes.agentAccess`를 통한 에이전트 수준 접근 제어 +- 기본값: 각 에이전트가 `global` + 자체 `agent:` 스코프에 접근 + +### Auto-Capture 및 Auto-Recall + +- **Auto-Capture** (`agent_end`): 대화에서 선호도/사실/결정/엔티티를 추출, 중복 제거, 턴당 최대 3개 저장 +- **Auto-Recall** (`before_agent_start`): `` 컨텍스트 주입 (최대 3개 항목) + +### 노이즈 필터링 및 적응형 검색 + +- 저품질 콘텐츠 필터링: 에이전트 거절, 메타 질문, 인사 +- 인사, 슬래시 명령어, 간단한 확인, 이모지에 대해서는 검색 건너뜀 +- 기억 키워드에 대해서는 검색 강제 실행 ("기억해", "이전에", "지난번에") +- CJK 인식 임계값 (중국어: 6자 vs 영어: 15자) + +--- + +
+내장 memory-lancedb와의 비교 (클릭하여 펼치기) + +| 기능 | 내장 `memory-lancedb` | **memory-lancedb-pro** | +| --- | :---: | :---: | +| 벡터 검색 | 예 | 예 | +| BM25 전문 검색 | - | 예 | +| 하이브리드 퓨전 (벡터 + BM25) | - | 예 | +| Cross-Encoder 리랭크 (멀티 프로바이더) | - | 예 | +| 최신성 부스트 및 시간 감쇠 | - | 예 | +| 길이 정규화 | - | 예 | +| MMR 다양성 | - | 예 | +| 멀티 스코프 격리 | - | 예 | +| 노이즈 필터링 | - | 예 | +| 적응형 검색 | - | 예 | +| 관리 CLI | - | 예 | +| 세션 메모리 | - | 예 | +| 태스크 인식 임베딩 | - | 예 | +| **LLM Smart Extraction (6개 카테고리)** | - | 예 (v1.1.0) | +| **Weibull 감쇠 + 단계 승격** | - | 예 (v1.1.0) | +| OpenAI 호환 임베딩 | 제한적 | 예 | + +
+ +--- + +## 설정 + +
+전체 설정 예시 + +```json +{ + "embedding": { + "apiKey": "${JINA_API_KEY}", + "model": "jina-embeddings-v5-text-small", + "baseURL": "https://api.jina.ai/v1", + "dimensions": 1024, + "taskQuery": "retrieval.query", + "taskPassage": "retrieval.passage", + "normalized": true + }, + "dbPath": "~/.openclaw/memory/lancedb-pro", + "autoCapture": true, + "autoRecall": true, + "retrieval": { + "mode": "hybrid", + "vectorWeight": 0.7, + "bm25Weight": 0.3, + "minScore": 0.3, + "rerank": "cross-encoder", + "rerankApiKey": "${JINA_API_KEY}", + "rerankModel": "jina-reranker-v3", + "rerankEndpoint": "https://api.jina.ai/v1/rerank", + "rerankProvider": "jina", + "candidatePoolSize": 20, + "recencyHalfLifeDays": 14, + "recencyWeight": 0.1, + "filterNoise": true, + "lengthNormAnchor": 500, + "hardMinScore": 0.35, + "timeDecayHalfLifeDays": 60, + "reinforcementFactor": 0.5, + "maxHalfLifeMultiplier": 3 + }, + "enableManagementTools": false, + "scopes": { + "default": "global", + "definitions": { + "global": { "description": "Shared knowledge" }, + "agent:discord-bot": { "description": "Discord bot private" } + }, + "agentAccess": { + "discord-bot": ["global", "agent:discord-bot"] + } + }, + "sessionMemory": { + "enabled": false, + "messageCount": 15 + }, + "smartExtraction": true, + "llm": { + "apiKey": "${OPENAI_API_KEY}", + "model": "gpt-4o-mini", + "baseURL": "https://api.openai.com/v1" + }, + "extractMinMessages": 2, + "extractMaxChars": 8000 +} +``` + +
+ +
+임베딩 프로바이더 + +**OpenAI 호환 임베딩 API**와 모두 동작합니다: + +| 프로바이더 | 모델 | Base URL | 차원 | +| --- | --- | --- | --- | +| **Jina** (권장) | `jina-embeddings-v5-text-small` | `https://api.jina.ai/v1` | 1024 | +| **OpenAI** | `text-embedding-3-small` | `https://api.openai.com/v1` | 1536 | +| **Voyage** | `voyage-4-lite` / `voyage-4` | `https://api.voyageai.com/v1` | 1024 / 1024 | +| **Google Gemini** | `gemini-embedding-001` | `https://generativelanguage.googleapis.com/v1beta/openai/` | 3072 | +| **Ollama** (로컬) | `nomic-embed-text` | `http://localhost:11434/v1` | 프로바이더별 상이 | + +
+ +
+리랭크 프로바이더 + +Cross-Encoder 리랭킹은 `rerankProvider`를 통해 여러 프로바이더를 지원합니다: + +| 프로바이더 | `rerankProvider` | 예시 모델 | +| --- | --- | --- | +| **Jina** (기본값) | `jina` | `jina-reranker-v3` | +| **SiliconFlow** (무료 티어 제공) | `siliconflow` | `BAAI/bge-reranker-v2-m3` | +| **Voyage AI** | `voyage` | `rerank-2.5` | +| **Pinecone** | `pinecone` | `bge-reranker-v2-m3` | + +Jina 호환 리랭크 엔드포인트도 사용 가능합니다 — `rerankProvider: "jina"`로 설정하고 `rerankEndpoint`를 해당 서비스로 지정하세요 (예: Hugging Face TEI, DashScope `qwen3-rerank`). + +
+ +
+Smart Extraction (LLM) — v1.1.0 + +`smartExtraction`이 활성화되면 (기본값: `true`), 플러그인이 정규식 기반 트리거 대신 LLM을 사용하여 기억을 지능적으로 추출하고 분류합니다. + +| 필드 | 타입 | 기본값 | 설명 | +|-------|------|---------|-------------| +| `smartExtraction` | boolean | `true` | LLM 기반 6개 카테고리 추출 활성화/비활성화 | +| `llm.auth` | string | `api-key` | `api-key`는 `llm.apiKey` / `embedding.apiKey`를 사용; `oauth`는 기본적으로 플러그인 범위의 OAuth 토큰 파일을 사용 | +| `llm.apiKey` | string | *(`embedding.apiKey`로 폴백)* | LLM 프로바이더용 API 키 | +| `llm.model` | string | `openai/gpt-oss-120b` | LLM 모델명 | +| `llm.baseURL` | string | *(`embedding.baseURL`로 폴백)* | LLM API 엔드포인트 | +| `llm.oauthProvider` | string | `openai-codex` | `llm.auth`가 `oauth`일 때 사용되는 OAuth 프로바이더 ID | +| `llm.oauthPath` | string | `~/.openclaw/.memory-lancedb-pro/oauth.json` | `llm.auth`가 `oauth`일 때 사용되는 OAuth 토큰 파일 | +| `llm.timeoutMs` | number | `30000` | LLM 요청 타임아웃 (밀리초) | +| `extractMinMessages` | number | `2` | 추출이 시작되는 최소 메시지 수 | +| `extractMaxChars` | number | `8000` | LLM에 전송되는 최대 문자 수 | + + +OAuth `llm` 설정 (기존 Codex / ChatGPT 로그인 캐시를 LLM 호출에 사용): +```json +{ + "llm": { + "auth": "oauth", + "oauthProvider": "openai-codex", + "model": "gpt-5.4", + "oauthPath": "${HOME}/.openclaw/.memory-lancedb-pro/oauth.json", + "timeoutMs": 30000 + } +} +``` + +`llm.auth: "oauth"` 참고사항: + +- `llm.oauthProvider`는 현재 `openai-codex`입니다. +- OAuth 토큰은 기본적으로 `~/.openclaw/.memory-lancedb-pro/oauth.json`에 저장됩니다. +- 파일을 다른 곳에 저장하려면 `llm.oauthPath`를 설정하세요. +- `auth login`은 OAuth 파일 옆에 이전 api-key `llm` 설정의 스냅샷을 저장하며, `auth logout`은 해당 스냅샷이 있을 때 복원합니다. +- `api-key`에서 `oauth`로 전환할 때 `llm.baseURL`이 자동으로 이전되지 않습니다. OAuth 모드에서 의도적으로 사용자 정의 ChatGPT/Codex 호환 백엔드를 원하는 경우에만 수동으로 설정하세요. + +
+ +
+라이프사이클 설정 (감쇠 + 단계) + +| 필드 | 기본값 | 설명 | +|-------|---------|-------------| +| `decay.recencyHalfLifeDays` | `30` | Weibull 최신성 감쇠의 기본 반감기 | +| `decay.frequencyWeight` | `0.3` | 복합 점수에서 접근 빈도의 가중치 | +| `decay.intrinsicWeight` | `0.3` | `importance × confidence`의 가중치 | +| `decay.betaCore` | `0.8` | `core` 기억의 Weibull 베타 | +| `decay.betaWorking` | `1.0` | `working` 기억의 Weibull 베타 | +| `decay.betaPeripheral` | `1.3` | `peripheral` 기억의 Weibull 베타 | +| `tier.coreAccessThreshold` | `10` | `core`로 승격하기 위한 최소 호출 횟수 | +| `tier.peripheralAgeDays` | `60` | 오래된 기억을 강등하기 위한 경과 일수 임계값 | + +
+ +
+접근 강화 + +자주 불러오는 기억은 더 느리게 감쇠합니다 (간격 반복 학습 방식). + +설정 키 (`retrieval` 하위): +- `reinforcementFactor` (0-2, 기본값: `0.5`) — `0`으로 설정하면 비활성화 +- `maxHalfLifeMultiplier` (1-10, 기본값: `3`) — 유효 반감기의 하드 캡 + +
+ +--- + +## CLI 명령어 + +```bash +openclaw memory-pro list [--scope global] [--category fact] [--limit 20] [--json] +openclaw memory-pro search "query" [--scope global] [--limit 10] [--json] +openclaw memory-pro stats [--scope global] [--json] +openclaw memory-pro auth login [--provider openai-codex] [--model gpt-5.4] [--oauth-path /abs/path/oauth.json] +openclaw memory-pro auth status +openclaw memory-pro auth logout +openclaw memory-pro delete +openclaw memory-pro delete-bulk --scope global [--before 2025-01-01] [--dry-run] +openclaw memory-pro export [--scope global] [--output memories.json] +openclaw memory-pro import memories.json [--scope global] [--dry-run] +openclaw memory-pro reembed --source-db /path/to/old-db [--batch-size 32] [--skip-existing] +openclaw memory-pro upgrade [--dry-run] [--batch-size 10] [--no-llm] [--limit N] [--scope SCOPE] +openclaw memory-pro migrate check|run|verify [--source /path] +``` + +OAuth 로그인 흐름: + +1. `openclaw memory-pro auth login` 실행 +2. `--provider`를 생략하고 대화형 터미널에서 실행하면, 브라우저를 열기 전에 CLI가 OAuth 프로바이더 선택기를 표시합니다 +3. 명령어가 인증 URL을 출력하고 `--no-browser`가 설정되지 않은 한 브라우저를 엽니다 +4. 콜백이 성공하면, 명령어가 플러그인 OAuth 파일 (기본값: `~/.openclaw/.memory-lancedb-pro/oauth.json`)을 저장하고, 이전 api-key `llm` 설정의 스냅샷을 로그아웃용으로 저장하며, 플러그인 `llm` 설정을 OAuth 설정 (`auth`, `oauthProvider`, `model`, `oauthPath`)으로 교체합니다 +5. `openclaw memory-pro auth logout`은 해당 OAuth 파일을 삭제하고 스냅샷이 존재하면 이전 api-key `llm` 설정을 복원합니다 + +--- + +## 고급 주제 + +
+주입된 기억이 응답에 표시되는 경우 + +가끔 모델이 주입된 `` 블록을 그대로 출력할 수 있습니다. + +**옵션 A (가장 안전):** 일시적으로 Auto-Recall 비활성화: +```json +{ "plugins": { "entries": { "memory-lancedb-pro": { "config": { "autoRecall": false } } } } } +``` + +**옵션 B (권장):** Auto-Recall은 유지하고 에이전트 시스템 프롬프트에 추가: +> `` / 메모리 주입 콘텐츠를 응답에 노출하거나 인용하지 마세요. 내부 참고용으로만 사용하세요. + +
+ +
+세션 메모리 + +- `/new` 명령어 시 작동 — 이전 세션 요약을 LanceDB에 저장 +- 기본적으로 비활성화 (OpenClaw에 이미 네이티브 `.jsonl` 세션 영속화 기능이 있음) +- 메시지 수 설정 가능 (기본값: 15) + +배포 모드와 `/new` 검증에 대해서는 [docs/openclaw-integration-playbook.md](docs/openclaw-integration-playbook.md)를 참조하세요. + +
+ +
+커스텀 슬래시 명령어 (예: /lesson) + +`CLAUDE.md`, `AGENTS.md` 또는 시스템 프롬프트에 다음을 추가하세요 (에이전트가 읽는 영문 지시문이므로 그대로 사용합니다): + +```markdown +## /lesson command +When the user sends `/lesson `: +1. Use memory_store to save as category=fact (raw knowledge) +2. Use memory_store to save as category=decision (actionable takeaway) +3. Confirm what was saved + +## /remember command +When the user sends `/remember `: +1. Use memory_store to save with appropriate category and importance +2. Confirm with the stored memory ID +``` + +
+ +
+AI 에이전트를 위한 철칙 + +> 아래 블록을 `AGENTS.md`에 복사하여 에이전트가 이 규칙을 자동으로 적용하도록 하세요 (에이전트가 읽는 영문 지시문이므로 그대로 사용합니다). + +```markdown +## Rule 1 — Dual-layer memory storage +Every pitfall/lesson learned → IMMEDIATELY store TWO memories: +- Technical layer: Pitfall: [symptom]. Cause: [root cause]. Fix: [solution]. Prevention: [how to avoid] + (category: fact, importance >= 0.8) +- Principle layer: Decision principle ([tag]): [behavioral rule]. Trigger: [when]. Action: [what to do] + (category: decision, importance >= 0.85) + +## Rule 2 — LanceDB hygiene +Entries must be short and atomic (< 500 chars). No raw conversation summaries or duplicates. + +## Rule 3 — Recall before retry +On ANY tool failure, ALWAYS memory_recall with relevant keywords BEFORE retrying. + +## Rule 4 — Confirm target codebase +Confirm you are editing memory-lancedb-pro vs built-in memory-lancedb before changes. + +## Rule 5 — Clear jiti cache after plugin code changes +After modifying .ts files under plugins/, MUST run rm -rf /tmp/jiti/ BEFORE openclaw gateway restart. +``` + +
+ +
+데이터베이스 스키마 + +LanceDB 테이블 `memories`: + +| 필드 | 타입 | 설명 | +| --- | --- | --- | +| `id` | string (UUID) | 기본 키 | +| `text` | string | 기억 텍스트 (FTS 인덱싱됨) | +| `vector` | float[] | 임베딩 벡터 | +| `category` | string | 저장 카테고리: `preference` / `fact` / `decision` / `entity` / `reflection` / `other` | +| `scope` | string | 스코프 식별자 (예: `global`, `agent:main`) | +| `importance` | float | 중요도 점수 0-1 | +| `timestamp` | int64 | 생성 타임스탬프 (ms) | +| `metadata` | string (JSON) | 확장 메타데이터 | + +v1.1.0의 주요 `metadata` 키: `l0_abstract`, `l1_overview`, `l2_content`, `memory_category`, `tier`, `access_count`, `confidence`, `last_accessed_at` + +> **카테고리 참고:** 최상위 `category` 필드는 6개 저장 카테고리를 사용합니다. Smart Extraction의 6개 카테고리 의미 라벨 (`profile` / `preferences` / `entities` / `events` / `cases` / `patterns`)은 `metadata.memory_category`에 저장됩니다. + +
+ +
+문제 해결 + +### "Cannot mix BigInt and other types" (LanceDB / Apache Arrow) + +LanceDB 0.26 이상에서 일부 숫자 열이 `BigInt`로 반환될 수 있습니다. **memory-lancedb-pro >= 1.0.14**로 업그레이드하세요 — 이 플러그인은 이제 산술 연산 전에 `Number(...)`를 사용하여 값을 변환합니다. + +
+ +--- + +## 문서 + +| 문서 | 설명 | +| --- | --- | +| [OpenClaw 통합 플레이북](docs/openclaw-integration-playbook.md) | 배포 모드, 검증, 회귀 매트릭스 | +| [메모리 아키텍처 분석](docs/memory_architecture_analysis.md) | 전체 아키텍처 심층 분석 | +| [CHANGELOG v1.1.0](docs/CHANGELOG-v1.1.0.md) | v1.1.0 동작 변경사항 및 업그레이드 근거 | +| [장문 컨텍스트 청킹](docs/long-context-chunking.md) | 긴 문서를 위한 청킹 전략 | + +--- + +## 베타: Smart Memory v1.1.0 + +> 상태: 베타 — `npm i memory-lancedb-pro@beta`로 사용 가능. `latest`를 사용하는 안정 버전 사용자는 영향 없음. + +| 기능 | 설명 | +|---------|-------------| +| **Smart Extraction** | LLM 기반 6개 카테고리 추출 + L0/L1/L2 메타데이터. 비활성화 시 정규식으로 폴백. | +| **라이프사이클 스코어링** | 검색에 Weibull 감쇠 통합 — 높은 빈도와 높은 중요도의 기억이 상위에 랭크. | +| **단계 관리** | 3단계 시스템 (Core → Working → Peripheral), 자동 승격/강등. | + +피드백: [GitHub Issues](https://github.com/CortexReach/memory-lancedb-pro/issues) · 되돌리기: `npm i memory-lancedb-pro@latest` + +--- + +## 의존성 + +| 패키지 | 용도 | +| --- | --- | +| `@lancedb/lancedb` ≥0.26.2 | 벡터 데이터베이스 (ANN + FTS) | +| `openai` ≥6.21.0 | OpenAI 호환 Embedding API 클라이언트 | +| `@sinclair/typebox` 0.34.48 | JSON Schema 타입 정의 | + +--- + +## Contributors + +

+@win4r +@kctony +@Akatsuki-Ryu +@JasonSuz +@Minidoracat +@furedericca-lab +@joe2643 +@AliceLJY +@chenjiyong +

+ +Full list: [Contributors](https://github.com/CortexReach/memory-lancedb-pro/graphs/contributors) + +## Star History + + + + + + Star History Chart + + + +## 라이선스 + +MIT + +--- + +## WeChat QR 코드 + + diff --git a/README_PT-BR.md b/README_PT-BR.md new file mode 100644 index 00000000..65d721f8 --- /dev/null +++ b/README_PT-BR.md @@ -0,0 +1,773 @@ +
+ +# 🧠 memory-lancedb-pro · 🦞OpenClaw Plugin + +**Assistente de Memória IA para Agentes [OpenClaw](https://github.com/openclaw/openclaw)** + +*Dê ao seu agente de IA um cérebro que realmente lembra — entre sessões, entre agentes, ao longo do tempo.* + +Um plugin de memória de longo prazo para OpenClaw baseado em LanceDB que armazena preferências, decisões e contexto de projetos, e os recupera automaticamente em sessões futuras. + +[![OpenClaw Plugin](https://img.shields.io/badge/OpenClaw-Plugin-blue)](https://github.com/openclaw/openclaw) +[![npm version](https://img.shields.io/npm/v/memory-lancedb-pro)](https://www.npmjs.com/package/memory-lancedb-pro) +[![LanceDB](https://img.shields.io/badge/LanceDB-Vectorstore-orange)](https://lancedb.com) +[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE) + +[English](README.md) | [简体中文](README_CN.md) | [繁體中文](README_TW.md) | [日本語](README_JA.md) | [한국어](README_KO.md) | [Français](README_FR.md) | [Español](README_ES.md) | [Deutsch](README_DE.md) | [Italiano](README_IT.md) | [Русский](README_RU.md) | [Português (Brasil)](README_PT-BR.md) + +
+ +--- + +## Por que memory-lancedb-pro? + +A maioria dos agentes de IA sofre de amnésia. Eles esquecem tudo no momento em que você inicia um novo chat. + +**memory-lancedb-pro** é um plugin de memória de longo prazo de nível de produção para OpenClaw que transforma seu agente em um verdadeiro **Assistente de Memória IA** — captura automaticamente o que importa, deixa o ruído desaparecer naturalmente e recupera a memória certa no momento certo. Sem tags manuais, sem dores de cabeça com configuração. + +### Seu Assistente de Memória IA em ação + +**Sem memória — cada sessão começa do zero:** + +> **Você:** "Use tabs para indentação, sempre adicione tratamento de erros." +> *(próxima sessão)* +> **Você:** "Eu já te disse — tabs, não espaços!" 😤 +> *(próxima sessão)* +> **Você:** "…sério, tabs. E tratamento de erros. De novo." + +**Com memory-lancedb-pro — seu agente aprende e lembra:** + +> **Você:** "Use tabs para indentação, sempre adicione tratamento de erros." +> *(próxima sessão — agente recupera automaticamente suas preferências)* +> **Agente:** *(aplica silenciosamente tabs + tratamento de erros)* ✅ +> **Você:** "Por que escolhemos PostgreSQL em vez de MongoDB no mês passado?" +> **Agente:** "Com base na nossa discussão de 12 de fevereiro, os principais motivos foram…" ✅ + +Essa é a diferença que um **Assistente de Memória IA** faz — aprende seu estilo, lembra decisões passadas e entrega respostas personalizadas sem você precisar se repetir. + +### O que mais ele pode fazer? + +| | O que você obtém | +|---|---| +| **Auto-Capture** | Seu agente aprende de cada conversa — sem necessidade de `memory_store` manual | +| **Extração inteligente** | Classificação LLM em 6 categorias: perfis, preferências, entidades, eventos, casos, padrões | +| **Esquecimento inteligente** | Modelo de decaimento Weibull — memórias importantes permanecem, ruído desaparece | +| **Busca híbrida** | Busca vetorial + BM25 full-text, fundida com reranking cross-encoder | +| **Injeção de contexto** | Memórias relevantes aparecem automaticamente antes de cada resposta | +| **Isolamento multi-scope** | Limites de memória por agente, por usuário, por projeto | +| **Qualquer provedor** | OpenAI, Jina, Gemini, Ollama ou qualquer API compatível com OpenAI | +| **Toolkit completo** | CLI, backup, migração, upgrade, exportação/importação — pronto para produção | + +--- + +## Início rápido + +### Opção A: Script de instalação com um clique (recomendado) + +O **[script de instalação](https://github.com/CortexReach/toolbox/tree/main/memory-lancedb-pro-setup)** mantido pela comunidade gerencia instalação, atualização e reparo em um único comando: + +```bash +curl -fsSL https://raw.githubusercontent.com/CortexReach/toolbox/main/memory-lancedb-pro-setup/setup-memory.sh -o setup-memory.sh +bash setup-memory.sh +``` + +> Veja [Ecossistema](#ecossistema) abaixo para a lista completa de cenários cobertos e outras ferramentas da comunidade. + +### Opção B: Instalação manual + +**Via OpenClaw CLI (recomendado):** +```bash +openclaw plugins install memory-lancedb-pro@beta +``` + +**Ou via npm:** +```bash +npm i memory-lancedb-pro@beta +``` +> Se usar npm, você também precisará adicionar o diretório de instalação do plugin como caminho **absoluto** em `plugins.load.paths` no seu `openclaw.json`. Este é o problema de configuração mais comum. + +Adicione ao seu `openclaw.json`: + +```json +{ + "plugins": { + "slots": { "memory": "memory-lancedb-pro" }, + "entries": { + "memory-lancedb-pro": { + "enabled": true, + "config": { + "embedding": { + "provider": "openai-compatible", + "apiKey": "${OPENAI_API_KEY}", + "model": "text-embedding-3-small" + }, + "autoCapture": true, + "autoRecall": true, + "smartExtraction": true, + "extractMinMessages": 2, + "extractMaxChars": 8000, + "sessionMemory": { "enabled": false } + } + } + } + } +} +``` + +**Por que esses valores padrão?** +- `autoCapture` + `smartExtraction` → seu agente aprende automaticamente de cada conversa +- `autoRecall` → memórias relevantes são injetadas antes de cada resposta +- `extractMinMessages: 2` → a extração é acionada em chats normais de dois turnos +- `sessionMemory.enabled: false` → evita poluir a busca com resumos de sessão no início + +Valide e reinicie: + +```bash +openclaw config validate +openclaw gateway restart +openclaw logs --follow --plain | grep "memory-lancedb-pro" +``` + +Você deve ver: +- `memory-lancedb-pro: smart extraction enabled` +- `memory-lancedb-pro@...: plugin registered` + +Pronto! Seu agente agora tem memória de longo prazo. + +
+Mais caminhos de instalação (usuários existentes, upgrades) + +**Já está usando OpenClaw?** + +1. Adicione o plugin com um caminho **absoluto** em `plugins.load.paths` +2. Vincule o slot de memória: `plugins.slots.memory = "memory-lancedb-pro"` +3. Verifique: `openclaw plugins info memory-lancedb-pro && openclaw memory-pro stats` + +**Atualizando de versões anteriores ao v1.1.0?** + +```bash +# 1) Backup +openclaw memory-pro export --scope global --output memories-backup.json +# 2) Dry run +openclaw memory-pro upgrade --dry-run +# 3) Run upgrade +openclaw memory-pro upgrade +# 4) Verify +openclaw memory-pro stats +``` + +Veja `CHANGELOG-v1.1.0.md` para mudanças de comportamento e justificativa de upgrade. + +
+ +
+Importação rápida via Telegram Bot (clique para expandir) + +Se você está usando a integração Telegram do OpenClaw, a maneira mais fácil é enviar um comando de importação diretamente para o Bot principal em vez de editar a configuração manualmente. + +Envie esta mensagem: + +```text +Help me connect this memory plugin with the most user-friendly configuration: https://github.com/CortexReach/memory-lancedb-pro + +Requirements: +1. Set it as the only active memory plugin +2. Use Jina for embedding +3. Use Jina for reranker +4. Use gpt-4o-mini for the smart-extraction LLM +5. Enable autoCapture, autoRecall, smartExtraction +6. extractMinMessages=2 +7. sessionMemory.enabled=false +8. captureAssistant=false +9. retrieval mode=hybrid, vectorWeight=0.7, bm25Weight=0.3 +10. rerank=cross-encoder, candidatePoolSize=12, minScore=0.6, hardMinScore=0.62 +11. Generate the final openclaw.json config directly, not just an explanation +``` + +
+ +--- + +## Ecossistema + +memory-lancedb-pro é o plugin principal. A comunidade construiu ferramentas ao redor dele para tornar a configuração e o uso diário ainda mais suaves: + +### Script de instalação — Instalação, atualização e reparo com um clique + +> **[CortexReach/toolbox/memory-lancedb-pro-setup](https://github.com/CortexReach/toolbox/tree/main/memory-lancedb-pro-setup)** + +Não é apenas um instalador simples — o script lida inteligentemente com diversos cenários reais: + +| Sua situação | O que o script faz | +|---|---| +| Nunca instalou | Download → instalar dependências → escolher config → gravar em openclaw.json → reiniciar | +| Instalado via `git clone`, preso em um commit antigo | `git fetch` + `checkout` automático para a versão mais recente → reinstalar dependências → verificar | +| Config tem campos inválidos | Detecção automática via filtro de schema, remoção de campos não suportados | +| Instalado via `npm` | Pula atualização git, lembra de executar `npm update` por conta própria | +| CLI `openclaw` quebrado por config inválida | Fallback: ler caminho do workspace diretamente do arquivo `openclaw.json` | +| `extensions/` em vez de `plugins/` | Detecção automática da localização do plugin a partir da config ou sistema de arquivos | +| Já está atualizado | Executa apenas verificações de saúde, sem alterações | + +```bash +bash setup-memory.sh # Install or upgrade +bash setup-memory.sh --dry-run # Preview only +bash setup-memory.sh --beta # Include pre-release versions +bash setup-memory.sh --uninstall # Revert config and remove plugin +``` + +Presets de provedores integrados: **Jina / DashScope / SiliconFlow / OpenAI / Ollama**, ou traga sua própria API compatível com OpenAI. Para uso completo (incluindo `--ref`, `--selfcheck-only` e mais), veja o [README do script de instalação](https://github.com/CortexReach/toolbox/tree/main/memory-lancedb-pro-setup). + +### Claude Code / OpenClaw Skill — Configuração guiada por IA + +> **[CortexReach/memory-lancedb-pro-skill](https://github.com/CortexReach/memory-lancedb-pro-skill)** + +Instale esta Skill e seu agente de IA (Claude Code ou OpenClaw) ganha conhecimento profundo de todas as funcionalidades do memory-lancedb-pro. Basta dizer **"me ajude a ativar a melhor configuração"** e obtenha: + +- **Workflow de configuração guiado em 7 etapas** com 4 planos de implantação: + - Full Power (Jina + OpenAI) / Budget (reranker SiliconFlow gratuito) / Simple (apenas OpenAI) / Totalmente local (Ollama, custo API zero) +- **Todas as 9 ferramentas MCP** usadas corretamente: `memory_recall`, `memory_store`, `memory_forget`, `memory_update`, `memory_stats`, `memory_list`, `self_improvement_log`, `self_improvement_extract_skill`, `self_improvement_review` *(o toolkit completo requer `enableManagementTools: true` — a configuração padrão do Quick Start expõe as 4 ferramentas principais)* +- **Prevenção de armadilhas comuns**: ativação de plugin workspace, `autoRecall` padrão false, cache jiti, variáveis de ambiente, isolamento de scope, etc. + +**Instalação para Claude Code:** +```bash +git clone https://github.com/CortexReach/memory-lancedb-pro-skill.git ~/.claude/skills/memory-lancedb-pro +``` + +**Instalação para OpenClaw:** +```bash +git clone https://github.com/CortexReach/memory-lancedb-pro-skill.git ~/.openclaw/workspace/skills/memory-lancedb-pro-skill +``` + +--- + +## Tutorial em vídeo + +> Guia completo: instalação, configuração e funcionamento interno da busca híbrida. + +[![YouTube Video](https://img.shields.io/badge/YouTube-Watch%20Now-red?style=for-the-badge&logo=youtube)](https://youtu.be/MtukF1C8epQ) +**https://youtu.be/MtukF1C8epQ** + +[![Bilibili Video](https://img.shields.io/badge/Bilibili-Watch%20Now-00A1D6?style=for-the-badge&logo=bilibili&logoColor=white)](https://www.bilibili.com/video/BV1zUf2BGEgn/) +**https://www.bilibili.com/video/BV1zUf2BGEgn/** + +--- + +## Arquitetura + +``` +┌─────────────────────────────────────────────────────────┐ +│ index.ts (Entry Point) │ +│ Plugin Registration · Config Parsing · Lifecycle Hooks │ +└────────┬──────────┬──────────┬──────────┬───────────────┘ + │ │ │ │ + ┌────▼───┐ ┌────▼───┐ ┌───▼────┐ ┌──▼──────────┐ + │ store │ │embedder│ │retriever│ │ scopes │ + │ .ts │ │ .ts │ │ .ts │ │ .ts │ + └────────┘ └────────┘ └────────┘ └─────────────┘ + │ │ + ┌────▼───┐ ┌─────▼──────────┐ + │migrate │ │noise-filter.ts │ + │ .ts │ │adaptive- │ + └────────┘ │retrieval.ts │ + └────────────────┘ + ┌─────────────┐ ┌──────────┐ + │ tools.ts │ │ cli.ts │ + │ (Agent API) │ │ (CLI) │ + └─────────────┘ └──────────┘ +``` + +> Para um mergulho profundo na arquitetura completa, veja [docs/memory_architecture_analysis.md](docs/memory_architecture_analysis.md). + +
+Referência de arquivos (clique para expandir) + +| Arquivo | Finalidade | +| --- | --- | +| `index.ts` | Ponto de entrada do plugin. Registra na API de Plugin do OpenClaw, analisa config, monta lifecycle hooks | +| `openclaw.plugin.json` | Metadados do plugin + declaração completa de config via JSON Schema | +| `cli.ts` | Comandos CLI: `memory-pro list/search/stats/delete/delete-bulk/export/import/reembed/upgrade/migrate` | +| `src/store.ts` | Camada de armazenamento LanceDB. Criação de tabelas / Indexação FTS / Busca vetorial / Busca BM25 / CRUD | +| `src/embedder.ts` | Abstração de embedding. Compatível com qualquer provedor de API compatível com OpenAI | +| `src/retriever.ts` | Motor de busca híbrida. Vector + BM25 → Fusão Híbrida → Rerank → Decaimento do Ciclo de Vida → Filtro | +| `src/scopes.ts` | Controle de acesso multi-scope | +| `src/tools.ts` | Definições de ferramentas do agente: `memory_recall`, `memory_store`, `memory_forget`, `memory_update` + ferramentas de gerenciamento | +| `src/noise-filter.ts` | Filtra recusas do agente, meta-perguntas, saudações e conteúdo de baixa qualidade | +| `src/adaptive-retrieval.ts` | Determina se uma consulta precisa de busca na memória | +| `src/migrate.ts` | Migração do `memory-lancedb` integrado para o Pro | +| `src/smart-extractor.ts` | Extração LLM em 6 categorias com armazenamento em camadas L0/L1/L2 e deduplicação em dois estágios | +| `src/decay-engine.ts` | Modelo de decaimento exponencial esticado Weibull | +| `src/tier-manager.ts` | Promoção/rebaixamento em três níveis: Peripheral ↔ Working ↔ Core | + +
+ +--- + +## Funcionalidades principais + +### Busca híbrida + +``` +Query → embedQuery() ─┐ + ├─→ Hybrid Fusion → Rerank → Lifecycle Decay Boost → Length Norm → Filter +Query → BM25 FTS ─────┘ +``` + +- **Busca vetorial** — similaridade semântica via LanceDB ANN (distância cosseno) +- **Busca full-text BM25** — correspondência exata de palavras-chave via índice FTS do LanceDB +- **Fusão híbrida** — pontuação vetorial como base, resultados BM25 recebem boost ponderado (não é RRF padrão — ajustado para qualidade de recall no mundo real) +- **Pesos configuráveis** — `vectorWeight`, `bm25Weight`, `minScore` + +### Reranking Cross-Encoder + +- Adaptadores integrados para **Jina**, **SiliconFlow**, **Voyage AI** e **Pinecone** +- Compatível com qualquer endpoint compatível com Jina (ex.: Hugging Face TEI, DashScope) +- Pontuação híbrida: 60% cross-encoder + 40% pontuação fundida original +- Degradação elegante: fallback para similaridade cosseno em caso de falha da API + +### Pipeline de pontuação multi-estágio + +| Estágio | Efeito | +| --- | --- | +| **Fusão híbrida** | Combina recall semântico e correspondência exata | +| **Rerank Cross-Encoder** | Promove resultados semanticamente precisos | +| **Boost de decaimento do ciclo de vida** | Frescor Weibull + frequência de acesso + importância × confiança | +| **Normalização de comprimento** | Impede que entradas longas dominem (âncora: 500 caracteres) | +| **Pontuação mínima rígida** | Remove resultados irrelevantes (padrão: 0.35) | +| **Diversidade MMR** | Similaridade cosseno > 0.85 → rebaixado | + +### Extração inteligente de memória (v1.1.0) + +- **Extração LLM em 6 categorias**: perfil, preferências, entidades, eventos, casos, padrões +- **Armazenamento em camadas L0/L1/L2**: L0 (índice em uma frase) → L1 (resumo estruturado) → L2 (narrativa completa) +- **Deduplicação em dois estágios**: pré-filtro de similaridade vetorial (≥0.7) → decisão semântica LLM (CREATE/MERGE/SKIP) +- **Fusão consciente de categorias**: `profile` sempre funde, `events`/`cases` apenas adicionam + +### Gerenciamento do ciclo de vida da memória (v1.1.0) + +- **Motor de decaimento Weibull**: pontuação composta = frescor + frequência + valor intrínseco +- **Promoção em três níveis**: `Peripheral ↔ Working ↔ Core` com limiares configuráveis +- **Reforço por acesso**: memórias recuperadas frequentemente decaem mais lentamente (estilo repetição espaçada) +- **Meia-vida modulada pela importância**: memórias importantes decaem mais lentamente + +### Isolamento multi-scope + +- Scopes integrados: `global`, `agent:`, `custom:`, `project:`, `user:` +- Controle de acesso no nível do agente via `scopes.agentAccess` +- Padrão: cada agente acessa `global` + seu próprio scope `agent:` + +### Auto-Capture e Auto-Recall + +- **Auto-Capture** (`agent_end`): extrai preferências/fatos/decisões/entidades das conversas, deduplica, armazena até 3 por turno +- **Auto-Recall** (`before_agent_start`): injeta contexto `` (até 3 entradas) + +### Filtragem de ruído e busca adaptativa + +- Filtra conteúdo de baixa qualidade: recusas do agente, meta-perguntas, saudações +- Pula a busca para: saudações, comandos slash, confirmações simples, emoji +- Força a busca para palavras-chave de memória ("lembra", "anteriormente", "da última vez") +- Limiares CJK (chinês: 6 caracteres vs inglês: 15 caracteres) + +--- + +
+Comparação com o memory-lancedb integrado (clique para expandir) + +| Funcionalidade | `memory-lancedb` integrado | **memory-lancedb-pro** | +| --- | :---: | :---: | +| Busca vetorial | Sim | Sim | +| Busca full-text BM25 | - | Sim | +| Fusão híbrida (Vector + BM25) | - | Sim | +| Rerank cross-encoder (multi-provedor) | - | Sim | +| Boost de frescor e decaimento temporal | - | Sim | +| Normalização de comprimento | - | Sim | +| Diversidade MMR | - | Sim | +| Isolamento multi-scope | - | Sim | +| Filtragem de ruído | - | Sim | +| Busca adaptativa | - | Sim | +| CLI de gerenciamento | - | Sim | +| Memória de sessão | - | Sim | +| Embeddings conscientes de tarefa | - | Sim | +| **Extração inteligente LLM (6 categorias)** | - | Sim (v1.1.0) | +| **Decaimento Weibull + Promoção de nível** | - | Sim (v1.1.0) | +| Qualquer embedding compatível com OpenAI | Limitado | Sim | + +
+ +--- + +## Configuração + +
+Exemplo de configuração completa + +```json +{ + "embedding": { + "apiKey": "${JINA_API_KEY}", + "model": "jina-embeddings-v5-text-small", + "baseURL": "https://api.jina.ai/v1", + "dimensions": 1024, + "taskQuery": "retrieval.query", + "taskPassage": "retrieval.passage", + "normalized": true + }, + "dbPath": "~/.openclaw/memory/lancedb-pro", + "autoCapture": true, + "autoRecall": true, + "retrieval": { + "mode": "hybrid", + "vectorWeight": 0.7, + "bm25Weight": 0.3, + "minScore": 0.3, + "rerank": "cross-encoder", + "rerankApiKey": "${JINA_API_KEY}", + "rerankModel": "jina-reranker-v3", + "rerankEndpoint": "https://api.jina.ai/v1/rerank", + "rerankProvider": "jina", + "candidatePoolSize": 20, + "recencyHalfLifeDays": 14, + "recencyWeight": 0.1, + "filterNoise": true, + "lengthNormAnchor": 500, + "hardMinScore": 0.35, + "timeDecayHalfLifeDays": 60, + "reinforcementFactor": 0.5, + "maxHalfLifeMultiplier": 3 + }, + "enableManagementTools": false, + "scopes": { + "default": "global", + "definitions": { + "global": { "description": "Shared knowledge" }, + "agent:discord-bot": { "description": "Discord bot private" } + }, + "agentAccess": { + "discord-bot": ["global", "agent:discord-bot"] + } + }, + "sessionMemory": { + "enabled": false, + "messageCount": 15 + }, + "smartExtraction": true, + "llm": { + "apiKey": "${OPENAI_API_KEY}", + "model": "gpt-4o-mini", + "baseURL": "https://api.openai.com/v1" + }, + "extractMinMessages": 2, + "extractMaxChars": 8000 +} +``` + +
+ +
+Provedores de Embedding + +Funciona com **qualquer API de embedding compatível com OpenAI**: + +| Provedor | Modelo | Base URL | Dimensões | +| --- | --- | --- | --- | +| **Jina** (recomendado) | `jina-embeddings-v5-text-small` | `https://api.jina.ai/v1` | 1024 | +| **OpenAI** | `text-embedding-3-small` | `https://api.openai.com/v1` | 1536 | +| **Voyage** | `voyage-4-lite` / `voyage-4` | `https://api.voyageai.com/v1` | 1024 / 1024 | +| **Google Gemini** | `gemini-embedding-001` | `https://generativelanguage.googleapis.com/v1beta/openai/` | 3072 | +| **Ollama** (local) | `nomic-embed-text` | `http://localhost:11434/v1` | específico do provedor | + +
+ +
+Provedores de Rerank + +O reranking cross-encoder suporta múltiplos provedores via `rerankProvider`: + +| Provedor | `rerankProvider` | Modelo de exemplo | +| --- | --- | --- | +| **Jina** (padrão) | `jina` | `jina-reranker-v3` | +| **SiliconFlow** (plano gratuito disponível) | `siliconflow` | `BAAI/bge-reranker-v2-m3` | +| **Voyage AI** | `voyage` | `rerank-2.5` | +| **Pinecone** | `pinecone` | `bge-reranker-v2-m3` | + +Qualquer endpoint de rerank compatível com Jina também funciona — defina `rerankProvider: "jina"` e aponte `rerankEndpoint` para seu serviço (ex.: Hugging Face TEI, DashScope `qwen3-rerank`). + +
+ +
+Extração inteligente (LLM) — v1.1.0 + +Quando `smartExtraction` está habilitado (padrão: `true`), o plugin usa um LLM para extrair e classificar memórias de forma inteligente em vez de gatilhos baseados em regex. + +| Campo | Tipo | Padrão | Descrição | +|-------|------|--------|-----------| +| `smartExtraction` | boolean | `true` | Habilitar/desabilitar extração LLM em 6 categorias | +| `llm.auth` | string | `api-key` | `api-key` usa `llm.apiKey` / `embedding.apiKey`; `oauth` usa um arquivo de token OAuth com escopo de plugin por padrão | +| `llm.apiKey` | string | *(fallback para `embedding.apiKey`)* | Chave de API para o provedor LLM | +| `llm.model` | string | `openai/gpt-oss-120b` | Nome do modelo LLM | +| `llm.baseURL` | string | *(fallback para `embedding.baseURL`)* | Endpoint da API LLM | +| `llm.oauthProvider` | string | `openai-codex` | ID do provedor OAuth usado quando `llm.auth` é `oauth` | +| `llm.oauthPath` | string | `~/.openclaw/.memory-lancedb-pro/oauth.json` | Arquivo de token OAuth usado quando `llm.auth` é `oauth` | +| `llm.timeoutMs` | number | `30000` | Timeout da requisição LLM em milissegundos | +| `extractMinMessages` | number | `2` | Mensagens mínimas antes da extração ser acionada | +| `extractMaxChars` | number | `8000` | Máximo de caracteres enviados ao LLM | + + +Configuração `llm` com OAuth (usa cache de login existente do Codex / ChatGPT para chamadas LLM): +```json +{ + "llm": { + "auth": "oauth", + "oauthProvider": "openai-codex", + "model": "gpt-5.4", + "oauthPath": "${HOME}/.openclaw/.memory-lancedb-pro/oauth.json", + "timeoutMs": 30000 + } +} +``` + +Notas para `llm.auth: "oauth"`: + +- `llm.oauthProvider` é atualmente `openai-codex`. +- Tokens OAuth têm como padrão `~/.openclaw/.memory-lancedb-pro/oauth.json`. +- Você pode definir `llm.oauthPath` se quiser armazenar esse arquivo em outro lugar. +- `auth login` faz snapshot da configuração `llm` anterior (api-key) ao lado do arquivo OAuth, e `auth logout` restaura esse snapshot quando disponível. +- Mudar de `api-key` para `oauth` não transfere automaticamente `llm.baseURL`. Defina-o manualmente no modo OAuth apenas quando você intencionalmente quiser um backend personalizado compatível com ChatGPT/Codex. + +
+ +
+Configuração do ciclo de vida (Decaimento + Nível) + +| Campo | Padrão | Descrição | +|-------|--------|-----------| +| `decay.recencyHalfLifeDays` | `30` | Meia-vida base para decaimento de frescor Weibull | +| `decay.frequencyWeight` | `0.3` | Peso da frequência de acesso na pontuação composta | +| `decay.intrinsicWeight` | `0.3` | Peso de `importance × confidence` | +| `decay.betaCore` | `0.8` | Beta Weibull para memórias `core` | +| `decay.betaWorking` | `1.0` | Beta Weibull para memórias `working` | +| `decay.betaPeripheral` | `1.3` | Beta Weibull para memórias `peripheral` | +| `tier.coreAccessThreshold` | `10` | Contagem mínima de recall antes de promover para `core` | +| `tier.peripheralAgeDays` | `60` | Limiar de idade para rebaixar memórias inativas | + +
+ +
+Reforço por acesso + +Memórias recuperadas com frequência decaem mais lentamente (estilo repetição espaçada). + +Chaves de configuração (em `retrieval`): +- `reinforcementFactor` (0-2, padrão: `0.5`) — defina `0` para desabilitar +- `maxHalfLifeMultiplier` (1-10, padrão: `3`) — limite rígido na meia-vida efetiva + +
+ +--- + +## Comandos CLI + +```bash +openclaw memory-pro list [--scope global] [--category fact] [--limit 20] [--json] +openclaw memory-pro search "query" [--scope global] [--limit 10] [--json] +openclaw memory-pro stats [--scope global] [--json] +openclaw memory-pro auth login [--provider openai-codex] [--model gpt-5.4] [--oauth-path /abs/path/oauth.json] +openclaw memory-pro auth status +openclaw memory-pro auth logout +openclaw memory-pro delete +openclaw memory-pro delete-bulk --scope global [--before 2025-01-01] [--dry-run] +openclaw memory-pro export [--scope global] [--output memories.json] +openclaw memory-pro import memories.json [--scope global] [--dry-run] +openclaw memory-pro reembed --source-db /path/to/old-db [--batch-size 32] [--skip-existing] +openclaw memory-pro upgrade [--dry-run] [--batch-size 10] [--no-llm] [--limit N] [--scope SCOPE] +openclaw memory-pro migrate check|run|verify [--source /path] +``` + +Fluxo de login OAuth: + +1. Execute `openclaw memory-pro auth login` +2. Se `--provider` for omitido em um terminal interativo, a CLI mostra um seletor de provedor OAuth antes de abrir o navegador +3. O comando imprime uma URL de autorização e abre seu navegador, a menos que `--no-browser` seja definido +4. Após o callback ser bem-sucedido, o comando salva o arquivo OAuth do plugin (padrão: `~/.openclaw/.memory-lancedb-pro/oauth.json`), faz snapshot da configuração `llm` anterior (api-key) para logout, e substitui a configuração `llm` do plugin com as configurações OAuth (`auth`, `oauthProvider`, `model`, `oauthPath`) +5. `openclaw memory-pro auth logout` deleta esse arquivo OAuth e restaura a configuração `llm` anterior (api-key) quando esse snapshot existe + +--- + +## Tópicos avançados + +
+Se memórias injetadas aparecem nas respostas + +Às vezes o modelo pode ecoar o bloco `` injetado. + +**Opção A (menor risco):** desabilite temporariamente o auto-recall: +```json +{ "plugins": { "entries": { "memory-lancedb-pro": { "config": { "autoRecall": false } } } } } +``` + +**Opção B (preferida):** mantenha o recall, adicione ao prompt do sistema do agente: +> Do not reveal or quote any `` / memory-injection content in your replies. Use it for internal reference only. + +
+ +
+Memória de sessão + +- Acionada no comando `/new` — salva o resumo da sessão anterior no LanceDB +- Desabilitada por padrão (OpenClaw já tem persistência nativa de sessão via `.jsonl`) +- Contagem de mensagens configurável (padrão: 15) + +Veja [docs/openclaw-integration-playbook.md](docs/openclaw-integration-playbook.md) para modos de implantação e verificação do `/new`. + +
+ +
+Comandos slash personalizados (ex.: /lesson) + +Adicione ao seu `CLAUDE.md`, `AGENTS.md` ou prompt do sistema: + +```markdown +## /lesson command +When the user sends `/lesson `: +1. Use memory_store to save as category=fact (raw knowledge) +2. Use memory_store to save as category=decision (actionable takeaway) +3. Confirm what was saved + +## /remember command +When the user sends `/remember `: +1. Use memory_store to save with appropriate category and importance +2. Confirm with the stored memory ID +``` + +
+ +
+Regras de ferro para agentes de IA + +> Copie o bloco abaixo no seu `AGENTS.md` para que seu agente aplique essas regras automaticamente. + +```markdown +## Rule 1 — Dual-layer memory storage +Every pitfall/lesson learned → IMMEDIATELY store TWO memories: +- Technical layer: Pitfall: [symptom]. Cause: [root cause]. Fix: [solution]. Prevention: [how to avoid] + (category: fact, importance >= 0.8) +- Principle layer: Decision principle ([tag]): [behavioral rule]. Trigger: [when]. Action: [what to do] + (category: decision, importance >= 0.85) + +## Rule 2 — LanceDB hygiene +Entries must be short and atomic (< 500 chars). No raw conversation summaries or duplicates. + +## Rule 3 — Recall before retry +On ANY tool failure, ALWAYS memory_recall with relevant keywords BEFORE retrying. + +## Rule 4 — Confirm target codebase +Confirm you are editing memory-lancedb-pro vs built-in memory-lancedb before changes. + +## Rule 5 — Clear jiti cache after plugin code changes +After modifying .ts files under plugins/, MUST run rm -rf /tmp/jiti/ BEFORE openclaw gateway restart. +``` + +
+ +
+Schema do banco de dados + +Tabela LanceDB `memories`: + +| Campo | Tipo | Descrição | +| --- | --- | --- | +| `id` | string (UUID) | Chave primária | +| `text` | string | Texto da memória (indexado FTS) | +| `vector` | float[] | Vetor de embedding | +| `category` | string | Categoria de armazenamento: `preference` / `fact` / `decision` / `entity` / `reflection` / `other` | +| `scope` | string | Identificador de scope (ex.: `global`, `agent:main`) | +| `importance` | float | Pontuação de importância 0-1 | +| `timestamp` | int64 | Timestamp de criação (ms) | +| `metadata` | string (JSON) | Metadados estendidos | + +Chaves `metadata` comuns no v1.1.0: `l0_abstract`, `l1_overview`, `l2_content`, `memory_category`, `tier`, `access_count`, `confidence`, `last_accessed_at` + +> **Nota sobre categorias:** O campo `category` de nível superior usa 6 categorias de armazenamento. As 6 categorias semânticas da Extração Inteligente (`profile` / `preferences` / `entities` / `events` / `cases` / `patterns`) são armazenadas em `metadata.memory_category`. + +
+ +
+Solução de problemas + +### "Cannot mix BigInt and other types" (LanceDB / Apache Arrow) + +No LanceDB 0.26+, algumas colunas numéricas podem ser retornadas como `BigInt`. Atualize para **memory-lancedb-pro >= 1.0.14** — este plugin agora converte valores usando `Number(...)` antes de operações aritméticas. + +
+ +--- + +## Documentação + +| Documento | Descrição | +| --- | --- | +| [Playbook de integração OpenClaw](docs/openclaw-integration-playbook.md) | Modos de implantação, verificação, matriz de regressão | +| [Análise da arquitetura de memória](docs/memory_architecture_analysis.md) | Análise aprofundada da arquitetura completa | +| [CHANGELOG v1.1.0](docs/CHANGELOG-v1.1.0.md) | Mudanças de comportamento v1.1.0 e justificativa de upgrade | +| [Chunking de contexto longo](docs/long-context-chunking.md) | Estratégia de chunking para documentos longos | + +--- + +## Beta: Smart Memory v1.1.0 + +> Status: Beta — disponível via `npm i memory-lancedb-pro@beta`. Usuários estáveis no `latest` não são afetados. + +| Funcionalidade | Descrição | +|---------|-------------| +| **Extração inteligente** | Extração LLM em 6 categorias com metadados L0/L1/L2. Fallback para regex quando desabilitado. | +| **Pontuação do ciclo de vida** | Decaimento Weibull integrado à busca — memórias frequentes e importantes ficam mais bem ranqueadas. | +| **Gerenciamento de níveis** | Sistema de três níveis (Core → Working → Peripheral) com promoção/rebaixamento automático. | + +Feedback: [GitHub Issues](https://github.com/CortexReach/memory-lancedb-pro/issues) · Reverter: `npm i memory-lancedb-pro@latest` + +--- + +## Dependências + +| Pacote | Finalidade | +| --- | --- | +| `@lancedb/lancedb` ≥0.26.2 | Banco de dados vetorial (ANN + FTS) | +| `openai` ≥6.21.0 | Cliente de API de Embedding compatível com OpenAI | +| `@sinclair/typebox` 0.34.48 | Definições de tipo JSON Schema | + +--- + +## Contributors + +

+@win4r +@kctony +@Akatsuki-Ryu +@JasonSuz +@Minidoracat +@furedericca-lab +@joe2643 +@AliceLJY +@chenjiyong +

+ +Full list: [Contributors](https://github.com/CortexReach/memory-lancedb-pro/graphs/contributors) + +## Star History + + + + + + Star History Chart + + + +## Licença + +MIT + +--- + +## Meu QR Code WeChat + + diff --git a/README_RU.md b/README_RU.md new file mode 100644 index 00000000..8fcb1031 --- /dev/null +++ b/README_RU.md @@ -0,0 +1,773 @@ +
+ +# 🧠 memory-lancedb-pro · 🦞OpenClaw Plugin + +**ИИ-ассистент памяти для агентов [OpenClaw](https://github.com/openclaw/openclaw)** + +*Дайте вашему ИИ-агенту мозг, который действительно помнит: между сессиями, между агентами и с течением времени.* + +Плагин долгосрочной памяти для OpenClaw на базе LanceDB, который сохраняет предпочтения, решения и контекст проекта, а затем автоматически вспоминает их в будущих сессиях. + +[![OpenClaw Plugin](https://img.shields.io/badge/OpenClaw-Plugin-blue)](https://github.com/openclaw/openclaw) +[![npm version](https://img.shields.io/npm/v/memory-lancedb-pro)](https://www.npmjs.com/package/memory-lancedb-pro) +[![LanceDB](https://img.shields.io/badge/LanceDB-Vectorstore-orange)](https://lancedb.com) +[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE) + +[English](README.md) | [简体中文](README_CN.md) | [繁體中文](README_TW.md) | [日本語](README_JA.md) | [한국어](README_KO.md) | [Français](README_FR.md) | [Español](README_ES.md) | [Deutsch](README_DE.md) | [Italiano](README_IT.md) | [Русский](README_RU.md) | [Português (Brasil)](README_PT-BR.md) + +
+ +--- + +## Почему memory-lancedb-pro? + +Большинство ИИ-агентов страдают амнезией. Они забывают все, как только вы начинаете новый чат. + +**memory-lancedb-pro** — это production-grade плагин долгосрочной памяти для OpenClaw, который превращает вашего агента в настоящего **ИИ-ассистента памяти**. Он автоматически фиксирует важное, позволяет шуму естественно угасать и поднимает нужное воспоминание в нужный момент. Никаких ручных тегов, никаких мучений с конфигурацией. + +### Как это выглядит на практике + +**Без памяти: каждая сессия начинается с нуля** + +> **Вы:** "Используй табы для отступов и всегда добавляй обработку ошибок." +> *(следующая сессия)* +> **Вы:** "Я же уже говорил: табы, а не пробелы!" 😤 +> *(еще одна сессия)* +> **Вы:** "...серьезно, табы. И обработка ошибок. Снова." + +**С memory-lancedb-pro агент учится и помнит** + +> **Вы:** "Используй табы для отступов и всегда добавляй обработку ошибок." +> *(следующая сессия: агент автоматически вспоминает ваши предпочтения)* +> **Агент:** *(молча применяет табы + обработку ошибок)* ✅ +> **Вы:** "Почему в прошлом месяце мы выбрали PostgreSQL, а не MongoDB?" +> **Агент:** "Судя по нашему обсуждению 12 февраля, основные причины были..." ✅ + +В этом и есть разница: **ИИ-ассистент памяти** изучает ваш стиль, вспоминает прошлые решения и дает персонализированные ответы без необходимости повторять одно и то же. + +### Что еще он умеет? + +| | Что вы получаете | +|---|---| +| **Автозахват** | Агент учится на каждом разговоре, без ручного `memory_store` | +| **Умное извлечение** | Классификация на основе LLM по 6 категориям: профили, предпочтения, сущности, события, кейсы, паттерны | +| **Интеллектуальное забывание** | Модель затухания Weibull: важные воспоминания остаются, шум естественно исчезает | +| **Гибридный поиск** | Векторный поиск + полнотекстовый BM25 с объединением и cross-encoder rerank | +| **Инъекция контекста** | Релевантные воспоминания автоматически подаются перед каждым ответом | +| **Изоляция областей памяти** | Границы памяти на уровне агента, пользователя и проекта | +| **Любой провайдер** | OpenAI, Jina, Gemini, Ollama или любой OpenAI-compatible API | +| **Полный набор инструментов** | CLI, backup, migration, upgrade, export/import — готово к продакшену | + +--- + +## Быстрый старт + +### Вариант A: скрипт установки в один клик (рекомендуется) + +Поддерживаемый сообществом **[скрипт установки](https://github.com/CortexReach/toolbox/tree/main/memory-lancedb-pro-setup)** берет на себя установку, обновление и восстановление одной командой: + +```bash +curl -fsSL https://raw.githubusercontent.com/CortexReach/toolbox/main/memory-lancedb-pro-setup/setup-memory.sh -o setup-memory.sh +bash setup-memory.sh +``` + +> Полный список сценариев, которые покрывает скрипт, и другие инструменты сообщества смотрите ниже в разделе [Экосистема](#экосистема). + +### Вариант B: ручная установка + +**Через OpenClaw CLI (рекомендуется):** +```bash +openclaw plugins install memory-lancedb-pro@beta +``` + +**Или через npm:** +```bash +npm i memory-lancedb-pro@beta +``` +> Если используете npm, вам также нужно добавить директорию установки плагина как **абсолютный** путь в `plugins.load.paths` вашего `openclaw.json`. Это самая частая проблема при настройке. + +Добавьте в `openclaw.json`: + +```json +{ + "plugins": { + "slots": { "memory": "memory-lancedb-pro" }, + "entries": { + "memory-lancedb-pro": { + "enabled": true, + "config": { + "embedding": { + "provider": "openai-compatible", + "apiKey": "${OPENAI_API_KEY}", + "model": "text-embedding-3-small" + }, + "autoCapture": true, + "autoRecall": true, + "smartExtraction": true, + "extractMinMessages": 2, + "extractMaxChars": 8000, + "sessionMemory": { "enabled": false } + } + } + } + } +} +``` + +**Почему именно такие значения по умолчанию?** +- `autoCapture` + `smartExtraction` → агент автоматически учится на каждом разговоре +- `autoRecall` → релевантные воспоминания подставляются перед каждым ответом +- `extractMinMessages: 2` → извлечение срабатывает в обычном двухходовом диалоге +- `sessionMemory.enabled: false` → поиск не засоряется сводками сессий с первого дня + +Проверьте и перезапустите: + +```bash +openclaw config validate +openclaw gateway restart +openclaw logs --follow --plain | grep "memory-lancedb-pro" +``` + +Вы должны увидеть: +- `memory-lancedb-pro: smart extraction enabled` +- `memory-lancedb-pro@...: plugin registered` + +Готово. Теперь у вашего агента есть долгосрочная память. + +
+Дополнительные варианты установки (для действующих пользователей и апгрейдов) + +**Уже используете OpenClaw?** + +1. Добавьте плагин в `plugins.load.paths` как **абсолютный** путь +2. Привяжите memory slot: `plugins.slots.memory = "memory-lancedb-pro"` +3. Проверьте: `openclaw plugins info memory-lancedb-pro && openclaw memory-pro stats` + +**Обновляетесь с версии до v1.1.0?** + +```bash +# 1) Резервная копия +openclaw memory-pro export --scope global --output memories-backup.json +# 2) Пробный запуск +openclaw memory-pro upgrade --dry-run +# 3) Выполнить апгрейд +openclaw memory-pro upgrade +# 4) Проверка +openclaw memory-pro stats +``` + +Изменения поведения и причины апгрейда описаны в `CHANGELOG-v1.1.0.md`. + +
+ +
+Быстрый импорт для Telegram Bot (нажмите, чтобы раскрыть) + +Если вы используете Telegram-интеграцию OpenClaw, самый простой путь — отправить команду импорта прямо основному боту вместо ручного редактирования конфига. + +Отправьте такое сообщение: + +```text +Help me connect this memory plugin with the most user-friendly configuration: https://github.com/CortexReach/memory-lancedb-pro + +Requirements: +1. Set it as the only active memory plugin +2. Use Jina for embedding +3. Use Jina for reranker +4. Use gpt-4o-mini for the smart-extraction LLM +5. Enable autoCapture, autoRecall, smartExtraction +6. extractMinMessages=2 +7. sessionMemory.enabled=false +8. captureAssistant=false +9. retrieval mode=hybrid, vectorWeight=0.7, bm25Weight=0.3 +10. rerank=cross-encoder, candidatePoolSize=12, minScore=0.6, hardMinScore=0.62 +11. Generate the final openclaw.json config directly, not just an explanation +``` + +
+ +--- + +## Экосистема + +memory-lancedb-pro — это основной плагин. Сообщество построило вокруг него инструменты, чтобы установка и ежедневная работа были еще проще. + +### Скрипт установки: установка, апгрейд и ремонт в один клик + +> **[CortexReach/toolbox/memory-lancedb-pro-setup](https://github.com/CortexReach/toolbox/tree/main/memory-lancedb-pro-setup)** + +Это не просто установщик: скрипт грамотно обрабатывает широкий набор реальных сценариев. + +| Ваша ситуация | Что делает скрипт | +|---|---| +| Никогда не устанавливали | Скачивает заново → ставит зависимости → помогает выбрать конфиг → записывает в `openclaw.json` → перезапускает | +| Установлено через `git clone`, но застряли на старом коммите | Автоматически делает `git fetch` + `checkout` на актуальную версию → переустанавливает зависимости → проверяет | +| В конфиге есть невалидные поля | Автоматически находит их через schema filter и удаляет неподдерживаемые значения | +| Установлено через `npm` | Пропускает git-обновление и напоминает вручную запустить `npm update` | +| `openclaw` CLI сломан из-за невалидного конфига | Фолбэк: читает путь workspace напрямую из файла `openclaw.json` | +| Используется `extensions/`, а не `plugins/` | Автоматически определяет расположение плагина по конфигу или файловой системе | +| Уже актуальная версия | Запускает только health checks, без изменений | + +```bash +bash setup-memory.sh # Установить или обновить +bash setup-memory.sh --dry-run # Только предпросмотр +bash setup-memory.sh --beta # Включить pre-release версии +bash setup-memory.sh --uninstall # Откатить конфиг и удалить плагин +``` + +Встроенные пресеты провайдеров: **Jina / DashScope / SiliconFlow / OpenAI / Ollama**, либо любой собственный OpenAI-compatible API. Полное использование (включая `--ref`, `--selfcheck-only` и другое) смотрите в [README скрипта установки](https://github.com/CortexReach/toolbox/tree/main/memory-lancedb-pro-setup). + +### Навык Claude Code / OpenClaw: настройка под управлением ИИ + +> **[CortexReach/memory-lancedb-pro-skill](https://github.com/CortexReach/memory-lancedb-pro-skill)** + +Установите этот навык, и ваш ИИ-агент (Claude Code или OpenClaw) получит глубокое знание всех возможностей memory-lancedb-pro. Достаточно сказать **"help me enable the best config"**, и вы получите: + +- **Пошаговый процесс настройки из 7 шагов** с 4 вариантами деплоя: + - Полная мощность (Jina + OpenAI) / Экономный (бесплатный reranker от SiliconFlow) / Простой (только OpenAI) / Полностью локальный (Ollama, нулевая стоимость API) +- **Корректное использование всех 9 инструментов MCP**: `memory_recall`, `memory_store`, `memory_forget`, `memory_update`, `memory_stats`, `memory_list`, `self_improvement_log`, `self_improvement_extract_skill`, `self_improvement_review` *(полный набор доступен при `enableManagementTools: true` — стандартный Quick Start открывает только 4 базовых инструмента)* +- **Защиту от типичных ошибок**: включение плагина в workspace, `autoRecall` со значением false по умолчанию, кэш jiti, переменные окружения, изоляция областей памяти и другое + +**Установка для Claude Code:** +```bash +git clone https://github.com/CortexReach/memory-lancedb-pro-skill.git ~/.claude/skills/memory-lancedb-pro +``` + +**Установка для OpenClaw:** +```bash +git clone https://github.com/CortexReach/memory-lancedb-pro-skill.git ~/.openclaw/workspace/skills/memory-lancedb-pro-skill +``` + +--- + +## Видеоруководство + +> Полный разбор: установка, настройка и внутреннее устройство гибридного поиска. + +[![YouTube Video](https://img.shields.io/badge/YouTube-Watch%20Now-red?style=for-the-badge&logo=youtube)](https://youtu.be/MtukF1C8epQ) +**https://youtu.be/MtukF1C8epQ** + +[![Bilibili Video](https://img.shields.io/badge/Bilibili-Watch%20Now-00A1D6?style=for-the-badge&logo=bilibili&logoColor=white)](https://www.bilibili.com/video/BV1zUf2BGEgn/) +**https://www.bilibili.com/video/BV1zUf2BGEgn/** + +--- + +## Архитектура + +``` +┌─────────────────────────────────────────────────────────┐ +│ index.ts (Entry Point) │ +│ Plugin Registration · Config Parsing · Lifecycle Hooks │ +└────────┬──────────┬──────────┬──────────┬───────────────┘ + │ │ │ │ + ┌────▼───┐ ┌────▼───┐ ┌───▼────┐ ┌──▼──────────┐ + │ store │ │embedder│ │retriever│ │ scopes │ + │ .ts │ │ .ts │ │ .ts │ │ .ts │ + └────────┘ └────────┘ └────────┘ └─────────────┘ + │ │ + ┌────▼───┐ ┌─────▼──────────┐ + │migrate │ │noise-filter.ts │ + │ .ts │ │adaptive- │ + └────────┘ │retrieval.ts │ + └────────────────┘ + ┌─────────────┐ ┌──────────┐ + │ tools.ts │ │ cli.ts │ + │ (Agent API) │ │ (CLI) │ + └─────────────┘ └──────────┘ +``` + +> Для глубокого разбора полной архитектуры смотрите [docs/memory_architecture_analysis.md](docs/memory_architecture_analysis.md). + +
+Справочник по файлам (нажмите, чтобы раскрыть) + +| Файл | Назначение | +| --- | --- | +| `index.ts` | Точка входа плагина. Регистрация в API плагинов OpenClaw, разбор конфига, подключение хуков жизненного цикла | +| `openclaw.plugin.json` | Метаданные плагина + полная декларация JSON Schema для конфига | +| `cli.ts` | CLI-команды: `memory-pro list/search/stats/delete/delete-bulk/export/import/reembed/upgrade/migrate` | +| `src/store.ts` | Слой хранения LanceDB. Создание таблиц / FTS-индекс / векторный поиск / BM25-поиск / CRUD | +| `src/embedder.ts` | Абстракция эмбеддингов. Совместима с любым провайдером OpenAI-compatible API | +| `src/retriever.ts` | Движок гибридного поиска. Векторный поиск + BM25 → гибридное объединение → реранжирование → затухание жизненного цикла → фильтрация | +| `src/scopes.ts` | Контроль доступа для нескольких областей памяти | +| `src/tools.ts` | Определения инструментов агента: `memory_recall`, `memory_store`, `memory_forget`, `memory_update` + административные инструменты | +| `src/noise-filter.ts` | Фильтрует отказы агента, мета-вопросы, приветствия и низкокачественный контент | +| `src/adaptive-retrieval.ts` | Определяет, нужен ли конкретному запросу поиск по памяти | +| `src/migrate.ts` | Миграция со встроенного `memory-lancedb` на Pro | +| `src/smart-extractor.ts` | Извлечение по 6 категориям на базе LLM с многослойным хранением L0/L1/L2 и двухэтапной дедупликацией | +| `src/decay-engine.ts` | Модель растянутого экспоненциального затухания Weibull | +| `src/tier-manager.ts` | Трехуровневое продвижение/понижение: Peripheral ↔ Working ↔ Core | + +
+ +--- + +## Ключевые возможности + +### Гибридный поиск + +``` +Query → embedQuery() ─┐ + ├─→ Hybrid Fusion → Rerank → Lifecycle Decay Boost → Length Norm → Filter +Query → BM25 FTS ─────┘ +``` + +- **Векторный поиск** — семантическая близость через LanceDB ANN (cosine distance) +- **Полнотекстовый BM25** — точное совпадение по ключевым словам через LanceDB FTS index +- **Hybrid Fusion** — векторный score служит базой, а BM25-попадания получают взвешенный буст (это не стандартный RRF, а вариант, настроенный под качество реального recall) +- **Настраиваемые веса** — `vectorWeight`, `bm25Weight`, `minScore` + +### Кросс-энкодерное реранжирование + +- Встроенные адаптеры для **Jina**, **SiliconFlow**, **Voyage AI** и **Pinecone** +- Совместимо с любым Jina-compatible endpoint (например, Hugging Face TEI, DashScope) +- Гибридный скоринг: 60% cross-encoder + 40% исходный fused score +- Graceful degradation: при сбое API откатывается к cosine similarity + +### Многоэтапный пайплайн скоринга + +| Этап | Эффект | +| --- | --- | +| **Hybrid Fusion** | Комбинирует семантический recall и точное совпадение | +| **Cross-Encoder Rerank** | Продвигает семантически точные попадания | +| **Lifecycle Decay Boost** | Свежесть по Weibull + частота доступа + важность × уверенность | +| **Length Normalization** | Не дает длинным записям доминировать (anchor: 500 chars) | +| **Hard Min Score** | Убирает нерелевантные результаты (по умолчанию: 0.35) | +| **MMR Diversity** | Cosine similarity > 0.85 → понижается | + +### Умное извлечение памяти (v1.1.0) + +- **LLM-powered извлечение по 6 категориям**: profile, preferences, entities, events, cases, patterns +- **Многослойное хранение L0/L1/L2**: L0 (одно предложение-индекс) → L1 (структурированное summary) → L2 (полный narrative) +- **Двухэтапная дедупликация**: предварительный фильтр по векторному сходству (≥0.7) → LLM-решение по смыслу (CREATE/MERGE/SKIP) +- **Слияние с учетом категории**: `profile` всегда merge, `events` и `cases` добавляются append-only + +### Управление жизненным циклом памяти (v1.1.0) + +- **Weibull Decay Engine**: composite score = recency + frequency + intrinsic value +- **Трехуровневое продвижение**: `Peripheral ↔ Working ↔ Core` с настраиваемыми порогами +- **Усиление при доступе**: часто вспоминаемые записи затухают медленнее (в духе spaced repetition) +- **Half-life с учетом важности**: важные воспоминания живут дольше + +### Изоляция между областями памяти + +- Встроенные области памяти: `global`, `agent:`, `custom:`, `project:`, `user:` +- Контроль доступа агента через `scopes.agentAccess` +- По умолчанию каждый агент видит `global` + собственную область `agent:` + +### Auto-Capture и Auto-Recall + +- **Auto-Capture** (`agent_end`): извлекает preference/fact/decision/entity из диалога, дедуплицирует и сохраняет до 3 записей за ход +- **Auto-Recall** (`before_agent_start`): внедряет контекст `` (до 3 записей) + +### Фильтрация шума и адаптивный поиск по памяти + +- Фильтрует низкокачественный контент: отказы агента, мета-вопросы, приветствия +- Пропускает поиск по памяти для приветствий, slash-команд, простых подтверждений и emoji +- Принудительно включает поиск по памяти по ключевым словам ("remember", "previously", "last time") +- Пороги с учетом CJK (китайский: 6 символов против английского: 15 символов) + +--- + +
+Сравнение со встроенным memory-lancedb (нажмите, чтобы раскрыть) + +| Возможность | Встроенный `memory-lancedb` | **memory-lancedb-pro** | +| --- | :---: | :---: | +| Векторный поиск | Yes | Yes | +| Полнотекстовый BM25 | - | Yes | +| Гибридное объединение (Vector + BM25) | - | Yes | +| Реранжирование cross-encoder (несколько провайдеров) | - | Yes | +| Буст по свежести и затухание во времени | - | Yes | +| Нормализация по длине | - | Yes | +| MMR-диверсификация | - | Yes | +| Изоляция областей памяти | - | Yes | +| Фильтрация шума | - | Yes | +| Адаптивный поиск по памяти | - | Yes | +| Административный CLI | - | Yes | +| Память сессий | - | Yes | +| Эмбеддинги с учетом задачи | - | Yes | +| **Умное извлечение LLM (6 категорий)** | - | Yes (v1.1.0) | +| **Затухание Weibull + продвижение по уровням** | - | Yes (v1.1.0) | +| Любые OpenAI-compatible эмбеддинги | Limited | Yes | + +
+ +--- + +## Конфигурация + +
+Полный пример конфигурации + +```json +{ + "embedding": { + "apiKey": "${JINA_API_KEY}", + "model": "jina-embeddings-v5-text-small", + "baseURL": "https://api.jina.ai/v1", + "dimensions": 1024, + "taskQuery": "retrieval.query", + "taskPassage": "retrieval.passage", + "normalized": true + }, + "dbPath": "~/.openclaw/memory/lancedb-pro", + "autoCapture": true, + "autoRecall": true, + "retrieval": { + "mode": "hybrid", + "vectorWeight": 0.7, + "bm25Weight": 0.3, + "minScore": 0.3, + "rerank": "cross-encoder", + "rerankApiKey": "${JINA_API_KEY}", + "rerankModel": "jina-reranker-v3", + "rerankEndpoint": "https://api.jina.ai/v1/rerank", + "rerankProvider": "jina", + "candidatePoolSize": 20, + "recencyHalfLifeDays": 14, + "recencyWeight": 0.1, + "filterNoise": true, + "lengthNormAnchor": 500, + "hardMinScore": 0.35, + "timeDecayHalfLifeDays": 60, + "reinforcementFactor": 0.5, + "maxHalfLifeMultiplier": 3 + }, + "enableManagementTools": false, + "scopes": { + "default": "global", + "definitions": { + "global": { "description": "Shared knowledge" }, + "agent:discord-bot": { "description": "Discord bot private" } + }, + "agentAccess": { + "discord-bot": ["global", "agent:discord-bot"] + } + }, + "sessionMemory": { + "enabled": false, + "messageCount": 15 + }, + "smartExtraction": true, + "llm": { + "apiKey": "${OPENAI_API_KEY}", + "model": "gpt-4o-mini", + "baseURL": "https://api.openai.com/v1" + }, + "extractMinMessages": 2, + "extractMaxChars": 8000 +} +``` + +
+ +
+Провайдеры эмбеддингов + +Работает с **любым OpenAI-compatible API для эмбеддингов**: + +| Provider | Model | Base URL | Dimensions | +| --- | --- | --- | --- | +| **Jina** (recommended) | `jina-embeddings-v5-text-small` | `https://api.jina.ai/v1` | 1024 | +| **OpenAI** | `text-embedding-3-small` | `https://api.openai.com/v1` | 1536 | +| **Voyage** | `voyage-4-lite` / `voyage-4` | `https://api.voyageai.com/v1` | 1024 / 1024 | +| **Google Gemini** | `gemini-embedding-001` | `https://generativelanguage.googleapis.com/v1beta/openai/` | 3072 | +| **Ollama** (local) | `nomic-embed-text` | `http://localhost:11434/v1` | зависит от провайдера | + +
+ +
+Провайдеры реранжирования + +Кросс-энкодерное реранжирование поддерживает несколько провайдеров через `rerankProvider`: + +| Provider | `rerankProvider` | Example Model | +| --- | --- | --- | +| **Jina** (default) | `jina` | `jina-reranker-v3` | +| **SiliconFlow** (есть бесплатный тариф) | `siliconflow` | `BAAI/bge-reranker-v2-m3` | +| **Voyage AI** | `voyage` | `rerank-2.5` | +| **Pinecone** | `pinecone` | `bge-reranker-v2-m3` | + +Подойдет и любой Jina-compatible rerank endpoint: задайте `rerankProvider: "jina"` и укажите ваш `rerankEndpoint` (например, Hugging Face TEI, DashScope `qwen3-rerank`). + +
+ +
+Smart Extraction (LLM) — v1.1.0 + +Когда включен `smartExtraction` (по умолчанию: `true`), плагин использует LLM для интеллектуального извлечения и классификации воспоминаний вместо правил на регулярных выражениях. + +| Поле | Тип | По умолчанию | Описание | +|-------|------|---------|-------------| +| `smartExtraction` | boolean | `true` | Включить/выключить извлечение по 6 категориям на базе LLM | +| `llm.auth` | string | `api-key` | `api-key` использует `llm.apiKey` / `embedding.apiKey`; `oauth` по умолчанию использует OAuth-файл токена в области плагина | +| `llm.apiKey` | string | *(по умолчанию берется из `embedding.apiKey`)* | API-ключ провайдера LLM | +| `llm.model` | string | `openai/gpt-oss-120b` | Имя модели LLM | +| `llm.baseURL` | string | *(по умолчанию берется из `embedding.baseURL`)* | URL LLM API | +| `llm.oauthProvider` | string | `openai-codex` | Идентификатор OAuth-провайдера, используемый при `llm.auth = "oauth"` | +| `llm.oauthPath` | string | `~/.openclaw/.memory-lancedb-pro/oauth.json` | Путь к OAuth-файлу токена при `llm.auth = "oauth"` | +| `llm.timeoutMs` | number | `30000` | Таймаут запроса к LLM в миллисекундах | +| `extractMinMessages` | number | `2` | Минимум сообщений до срабатывания извлечения | +| `extractMaxChars` | number | `8000` | Максимум символов, отправляемых в LLM | + + +OAuth `llm` config (использует существующий кэш логина Codex / ChatGPT для LLM-запросов): +```json +{ + "llm": { + "auth": "oauth", + "oauthProvider": "openai-codex", + "model": "gpt-5.4", + "oauthPath": "${HOME}/.openclaw/.memory-lancedb-pro/oauth.json", + "timeoutMs": 30000 + } +} +``` + +Примечания для `llm.auth: "oauth"`: + +- `llm.oauthProvider` сейчас равен `openai-codex`. +- По умолчанию OAuth token хранится в `~/.openclaw/.memory-lancedb-pro/oauth.json`. +- Если хотите хранить этот файл в другом месте, можно задать `llm.oauthPath`. +- `auth login` сохраняет снимок предыдущего `llm` конфига в режиме api-key рядом с OAuth-файлом, а `auth logout` восстанавливает этот снимок, если он доступен. +- При переключении с `api-key` на `oauth` значение `llm.baseURL` автоматически не переносится. Указывайте его вручную в OAuth-режиме только если вам действительно нужен кастомный ChatGPT/Codex-compatible backend. + +
+ +
+Конфигурация жизненного цикла (Decay + Tier) + +| Поле | По умолчанию | Описание | +|-------|---------|-------------| +| `decay.recencyHalfLifeDays` | `30` | Базовый период полураспада для Weibull recency decay | +| `decay.frequencyWeight` | `0.3` | Вес частоты доступа в composite score | +| `decay.intrinsicWeight` | `0.3` | Вес `importance × confidence` | +| `decay.betaCore` | `0.8` | Weibull beta для воспоминаний уровня `core` | +| `decay.betaWorking` | `1.0` | Weibull beta для `working` | +| `decay.betaPeripheral` | `1.3` | Weibull beta для `peripheral` | +| `tier.coreAccessThreshold` | `10` | Минимальное число recall перед повышением в `core` | +| `tier.peripheralAgeDays` | `60` | Порог возраста для понижения устаревших воспоминаний | + +
+ +
+Усиление за счет доступа + +Часто вспоминаемые записи затухают медленнее (в духе spaced repetition). + +Ключи конфига (в разделе `retrieval`): +- `reinforcementFactor` (0-2, по умолчанию: `0.5`) — задайте `0`, чтобы отключить +- `maxHalfLifeMultiplier` (1-10, по умолчанию: `3`) — жесткий потолок эффективного периода полураспада + +
+ +--- + +## CLI-команды + +```bash +openclaw memory-pro list [--scope global] [--category fact] [--limit 20] [--json] +openclaw memory-pro search "запрос" [--scope global] [--limit 10] [--json] +openclaw memory-pro stats [--scope global] [--json] +openclaw memory-pro auth login [--provider openai-codex] [--model gpt-5.4] [--oauth-path /abs/path/oauth.json] +openclaw memory-pro auth status +openclaw memory-pro auth logout +openclaw memory-pro delete +openclaw memory-pro delete-bulk --scope global [--before 2025-01-01] [--dry-run] +openclaw memory-pro export [--scope global] [--output memories.json] +openclaw memory-pro import memories.json [--scope global] [--dry-run] +openclaw memory-pro reembed --source-db /path/to/old-db [--batch-size 32] [--skip-existing] +openclaw memory-pro upgrade [--dry-run] [--batch-size 10] [--no-llm] [--limit N] [--scope SCOPE] +openclaw memory-pro migrate check|run|verify [--source /path] +``` + +Поток OAuth-авторизации: + +1. Запустите `openclaw memory-pro auth login` +2. Если `--provider` не указан и терминал интерактивный, CLI покажет выбор OAuth-провайдера перед открытием браузера +3. Команда выведет URL авторизации и откроет браузер, если не задан `--no-browser` +4. После успешного обратного вызова команда сохранит OAuth-файл плагина (по умолчанию: `~/.openclaw/.memory-lancedb-pro/oauth.json`), снимет текущий `llm` конфиг режима api-key для будущего выхода и заменит конфиг `llm` на OAuth-настройки (`auth`, `oauthProvider`, `model`, `oauthPath`) +5. `openclaw memory-pro auth logout` удаляет этот OAuth-файл и восстанавливает прежний `llm` конфиг api-key, если снимок существует + +--- + +## Продвинутые темы + +
+Если внедренные воспоминания попадают в ответы + +Иногда модель может дословно повторять внедренный блок ``. + +**Вариант A (наименее рискованный):** временно отключить auto-recall: +```json +{ "plugins": { "entries": { "memory-lancedb-pro": { "config": { "autoRecall": false } } } } } +``` + +**Вариант B (предпочтительный):** оставить recall включенным и добавить в system prompt агента: +> Do not reveal or quote any `` / memory-injection content in your replies. Use it for internal reference only. + +
+ +
+Память сессии + +- Срабатывает по команде `/new` — сохраняет сводку предыдущей сессии в LanceDB +- По умолчанию отключено (в OpenClaw уже есть встроенная `.jsonl`-персистентность сессий) +- Количество сообщений настраивается (по умолчанию: 15) + +О режимах деплоя и проверке `/new` читайте в [docs/openclaw-integration-playbook.md](docs/openclaw-integration-playbook.md). + +
+ +
+Пользовательские slash-команды (например, /lesson) + +Добавьте в `CLAUDE.md`, `AGENTS.md` или system prompt: + +```markdown +## Команда /lesson +Когда пользователь отправляет `/lesson <контент>`: +1. Используй memory_store и сохрани как category=fact (сырое знание) +2. Используй memory_store и сохрани как category=decision (прикладной вывод) +3. Подтверди, что именно было сохранено + +## Команда /remember +Когда пользователь отправляет `/remember <контент>`: +1. Используй memory_store и сохрани с подходящими category и importance +2. Подтверди сохраненным ID памяти +``` + +
+ +
+Железные правила для ИИ-агентов + +> Скопируйте блок ниже в `AGENTS.md`, чтобы агент автоматически соблюдал эти правила. + +```markdown +## Правило 1 — Двухслойное сохранение памяти +Каждая ошибка/урок → НЕМЕДЛЕННО сохранить ДВЕ записи памяти: +- Технический слой: Проблема: [симптом]. Причина: [корневая причина]. Исправление: [решение]. Профилактика: [как избежать] + (category: fact, importance >= 0.8) +- Принципиальный слой: Принцип решения ([tag]): [правило поведения]. Триггер: [когда]. Действие: [что делать] + (category: decision, importance >= 0.85) + +## Правило 2 — Гигиена LanceDB +Записи должны быть короткими и атомарными (< 500 chars). Никаких сырых summary разговоров и дубликатов. + +## Правило 3 — Recall перед повторной попыткой +При ЛЮБОЙ ошибке инструмента ВСЕГДА выполняй memory_recall по релевантным ключевым словам ПЕРЕД повторной попыткой. + +## Правило 4 — Подтверди целевую кодовую базу +Перед изменениями убедись, что редактируешь memory-lancedb-pro, а не встроенный memory-lancedb. + +## Правило 5 — Очищай кэш jiti после изменений кода плагина +После изменения .ts-файлов в plugins/ ОБЯЗАТЕЛЬНО выполни rm -rf /tmp/jiti/ перед openclaw gateway restart. +``` + +
+ +
+Схема базы данных + +Таблица LanceDB `memories`: + +| Поле | Тип | Описание | +| --- | --- | --- | +| `id` | string (UUID) | Первичный ключ | +| `text` | string | Текст памяти (индексируется для FTS) | +| `vector` | float[] | Вектор эмбеддинга | +| `category` | string | Категория хранения: `preference` / `fact` / `decision` / `entity` / `reflection` / `other` | +| `scope` | string | Идентификатор области памяти (например, `global`, `agent:main`) | +| `importance` | float | Оценка важности от 0 до 1 | +| `timestamp` | int64 | Временная метка создания (мс) | +| `metadata` | string (JSON) | Расширенные метаданные | + +Обычные ключи `metadata` в v1.1.0: `l0_abstract`, `l1_overview`, `l2_content`, `memory_category`, `tier`, `access_count`, `confidence`, `last_accessed_at` + +> **Примечание о категориях:** поле верхнего уровня `category` использует 6 storage categories. Семантические метки Smart Extraction (`profile` / `preferences` / `entities` / `events` / `cases` / `patterns`) сохраняются в `metadata.memory_category`. + +
+ +
+Устранение неполадок + +### "Cannot mix BigInt and other types" (LanceDB / Apache Arrow) + +Начиная с LanceDB 0.26+, некоторые числовые колонки могут возвращаться как `BigInt`. Обновитесь до **memory-lancedb-pro >= 1.0.14**: теперь плагин приводит такие значения через `Number(...)` перед арифметикой. + +
+ +--- + +## Документация + +| Документ | Описание | +| --- | --- | +| [OpenClaw Integration Playbook](docs/openclaw-integration-playbook.md) | Режимы деплоя, проверка, матрица регрессии | +| [Memory Architecture Analysis](docs/memory_architecture_analysis.md) | Глубокий разбор полной архитектуры | +| [CHANGELOG v1.1.0](docs/CHANGELOG-v1.1.0.md) | Изменения поведения в v1.1.0 и причины апгрейда | +| [Long-Context Chunking](docs/long-context-chunking.md) | Стратегия разбиения длинных документов | + +--- + +## Бета: Smart Memory v1.1.0 + +> Статус: Beta — доступно через `npm i memory-lancedb-pro@beta`. Пользователи стабильного `latest` не затронуты. + +| Возможность | Описание | +|---------|-------------| +| **Умное извлечение** | Извлечение по 6 категориям на базе LLM с метаданными L0/L1/L2. При отключении откатывается к регулярным правилам. | +| **Оценка жизненного цикла** | Затухание Weibull встроено в поиск по памяти: записи с высокой частотой и важностью ранжируются выше. | +| **Управление уровнями** | Трехуровневая система (Core → Working → Peripheral) с автоматическим повышением и понижением. | + +Обратная связь: [GitHub Issues](https://github.com/CortexReach/memory-lancedb-pro/issues) · Откат: `npm i memory-lancedb-pro@latest` + +--- + +## Зависимости + +| Пакет | Назначение | +| --- | --- | +| `@lancedb/lancedb` ≥0.26.2 | Векторная база данных (ANN + FTS) | +| `openai` ≥6.21.0 | Клиент OpenAI-compatible Embedding API | +| `@sinclair/typebox` 0.34.48 | Определения типов для JSON Schema | + +--- + +## Участники + +

+@win4r +@kctony +@Akatsuki-Ryu +@JasonSuz +@Minidoracat +@furedericca-lab +@joe2643 +@AliceLJY +@chenjiyong +

+ +Полный список: [Участники](https://github.com/CortexReach/memory-lancedb-pro/graphs/contributors) + +## История звезд + + + + + + Star History Chart + + + +## Лицензия + +MIT + +--- + +## Мой QR-код WeChat + + diff --git a/README_TW.md b/README_TW.md new file mode 100644 index 00000000..b582d40f --- /dev/null +++ b/README_TW.md @@ -0,0 +1,773 @@ +
+ +# 🧠 memory-lancedb-pro · 🦞OpenClaw Plugin + +**[OpenClaw](https://github.com/openclaw/openclaw) 智慧體的 AI 記憶助理** + +*讓你的 AI 智慧體擁有真正的記憶力——跨工作階段、跨智慧體、跨時間。* + +基於 LanceDB 的 OpenClaw 長期記憶外掛，自動儲存偏好、決策和專案上下文，在後續工作階段中自動回憶。 + +[![OpenClaw Plugin](https://img.shields.io/badge/OpenClaw-Plugin-blue)](https://github.com/openclaw/openclaw) +[![npm version](https://img.shields.io/npm/v/memory-lancedb-pro)](https://www.npmjs.com/package/memory-lancedb-pro) +[![LanceDB](https://img.shields.io/badge/LanceDB-Vectorstore-orange)](https://lancedb.com) +[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE) + +[English](README.md) | [简体中文](README_CN.md) | [繁體中文](README_TW.md) | [日本語](README_JA.md) | [한국어](README_KO.md) | [Français](README_FR.md) | [Español](README_ES.md) | [Deutsch](README_DE.md) | [Italiano](README_IT.md) | [Русский](README_RU.md) | [Português (Brasil)](README_PT-BR.md) + +
+ +--- + +## 為什麼選 memory-lancedb-pro？ + +大多數 AI 智慧體都有「失憶症」——每次新對話，之前聊過的全部清零。 + +**memory-lancedb-pro** 是 OpenClaw 的生產級長期記憶外掛，把你的智慧體變成一個真正的 **AI 記憶助理**——自動擷取重要資訊，讓雜訊自然衰減，在恰當的時候回憶起恰當的內容。無需手動標記，無需複雜設定。 + +### AI 記憶助理實際效果 + +**沒有記憶——每次都從零開始：** + +> **你：** 「縮排用 tab，所有函式都要加錯誤處理。」 +> *（下一次工作階段）* +> **你：** 「我都說了用 tab 不是空格！」 😤 +> *（再下一次工作階段）* +> **你：** 「……我真的說了第三遍了，tab，還有錯誤處理。」 + +**有了 memory-lancedb-pro——你的智慧體學會了、記住了：** + +> **你：** 「縮排用 tab，所有函式都要加錯誤處理。」 +> *（下一次工作階段——智慧體自動回憶你的偏好）* +> **智慧體：** *（默默改成 tab 縮排，並補上錯誤處理）* ✅ +> **你：** 「上個月我們為什麼選了 PostgreSQL 而不是 MongoDB？」 +> **智慧體：** 「根據我們 2 月 12 日的討論，主要原因是……」 ✅ + +這就是 **AI 記憶助理** 的價值——學習你的風格，回憶過去的決策，提供個人化的回應，不再讓你重複自己。 + +### 還能做什麼？ + +| | 你能得到的 | +|---|---| +| **自動擷取** | 智慧體從每次對話中學習——不需要手動呼叫 `memory_store` | +| **智慧擷取** | LLM 驅動的 6 類分類：使用者輪廓、偏好、實體、事件、案例、模式 | +| **智慧遺忘** | Weibull 衰減模型——重要記憶留存，雜訊自然消退 | +| **混合檢索** | 向量 + BM25 全文搜尋，融合交叉編碼器重排序 | +| **上下文注入** | 相關記憶在每次回覆前自動浮現 | +| **多作用域隔離** | 按智慧體、按使用者、按專案隔離記憶邊界 | +| **任意服務商** | OpenAI、Jina、Gemini、Ollama 或任意 OpenAI 相容 API | +| **完整工具鏈** | CLI、備份、遷移、升級、匯入匯出——生產可用 | + +--- + +## 快速開始 + +### 方式 A：一鍵安裝指令碼（推薦） + +社群維護的 **[安裝指令碼](https://github.com/CortexReach/toolbox/tree/main/memory-lancedb-pro-setup)** 一條指令搞定安裝、升級和修復： + +```bash +curl -fsSL https://raw.githubusercontent.com/CortexReach/toolbox/main/memory-lancedb-pro-setup/setup-memory.sh -o setup-memory.sh +bash setup-memory.sh +``` + +> 指令碼涵蓋的完整場景和其他社群工具，詳見下方 [生態工具](#生態工具)。 + +### 方式 B：手動安裝 + +**透過 OpenClaw CLI（推薦）：** +```bash +openclaw plugins install memory-lancedb-pro@beta +``` + +**或透過 npm：** +```bash +npm i memory-lancedb-pro@beta +``` +> 如果用 npm 安裝，你還需要在 `openclaw.json` 的 `plugins.load.paths` 中新增外掛安裝目錄的 **絕對路徑**。這是最常見的安裝問題。 + +在 `openclaw.json` 中新增設定： + +```json +{ + "plugins": { + "slots": { "memory": "memory-lancedb-pro" }, + "entries": { + "memory-lancedb-pro": { + "enabled": true, + "config": { + "embedding": { + "provider": "openai-compatible", + "apiKey": "${OPENAI_API_KEY}", + "model": "text-embedding-3-small" + }, + "autoCapture": true, + "autoRecall": true, + "smartExtraction": true, + "extractMinMessages": 2, + "extractMaxChars": 8000, + "sessionMemory": { "enabled": false } + } + } + } + } +} +``` + +**為什麼用這些預設值？** +- `autoCapture` + `smartExtraction` → 智慧體自動從每次對話中學習 +- `autoRecall` → 相關記憶在每次回覆前自動注入 +- `extractMinMessages: 2` → 正常兩輪對話即觸發擷取 +- `sessionMemory.enabled: false` → 避免工作階段摘要在初期汙染檢索結果 + +驗證並重啟： + +```bash +openclaw config validate +openclaw gateway restart +openclaw logs --follow --plain | grep "memory-lancedb-pro" +``` + +你應該能看到： +- `memory-lancedb-pro: smart extraction enabled` +- `memory-lancedb-pro@...: plugin registered` + +完成！你的智慧體現在擁有長期記憶了。 + +
+更多安裝路徑（現有使用者、升級） + +**已在使用 OpenClaw？** + +1. 在 `plugins.load.paths` 中新增外掛的 **絕對路徑** +2. 繫結記憶插槽：`plugins.slots.memory = "memory-lancedb-pro"` +3. 驗證：`openclaw plugins info memory-lancedb-pro && openclaw memory-pro stats` + +**從 v1.1.0 之前的版本升級？** + +```bash +# 1) 備份 +openclaw memory-pro export --scope global --output memories-backup.json +# 2) 試執行 +openclaw memory-pro upgrade --dry-run +# 3) 執行升級 +openclaw memory-pro upgrade +# 4) 驗證 +openclaw memory-pro stats +``` + +詳見 [`CHANGELOG-v1.1.0.md`](docs/CHANGELOG-v1.1.0.md) 了解行為變更和升級說明。 + +
+ +
+Telegram Bot 快速匯入（點選展開） + +如果你在使用 OpenClaw 的 Telegram 整合，最簡單的方式是直接給主 Bot 發訊息，而不是手動編輯設定檔。 + +以下為英文原文，方便直接複製傳送給 Bot： + +```text +Help me connect this memory plugin with the most user-friendly configuration: https://github.com/CortexReach/memory-lancedb-pro + +Requirements: +1. Set it as the only active memory plugin +2. Use Jina for embedding +3. Use Jina for reranker +4. Use gpt-4o-mini for the smart-extraction LLM +5. Enable autoCapture, autoRecall, smartExtraction +6. extractMinMessages=2 +7. sessionMemory.enabled=false +8. captureAssistant=false +9. retrieval mode=hybrid, vectorWeight=0.7, bm25Weight=0.3 +10. rerank=cross-encoder, candidatePoolSize=12, minScore=0.6, hardMinScore=0.62 +11. Generate the final openclaw.json config directly, not just an explanation +``` + +
+ +--- + +## 生態工具 + +memory-lancedb-pro 是核心外掛。社群圍繞它建構了配套工具，讓安裝和日常使用更加順暢： + +### 安裝指令碼——一鍵安裝、升級和修復 + +> **[CortexReach/toolbox/memory-lancedb-pro-setup](https://github.com/CortexReach/toolbox/tree/main/memory-lancedb-pro-setup)** + +不只是簡單的安裝器——指令碼能智慧處理各種常見場景： + +| 你的情況 | 指令碼會做什麼 | +|---|---| +| 從未安裝 | 全新下載 → 安裝依賴 → 選擇設定 → 寫入 openclaw.json → 重啟 | +| 透過 `git clone` 安裝，卡在舊版本 | 自動 `git fetch` + `checkout` 到最新 → 重裝依賴 → 驗證 | +| 設定中有無效欄位 | 自動偵測並透過 schema 過濾移除不支援的欄位 | +| 透過 `npm` 安裝 | 跳過 git 更新，提醒你自行執行 `npm update` | +| `openclaw` CLI 因無效設定崩潰 | 降級方案：直接從 `openclaw.json` 檔案讀取工作目錄路徑 | +| `extensions/` 而非 `plugins/` | 從設定或檔案系統自動偵測外掛位置 | +| 已是最新版 | 僅執行健康檢查，不做變動 | + +```bash +bash setup-memory.sh # 安裝或升級 +bash setup-memory.sh --dry-run # 僅預覽 +bash setup-memory.sh --beta # 包含預發布版本 +bash setup-memory.sh --uninstall # 還原設定並移除外掛 +``` + +內建服務商預設：**Jina / DashScope / SiliconFlow / OpenAI / Ollama**，或自帶任意 OpenAI 相容 API。完整用法（含 `--ref`、`--selfcheck-only` 等）詳見[安裝指令碼 README](https://github.com/CortexReach/toolbox/tree/main/memory-lancedb-pro-setup)。 + +### Claude Code / OpenClaw Skill——AI 引導式設定 + +> **[CortexReach/memory-lancedb-pro-skill](https://github.com/CortexReach/memory-lancedb-pro-skill)** + +安裝這個 Skill，你的 AI 智慧體（Claude Code 或 OpenClaw）就能深度掌握 memory-lancedb-pro 的所有功能。只需說 **「help me enable the best config」** 即可獲得： + +- **7 步引導式設定流程**，提供 4 套部署方案： + - 滿血版（Jina + OpenAI）/ 省錢版（免費 SiliconFlow 重排序）/ 簡約版（僅 OpenAI）/ 全本機版（Ollama，零 API 成本） +- **全部 9 個 MCP 工具** 的正確用法：`memory_recall`、`memory_store`、`memory_forget`、`memory_update`、`memory_stats`、`memory_list`、`self_improvement_log`、`self_improvement_extract_skill`、`self_improvement_review` *（完整工具集需要設定 `enableManagementTools: true`——預設快速設定僅公開 4 個核心工具）* +- **避開常見陷阱**：workspace 外掛啟用、`autoRecall` 預設 false、jiti 快取、環境變數、作用域隔離等 + +**Claude Code 安裝：** +```bash +git clone https://github.com/CortexReach/memory-lancedb-pro-skill.git ~/.claude/skills/memory-lancedb-pro +``` + +**OpenClaw 安裝：** +```bash +git clone https://github.com/CortexReach/memory-lancedb-pro-skill.git ~/.openclaw/workspace/skills/memory-lancedb-pro-skill +``` + +--- + +## 影片教學 + +> 完整演示：安裝、設定、混合檢索內部原理。 + +[![YouTube Video](https://img.shields.io/badge/YouTube-Watch%20Now-red?style=for-the-badge&logo=youtube)](https://youtu.be/MtukF1C8epQ) +**https://youtu.be/MtukF1C8epQ** + +[![Bilibili Video](https://img.shields.io/badge/Bilibili-Watch%20Now-00A1D6?style=for-the-badge&logo=bilibili&logoColor=white)](https://www.bilibili.com/video/BV1zUf2BGEgn/) +**https://www.bilibili.com/video/BV1zUf2BGEgn/** + +--- + +## 架構 + +``` +┌─────────────────────────────────────────────────────────┐ +│ index.ts (入口) │ +│ 外掛註冊 · 設定解析 · 生命週期鉤子 │ +└────────┬──────────┬──────────┬──────────┬───────────────┘ + │ │ │ │ + ┌────▼───┐ ┌────▼───┐ ┌───▼────┐ ┌──▼──────────┐ + │ store │ │embedder│ │retriever│ │ scopes │ + │ .ts │ │ .ts │ │ .ts │ │ .ts │ + └────────┘ └────────┘ └────────┘ └─────────────┘ + │ │ + ┌────▼───┐ ┌─────▼──────────┐ + │migrate │ │noise-filter.ts │ + │ .ts │ │adaptive- │ + └────────┘ │retrieval.ts │ + └────────────────┘ + ┌─────────────┐ ┌──────────┐ + │ tools.ts │ │ cli.ts │ + │ (智慧體 API)│ │ (CLI) │ + └─────────────┘ └──────────┘ +``` + +> 完整架構解析見 [docs/memory_architecture_analysis.md](docs/memory_architecture_analysis.md)。 + +
+檔案說明（點選展開） + +| 檔案 | 用途 | +| --- | --- | +| `index.ts` | 外掛入口，註冊 OpenClaw Plugin API、解析設定、掛載生命週期鉤子 | +| `openclaw.plugin.json` | 外掛中繼資料 + 完整 JSON Schema 設定宣告 | +| `cli.ts` | CLI 指令：`memory-pro list/search/stats/delete/delete-bulk/export/import/reembed/upgrade/migrate` | +| `src/store.ts` | LanceDB 儲存層：建表 / 全文索引 / 向量搜尋 / BM25 搜尋 / CRUD | +| `src/embedder.ts` | Embedding 抽象層，相容任意 OpenAI 相容 API | +| `src/retriever.ts` | 混合檢索引擎：向量 + BM25 → 混合融合 → 重排序 → 生命週期衰減 → 過濾 | +| `src/scopes.ts` | 多作用域存取控制 | +| `src/tools.ts` | 智慧體工具定義：`memory_recall`、`memory_store`、`memory_forget`、`memory_update` + 管理工具 | +| `src/noise-filter.ts` | 過濾智慧體拒絕回覆、元問題、打招呼等低品質內容 | +| `src/adaptive-retrieval.ts` | 判斷查詢是否需要記憶檢索 | +| `src/migrate.ts` | 從內建 `memory-lancedb` 遷移到 Pro | +| `src/smart-extractor.ts` | LLM 驅動的 6 類擷取，支援 L0/L1/L2 分層儲存和兩階段去重 | +| `src/decay-engine.ts` | Weibull 拉伸指數衰減模型 | +| `src/tier-manager.ts` | 三級晉升/降級：外圍 ↔ 工作 ↔ 核心 | + +
+ +--- + +## 核心功能 + +### 混合檢索 + +``` +查詢 → embedQuery() ─┐ + ├─→ 混合融合 → 重排序 → 生命週期衰減加權 → 長度正規化 → 過濾 +查詢 → BM25 全文 ─────┘ +``` + +- **向量搜尋** — 基於 LanceDB ANN 的語意相似度（餘弦距離） +- **BM25 全文搜尋** — 透過 LanceDB FTS 索引進行精確關鍵字比對 +- **混合融合** — 以向量分數為基礎，BM25 命中結果獲得加權提升（非標準 RRF——針對實際召回品質調優） +- **可設定權重** — `vectorWeight`、`bm25Weight`、`minScore` + +### 交叉編碼器重排序 + +- 內建 **Jina**、**SiliconFlow**、**Voyage AI** 和 **Pinecone** 適配器 +- 相容任意 Jina 相容端點（如 Hugging Face TEI、DashScope） +- 混合打分：60% 交叉編碼器 + 40% 原始融合分數 +- 優雅降級：API 失敗時回退到餘弦相似度 + +### 多階段評分管線 + +| 階段 | 效果 | +| --- | --- | +| **混合融合** | 結合語意召回和精確比對召回 | +| **交叉編碼器重排序** | 提升語意精確命中的排名 | +| **生命週期衰減加權** | Weibull 時效性 + 存取頻率 + 重要性 × 置信度 | +| **長度正規化** | 防止長條目主導結果（錨點：500 字元） | +| **硬最低分** | 移除無關結果（預設：0.35） | +| **MMR 多樣性** | 餘弦相似度 > 0.85 → 降權 | + +### 智慧記憶擷取（v1.1.0） + +- **LLM 驅動的 6 類擷取**：使用者輪廓、偏好、實體、事件、案例、模式 +- **L0/L1/L2 分層儲存**：L0（一句話索引）→ L1（結構化摘要）→ L2（完整敘述） +- **兩階段去重**：向量相似度預過濾（≥0.7）→ LLM 語意決策（CREATE/MERGE/SKIP） +- **類別感知合併**：`profile` 始終合併，`events`/`cases` 僅追加 + +### 記憶生命週期管理（v1.1.0） + +- **Weibull 衰減引擎**：綜合分數 = 時效性 + 頻率 + 內在價值 +- **三級晉升**：`外圍 ↔ 工作 ↔ 核心`，閾值可設定 +- **存取強化**：頻繁被召回的記憶衰減更慢（類似間隔重複機制） +- **重要性調制半衰期**：重要記憶衰減更慢 + +### 多作用域隔離 + +- 內建作用域：`global`、`agent:`、`custom:`、`project:`、`user:` +- 透過 `scopes.agentAccess` 實現智慧體級別的存取控制 +- 預設：每個智慧體存取 `global` + 自己的 `agent:` 作用域 + +### 自動擷取與自動回憶 + +- **自動擷取**（`agent_end`）：從對話中擷取偏好/事實/決策/實體，去重後每輪最多儲存 3 條 +- **自動回憶**（`before_agent_start`）：注入 `` 上下文（最多 3 條） + +### 雜訊過濾與自適應檢索 + +- 過濾低品質內容：智慧體拒絕回覆、元問題、打招呼 +- 跳過檢索：打招呼、斜線指令、簡單確認、表情符號 +- 強制檢索：記憶關鍵字（「記得」、「之前」、「上次」） +- CJK 感知閾值（中文：6 字元 vs 英文：15 字元） + +--- + +
+與內建 memory-lancedb 的對比（點選展開） + +| 功能 | 內建 `memory-lancedb` | **memory-lancedb-pro** | +| --- | :---: | :---: | +| 向量搜尋 | 有 | 有 | +| BM25 全文搜尋 | - | 有 | +| 混合融合（向量 + BM25） | - | 有 | +| 交叉編碼器重排序（多服務商） | - | 有 | +| 時效性提升和時間衰減 | - | 有 | +| 長度正規化 | - | 有 | +| MMR 多樣性 | - | 有 | +| 多作用域隔離 | - | 有 | +| 雜訊過濾 | - | 有 | +| 自適應檢索 | - | 有 | +| 管理 CLI | - | 有 | +| 工作階段記憶 | - | 有 | +| 任務感知 Embedding | - | 有 | +| **LLM 智慧擷取（6 類）** | - | 有（v1.1.0） | +| **Weibull 衰減 + 層級晉升** | - | 有（v1.1.0） | +| 任意 OpenAI 相容 Embedding | 有限 | 有 | + +
+ +--- + +## 設定 + +
+完整設定範例 + +```json +{ + "embedding": { + "apiKey": "${JINA_API_KEY}", + "model": "jina-embeddings-v5-text-small", + "baseURL": "https://api.jina.ai/v1", + "dimensions": 1024, + "taskQuery": "retrieval.query", + "taskPassage": "retrieval.passage", + "normalized": true + }, + "dbPath": "~/.openclaw/memory/lancedb-pro", + "autoCapture": true, + "autoRecall": true, + "retrieval": { + "mode": "hybrid", + "vectorWeight": 0.7, + "bm25Weight": 0.3, + "minScore": 0.3, + "rerank": "cross-encoder", + "rerankApiKey": "${JINA_API_KEY}", + "rerankModel": "jina-reranker-v3", + "rerankEndpoint": "https://api.jina.ai/v1/rerank", + "rerankProvider": "jina", + "candidatePoolSize": 20, + "recencyHalfLifeDays": 14, + "recencyWeight": 0.1, + "filterNoise": true, + "lengthNormAnchor": 500, + "hardMinScore": 0.35, + "timeDecayHalfLifeDays": 60, + "reinforcementFactor": 0.5, + "maxHalfLifeMultiplier": 3 + }, + "enableManagementTools": false, + "scopes": { + "default": "global", + "definitions": { + "global": { "description": "Shared knowledge" }, + "agent:discord-bot": { "description": "Discord bot private" } + }, + "agentAccess": { + "discord-bot": ["global", "agent:discord-bot"] + } + }, + "sessionMemory": { + "enabled": false, + "messageCount": 15 + }, + "smartExtraction": true, + "llm": { + "apiKey": "${OPENAI_API_KEY}", + "model": "gpt-4o-mini", + "baseURL": "https://api.openai.com/v1" + }, + "extractMinMessages": 2, + "extractMaxChars": 8000 +} +``` + +
+ +
+Embedding 服務商 + +相容 **任意 OpenAI 相容 Embedding API**： + +| 服務商 | 模型 | Base URL | 維度 | +| --- | --- | --- | --- | +| **Jina**（推薦） | `jina-embeddings-v5-text-small` | `https://api.jina.ai/v1` | 1024 | +| **OpenAI** | `text-embedding-3-small` | `https://api.openai.com/v1` | 1536 | +| **Voyage** | `voyage-4-lite` / `voyage-4` | `https://api.voyageai.com/v1` | 1024 / 1024 | +| **Google Gemini** | `gemini-embedding-001` | `https://generativelanguage.googleapis.com/v1beta/openai/` | 3072 | +| **Ollama**（本地） | `nomic-embed-text` | `http://localhost:11434/v1` | 取決於模型 | + +
+ +
+重排序服務商 + +交叉編碼器重排序透過 `rerankProvider` 支援多個服務商： + +| 服務商 | `rerankProvider` | 範例模型 | +| --- | --- | --- | +| **Jina**（預設） | `jina` | `jina-reranker-v3` | +| **SiliconFlow**（有免費額度） | `siliconflow` | `BAAI/bge-reranker-v2-m3` | +| **Voyage AI** | `voyage` | `rerank-2.5` | +| **Pinecone** | `pinecone` | `bge-reranker-v2-m3` | + +任何 Jina 相容的重排序端點也可以使用——設定 `rerankProvider: "jina"` 並將 `rerankEndpoint` 指向你的服務（如 Hugging Face TEI、DashScope `qwen3-rerank`）。 + +
+ +
+智慧擷取（LLM）— v1.1.0 + +當 `smartExtraction` 啟用（預設 `true`）時，外掛使用 LLM 智慧擷取和分類記憶，取代基於正則的觸發方式。 + +| 欄位 | 類型 | 預設值 | 說明 | +|------|------|--------|------| +| `smartExtraction` | boolean | `true` | 是否啟用 LLM 智慧 6 類別擷取 | +| `llm.auth` | string | `api-key` | `api-key` 使用 `llm.apiKey` / `embedding.apiKey`；`oauth` 預設使用外掛級 OAuth token 檔案 | +| `llm.apiKey` | string | *（複用 `embedding.apiKey`）* | LLM 服務商 API Key | +| `llm.model` | string | `openai/gpt-oss-120b` | LLM 模型名稱 | +| `llm.baseURL` | string | *（複用 `embedding.baseURL`）* | LLM API 端點 | +| `llm.oauthProvider` | string | `openai-codex` | `llm.auth` 為 `oauth` 時使用的 OAuth provider id | +| `llm.oauthPath` | string | `~/.openclaw/.memory-lancedb-pro/oauth.json` | `llm.auth` 為 `oauth` 時使用的 OAuth token 檔案 | +| `llm.timeoutMs` | number | `30000` | LLM 請求逾時（毫秒） | +| `extractMinMessages` | number | `2` | 觸發擷取的最小訊息數 | +| `extractMaxChars` | number | `8000` | 傳送給 LLM 的最大字元數 | + + +OAuth `llm` 設定（使用現有 Codex / ChatGPT 登入快取來發送 LLM 請求）： +```json +{ + "llm": { + "auth": "oauth", + "oauthProvider": "openai-codex", + "model": "gpt-5.4", + "oauthPath": "${HOME}/.openclaw/.memory-lancedb-pro/oauth.json", + "timeoutMs": 30000 + } +} +``` + +`llm.auth: "oauth"` 說明： + +- `llm.oauthProvider` 目前僅支援 `openai-codex`。 +- OAuth token 預設存放在 `~/.openclaw/.memory-lancedb-pro/oauth.json`。 +- 如需自訂路徑，可設定 `llm.oauthPath`。 +- `auth login` 會在 OAuth 檔案旁邊快照原來的 `api-key` 模式 `llm` 設定；`auth logout` 在可用時會恢復這份快照。 +- 從 `api-key` 切到 `oauth` 時不會自動沿用 `llm.baseURL`；只有在你明確需要自訂 ChatGPT/Codex 相容後端時，才應在 `oauth` 模式下手動設定。 + +
+ +
+生命週期設定（衰減 + 層級） + +| 欄位 | 預設值 | 說明 | +|------|--------|------| +| `decay.recencyHalfLifeDays` | `30` | Weibull 時效性衰減的基礎半衰期 | +| `decay.frequencyWeight` | `0.3` | 存取頻率在綜合分數中的權重 | +| `decay.intrinsicWeight` | `0.3` | `重要性 × 置信度` 的權重 | +| `decay.betaCore` | `0.8` | `核心` 記憶的 Weibull beta | +| `decay.betaWorking` | `1.0` | `工作` 記憶的 Weibull beta | +| `decay.betaPeripheral` | `1.3` | `外圍` 記憶的 Weibull beta | +| `tier.coreAccessThreshold` | `10` | 晉升到 `核心` 所需的最小召回次數 | +| `tier.peripheralAgeDays` | `60` | 降級過期記憶的天數閾值 | + +
+ +
+存取強化 + +頻繁被召回的記憶衰減更慢（類似間隔重複機制）。 + +設定項（在 `retrieval` 下）： +- `reinforcementFactor`（0-2，預設 `0.5`）— 設為 `0` 可停用 +- `maxHalfLifeMultiplier`（1-10，預設 `3`）— 有效半衰期的硬上限 + +
+ +--- + +## CLI 指令 + +```bash +openclaw memory-pro list [--scope global] [--category fact] [--limit 20] [--json] +openclaw memory-pro search "查詢" [--scope global] [--limit 10] [--json] +openclaw memory-pro stats [--scope global] [--json] +openclaw memory-pro auth login [--provider openai-codex] [--model gpt-5.4] [--oauth-path /abs/path/oauth.json] +openclaw memory-pro auth status +openclaw memory-pro auth logout +openclaw memory-pro delete +openclaw memory-pro delete-bulk --scope global [--before 2025-01-01] [--dry-run] +openclaw memory-pro export [--scope global] [--output memories.json] +openclaw memory-pro import memories.json [--scope global] [--dry-run] +openclaw memory-pro reembed --source-db /path/to/old-db [--batch-size 32] [--skip-existing] +openclaw memory-pro upgrade [--dry-run] [--batch-size 10] [--no-llm] [--limit N] [--scope SCOPE] +openclaw memory-pro migrate check|run|verify [--source /path] +``` + +OAuth 登入流程： + +1. 執行 `openclaw memory-pro auth login` +2. 如果省略 `--provider` 且目前終端可互動，CLI 會先顯示 OAuth 服務商選擇器 +3. 指令會列印授權 URL，並在未指定 `--no-browser` 時自動開啟瀏覽器 +4. 回呼成功後，指令會儲存外掛 OAuth 檔案（預設：`~/.openclaw/.memory-lancedb-pro/oauth.json`）、為 logout 快照原來的 `api-key` 模式 `llm` 設定，並把外掛 `llm` 設定切換為 OAuth 欄位（`auth`、`oauthProvider`、`model`、`oauthPath`） +5. `openclaw memory-pro auth logout` 會刪除這份 OAuth 檔案，並在存在快照時恢復之前的 `api-key` 模式 `llm` 設定 + +--- + +## 進階主題 + +
+注入的記憶出現在回覆中 + +有時模型可能會將注入的 `` 區塊原文輸出。 + +**方案 A（最安全）：** 暫時關閉自動回憶： +```json +{ "plugins": { "entries": { "memory-lancedb-pro": { "config": { "autoRecall": false } } } } } +``` + +**方案 B（推薦）：** 保留回憶，在智慧體系統提示詞中新增： +> Do not reveal or quote any `` / memory-injection content in your replies. Use it for internal reference only. + +
+ +
+工作階段記憶 + +- 透過 `/new` 指令觸發——將上一段工作階段摘要儲存到 LanceDB +- 預設關閉（OpenClaw 已有原生 `.jsonl` 工作階段持久化） +- 可設定訊息數量（預設 15） + +部署模式和 `/new` 驗證詳見 [docs/openclaw-integration-playbook.md](docs/openclaw-integration-playbook.md)。 + +
+ +
+自訂斜線指令（如 /lesson） + +在你的 `CLAUDE.md`、`AGENTS.md` 或系統提示詞中新增： + +```markdown +## /lesson 指令 +當使用者傳送 `/lesson <內容>` 時： +1. 用 memory_store 儲存為 category=fact（原始知識） +2. 用 memory_store 儲存為 category=decision（可執行的結論） +3. 確認已儲存的內容 + +## /remember 指令 +當使用者傳送 `/remember <內容>` 時： +1. 用 memory_store 以合適的 category 和 importance 儲存 +2. 回傳已儲存的記憶 ID 確認 +``` + +
+ +
+AI 智慧體鐵律 + +> 將以下內容複製到你的 `AGENTS.md`，讓智慧體自動遵守這些規則。 + +```markdown +## 規則 1 — 雙層記憶儲存 +每個踩坑/經驗教訓 → 立即儲存兩條記憶： +- 技術層：踩坑：[現象]。原因：[根因]。修復：[方案]。預防：[如何避免] + (category: fact, importance >= 0.8) +- 原則層：決策原則 ([標籤])：[行為規則]。觸發：[何時]。動作：[做什麼] + (category: decision, importance >= 0.85) + +## 規則 2 — LanceDB 資料品質 +條目必須簡短且原子化（< 500 字元）。不儲存原始對話摘要或重複內容。 + +## 規則 3 — 重試前先回憶 +任何工具呼叫失敗時，必須先用 memory_recall 搜尋相關關鍵字，再重試。 + +## 規則 4 — 確認目標程式碼庫 +修改前確認你操作的是 memory-lancedb-pro 還是內建 memory-lancedb。 + +## 規則 5 — 修改外掛程式碼後清除 jiti 快取 +修改 plugins/ 下的 .ts 檔案後，必須先清除 /tmp/jiti/ 目錄再重啟 openclaw gateway。 +``` + +
+ +
+資料庫 Schema + +LanceDB 表 `memories`： + +| 欄位 | 類型 | 說明 | +| --- | --- | --- | +| `id` | string (UUID) | 主鍵 | +| `text` | string | 記憶文字（全文索引） | +| `vector` | float[] | Embedding 向量 | +| `category` | string | 儲存類別：`preference` / `fact` / `decision` / `entity` / `reflection` / `other` | +| `scope` | string | 作用域識別碼（如 `global`、`agent:main`） | +| `importance` | float | 重要性分數 0-1 | +| `timestamp` | int64 | 建立時間戳記（毫秒） | +| `metadata` | string (JSON) | 擴充中繼資料 | + +v1.1.0 常用 `metadata` 欄位：`l0_abstract`、`l1_overview`、`l2_content`、`memory_category`、`tier`、`access_count`、`confidence`、`last_accessed_at` + +> **關於分類的說明：** 頂層 `category` 欄位使用 6 個儲存類別。智慧擷取的 6 類語意標籤（`profile` / `preferences` / `entities` / `events` / `cases` / `patterns`）儲存在 `metadata.memory_category` 中。 + +
+ +
+故障排除 + +### "Cannot mix BigInt and other types"（LanceDB / Apache Arrow） + +在 LanceDB 0.26+ 上，某些數值欄位可能以 `BigInt` 形式回傳。升級到 **memory-lancedb-pro >= 1.0.14**——外掛現在會在運算前使用 `Number(...)` 進行類型轉換。 + +
+ +--- + +## 文件 + +| 文件 | 說明 | +| --- | --- | +| [OpenClaw 整合手冊](docs/openclaw-integration-playbook.md) | 部署模式、驗證、迴歸矩陣 | +| [記憶架構分析](docs/memory_architecture_analysis.md) | 完整架構深度解析 | +| [CHANGELOG v1.1.0](docs/CHANGELOG-v1.1.0.md) | v1.1.0 行為變更和升級說明 | +| [長上下文分塊](docs/long-context-chunking.md) | 長文件分塊策略 | + +--- + +## 測試版：智慧記憶 v1.1.0 + +> 狀態：Beta（測試版）——透過 `npm i memory-lancedb-pro@beta` 安裝。使用 `latest` 的穩定版使用者不受影響。 + +| 功能 | 說明 | +|------|------| +| **智慧擷取** | LLM 驅動的 6 類擷取，支援 L0/L1/L2 中繼資料。停用時回退到正則模式。 | +| **生命週期評分** | Weibull 衰減整合到檢索中——高頻和高重要性記憶排名更高。 | +| **層級管理** | 三級系統（核心 → 工作 → 外圍），自動晉升/降級。 | + +回饋：[GitHub Issues](https://github.com/CortexReach/memory-lancedb-pro/issues) · 回退：`npm i memory-lancedb-pro@latest` + +--- + +## 依賴 + +| 套件 | 用途 | +| --- | --- | +| `@lancedb/lancedb` ≥0.26.2 | 向量資料庫（ANN + FTS） | +| `openai` ≥6.21.0 | OpenAI 相容 Embedding API 客戶端 | +| `@sinclair/typebox` 0.34.48 | JSON Schema 類型定義 | + +--- + +## Contributors + +

+@win4r +@kctony +@Akatsuki-Ryu +@JasonSuz +@Minidoracat +@furedericca-lab +@joe2643 +@AliceLJY +@chenjiyong +

+ +Full list: [Contributors](https://github.com/CortexReach/memory-lancedb-pro/graphs/contributors) + +## Star History + + + + + + Star History Chart + + + +## 授權條款 + +MIT + +--- + +## 我的微信 QR Code + + diff --git a/scripts/governance-maintenance.mjs b/scripts/governance-maintenance.mjs new file mode 100644 index 00000000..7eb4a73c --- /dev/null +++ b/scripts/governance-maintenance.mjs @@ -0,0 +1,130 @@ +#!/usr/bin/env node +import { resolve } from "node:path"; +import jitiFactory from "jiti"; + +const jiti = jitiFactory(import.meta.url, { interopDefault: true }); +const { MemoryStore } = jiti("../src/store.ts"); +const { parseSmartMetadata, stringifySmartMetadata } = jiti("../src/smart-metadata.ts"); + +function parseArgs(argv) { + const args = { + dbPath: process.env.MEMORY_DB_PATH || "", + vectorDim: Number(process.env.MEMORY_VECTOR_DIM || "1536"), + scope: undefined, + apply: false, + pendingDays: 30, + limit: 1000, + }; + for (let i = 2; i < argv.length; i++) { + const a = argv[i]; + if (a === "--db-path") args.dbPath = argv[++i] || ""; + else if (a === "--vector-dim") args.vectorDim = Number(argv[++i] || "1536"); + else if (a === "--scope") args.scope = argv[++i] || undefined; + else if (a === "--apply") args.apply = true; + else if (a === "--pending-days") args.pendingDays = Number(argv[++i] || "30"); + else if (a === "--limit") args.limit = Number(argv[++i] || "1000"); + } + return args; +} + +async function loadAllEntries(store, scopeFilter, limit) { + const out = []; + let offset = 0; + const pageSize = 200; + while (out.length < limit) { + const page = await store.list(scopeFilter, undefined, Math.min(pageSize, limit - out.length), offset); + if (!page.length) break; + out.push(...page); + offset += page.length; + if (page.length < pageSize) break; + } + return out; +} + +function normalizeKey(text) { + return text.toLowerCase().replace(/\s+/g, " ").trim(); +} + +async function run() { + const args = parseArgs(process.argv); + if (!args.dbPath) throw new Error("Missing --db-path (or MEMORY_DB_PATH)"); + + const store = new MemoryStore({ + dbPath: resolve(args.dbPath), + vectorDim: Number.isFinite(args.vectorDim) ? args.vectorDim : 1536, + }); + const scopeFilter = args.scope ? [args.scope] : undefined; + const entries = await loadAllEntries(store, scopeFilter, args.limit); + + const now = Date.now(); + const pendingCutoff = now - Math.max(1, args.pendingDays) * 24 * 60 * 60 * 1000; + + const toArchivePending = []; + const canonicalByKey = new Map(); + const duplicateCandidates = []; + + for (const entry of entries) { + const meta = parseSmartMetadata(entry.metadata, entry); + + if (meta.state === "pending" && entry.timestamp < pendingCutoff) { + toArchivePending.push(entry.id); + } + + if (meta.state === "archived") continue; + const key = `${meta.memory_category}:${normalizeKey(meta.l0_abstract || entry.text)}`; + const existing = canonicalByKey.get(key); + if (!existing) { + canonicalByKey.set(key, entry); + continue; + } + const keep = existing.timestamp >= entry.timestamp ? existing : entry; + const drop = keep.id === existing.id ? entry : existing; + canonicalByKey.set(key, keep); + duplicateCandidates.push({ duplicateId: drop.id, canonicalId: keep.id }); + } + + if (!args.apply) { + console.log(`Dry run summary:`); + console.log(`- scanned: ${entries.length}`); + console.log(`- stale pending -> archive: ${toArchivePending.length}`); + console.log(`- duplicate compact candidates: ${duplicateCandidates.length}`); + return; + } + + let archivedPending = 0; + for (const id of toArchivePending) { + const existing = await store.getById(id, scopeFilter); + if (!existing) continue; + const meta = parseSmartMetadata(existing.metadata, existing); + meta.state = "archived"; + meta.memory_layer = "archive"; + meta.archive_reason = "pending_timeout"; + meta.archived_at = now; + await store.update(id, { metadata: stringifySmartMetadata(meta) }, scopeFilter); + archivedPending++; + } + + let compacted = 0; + for (const row of duplicateCandidates) { + const existing = await store.getById(row.duplicateId, scopeFilter); + if (!existing) continue; + const meta = parseSmartMetadata(existing.metadata, existing); + meta.state = "archived"; + meta.memory_layer = "archive"; + meta.archive_reason = "compact_duplicate"; + meta.canonical_id = row.canonicalId; + meta.archived_at = now; + await store.update(row.duplicateId, { metadata: stringifySmartMetadata(meta) }, scopeFilter); + compacted++; + } + + console.log(`Maintenance complete:`); + console.log(`- scanned: ${entries.length}`); + console.log(`- archived pending: ${archivedPending}`); + console.log(`- compacted duplicates: ${compacted}`); +} + +run().catch((err) => { + console.error(err instanceof Error ? err.message : String(err)); + process.exit(1); +}); diff --git a/scripts/migrate-governance-metadata.mjs b/scripts/migrate-governance-metadata.mjs new file mode 100644 index 00000000..fafb70d9 --- /dev/null +++ b/scripts/migrate-governance-metadata.mjs @@ -0,0 +1,110 @@ +#!/usr/bin/env node +import { createWriteStream, readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import jitiFactory from "jiti"; + +const jiti = jitiFactory(import.meta.url, { interopDefault: true }); +const { MemoryStore } = jiti("../src/store.ts"); +const { buildSmartMetadata, stringifySmartMetadata } = jiti("../src/smart-metadata.ts"); + +function parseArgs(argv) { + const args = { + dbPath: process.env.MEMORY_DB_PATH || "", + vectorDim: Number(process.env.MEMORY_VECTOR_DIM || "1536"), + scope: undefined, + apply: false, + limit: 1000, + rollbackFile: "", + }; + + for (let i = 2; i < argv.length; i++) { + const a = argv[i]; + if (a === "--db-path") args.dbPath = argv[++i] || ""; + else if (a === "--vector-dim") args.vectorDim = Number(argv[++i] || "1536"); + else if (a === "--scope") args.scope = argv[++i] || undefined; + else if (a === "--apply") args.apply = true; + else if (a === "--limit") args.limit = Number(argv[++i] || "1000"); + else if (a === "--rollback") args.rollbackFile = argv[++i] || ""; + } + + return args; +} + +async function loadAllEntries(store, scopeFilter, limit) { + const out = []; + let offset = 0; + const pageSize = 200; + while (out.length < limit) { + const page = await store.list(scopeFilter, undefined, Math.min(pageSize, limit - out.length), offset); + if (!page.length) break; + out.push(...page); + offset += page.length; + if (page.length < pageSize) break; + } + return out; +} + +async function run() { + const args = parseArgs(process.argv); + if (!args.dbPath) { + throw new Error("Missing --db-path (or MEMORY_DB_PATH)"); + } + + const store = new MemoryStore({ + dbPath: resolve(args.dbPath), + vectorDim: Number.isFinite(args.vectorDim) ? args.vectorDim : 1536, + }); + + const scopeFilter = args.scope ? [args.scope] : undefined; + + if (args.rollbackFile) { + const raw = readFileSync(resolve(args.rollbackFile), "utf8"); + const lines = raw.split(/\r?\n/).filter(Boolean); + let restored = 0; + for (const line of lines) { + const row = JSON.parse(line); + await store.update(row.id, { metadata: row.metadata }, scopeFilter); + restored++; + } + console.log(`Rollback complete. Restored ${restored} metadata entries.`); + return; + } + + const entries = await loadAllEntries(store, scopeFilter, args.limit); + const changed = []; + + for (const entry of entries) { + const normalized = buildSmartMetadata(entry, {}); + const next = stringifySmartMetadata(normalized); + const prev = typeof entry.metadata === "string" ? entry.metadata : "{}"; + if (next !== prev) { + changed.push({ id: entry.id, prev, next }); + } + } + + if (!args.apply) { + console.log(`Dry run complete. scanned=${entries.length} pending_updates=${changed.length}`); + return; + } + + const ts = new Date().toISOString().replace(/[:.]/g, "-"); + const backupPath = resolve(`governance-migration-backup-${ts}.jsonl`); + const backup = createWriteStream(backupPath, { flags: "wx" }); + + let applied = 0; + for (const row of changed) { + backup.write(`${JSON.stringify({ id: row.id, metadata: row.prev })}\n`); + await store.update(row.id, { metadata: row.next }, scopeFilter); + applied++; + } + backup.end(); + + console.log(`Migration complete. scanned=${entries.length} updated=${applied}`); + console.log(`Rollback file: ${backupPath}`); +} + +run().catch((err) => { + console.error(err instanceof Error ? err.message : String(err)); + process.exit(1); +}); diff --git a/src/admission-control.ts b/src/admission-control.ts new file mode 100644 index 00000000..eee44d35 --- /dev/null +++ b/src/admission-control.ts @@ -0,0 +1,748 @@ +import { join } from "node:path"; +import type { LlmClient } from "./llm-client.js"; +import type { CandidateMemory, MemoryCategory } from "./memory-categories.js"; +import type { MemorySearchResult, MemoryStore } from "./store.js"; +import { parseSmartMetadata } from "./smart-metadata.js"; + +export interface AdmissionWeights { + utility: number; + confidence: number; + novelty: number; + recency: number; + typePrior: number; +} + +export interface AdmissionTypePriors { + profile: number; + preferences: number; + entities: number; + events: number; + cases: number; + patterns: number; +} + +export interface AdmissionRecencyConfig { + halfLifeDays: number; +} + +export type AdmissionControlPreset = + | "balanced" + | "conservative" + | "high-recall"; + +export interface AdmissionControlConfig { + preset: AdmissionControlPreset; + enabled: boolean; + utilityMode: "standalone" | "off"; + weights: AdmissionWeights; + rejectThreshold: number; + admitThreshold: number; + noveltyCandidatePoolSize: number; + recency: AdmissionRecencyConfig; + typePriors: AdmissionTypePriors; + auditMetadata: boolean; + persistRejectedAudits: boolean; + rejectedAuditFilePath?: string; +} + +export interface AdmissionFeatureScores { + utility: number; + confidence: number; + novelty: number; + recency: number; + typePrior: number; +} + +export interface AdmissionAuditRecord { + version: "amac-v1"; + decision: "reject" | "pass_to_dedup"; + hint?: "add" | "update_or_merge"; + score: number; + reason: string; + utility_reason?: string; + thresholds: { + reject: number; + admit: number; + }; + weights: AdmissionWeights; + feature_scores: AdmissionFeatureScores; + matched_existing_memory_ids: string[]; + compared_existing_memory_ids: string[]; + max_similarity: number; + evaluated_at: number; +} + +export interface AdmissionEvaluation { + decision: "reject" | "pass_to_dedup"; + hint?: "add" | "update_or_merge"; + audit: AdmissionAuditRecord; +} + +export interface AdmissionRejectionAuditEntry { + version: "amac-v1"; + rejected_at: number; + session_key: string; + target_scope: string; + scope_filter: string[]; + candidate: CandidateMemory; + audit: AdmissionAuditRecord & { decision: "reject" }; + conversation_excerpt: string; +} + +export interface ConfidenceSupportBreakdown { + score: number; + bestSupport: number; + coverage: number; + unsupportedRatio: number; +} + +export interface NoveltyBreakdown { + score: number; + maxSimilarity: number; + matchedIds: string[]; + comparedIds: string[]; +} + +const DEFAULT_WEIGHTS: AdmissionWeights = { + utility: 0.1, + confidence: 0.1, + novelty: 0.1, + recency: 0.1, + typePrior: 0.6, +}; + +const DEFAULT_TYPE_PRIORS: AdmissionTypePriors = { + profile: 0.95, + preferences: 0.9, + entities: 0.75, + events: 0.45, + cases: 0.8, + patterns: 0.85, +}; + +function cloneAdmissionControlConfig(config: AdmissionControlConfig): AdmissionControlConfig { + return { + ...config, + recency: { ...config.recency }, + weights: { ...config.weights }, + typePriors: { ...config.typePriors }, + }; +} + +export const ADMISSION_CONTROL_PRESETS: Record = { + balanced: { + preset: "balanced", + enabled: false, + utilityMode: "standalone", + weights: DEFAULT_WEIGHTS, + rejectThreshold: 0.45, + admitThreshold: 0.6, + noveltyCandidatePoolSize: 8, + recency: { + halfLifeDays: 14, + }, + typePriors: DEFAULT_TYPE_PRIORS, + auditMetadata: true, + persistRejectedAudits: true, + rejectedAuditFilePath: undefined, + }, + conservative: { + preset: "conservative", + enabled: false, + utilityMode: "standalone", + weights: { + utility: 0.16, + confidence: 0.16, + novelty: 0.18, + recency: 0.08, + typePrior: 0.42, + }, + rejectThreshold: 0.52, + admitThreshold: 0.68, + noveltyCandidatePoolSize: 10, + recency: { + halfLifeDays: 10, + }, + typePriors: { + profile: 0.98, + preferences: 0.94, + entities: 0.78, + events: 0.28, + cases: 0.78, + patterns: 0.8, + }, + auditMetadata: true, + persistRejectedAudits: true, + rejectedAuditFilePath: undefined, + }, + "high-recall": { + preset: "high-recall", + enabled: false, + utilityMode: "standalone", + weights: { + utility: 0.08, + confidence: 0.1, + novelty: 0.08, + recency: 0.14, + typePrior: 0.6, + }, + rejectThreshold: 0.34, + admitThreshold: 0.52, + noveltyCandidatePoolSize: 6, + recency: { + halfLifeDays: 21, + }, + typePriors: { + profile: 0.96, + preferences: 0.92, + entities: 0.8, + events: 0.58, + cases: 0.84, + patterns: 0.88, + }, + auditMetadata: true, + persistRejectedAudits: true, + rejectedAuditFilePath: undefined, + }, +}; + +export const DEFAULT_ADMISSION_CONTROL_CONFIG = + ADMISSION_CONTROL_PRESETS.balanced; + +function parseAdmissionControlPreset(raw: unknown): AdmissionControlPreset { + switch (raw) { + case "conservative": + case "high-recall": + case "balanced": + return raw; + default: + return "balanced"; + } +} + +function clamp01(value: unknown, fallback: number): number { + const n = typeof value === "number" ? value : Number(value); + if (!Number.isFinite(n)) return fallback; + return Math.min(1, Math.max(0, n)); +} + +function clampPositiveInt(value: unknown, fallback: number, max: number): number { + const n = typeof value === "number" ? value : Number(value); + if (!Number.isFinite(n) || n <= 0) return fallback; + return Math.min(max, Math.max(1, Math.floor(n))); +} + +function normalizeWeights(raw: unknown, defaults: AdmissionWeights): AdmissionWeights { + if (!raw || typeof raw !== "object") { + return { ...defaults }; + } + + const obj = raw as Record; + const candidate: AdmissionWeights = { + utility: clamp01(obj.utility, defaults.utility), + confidence: clamp01(obj.confidence, defaults.confidence), + novelty: clamp01(obj.novelty, defaults.novelty), + recency: clamp01(obj.recency, defaults.recency), + typePrior: clamp01(obj.typePrior, defaults.typePrior), + }; + + const total = + candidate.utility + + candidate.confidence + + candidate.novelty + + candidate.recency + + candidate.typePrior; + + if (total <= 0) { + return { ...defaults }; + } + + return { + utility: candidate.utility / total, + confidence: candidate.confidence / total, + novelty: candidate.novelty / total, + recency: candidate.recency / total, + typePrior: candidate.typePrior / total, + }; +} + +function normalizeTypePriors(raw: unknown, defaults: AdmissionTypePriors): AdmissionTypePriors { + if (!raw || typeof raw !== "object") { + return { ...defaults }; + } + + const obj = raw as Record; + return { + profile: clamp01(obj.profile, defaults.profile), + preferences: clamp01(obj.preferences, defaults.preferences), + entities: clamp01(obj.entities, defaults.entities), + events: clamp01(obj.events, defaults.events), + cases: clamp01(obj.cases, defaults.cases), + patterns: clamp01(obj.patterns, defaults.patterns), + }; +} + +export function normalizeAdmissionControlConfig(raw: unknown): AdmissionControlConfig { + if (!raw || typeof raw !== "object") { + return cloneAdmissionControlConfig(DEFAULT_ADMISSION_CONTROL_CONFIG); + } + + const obj = raw as Record; + const preset = parseAdmissionControlPreset(obj.preset); + const base = cloneAdmissionControlConfig(ADMISSION_CONTROL_PRESETS[preset]); + const rejectThreshold = clamp01(obj.rejectThreshold, base.rejectThreshold); + const admitThreshold = clamp01(obj.admitThreshold, base.admitThreshold); + const normalizedAdmit = Math.max(admitThreshold, rejectThreshold); + const recencyRaw = + typeof obj.recency === "object" && obj.recency !== null + ? (obj.recency as Record) + : {}; + + return { + preset, + enabled: obj.enabled === true, + utilityMode: + obj.utilityMode === "off" + ? "off" + : obj.utilityMode === "standalone" + ? "standalone" + : base.utilityMode, + weights: normalizeWeights(obj.weights, base.weights), + rejectThreshold, + admitThreshold: normalizedAdmit, + noveltyCandidatePoolSize: clampPositiveInt( + obj.noveltyCandidatePoolSize, + base.noveltyCandidatePoolSize, + 20, + ), + recency: { + halfLifeDays: clampPositiveInt( + recencyRaw.halfLifeDays, + base.recency.halfLifeDays, + 365, + ), + }, + typePriors: normalizeTypePriors(obj.typePriors, base.typePriors), + auditMetadata: + typeof obj.auditMetadata === "boolean" + ? obj.auditMetadata + : base.auditMetadata, + persistRejectedAudits: + typeof obj.persistRejectedAudits === "boolean" + ? obj.persistRejectedAudits + : base.persistRejectedAudits, + rejectedAuditFilePath: + typeof obj.rejectedAuditFilePath === "string" && + obj.rejectedAuditFilePath.trim().length > 0 + ? obj.rejectedAuditFilePath.trim() + : undefined, + }; +} + +export function resolveRejectedAuditFilePath( + dbPath: string, + config?: Pick | null, +): string { + const explicitPath = config?.rejectedAuditFilePath; + if (typeof explicitPath === "string" && explicitPath.trim().length > 0) { + return explicitPath.trim(); + } + return join(dbPath, "..", "admission-audit", "rejections.jsonl"); +} + +function isHanChar(char: string): boolean { + return /\p{Script=Han}/u.test(char); +} + +function isWordChar(char: string): boolean { + return /[\p{Letter}\p{Number}]/u.test(char); +} + +function tokenizeText(value: string): string[] { + const normalized = value.toLowerCase().trim(); + const tokens: string[] = []; + let current = ""; + + for (const char of normalized) { + if (isHanChar(char)) { + if (current) { + tokens.push(current); + current = ""; + } + tokens.push(char); + continue; + } + + if (isWordChar(char)) { + current += char; + continue; + } + + if (current) { + tokens.push(current); + current = ""; + } + } + + if (current) { + tokens.push(current); + } + + return tokens; +} + +function lcsLength(left: string[], right: string[]): number { + if (left.length === 0 || right.length === 0) return 0; + const dp = Array.from({ length: left.length + 1 }, () => + Array(right.length + 1).fill(0), + ); + + for (let i = 1; i <= left.length; i++) { + for (let j = 1; j <= right.length; j++) { + if (left[i - 1] === right[j - 1]) { + dp[i][j] = dp[i - 1][j - 1] + 1; + } else { + dp[i][j] = Math.max(dp[i - 1][j], dp[i][j - 1]); + } + } + } + + return dp[left.length][right.length]; +} + +function rougeLikeF1(left: string[], right: string[]): number { + if (left.length === 0 || right.length === 0) return 0; + const lcs = lcsLength(left, right); + if (lcs === 0) return 0; + const precision = lcs / left.length; + const recall = lcs / right.length; + if (precision + recall === 0) return 0; + return (2 * precision * recall) / (precision + recall); +} + +function splitSupportSpans(conversationText: string): string[] { + const spans = new Set(); + for (const line of conversationText.split(/\n+/)) { + const trimmed = line.trim(); + if (!trimmed) continue; + spans.add(trimmed); + for (const sentence of trimmed.split(/[。！？!?]+/)) { + const candidate = sentence.trim(); + if (candidate.length >= 4) { + spans.add(candidate); + } + } + } + return Array.from(spans); +} + +function cosineSimilarity(left: number[], right: number[]): number { + if (!Array.isArray(left) || !Array.isArray(right) || left.length === 0 || right.length === 0) { + return 0; + } + + const size = Math.min(left.length, right.length); + let dot = 0; + let leftNorm = 0; + let rightNorm = 0; + + for (let i = 0; i < size; i++) { + const l = Number(left[i]) || 0; + const r = Number(right[i]) || 0; + dot += l * r; + leftNorm += l * l; + rightNorm += r * r; + } + + if (leftNorm === 0 || rightNorm === 0) return 0; + return dot / (Math.sqrt(leftNorm) * Math.sqrt(rightNorm)); +} + +function buildUtilityPrompt(candidate: CandidateMemory, conversationText: string): string { + const excerpt = + conversationText.length > 3000 + ? conversationText.slice(-3000) + : conversationText; + + return `Evaluate whether this candidate memory is worth keeping for future cross-session interactions. + +Conversation excerpt: +${excerpt} + +Candidate memory: +- Category: ${candidate.category} +- Abstract: ${candidate.abstract} +- Overview: ${candidate.overview} +- Content: ${candidate.content} + +Score future usefulness on a 0.0-1.0 scale. + +Use higher scores for durable preferences, profile facts, reusable procedures, and long-lived project/entity state. +Use lower scores for one-off chatter, low-signal situational remarks, thin restatements, and low-value transient details. + +Return JSON only: +{ + "utility": 0.0, + "reason": "short explanation" +}`; +} + +function buildReason(details: { + decision: "reject" | "pass_to_dedup"; + hint?: "add" | "update_or_merge"; + score: number; + rejectThreshold: number; + maxSimilarity: number; + utilityReason?: string; +}): string { + const scoreText = details.score.toFixed(3); + const similarityText = details.maxSimilarity.toFixed(3); + const utilityText = details.utilityReason ? ` Utility: ${details.utilityReason}` : ""; + if (details.decision === "reject") { + return `Admission rejected (${scoreText} < ${details.rejectThreshold.toFixed(3)}). maxSimilarity=${similarityText}.${utilityText}`.trim(); + } + const hintText = details.hint ? ` hint=${details.hint};` : ""; + return `Admission passed (${scoreText});${hintText} maxSimilarity=${similarityText}.${utilityText}`.trim(); +} + +export function scoreTypePrior( + category: MemoryCategory, + typePriors: AdmissionTypePriors, +): number { + return clamp01(typePriors[category], DEFAULT_TYPE_PRIORS[category]); +} + +export function scoreConfidenceSupport( + candidate: CandidateMemory, + conversationText: string, +): ConfidenceSupportBreakdown { + const candidateText = `${candidate.abstract}\n${candidate.content}`.trim(); + const candidateTokens = tokenizeText(candidateText); + if (candidateTokens.length === 0) { + return { score: 0, bestSupport: 0, coverage: 0, unsupportedRatio: 1 }; + } + + const spans = splitSupportSpans(conversationText); + const conversationTokens = new Set(tokenizeText(conversationText)); + let bestSupport = 0; + + for (const span of spans) { + const spanTokens = tokenizeText(span); + bestSupport = Math.max(bestSupport, rougeLikeF1(candidateTokens, spanTokens)); + } + + const uniqueCandidateTokens = Array.from(new Set(candidateTokens)); + const supportedTokenCount = uniqueCandidateTokens.filter((token) => conversationTokens.has(token)).length; + const coverage = uniqueCandidateTokens.length > 0 ? supportedTokenCount / uniqueCandidateTokens.length : 0; + const unsupportedRatio = uniqueCandidateTokens.length > 0 ? 1 - coverage : 1; + const score = clamp01((bestSupport * 0.7) + (coverage * 0.3) - (unsupportedRatio * 0.25), 0); + + return { score, bestSupport, coverage, unsupportedRatio }; +} + +export function scoreNoveltyFromMatches( + candidateVector: number[], + matches: MemorySearchResult[], +): NoveltyBreakdown { + if (!Array.isArray(candidateVector) || candidateVector.length === 0 || matches.length === 0) { + return { score: 1, maxSimilarity: 0, matchedIds: [], comparedIds: [] }; + } + + let maxSimilarity = 0; + const comparedIds: string[] = []; + const matchedIds: string[] = []; + + for (const match of matches) { + comparedIds.push(match.entry.id); + const similarity = Math.max(0, cosineSimilarity(candidateVector, match.entry.vector)); + if (similarity > maxSimilarity) { + maxSimilarity = similarity; + } + if (similarity >= 0.55) { + matchedIds.push(match.entry.id); + } + } + + return { + score: clamp01(1 - maxSimilarity, 1), + maxSimilarity, + matchedIds, + comparedIds, + }; +} + +export function scoreRecencyGap( + now: number, + matches: MemorySearchResult[], + halfLifeDays: number, +): number { + if (matches.length === 0 || halfLifeDays <= 0) { + return 1; + } + + const latestTimestamp = Math.max( + ...matches.map((match) => (Number.isFinite(match.entry.timestamp) ? match.entry.timestamp : 0)), + ); + if (!Number.isFinite(latestTimestamp) || latestTimestamp <= 0) { + return 1; + } + + const gapMs = Math.max(0, now - latestTimestamp); + const gapDays = gapMs / 86_400_000; + if (gapDays === 0) { + return 0; + } + + const lambda = Math.LN2 / halfLifeDays; + return clamp01(1 - Math.exp(-lambda * gapDays), 1); +} + +async function scoreUtility( + llm: LlmClient, + mode: AdmissionControlConfig["utilityMode"], + candidate: CandidateMemory, + conversationText: string, +): Promise<{ score: number; reason?: string }> { + if (mode === "off") { + return { score: 0.5, reason: "Utility scoring disabled" }; + } + + let response: { utility?: number; reason?: string } | null = null; + try { + response = await llm.completeJson<{ utility?: number; reason?: string }>( + buildUtilityPrompt(candidate, conversationText), + "admission-utility", + ); + } catch { + return { score: 0.5, reason: "Utility scoring failed" }; + } + + if (!response) { + return { score: 0.5, reason: "Utility scoring unavailable" }; + } + + return { + score: clamp01(response.utility, 0.5), + reason: typeof response.reason === "string" ? response.reason.trim() : undefined, + }; +} + +export class AdmissionController { + constructor( + private readonly store: MemoryStore, + private readonly llm: LlmClient, + private readonly config: AdmissionControlConfig, + private readonly debugLog: (msg: string) => void = () => {}, + ) {} + + private async loadRelevantMatches( + candidate: CandidateMemory, + candidateVector: number[], + scopeFilter: string[], + ): Promise { + if (!Array.isArray(candidateVector) || candidateVector.length === 0) { + return []; + } + + const rawMatches = await this.store.vectorSearch( + candidateVector, + this.config.noveltyCandidatePoolSize, + 0, + scopeFilter, + ); + + if (rawMatches.length === 0) { + return []; + } + + const sameCategoryMatches = rawMatches.filter((match) => { + const metadata = parseSmartMetadata(match.entry.metadata, match.entry); + return metadata.memory_category === candidate.category; + }); + + return sameCategoryMatches.length > 0 ? sameCategoryMatches : rawMatches; + } + + async evaluate(params: { + candidate: CandidateMemory; + candidateVector: number[]; + conversationText: string; + scopeFilter: string[]; + now?: number; + }): Promise { + const now = params.now ?? Date.now(); + const relevantMatches = await this.loadRelevantMatches( + params.candidate, + params.candidateVector, + params.scopeFilter, + ); + + const utility = await scoreUtility( + this.llm, + this.config.utilityMode, + params.candidate, + params.conversationText, + ); + const confidence = scoreConfidenceSupport(params.candidate, params.conversationText); + const novelty = scoreNoveltyFromMatches(params.candidateVector, relevantMatches); + const recency = scoreRecencyGap(now, relevantMatches, this.config.recency.halfLifeDays); + const typePrior = scoreTypePrior(params.candidate.category, this.config.typePriors); + + const featureScores: AdmissionFeatureScores = { + utility: utility.score, + confidence: confidence.score, + novelty: novelty.score, + recency, + typePrior, + }; + + const score = + (featureScores.utility * this.config.weights.utility) + + (featureScores.confidence * this.config.weights.confidence) + + (featureScores.novelty * this.config.weights.novelty) + + (featureScores.recency * this.config.weights.recency) + + (featureScores.typePrior * this.config.weights.typePrior); + + const decision = score < this.config.rejectThreshold ? "reject" : "pass_to_dedup"; + const hint = + decision === "reject" + ? undefined + : score >= this.config.admitThreshold && novelty.maxSimilarity < 0.55 + ? "add" + : "update_or_merge"; + + const reason = buildReason({ + decision, + hint, + score, + rejectThreshold: this.config.rejectThreshold, + maxSimilarity: novelty.maxSimilarity, + utilityReason: utility.reason, + }); + + const audit: AdmissionAuditRecord = { + version: "amac-v1", + decision, + hint, + score, + reason, + utility_reason: utility.reason, + thresholds: { + reject: this.config.rejectThreshold, + admit: this.config.admitThreshold, + }, + weights: this.config.weights, + feature_scores: featureScores, + matched_existing_memory_ids: novelty.matchedIds, + compared_existing_memory_ids: novelty.comparedIds, + max_similarity: novelty.maxSimilarity, + evaluated_at: now, + }; + + this.debugLog( + `memory-lancedb-pro: admission-control: decision=${audit.decision} hint=${audit.hint ?? "n/a"} score=${audit.score.toFixed(3)} candidate=${JSON.stringify(params.candidate.abstract.slice(0, 80))}`, + ); + + return { decision, hint, audit }; + } +} diff --git a/src/admission-stats.ts b/src/admission-stats.ts new file mode 100644 index 00000000..5dd60d8c --- /dev/null +++ b/src/admission-stats.ts @@ -0,0 +1,332 @@ +import { readFile } from "node:fs/promises"; +import type { AdmissionControlConfig, AdmissionRejectionAuditEntry } from "./admission-control.js"; +import { resolveRejectedAuditFilePath } from "./admission-control.js"; +import { parseSmartMetadata } from "./smart-metadata.js"; + +const DEFAULT_TOP_REJECTION_REASONS = 5; +const ADMISSION_WINDOWS = [ + { key: "last24h", durationMs: 24 * 60 * 60 * 1000 }, + { key: "last7d", durationMs: 7 * 24 * 60 * 60 * 1000 }, +] as const; + +export interface AdmissionAuditedMemoryLike { + metadata?: string; + timestamp?: number; + category?: string; + text?: string; + importance?: number; +} + +export interface AdmissionStatsStoreLike { + dbPath: string; + list?: ( + scopeFilter?: string[], + category?: string, + limit?: number, + offset?: number, + ) => Promise; +} + +export interface AdmissionCategoryBreakdown { + admittedCount: number | null; + rejectedCount: number; + totalObserved: number | null; + rejectRate: number | null; +} + +export interface AdmissionWindowBreakdown { + admittedCount: number | null; + rejectedCount: number; + totalObserved: number | null; + rejectRate: number | null; +} + +export interface AdmissionRejectionReasonCount { + label: string; + count: number; +} + +export interface AdmissionRejectionSummary { + total: number; + latestRejectedAt: number | null; + byCategory: Record; + byScope: Record; + topReasons: AdmissionRejectionReasonCount[]; +} + +export interface AdmissionStatsSummary { + enabled: boolean; + auditMetadataEnabled: boolean; + rejectedAuditFilePath: string; + rejectedCount: number; + admittedCount: number | null; + totalObserved: number | null; + rejectRate: number | null; + latestRejectedAt: number | null; + rejectedByCategory: Record; + rejectedByScope: Record; + categoryBreakdown: Record; + topReasons: AdmissionRejectionReasonCount[]; + windows: Record; + observedAuditedMemories: number; +} + +export async function readAdmissionRejectionAudits( + filePath: string, +): Promise { + try { + const raw = await readFile(filePath, "utf8"); + const entries: AdmissionRejectionAuditEntry[] = []; + for (const rawLine of raw.split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line) continue; + try { + entries.push(JSON.parse(line) as AdmissionRejectionAuditEntry); + } catch { + // Skip corrupt JSONL lines (truncated writes, disk errors, etc.) + } + } + return entries; + } catch (error) { + const err = error as NodeJS.ErrnoException; + if (err?.code === "ENOENT") { + return []; + } + throw error; + } +} + +export function normalizeReasonKey(reason: string): string { + return reason + .toLowerCase() + .replace(/\d+(?:\.\d+)?/g, "#") + .replace(/\s+/g, " ") + .trim(); +} + +export function extractAdmissionReasonLabel(entry: AdmissionRejectionAuditEntry): string { + const utilityReason = entry.audit.utility_reason?.trim(); + if (utilityReason) { + return utilityReason; + } + return entry.audit.reason.trim(); +} + +export function summarizeAdmissionRejections( + entries: AdmissionRejectionAuditEntry[], +): AdmissionRejectionSummary { + const byCategory: Record = {}; + const byScope: Record = {}; + const reasonCounts = new Map(); + + for (const entry of entries) { + byCategory[entry.candidate.category] = (byCategory[entry.candidate.category] ?? 0) + 1; + byScope[entry.target_scope] = (byScope[entry.target_scope] ?? 0) + 1; + const label = extractAdmissionReasonLabel(entry); + const key = normalizeReasonKey(label); + const current = reasonCounts.get(key); + if (current) { + current.count += 1; + } else { + reasonCounts.set(key, { label, count: 1 }); + } + } + + const latestRejectedAt = entries.length > 0 + ? Math.max(...entries.map((entry) => entry.rejected_at)) + : null; + const topReasons = Array.from(reasonCounts.values()) + .sort((left, right) => right.count - left.count || left.label.localeCompare(right.label)) + .slice(0, DEFAULT_TOP_REJECTION_REASONS); + + return { + total: entries.length, + latestRejectedAt, + byCategory, + byScope, + topReasons, + }; +} + +export function getAdmissionAuditDecision( + entry: { metadata?: string }, +): "pass_to_dedup" | "reject" | null { + try { + const parsed = JSON.parse(entry.metadata || "{}") as Record; + const audit = parsed.admission_control as Record | undefined; + const decision = audit?.decision; + return decision === "pass_to_dedup" || decision === "reject" ? decision : null; + } catch { + return null; + } +} + +export function getAdmittedDecisionTimestamp( + entry: { metadata?: string; timestamp?: number }, +): number | null { + try { + const parsed = JSON.parse(entry.metadata || "{}") as Record; + const audit = parsed.admission_control as Record | undefined; + const evaluatedAt = Number(audit?.evaluated_at); + if (Number.isFinite(evaluatedAt) && evaluatedAt > 0) { + return evaluatedAt; + } + } catch { + // ignore + } + + const timestamp = Number(entry.timestamp); + if (Number.isFinite(timestamp) && timestamp > 0) { + return timestamp; + } + return null; +} + +export function getObservedAdmissionCategory( + entry: AdmissionAuditedMemoryLike, +): string { + return parseSmartMetadata(entry.metadata, entry).memory_category || entry.category || "patterns"; +} + +export function buildAdmissionCategoryBreakdown( + admittedCategories: string[] | null, + rejectedEntries: AdmissionRejectionAuditEntry[], +): Record { + const admittedCounts: Record | null = admittedCategories ? {} : null; + const rejectedCounts: Record = {}; + + if (admittedCategories) { + for (const category of admittedCategories) { + admittedCounts[category] = (admittedCounts[category] ?? 0) + 1; + } + } + + for (const entry of rejectedEntries) { + const category = entry.candidate.category; + rejectedCounts[category] = (rejectedCounts[category] ?? 0) + 1; + } + + const categories = Array.from( + new Set([ + ...Object.keys(rejectedCounts), + ...(admittedCounts ? Object.keys(admittedCounts) : []), + ]), + ).sort((left, right) => left.localeCompare(right)); + + const breakdown: Record = {}; + for (const category of categories) { + const admittedCount = admittedCounts ? (admittedCounts[category] ?? 0) : null; + const rejectedCount = rejectedCounts[category] ?? 0; + const totalObserved = admittedCount !== null ? admittedCount + rejectedCount : null; + const rejectRate = + totalObserved && totalObserved > 0 ? rejectedCount / totalObserved : null; + + breakdown[category] = { + admittedCount, + rejectedCount, + totalObserved, + rejectRate, + }; + } + + return breakdown; +} + +export function buildAdmissionWindowSummary( + admittedTimestamps: number[] | null, + rejectedEntries: AdmissionRejectionAuditEntry[], + now = Date.now(), +): Record { + const windows: Record = {}; + + for (const windowDef of ADMISSION_WINDOWS) { + const since = now - windowDef.durationMs; + const rejectedCount = rejectedEntries.filter((entry) => entry.rejected_at >= since).length; + const admittedCount = admittedTimestamps + ? admittedTimestamps.filter((ts) => ts >= since).length + : null; + const totalObserved = admittedCount !== null ? admittedCount + rejectedCount : null; + const rejectRate = + totalObserved && totalObserved > 0 ? rejectedCount / totalObserved : null; + + windows[windowDef.key] = { + admittedCount, + rejectedCount, + totalObserved, + rejectRate, + }; + } + + return windows; +} + +export async function buildAdmissionStats(params: { + store: AdmissionStatsStoreLike; + admissionControl?: AdmissionControlConfig; + scopeFilter?: string[]; + memoryTotalCount: number; +}): Promise { + const rejectionFilePath = resolveRejectedAuditFilePath( + params.store.dbPath, + params.admissionControl, + ); + let rejectionEntries = await readAdmissionRejectionAudits(rejectionFilePath); + if (params.scopeFilter && params.scopeFilter.length > 0) { + const scopeSet = new Set(params.scopeFilter); + rejectionEntries = rejectionEntries.filter((entry) => scopeSet.has(entry.target_scope)); + } + + const rejectionSummary = summarizeAdmissionRejections(rejectionEntries); + const auditMetadataEnabled = params.admissionControl?.auditMetadata !== false; + let admittedCount: number | null = null; + let admittedTimestamps: number[] | null = null; + let admittedCategories: string[] | null = null; + let observedAuditedMemories = 0; + + if (auditMetadataEnabled && typeof params.store.list === "function") { + const memories = await params.store.list( + params.scopeFilter, + undefined, + Math.max(params.memoryTotalCount, 1), + 0, + ); + admittedCount = 0; + admittedTimestamps = []; + admittedCategories = []; + for (const memory of memories) { + const decision = getAdmissionAuditDecision(memory); + if (decision === "pass_to_dedup") { + admittedCount += 1; + observedAuditedMemories += 1; + admittedCategories.push(getObservedAdmissionCategory(memory)); + const admittedAt = getAdmittedDecisionTimestamp(memory); + if (admittedAt !== null) { + admittedTimestamps.push(admittedAt); + } + } else if (decision === "reject") { + observedAuditedMemories += 1; + } + } + } + + const totalObserved = admittedCount !== null ? admittedCount + rejectionSummary.total : null; + const rejectRate = + totalObserved && totalObserved > 0 ? rejectionSummary.total / totalObserved : null; + + return { + enabled: params.admissionControl?.enabled === true, + auditMetadataEnabled, + rejectedAuditFilePath: rejectionFilePath, + rejectedCount: rejectionSummary.total, + admittedCount, + totalObserved, + rejectRate, + latestRejectedAt: rejectionSummary.latestRejectedAt, + rejectedByCategory: rejectionSummary.byCategory, + rejectedByScope: rejectionSummary.byScope, + categoryBreakdown: buildAdmissionCategoryBreakdown(admittedCategories, rejectionEntries), + topReasons: rejectionSummary.topReasons, + windows: buildAdmissionWindowSummary(admittedTimestamps, rejectionEntries), + observedAuditedMemories, + }; +} diff --git a/src/auto-capture-cleanup.ts b/src/auto-capture-cleanup.ts new file mode 100644 index 00000000..b677ed2d --- /dev/null +++ b/src/auto-capture-cleanup.ts @@ -0,0 +1,94 @@ +const AUTO_CAPTURE_INBOUND_META_SENTINELS = [ + "Conversation info (untrusted metadata):", + "Sender (untrusted metadata):", + "Thread starter (untrusted, for context):", + "Replied message (untrusted, for context):", + "Forwarded message context (untrusted metadata):", + "Chat history since last reply (untrusted, for context):", +] as const; + +const AUTO_CAPTURE_SESSION_RESET_PREFIX = + "A new session was started via /new or /reset. Execute your Session Startup sequence now"; +const AUTO_CAPTURE_ADDRESSING_PREFIX_RE = /^(?:<@!?[0-9]+>|@[A-Za-z0-9_.-]+)\s*/; +const AUTO_CAPTURE_SYSTEM_EVENT_LINE_RE = /^System:\s*\[[^\n]*?\]\s*Exec\s+(?:completed|failed|started)\b.*$/gim; + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +const AUTO_CAPTURE_INBOUND_META_BLOCK_RE = new RegExp( + String.raw`(?:^|\n)\s*(?:${AUTO_CAPTURE_INBOUND_META_SENTINELS.map((sentinel) => escapeRegExp(sentinel)).join("|")})\s*\n\`\`\`json[\s\S]*?\n\`\`\`\s*`, + "g", +); + +function stripLeadingInboundMetadata(text: string): string { + if (!text) { + return text; + } + + let normalized = text; + for (let i = 0; i < 6; i++) { + const before = normalized; + normalized = normalized.replace(AUTO_CAPTURE_SYSTEM_EVENT_LINE_RE, "\n"); + normalized = normalized.replace(AUTO_CAPTURE_INBOUND_META_BLOCK_RE, "\n"); + normalized = normalized.replace(/\n{3,}/g, "\n\n").trim(); + if (normalized === before.trim()) { + break; + } + } + + return normalized.trim(); +} + +function stripAutoCaptureSessionResetPrefix(text: string): string { + const trimmed = text.trim(); + if (!trimmed.startsWith(AUTO_CAPTURE_SESSION_RESET_PREFIX)) { + return trimmed; + } + + const blankLineIndex = trimmed.indexOf("\n\n"); + if (blankLineIndex >= 0) { + return trimmed.slice(blankLineIndex + 2).trim(); + } + + const lines = trimmed.split("\n"); + if (lines.length <= 2) { + return ""; + } + return lines.slice(2).join("\n").trim(); +} + +function stripAutoCaptureAddressingPrefix(text: string): string { + return text.replace(AUTO_CAPTURE_ADDRESSING_PREFIX_RE, "").trim(); +} + +export function stripAutoCaptureInjectedPrefix(role: string, text: string): string { + if (role !== "user") { + return text.trim(); + } + + let normalized = text.trim(); + normalized = normalized.replace(/\s*[\s\S]*?<\/relevant-memories>\s*/gi, ""); + normalized = normalized.replace( + /\[UNTRUSTED DATA[^\n]*\][\s\S]*?\[END UNTRUSTED DATA\]\s*/gi, + "", + ); + normalized = stripAutoCaptureSessionResetPrefix(normalized); + normalized = stripLeadingInboundMetadata(normalized); + normalized = stripAutoCaptureAddressingPrefix(normalized); + normalized = stripLeadingInboundMetadata(normalized); + normalized = normalized.replace(/\n{3,}/g, "\n\n"); + return normalized.trim(); +} + +export function normalizeAutoCaptureText( + role: unknown, + text: string, + shouldSkipMessage?: (role: string, text: string) => boolean, +): string | null { + if (typeof role !== "string") return null; + const normalized = stripAutoCaptureInjectedPrefix(role, text); + if (!normalized) return null; + if (shouldSkipMessage?.(role, normalized)) return null; + return normalized; +} diff --git a/src/batch-dedup.ts b/src/batch-dedup.ts new file mode 100644 index 00000000..0cbd339f --- /dev/null +++ b/src/batch-dedup.ts @@ -0,0 +1,146 @@ +/** + * Batch-Internal Dedup — Cosine similarity dedup within extraction batches + * + * Before running expensive per-candidate LLM dedup calls, this module + * checks all candidates against each other using cosine similarity + * on their embedded abstracts. Candidates with similarity > threshold + * are marked as batch duplicates and skipped. + * + * For n <= 5 candidates, O(n^2) pairwise comparison is trivial. + */ + +// ============================================================================ +// Types +// ============================================================================ + +export interface BatchDedupCandidate { + /** Unique index within the batch */ + index: number; + /** L0 abstract text used for embedding */ + abstract: string; + /** Embedded vector of the abstract */ + vector?: number[]; + /** Whether this candidate was marked as a batch duplicate */ + isBatchDuplicate: boolean; + /** If duplicate, index of the surviving candidate it duplicates */ + duplicateOf?: number; +} + +export interface BatchDedupResult { + /** Indices of candidates that survived (not duplicates) */ + survivingIndices: number[]; + /** Indices of candidates marked as batch duplicates */ + duplicateIndices: number[]; + /** Number of candidates before dedup */ + inputCount: number; + /** Number of candidates after dedup */ + outputCount: number; +} + +export interface ExtractionCostStats { + /** Candidates dropped by batch dedup */ + batchDeduped: number; + /** Total extraction wall time in ms */ + durationMs: number; + /** Count of LLM invocations */ + llmCalls: number; +} + +// ============================================================================ +// Cosine Similarity +// ============================================================================ + +function cosineSimilarity(a: number[], b: number[]): number { + if (a.length !== b.length || a.length === 0) return 0; + + let dotProduct = 0; + let normA = 0; + let normB = 0; + + for (let i = 0; i < a.length; i++) { + dotProduct += a[i] * b[i]; + normA += a[i] * a[i]; + normB += b[i] * b[i]; + } + + const norm = Math.sqrt(normA) * Math.sqrt(normB); + return norm === 0 ? 0 : dotProduct / norm; +} + +// ============================================================================ +// Batch Dedup +// ============================================================================ + +/** + * Perform batch-internal cosine dedup on candidate abstracts. + * + * @param abstracts - Array of L0 abstract strings from extracted candidates + * @param vectors - Parallel array of embedded vectors for each abstract + * @param threshold - Cosine similarity threshold above which candidates are considered duplicates (default: 0.85) + * @returns BatchDedupResult with surviving and duplicate indices + */ +export function batchDedup( + abstracts: string[], + vectors: number[][], + threshold = 0.85, +): BatchDedupResult { + const n = abstracts.length; + if (n <= 1) { + return { + survivingIndices: n === 1 ? [0] : [], + duplicateIndices: [], + inputCount: n, + outputCount: n, + }; + } + + // Track which candidates are duplicates + const isDuplicate = new Array(n).fill(false); + const duplicateOf = new Array(n).fill(undefined); + + // Pairwise comparison: O(n^2) but n <= 5 typically + for (let i = 0; i < n; i++) { + if (isDuplicate[i]) continue; + for (let j = i + 1; j < n; j++) { + if (isDuplicate[j]) continue; + if (!vectors[i] || !vectors[j]) continue; + if (vectors[i].length === 0 || vectors[j].length === 0) continue; + + const sim = cosineSimilarity(vectors[i], vectors[j]); + if (sim > threshold) { + // Mark the later candidate as duplicate of the earlier one + isDuplicate[j] = true; + duplicateOf[j] = i; + } + } + } + + const survivingIndices: number[] = []; + const duplicateIndices: number[] = []; + + for (let i = 0; i < n; i++) { + if (isDuplicate[i]) { + duplicateIndices.push(i); + } else { + survivingIndices.push(i); + } + } + + return { + survivingIndices, + duplicateIndices, + inputCount: n, + outputCount: survivingIndices.length, + }; +} + +/** + * Create a fresh ExtractionCostStats tracker. + */ +export function createExtractionCostStats(): ExtractionCostStats { + return { + batchDeduped: 0, + durationMs: 0, + llmCalls: 0, + }; +} diff --git a/src/clawteam-scope.ts b/src/clawteam-scope.ts new file mode 100644 index 00000000..fa74ffb8 --- /dev/null +++ b/src/clawteam-scope.ts @@ -0,0 +1,63 @@ +/** + * ClawTeam Shared Memory Scope Integration + * + * Provides env-var-driven scope extension for ClawTeam multi-agent setups. + * When CLAWTEAM_MEMORY_SCOPE is set, agents gain access to the specified + * team scopes in addition to their own default scopes. + * + * Note: this extends `getAccessibleScopes()`, which MemoryScopeManager's + * `isAccessible()` and `getScopeFilter()` both delegate to. So the extra + * scopes affect both read and write access checks. The default *write target* + * (getDefaultScope) is NOT changed — agents still write to their own scope + * unless they explicitly specify a team scope. + */ + +import type { ScopeDefinition } from "./scopes.js"; +import type { MemoryScopeManager } from "./scopes.js"; + +/** + * Parse the CLAWTEAM_MEMORY_SCOPE env var value into a list of scope names. + * Supports comma-separated values, trims whitespace, and filters empty strings. + */ +export function parseClawteamScopes(envValue: string | undefined): string[] { + if (!envValue) return []; + return envValue.split(",").map(s => s.trim()).filter(Boolean); +} + +/** + * Register ClawTeam scopes and extend the scope manager's accessible scopes. + * + * 1. Registers scope definitions for any scopes not already defined. + * 2. Wraps `getAccessibleScopes()` to include the extra scopes for all agents. + * + * Designed for MemoryScopeManager specifically, where `isAccessible()` and + * `getScopeFilter()` delegate to `getAccessibleScopes()`. Custom ScopeManager + * implementations may need additional patching. + */ +export function applyClawteamScopes( + scopeManager: MemoryScopeManager, + scopes: string[], +): void { + if (scopes.length === 0) return; + + // Register scope definitions for unknown scopes + for (const scope of scopes) { + if (!scopeManager.getScopeDefinition(scope)) { + scopeManager.addScopeDefinition(scope, { + description: `ClawTeam shared scope: ${scope}`, + }); + } + } + + // Wrap getAccessibleScopes to include extra scopes + // Copy the base array to avoid mutating the manager's internal state + const originalGetAccessibleScopes = scopeManager.getAccessibleScopes.bind(scopeManager); + scopeManager.getAccessibleScopes = (agentId?: string): string[] => { + const base = originalGetAccessibleScopes(agentId); + const result = [...base]; + for (const s of scopes) { + if (!result.includes(s)) result.push(s); + } + return result; + }; +} diff --git a/src/identity-addressing.ts b/src/identity-addressing.ts new file mode 100644 index 00000000..5ac653f5 --- /dev/null +++ b/src/identity-addressing.ts @@ -0,0 +1,201 @@ +import type { CandidateMemory } from "./memory-categories.js"; + +export const CANONICAL_NAME_FACT_KEY = "entities:姓名"; +export const CANONICAL_ADDRESSING_FACT_KEY = "preferences:称呼偏好"; + +type IdentityKind = "name" | "addressing"; +export type IdentityAddressingSlot = "name" | "addressing"; + +type IdentityAddressingMemoryLike = { + factKey?: string; + text?: string; + abstract?: string; + overview?: string; + content?: string; +}; + +function trimCapturedValue(value: string): string { + return value + .replace(/^[\s"'“”‘’「」『』*`_]+/, "") + .replace(/[\s"'“”‘’「」『』*`_。！，、,.!?:：；;]+$/u, "") + .trim(); +} + +function extractFirst(patterns: RegExp[], text: string): string | undefined { + for (const pattern of patterns) { + const match = pattern.exec(text); + const captured = match?.[1] ? trimCapturedValue(match[1]) : ""; + if (captured) return captured; + } + return undefined; +} + +function combineIdentityTextProbe(params: IdentityAddressingMemoryLike): string { + return [ + params.text, + params.abstract, + params.overview, + params.content, + ] + .filter((value): value is string => typeof value === "string" && value.trim().length > 0) + .map((value) => value.trim()) + .join("\n"); +} + +const NAME_PATTERNS = [ + /(?:我的名字是|我(?:现在)?叫|本名是)\s*([^\s，。,.!！?？"'“”‘’「」『』]+)/iu, + /calls?\s+themselves\s+['"]([^'"]+)['"]/i, + /name\s+is\s+['"]?([^'".,\n]+)['"]?/i, +]; + +const ADDRESSING_PATTERNS = [ + /(?:以后你叫我|以后请叫我|请叫我|以后称呼我(?:为)?|称呼我(?:为)?|称呼其为|称呼他为)\s*([^\s，。,.!！?？"'“”‘’「」『』]+)/iu, + /(?:希望(?:在[^\n。]{0,20})?(?:以后)?(?:你)?(?:被)?称呼(?:我|其|他)?为)\s*([^\s，。,.!！?？"'“”‘’「」『』]+)/iu, + /(?:被称呼为|称呼偏好(?:是)?|Preferred address(?: is)?|be addressed as|addressed as)\s*['"]?([^'".,\n]+)['"]?/i, + /(?:addressive identifier is|preferred (?:and permanently assigned )?addressive identifier is)\s*['"]?([^'".,\n]+)['"]?/i, +]; + +const NAME_HINT_PATTERNS = [ + /^姓名[:：]/m, + /^## Identity$/m, + /(?:^|\n)-\s*Name:\s+/i, + /用户当前姓名\/自称为/u, +]; + +const ADDRESSING_HINT_PATTERNS = [ + /^称呼偏好[:：]/m, + /^## Addressing$/m, + /Preferred form of address/i, + /被称呼为/u, + /addressive identifier/i, +]; + +function makeCandidate(kind: IdentityKind, alias: string, sourceText: string): CandidateMemory { + if (kind === "name") { + return { + category: "entities", + abstract: `姓名：${alias}`, + overview: `## Identity\n- Name: ${alias}`, + content: `用户当前姓名/自称为“${alias}”。原始表述：${sourceText}`, + }; + } + + return { + category: "preferences", + abstract: `称呼偏好：${alias}`, + overview: `## Addressing\n- Preferred form of address: ${alias}`, + content: `用户希望以后被称呼为“${alias}”。原始表述：${sourceText}`, + }; +} + +export function createIdentityAndAddressingCandidates(text: string): CandidateMemory[] { + const sourceText = text.trim(); + if (!sourceText) return []; + + const name = extractFirst(NAME_PATTERNS, sourceText); + const addressing = extractFirst(ADDRESSING_PATTERNS, sourceText); + const candidates: CandidateMemory[] = []; + + if (name) { + candidates.push(makeCandidate("name", name, sourceText)); + } + if (addressing) { + const duplicateOfName = name && addressing === name; + if (!duplicateOfName || candidates.length === 0) { + candidates.push(makeCandidate("addressing", addressing, sourceText)); + } else { + candidates.push(makeCandidate("addressing", addressing, sourceText)); + } + } + + return candidates; +} + +export function extractIdentityAndAddressingValues(text: string): { + name?: string; + addressing?: string; +} { + const sourceText = text.trim(); + if (!sourceText) return {}; + + return { + name: extractFirst(NAME_PATTERNS, sourceText), + addressing: extractFirst(ADDRESSING_PATTERNS, sourceText), + }; +} + +export function classifyIdentityAndAddressingMemory( + params: IdentityAddressingMemoryLike, +): { + slots: Set; + name?: string; + addressing?: string; +} { + const slots = new Set(); + + if (params.factKey === CANONICAL_NAME_FACT_KEY) { + slots.add("name"); + } + if (params.factKey === CANONICAL_ADDRESSING_FACT_KEY) { + slots.add("addressing"); + } + + const probe = combineIdentityTextProbe(params); + if (!probe) { + return { slots }; + } + + const extracted = extractIdentityAndAddressingValues(probe); + + if (extracted.name || NAME_HINT_PATTERNS.some((pattern) => pattern.test(probe))) { + slots.add("name"); + } + if ( + extracted.addressing || + ADDRESSING_HINT_PATTERNS.some((pattern) => pattern.test(probe)) + ) { + slots.add("addressing"); + } + + return { + slots, + name: extracted.name, + addressing: extracted.addressing, + }; +} + +export function canonicalizeIdentityAndAddressingCandidate( + candidate: CandidateMemory, +): CandidateMemory { + const combined = [candidate.abstract, candidate.overview, candidate.content] + .filter(Boolean) + .join("\n"); + + if (candidate.category === "entities") { + const name = extractFirst(NAME_PATTERNS, combined); + if (name) { + return makeCandidate("name", name, candidate.content || candidate.abstract); + } + const addressing = extractFirst(ADDRESSING_PATTERNS, combined); + if (addressing) { + return makeCandidate("addressing", addressing, candidate.content || candidate.abstract); + } + return candidate; + } + + const addressing = extractFirst(ADDRESSING_PATTERNS, combined); + if (addressing) { + return makeCandidate("addressing", addressing, candidate.content || candidate.abstract); + } + + const name = extractFirst(NAME_PATTERNS, combined); + if (name) { + return makeCandidate("name", name, candidate.content || candidate.abstract); + } + + return candidate; +} + +export function isCanonicalIdentityOrAddressingFactKey(factKey: string | undefined): boolean { + return factKey === CANONICAL_NAME_FACT_KEY || factKey === CANONICAL_ADDRESSING_FACT_KEY; +} diff --git a/src/intent-analyzer.ts b/src/intent-analyzer.ts new file mode 100644 index 00000000..58c34281 --- /dev/null +++ b/src/intent-analyzer.ts @@ -0,0 +1,259 @@ +/** + * Intent Analyzer for Adaptive Recall + * + * Lightweight, rule-based intent analysis that determines which memory categories + * are most relevant for a given query and what recall depth to use. + * + * Inspired by OpenViking's hierarchical retrieval intent routing, adapted for + * memory-lancedb-pro's flat category model. No LLM calls — pure pattern matching + * for minimal latency impact on auto-recall. + * + * @see https://github.com/volcengine/OpenViking — hierarchical_retriever.py intent analysis + */ + +// ============================================================================ +// Types +// ============================================================================ + +/** + * Intent categories map to actual stored MemoryEntry categories. + * Note: "event" is NOT a stored category — event queries route to + * entity + decision (the categories most likely to contain timeline data). + */ +export type MemoryCategoryIntent = + | "preference" + | "fact" + | "decision" + | "entity" + | "other"; + +export type RecallDepth = "l0" | "l1" | "full"; + +export interface IntentSignal { + /** Categories to prioritize (ordered by relevance). */ + categories: MemoryCategoryIntent[]; + /** Recommended recall depth for this intent. */ + depth: RecallDepth; + /** Confidence level of the intent classification. */ + confidence: "high" | "medium" | "low"; + /** Short label for logging. */ + label: string; +} + +// ============================================================================ +// Intent Patterns +// ============================================================================ + +interface IntentRule { + label: string; + patterns: RegExp[]; + categories: MemoryCategoryIntent[]; + depth: RecallDepth; +} + +/** + * Intent rules ordered by specificity (most specific first). + * First match wins — keep high-confidence patterns at the top. + */ +const INTENT_RULES: IntentRule[] = [ + // --- Preference / Style queries --- + { + label: "preference", + patterns: [ + /\b(prefer|preference|style|convention|like|dislike|favorite|habit)\b/i, + /\b(how do (i|we) usually|what('s| is) (my|our) (style|convention|approach))\b/i, + /(偏好|喜欢|习惯|风格|惯例|常用|不喜欢|不要用|别用)/, + ], + categories: ["preference", "decision"], + depth: "l0", + }, + + // --- Decision / Rationale queries --- + { + label: "decision", + patterns: [ + /\b(why did (we|i)|decision|decided|chose|rationale|trade-?off|reason for)\b/i, + /\b(what was the (reason|rationale|decision))\b/i, + /(为什么选|决定|选择了|取舍|权衡|原因是|当时决定)/, + ], + categories: ["decision", "fact"], + depth: "l1", + }, + + // --- Entity / People / Project queries --- + // Narrowed patterns to avoid over-matching: require "who is" / "tell me about" + // style phrasing, not bare nouns like "tool" or "component". + { + label: "entity", + patterns: [ + /\b(who is|who are|tell me about|info on|details about|contact info)\b/i, + /\b(who('s| is) (the|our|my)|what team|which (person|team))\b/i, + /(谁是|告诉我关于|详情|联系方式|哪个团队)/, + ], + categories: ["entity", "fact"], + depth: "l1", + }, + + // --- Event / Timeline queries --- + // Note: "event" is not a stored category. Route to entity + decision + // (the categories most likely to contain timeline/incident data). + { + label: "event", + patterns: [ + /\b(when did|what happened|timeline|incident|outage|deploy|release|shipped)\b/i, + /\b(last (week|month|time|sprint)|recently|yesterday|today)\b/i, + /(什么时候|发生了什么|时间线|事件|上线|部署|发布|上次|最近)/, + ], + categories: ["entity", "decision"], + depth: "full", + }, + + // --- Fact / Knowledge queries --- + { + label: "fact", + patterns: [ + /\b(how (does|do|to)|what (does|do|is)|explain|documentation|spec)\b/i, + /\b(config|configuration|setup|install|architecture|api|endpoint)\b/i, + /(怎么|如何|是什么|解释|文档|规范|配置|安装|架构|接口)/, + ], + categories: ["fact", "entity"], + depth: "l1", + }, +]; + +// ============================================================================ +// Analyzer +// ============================================================================ + +/** + * Analyze a query to determine which memory categories and recall depth + * are most appropriate. + * + * Returns a default "broad" signal if no specific intent is detected, + * so callers can always use the result without null checks. + */ +export function analyzeIntent(query: string): IntentSignal { + const trimmed = query.trim(); + if (!trimmed) { + return { + categories: [], + depth: "l0", + confidence: "low", + label: "empty", + }; + } + + for (const rule of INTENT_RULES) { + if (rule.patterns.some((p) => p.test(trimmed))) { + return { + categories: rule.categories, + depth: rule.depth, + confidence: "high", + label: rule.label, + }; + } + } + + // No specific intent detected — return broad signal. + // All categories are eligible; use L0 to minimize token cost. + return { + categories: [], + depth: "l0", + confidence: "low", + label: "broad", + }; +} + +/** + * Apply intent-based category boost to retrieval results. + * + * Instead of filtering (which would lose potentially relevant results), + * this boosts scores of results matching the detected intent categories. + * Non-matching results are kept but ranked lower. + * + * @param results - Retrieval results with scores + * @param intent - Detected intent signal + * @param boostFactor - Score multiplier for matching categories (default: 1.15) + * @returns Results with adjusted scores, re-sorted + */ +export function applyCategoryBoost< + T extends { entry: { category: string }; score: number }, +>(results: T[], intent: IntentSignal, boostFactor = 1.15): T[] { + if (intent.categories.length === 0 || intent.confidence === "low") { + return results; // No intent signal — return as-is + } + + const prioritySet = new Set(intent.categories); + + const boosted = results.map((r) => { + if (prioritySet.has(r.entry.category)) { + return { ...r, score: Math.min(1, r.score * boostFactor) }; + } + return r; + }); + + return boosted.sort((a, b) => b.score - a.score); +} + +/** + * Format a memory entry for context injection at the specified depth level. + * + * - l0: One-line summary (category + scope + truncated text) + * - l1: Medium detail (category + scope + text up to ~300 chars) + * - full: Complete text (existing behavior) + */ +export function formatAtDepth( + entry: { text: string; category: string; scope: string }, + depth: RecallDepth, + score: number, + index: number, + extra?: { bm25Hit?: boolean; reranked?: boolean; sanitize?: (text: string) => string }, +): string { + const scoreStr = `${(score * 100).toFixed(0)}%`; + const sourceSuffix = [ + extra?.bm25Hit ? "vector+BM25" : null, + extra?.reranked ? "+reranked" : null, + ] + .filter(Boolean) + .join(""); + const sourceTag = sourceSuffix ? `, ${sourceSuffix}` : ""; + + // Apply sanitization if provided (prevents prompt injection from stored memories) + const safe = extra?.sanitize ? extra.sanitize(entry.text) : entry.text; + + switch (depth) { + case "l0": { + // Ultra-compact: first sentence or first 80 chars + const brief = extractFirstSentence(safe, 80); + return `- [${entry.category}] ${brief} (${scoreStr}${sourceTag})`; + } + case "l1": { + // Medium: up to 300 chars + const medium = + safe.length > 300 + ? safe.slice(0, 297) + "..." + : safe; + return `- [${entry.category}:${entry.scope}] ${medium} (${scoreStr}${sourceTag})`; + } + case "full": + default: + return `- [${entry.category}:${entry.scope}] ${safe} (${scoreStr}${sourceTag})`; + } +} + +// ============================================================================ +// Helpers +// ============================================================================ + +function extractFirstSentence(text: string, maxLen: number): string { + // Try to find a sentence boundary (CJK punctuation may not be followed by space) + const sentenceEnd = text.search(/[.!?]\s|[。！？]/); + if (sentenceEnd > 0 && sentenceEnd < maxLen) { + return text.slice(0, sentenceEnd + 1); + } + if (text.length <= maxLen) return text; + // Fall back to truncation at word boundary + const truncated = text.slice(0, maxLen); + const lastSpace = truncated.lastIndexOf(" "); + return (lastSpace > maxLen * 0.6 ? truncated.slice(0, lastSpace) : truncated) + "..."; +} diff --git a/src/llm-oauth.ts b/src/llm-oauth.ts new file mode 100644 index 00000000..65bd650b --- /dev/null +++ b/src/llm-oauth.ts @@ -0,0 +1,675 @@ +import { createHash, randomBytes } from "node:crypto"; +import { createServer } from "node:http"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { dirname } from "node:path"; +import { platform } from "node:os"; +import { spawn } from "node:child_process"; + +export interface OAuthLoginOptions { + authPath: string; + timeoutMs?: number; + noBrowser?: boolean; + model?: string; + providerId?: string; + onOpenUrl?: (url: string) => void | Promise; + onAuthorizeUrl?: (url: string) => void | Promise; +} +const EXPIRY_SKEW_MS = 60_000; + +export type OAuthProviderId = "openai-codex"; + +interface OAuthProviderDefinition { + id: OAuthProviderId; + label: string; + authorizeUrl: string; + tokenUrl: string; + clientId: string; + redirectUri: string; + scope: string; + accountIdClaim: string; + backendBaseUrl: string; + defaultModel: string; + modelPattern: RegExp; + extraAuthorizeParams?: Record; +} + +const DEFAULT_OAUTH_PROVIDER_ID: OAuthProviderId = "openai-codex"; +const OAUTH_PROVIDER_ALIASES: Record = { + openai: "openai-codex", + codex: "openai-codex", + "openai-codex": "openai-codex", +}; +const OAUTH_PROVIDERS: Record = { + "openai-codex": { + id: "openai-codex", + label: "OpenAI Codex", + authorizeUrl: "https://auth.openai.com/oauth/authorize", + tokenUrl: "https://auth.openai.com/oauth/token", + clientId: "app_EMoamEEZ73f0CkXaXp7hrann", + redirectUri: "http://localhost:1455/auth/callback", + scope: "openid profile email offline_access", + accountIdClaim: "https://api.openai.com/auth", + backendBaseUrl: "https://chatgpt.com/backend-api", + defaultModel: "gpt-5.4", + modelPattern: /^(gpt-|o[1345]\b|o\d-mini\b|gpt-5|gpt-4|gpt-4o|gpt-5-codex|gpt-5\.1-codex)/i, + extraAuthorizeParams: { + id_token_add_organizations: "true", + codex_cli_simplified_flow: "true", + originator: "codex_cli_rs", + }, + }, +}; + +export interface OAuthSession { + accessToken: string; + refreshToken?: string; + expiresAt?: number; + accountId: string; + providerId: OAuthProviderId; + authPath: string; +} + +interface TokenRefreshResponse { + access_token?: string; + refresh_token?: string; + expires_in?: number; +} + +function parseNumericTimestamp(value: unknown): number | undefined { + if (typeof value === "number" && Number.isFinite(value) && value > 0) { + return value > 1_000_000_000_000 ? value : value * 1000; + } + + if (typeof value === "string") { + const trimmed = value.trim(); + if (!trimmed) return undefined; + const parsed = Number(trimmed); + if (Number.isFinite(parsed) && parsed > 0) { + return parsed > 1_000_000_000_000 ? parsed : parsed * 1000; + } + } + + return undefined; +} + +function toBase64Url(value: Buffer): string { + return value.toString("base64url"); +} + +function createState(): string { + return randomBytes(16).toString("hex"); +} + +function createPkceVerifier(): string { + return toBase64Url(randomBytes(32)); +} + +function createPkceChallenge(verifier: string): string { + return createHash("sha256").update(verifier).digest("base64url"); +} + +export function listOAuthProviders(): Array> { + return Object.values(OAUTH_PROVIDERS).map((provider) => ({ + id: provider.id, + label: provider.label, + defaultModel: provider.defaultModel, + })); +} + +export function normalizeOAuthProviderId(providerId?: string): OAuthProviderId { + const raw = providerId?.trim().toLowerCase(); + if (!raw) return DEFAULT_OAUTH_PROVIDER_ID; + const resolved = OAUTH_PROVIDER_ALIASES[raw]; + if (resolved) return resolved; + const available = listOAuthProviders().map((provider) => provider.id).join(", "); + throw new Error(`Unsupported OAuth provider "${providerId}". Available providers: ${available}`); +} + +export function getOAuthProvider(providerId?: string): OAuthProviderDefinition { + return OAUTH_PROVIDERS[normalizeOAuthProviderId(providerId)]; +} + +export function getOAuthProviderLabel(providerId?: string): string { + return getOAuthProvider(providerId).label; +} + +export function getDefaultOauthModelForProvider(providerId?: string): string { + return getOAuthProvider(providerId).defaultModel; +} + +export function isOauthModelSupported(providerId: string | undefined, value: string | undefined): boolean { + if (!value || !value.trim()) return false; + const provider = getOAuthProvider(providerId); + const trimmed = value.trim(); + const slashIndex = trimmed.indexOf("/"); + if (slashIndex !== -1) { + const modelProvider = trimmed.slice(0, slashIndex).trim().toLowerCase(); + if (provider.id === "openai-codex" && modelProvider !== "openai" && modelProvider !== "openai-codex") { + return false; + } + } + + return provider.modelPattern.test(normalizeOauthModel(trimmed)); +} + +function resolveOauthClientId(providerId?: string): string { + return process.env.MEMORY_PRO_OAUTH_CLIENT_ID?.trim() || getOAuthProvider(providerId).clientId; +} + +function resolveOauthAuthorizeUrl(providerId?: string): string { + return process.env.MEMORY_PRO_OAUTH_AUTHORIZE_URL?.trim() || getOAuthProvider(providerId).authorizeUrl; +} + +function resolveOauthTokenUrl(providerId?: string): string { + return process.env.MEMORY_PRO_OAUTH_TOKEN_URL?.trim() || getOAuthProvider(providerId).tokenUrl; +} + +function resolveOauthRedirectUri(providerId?: string): string { + return process.env.MEMORY_PRO_OAUTH_REDIRECT_URI?.trim() || getOAuthProvider(providerId).redirectUri; +} + +function buildAuthorizationUrl(state: string, verifier: string, providerId?: string): string { + const provider = getOAuthProvider(providerId); + const url = new URL(resolveOauthAuthorizeUrl(provider.id)); + url.searchParams.set("response_type", "code"); + url.searchParams.set("client_id", resolveOauthClientId(provider.id)); + url.searchParams.set("redirect_uri", resolveOauthRedirectUri(provider.id)); + url.searchParams.set("scope", provider.scope); + url.searchParams.set("code_challenge", createPkceChallenge(verifier)); + url.searchParams.set("code_challenge_method", "S256"); + url.searchParams.set("state", state); + for (const [key, value] of Object.entries(provider.extraAuthorizeParams || {})) { + url.searchParams.set(key, value); + } + return url.toString(); +} + +function buildSuccessHtml(): string { + return [ + "", + "", + "

memory-pro OAuth complete

", + "

You can close this window and return to your terminal.

", + "", + ].join(""); +} + +function buildErrorHtml(message: string): string { + return [ + "", + "", + "

memory-pro OAuth failed

", + `

${message}

`, + "", + ].join(""); +} + +function decodeJwtPayload(token: string): Record | null { + try { + const parts = token.split("."); + if (parts.length !== 3) return null; + return JSON.parse(Buffer.from(parts[1], "base64").toString("utf8")) as Record; + } catch { + return null; + } +} + +function getJwtExpiry(token: string): number | undefined { + const payload = decodeJwtPayload(token); + return parseNumericTimestamp(payload?.exp); +} + +function getJwtAccountId(token: string, providerId?: string): string | undefined { + const provider = getOAuthProvider(providerId); + const payload = decodeJwtPayload(token); + const claims = payload?.[provider.accountIdClaim]; + if (!claims || typeof claims !== "object") return undefined; + + const accountId = (claims as Record).chatgpt_account_id; + return typeof accountId === "string" && accountId.trim() ? accountId : undefined; +} + +function pickString(container: Record, keys: string[]): string | undefined { + for (const key of keys) { + const value = container[key]; + if (typeof value === "string" && value.trim()) { + return value.trim(); + } + } + return undefined; +} + +function pickTimestamp(container: Record, keys: string[]): number | undefined { + for (const key of keys) { + const parsed = parseNumericTimestamp(container[key]); + if (parsed) return parsed; + } + return undefined; +} + +function extractSessionFromObject(source: Record, authPath: string): OAuthSession | null { + const scopes: Record[] = [ + source, + typeof source.tokens === "object" && source.tokens ? source.tokens as Record : {}, + typeof source.oauth === "object" && source.oauth ? source.oauth as Record : {}, + typeof source.openai === "object" && source.openai ? source.openai as Record : {}, + typeof source.chatgpt === "object" && source.chatgpt ? source.chatgpt as Record : {}, + typeof source.auth === "object" && source.auth ? source.auth as Record : {}, + typeof source.credentials === "object" && source.credentials ? source.credentials as Record : {}, + ]; + + let accessToken: string | undefined; + let refreshToken: string | undefined; + let expiresAt: number | undefined; + let accountId: string | undefined; + const providerRaw = pickString(source, ["provider", "oauth_provider", "oauthProvider"]); + let providerId: OAuthProviderId; + try { + providerId = normalizeOAuthProviderId(providerRaw); + } catch { + return null; + } + + for (const scope of scopes) { + accessToken ||= pickString(scope, ["access_token", "accessToken", "access", "token"]); + refreshToken ||= pickString(scope, ["refresh_token", "refreshToken", "refresh"]); + expiresAt ||= pickTimestamp(scope, ["expires_at", "expiresAt", "expires", "expires_on"]); + accountId ||= pickString(scope, ["account_id", "accountId", "chatgpt_account_id", "chatgptAccountId"]); + } + + const apiKey = pickString(source, ["OPENAI_API_KEY", "api_key", "apiKey"]); + if (!accessToken && apiKey) { + return null; + } + + if (!accessToken) return null; + + accountId ||= getJwtAccountId(accessToken, providerId); + if (!accountId) return null; + + expiresAt ||= getJwtExpiry(accessToken); + + return { + accessToken, + refreshToken, + expiresAt, + accountId, + providerId, + authPath, + }; +} + +export async function loadOAuthSession(authPath: string): Promise { + let raw: string; + try { + raw = await readFile(authPath, "utf8"); + } catch (err) { + const reason = err instanceof Error ? err.message : String(err); + throw new Error( + `LLM OAuth requires a project OAuth file. Expected ${authPath}. Read failed: ${reason}`, + ); + } + + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (err) { + const reason = err instanceof Error ? err.message : String(err); + throw new Error(`Invalid project OAuth JSON at ${authPath}: ${reason}`); + } + + if (!parsed || typeof parsed !== "object") { + throw new Error(`Invalid project OAuth file at ${authPath}: expected a JSON object`); + } + + const session = extractSessionFromObject(parsed as Record, authPath); + if (!session) { + throw new Error( + `Project OAuth file at ${authPath} does not contain an OAuth access token and ChatGPT account id.`, + ); + } + + return session; +} + +export function needsRefresh(session: OAuthSession): boolean { + return !!session.refreshToken && !!session.expiresAt && session.expiresAt - EXPIRY_SKEW_MS <= Date.now(); +} + +function createTimeoutSignal(timeoutMs?: number): { signal: AbortSignal; dispose: () => void } { + const effectiveTimeoutMs = + typeof timeoutMs === "number" && Number.isFinite(timeoutMs) && timeoutMs > 0 ? timeoutMs : 30_000; + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), effectiveTimeoutMs); + return { + signal: controller.signal, + dispose: () => clearTimeout(timer), + }; +} + +export async function refreshOAuthSession(session: OAuthSession, timeoutMs?: number): Promise { + if (!session.refreshToken) { + throw new Error( + `OAuth session from ${session.authPath} is expired and has no refresh token. Re-run \`codex login\`.`, + ); + } + + const { signal, dispose } = createTimeoutSignal(timeoutMs); + try { + const response = await fetch(resolveOauthTokenUrl(session.providerId), { + method: "POST", + headers: { + "Content-Type": "application/x-www-form-urlencoded", + }, + body: new URLSearchParams({ + grant_type: "refresh_token", + refresh_token: session.refreshToken, + client_id: resolveOauthClientId(session.providerId), + }), + signal, + }); + + if (!response.ok) { + const detail = await response.text().catch(() => ""); + throw new Error(`OAuth refresh failed (${response.status}): ${detail.slice(0, 500)}`); + } + + const payload = await response.json() as TokenRefreshResponse; + if (!payload.access_token) { + throw new Error("OAuth refresh returned no access token"); + } + + const accessToken = payload.access_token; + const refreshToken = payload.refresh_token || session.refreshToken; + const expiresAt = + typeof payload.expires_in === "number" + ? Date.now() + payload.expires_in * 1000 + : getJwtExpiry(accessToken); + const accountId = getJwtAccountId(accessToken, session.providerId) || session.accountId; + + if (!accountId) { + throw new Error("OAuth refresh returned a token without a ChatGPT account id"); + } + + return { + accessToken, + refreshToken, + expiresAt, + accountId, + providerId: session.providerId, + authPath: session.authPath, + }; + } finally { + dispose(); + } +} + +async function exchangeAuthorizationCode(code: string, verifier: string, providerId?: string): Promise { + const resolvedProviderId = normalizeOAuthProviderId(providerId); + const response = await fetch(resolveOauthTokenUrl(resolvedProviderId), { + method: "POST", + headers: { + "Content-Type": "application/x-www-form-urlencoded", + }, + body: new URLSearchParams({ + grant_type: "authorization_code", + client_id: resolveOauthClientId(resolvedProviderId), + code, + code_verifier: verifier, + redirect_uri: resolveOauthRedirectUri(resolvedProviderId), + }), + }); + + if (!response.ok) { + const detail = await response.text().catch(() => ""); + throw new Error(`OAuth token exchange failed (${response.status}): ${detail.slice(0, 500)}`); + } + + const payload = await response.json() as TokenRefreshResponse; + if (!payload.access_token) { + throw new Error("OAuth token exchange returned no access token"); + } + + const accountId = getJwtAccountId(payload.access_token, resolvedProviderId); + if (!accountId) { + throw new Error("OAuth token exchange returned a token without a ChatGPT account id"); + } + + return { + accessToken: payload.access_token, + refreshToken: payload.refresh_token, + expiresAt: + typeof payload.expires_in === "number" + ? Date.now() + payload.expires_in * 1000 + : getJwtExpiry(payload.access_token), + accountId, + providerId: resolvedProviderId, + authPath: "", + }; +} + +export async function saveOAuthSession(authPath: string, session: OAuthSession): Promise { + await mkdir(dirname(authPath), { recursive: true }); + const payload = { + provider: session.providerId, + type: "oauth", + access_token: session.accessToken, + refresh_token: session.refreshToken, + expires_at: session.expiresAt, + account_id: session.accountId, + updated_at: new Date().toISOString(), + }; + await writeFile(authPath, JSON.stringify(payload, null, 2) + "\n", { + encoding: "utf8", + mode: 0o600, + }); +} + +function tryOpenBrowser(url: string): void { + const targetPlatform = platform(); + if (targetPlatform === "darwin") { + const child = spawn("open", [url], { detached: true, stdio: "ignore" }); + child.unref(); + return; + } + + if (targetPlatform === "win32") { + const child = spawn("cmd", ["/c", "start", "", url], { detached: true, stdio: "ignore" }); + child.unref(); + return; + } + + const child = spawn("xdg-open", [url], { detached: true, stdio: "ignore" }); + child.unref(); +} + +async function waitForAuthorizationCode(state: string, timeoutMs: number, providerId?: string): Promise { + const redirectUri = new URL(resolveOauthRedirectUri(providerId)); + const listenPort = Number(redirectUri.port || 80); + const callbackPath = redirectUri.pathname || "/"; + const listenHost = resolveOAuthCallbackListenHost(redirectUri); + + return await new Promise((resolve, reject) => { + const timer = setTimeout(() => { + server.close(); + reject(new Error(`Timed out waiting for OAuth callback on ${redirectUri.origin}${callbackPath}`)); + }, timeoutMs); + + const server = createServer((req, res) => { + if (!req.url) { + res.writeHead(400, { "Content-Type": "text/html; charset=utf-8" }); + res.end(buildErrorHtml("Missing callback URL.")); + return; + } + + const url = new URL(req.url, redirectUri.origin); + if (url.pathname !== callbackPath) { + res.writeHead(404, { "Content-Type": "text/html; charset=utf-8" }); + res.end(buildErrorHtml("Unknown callback path.")); + return; + } + + const returnedState = url.searchParams.get("state"); + const code = url.searchParams.get("code"); + const error = url.searchParams.get("error"); + + if (error) { + clearTimeout(timer); + server.close(); + res.writeHead(400, { "Content-Type": "text/html; charset=utf-8" }); + res.end(buildErrorHtml(`Authorization failed: ${error}`)); + reject(new Error(`OAuth authorization failed: ${error}`)); + return; + } + + if (!code || returnedState !== state) { + clearTimeout(timer); + server.close(); + res.writeHead(400, { "Content-Type": "text/html; charset=utf-8" }); + res.end(buildErrorHtml("Invalid authorization callback.")); + reject(new Error("OAuth callback did not include a valid code/state pair")); + return; + } + + clearTimeout(timer); + server.close(); + res.writeHead(200, { "Content-Type": "text/html; charset=utf-8" }); + res.end(buildSuccessHtml()); + resolve(code); + }); + + server.on("error", (err) => { + clearTimeout(timer); + reject(err); + }); + + server.listen(listenPort, listenHost); + }); +} + +export function resolveOAuthCallbackListenHost(redirectUri: URL | string): string { + const parsed = typeof redirectUri === "string" ? new URL(redirectUri) : redirectUri; + const hostname = parsed.hostname.trim(); + if (!hostname) return "127.0.0.1"; + return hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname; +} + +export async function performOAuthLogin(options: OAuthLoginOptions): Promise<{ session: OAuthSession; authorizeUrl: string }> { + const provider = getOAuthProvider(options.providerId); + const verifier = createPkceVerifier(); + const state = createState(); + const authorizeUrl = buildAuthorizationUrl(state, verifier, provider.id); + + await options.onAuthorizeUrl?.(authorizeUrl); + if (!options.noBrowser) { + if (options.onOpenUrl) { + await options.onOpenUrl(authorizeUrl); + } else { + try { + tryOpenBrowser(authorizeUrl); + } catch { + // Browser opening is best-effort; caller still receives the URL. + } + } + } + + const code = await waitForAuthorizationCode(state, options.timeoutMs ?? 120_000, provider.id); + const session = await exchangeAuthorizationCode(code, verifier, provider.id); + session.authPath = options.authPath; + await saveOAuthSession(options.authPath, session); + return { session, authorizeUrl }; +} + +export function normalizeOauthModel(model: string): string { + const trimmed = model.trim(); + if (!trimmed) return trimmed; + + const slashIndex = trimmed.indexOf("/"); + if (slashIndex === -1) return trimmed; + + const provider = trimmed.slice(0, slashIndex).trim().toLowerCase(); + const modelName = trimmed.slice(slashIndex + 1).trim(); + if (!modelName) return trimmed; + + if (provider === "openai" || provider === "openai-codex") { + return modelName; + } + + return trimmed; +} + +export function buildOauthEndpoint(baseURL?: string, providerId?: string): string { + const root = (baseURL?.trim() || getOAuthProvider(providerId).backendBaseUrl).replace(/\/+$/, ""); + if (root.endsWith("/codex/responses")) return root; + if (root.endsWith("/responses")) return root.replace(/\/responses$/, "/codex/responses"); + return `${root}/codex/responses`; +} + +function extractOutputTextFromResponsePayload(payload: unknown): string | null { + if (!payload || typeof payload !== "object") return null; + + const response = payload as Record; + const output = Array.isArray(response.output) ? response.output : null; + if (!output) return null; + + const texts: string[] = []; + for (const item of output) { + if (!item || typeof item !== "object") continue; + const content = Array.isArray((item as Record).content) + ? (item as Record).content as Array> + : []; + for (const part of content) { + if (part?.type === "output_text" && typeof part.text === "string") { + texts.push(part.text); + } + } + } + + return texts.length ? texts.join("\n") : null; +} + +export function extractOutputTextFromSse(bodyText: string): string | null { + const chunks = bodyText.split(/\r?\n\r?\n/); + let deltas = ""; + + for (const chunk of chunks) { + const dataLines = chunk + .split(/\r?\n/) + .filter((line) => line.startsWith("data:")) + .map((line) => line.slice(5).trim()); + + if (!dataLines.length) continue; + + const data = dataLines.join("\n"); + if (!data || data === "[DONE]") continue; + + let payload: unknown; + try { + payload = JSON.parse(data); + } catch { + continue; + } + + if (!payload || typeof payload !== "object") continue; + + const event = payload as Record; + if (event.type === "response.output_text.delta" && typeof event.delta === "string") { + deltas += event.delta; + continue; + } + + if (event.type === "response.output_text.done" && typeof event.text === "string") { + return event.text; + } + + const nested = typeof event.response === "object" && event.response + ? extractOutputTextFromResponsePayload(event.response) + : null; + if (nested) return nested; + + const direct = extractOutputTextFromResponsePayload(event); + if (direct) return direct; + } + + return deltas || null; +} diff --git a/src/memory-compactor.ts b/src/memory-compactor.ts new file mode 100644 index 00000000..1c0b1ead --- /dev/null +++ b/src/memory-compactor.ts @@ -0,0 +1,403 @@ +/** + * Memory Compactor — Progressive Summarization + * + * Identifies clusters of semantically similar memories older than a configured + * age threshold and merges each cluster into a single, higher-quality entry. + * + * Implements the "progressive summarization" pattern: memories get more refined + * over time as related fragments are consolidated, reducing noise and improving + * retrieval quality without requiring an external LLM call. + * + * Algorithm: + * 1. Load memories older than `minAgeDays` (with vectors). + * 2. Build similarity clusters using greedy cosine-similarity expansion. + * 3. For each cluster >= `minClusterSize`, merge into one entry: + * - text: deduplicated lines joined with newlines + * - importance: max of cluster members (never downgrade) + * - category: plurality vote + * - scope: shared scope (all members must share one) + * - metadata: marked { compacted: true, sourceCount: N } + * 4. Delete source entries, store merged entry. + */ + +import type { MemoryEntry } from "./store.js"; + +// ============================================================================ +// Types +// ============================================================================ + +export interface CompactionConfig { + /** Enable automatic compaction. Default: false */ + enabled: boolean; + /** Only compact memories at least this many days old. Default: 7 */ + minAgeDays: number; + /** Cosine similarity threshold for clustering [0, 1]. Default: 0.88 */ + similarityThreshold: number; + /** Minimum number of memories in a cluster to trigger merge. Default: 2 */ + minClusterSize: number; + /** Maximum memories to scan per compaction run. Default: 200 */ + maxMemoriesToScan: number; + /** Report plan without writing changes. Default: false */ + dryRun: boolean; + /** Run at most once per N hours (gateway_start guard). Default: 24 */ + cooldownHours: number; +} + +export interface CompactionEntry { + id: string; + text: string; + vector: number[]; + category: MemoryEntry["category"]; + scope: string; + importance: number; + timestamp: number; + metadata: string; +} + +export interface ClusterPlan { + /** Indices into the input entries array */ + memberIndices: number[]; + /** Proposed merged entry (without id/vector — computed by caller) */ + merged: { + text: string; + importance: number; + category: MemoryEntry["category"]; + scope: string; + metadata: string; + }; +} + +export interface CompactionResult { + /** Memories scanned (limited by maxMemoriesToScan) */ + scanned: number; + /** Clusters found with >= minClusterSize members */ + clustersFound: number; + /** Source memories deleted (0 when dryRun) */ + memoriesDeleted: number; + /** Merged memories created (0 when dryRun) */ + memoriesCreated: number; + /** Whether this was a dry run */ + dryRun: boolean; +} + +// ============================================================================ +// Math helpers +// ============================================================================ + +/** Dot product of two equal-length vectors. */ +function dot(a: number[], b: number[]): number { + let s = 0; + for (let i = 0; i < a.length; i++) s += a[i] * b[i]; + return s; +} + +/** L2 norm of a vector. */ +function norm(v: number[]): number { + return Math.sqrt(dot(v, v)); +} + +/** + * Cosine similarity in [0, 1]. + * Returns 0 if either vector has zero norm (avoids NaN). + */ +export function cosineSimilarity(a: number[], b: number[]): number { + if (a.length === 0 || a.length !== b.length) return 0; + const na = norm(a); + const nb = norm(b); + if (na === 0 || nb === 0) return 0; + return Math.max(0, Math.min(1, dot(a, b) / (na * nb))); +} + +// ============================================================================ +// Cluster building +// ============================================================================ + +/** + * Greedy cluster expansion. + * + * Sort entries by importance DESC so the most valuable memory seeds each + * cluster. Expand each seed by collecting every unassigned entry whose + * cosine similarity with the seed is >= threshold. + * + * Returns an array of index-arrays (each inner array = one cluster). + * Only clusters with >= minClusterSize entries are returned. + */ +export function buildClusters( + entries: CompactionEntry[], + threshold: number, + minClusterSize: number, +): ClusterPlan[] { + if (entries.length < minClusterSize) return []; + + // Sort indices by importance desc (highest importance seeds first) + const order = entries + .map((_, i) => i) + .sort((a, b) => entries[b].importance - entries[a].importance); + + const assigned = new Uint8Array(entries.length); // 0 = unassigned + const plans: ClusterPlan[] = []; + + for (const seedIdx of order) { + if (assigned[seedIdx]) continue; + + const cluster: number[] = [seedIdx]; + assigned[seedIdx] = 1; + + const seedVec = entries[seedIdx].vector; + if (seedVec.length === 0) continue; // skip entries without vectors + + for (let j = 0; j < entries.length; j++) { + if (assigned[j]) continue; + const jVec = entries[j].vector; + if (jVec.length === 0) continue; + if (cosineSimilarity(seedVec, jVec) >= threshold) { + cluster.push(j); + assigned[j] = 1; + } + } + + if (cluster.length >= minClusterSize) { + const members = cluster.map((i) => entries[i]); + plans.push({ + memberIndices: cluster, + merged: buildMergedEntry(members), + }); + } + } + + return plans; +} + +// ============================================================================ +// Merge strategy +// ============================================================================ + +/** + * Merge a cluster of entries into a single proposed entry. + * + * Text strategy: deduplicate lines across all member texts, join with newline. + * This preserves all unique information while removing redundancy. + * + * Importance: max across cluster (never downgrade). + * Category: plurality vote; ties broken by member with highest importance. + * Scope: all members must share a scope (validated upstream). + */ +export function buildMergedEntry( + members: CompactionEntry[], +): ClusterPlan["merged"] { + // --- text: deduplicate lines --- + const seen = new Set(); + const lines: string[] = []; + for (const m of members) { + for (const line of m.text.split("\n")) { + const trimmed = line.trim(); + if (trimmed && !seen.has(trimmed.toLowerCase())) { + seen.add(trimmed.toLowerCase()); + lines.push(trimmed); + } + } + } + const text = lines.join("\n"); + + // --- importance: max --- + const importance = Math.min( + 1.0, + Math.max(...members.map((m) => m.importance)), + ); + + // --- category: plurality vote --- + const counts = new Map(); + for (const m of members) { + counts.set(m.category, (counts.get(m.category) ?? 0) + 1); + } + let category: MemoryEntry["category"] = "other"; + let best = 0; + for (const [cat, count] of counts) { + if (count > best) { + best = count; + category = cat as MemoryEntry["category"]; + } + } + + // --- scope: use the first (all should match) --- + const scope = members[0].scope; + + // --- metadata --- + const metadata = JSON.stringify({ + compacted: true, + sourceCount: members.length, + compactedAt: Date.now(), + }); + + return { text, importance, category, scope, metadata }; +} + +// ============================================================================ +// Minimal store interface (duck-typed so no circular import) +// ============================================================================ + +export interface CompactorStore { + fetchForCompaction( + maxTimestamp: number, + scopeFilter?: string[], + limit?: number, + ): Promise; + store(entry: { + text: string; + vector: number[]; + importance: number; + category: MemoryEntry["category"]; + scope: string; + metadata?: string; + }): Promise; + delete(id: string, scopeFilter?: string[]): Promise; +} + +export interface CompactorEmbedder { + embedPassage(text: string): Promise; +} + +export interface CompactorLogger { + info(msg: string): void; + warn(msg: string): void; +} + +// ============================================================================ +// Main runner +// ============================================================================ + +/** + * Run a single compaction pass over memories in the given scopes. + * + * @param store Storage backend (must support fetchForCompaction + store + delete) + * @param embedder Used to embed merged text before storage + * @param config Compaction configuration + * @param scopes Scope filter; undefined = all scopes + * @param logger Optional logger + */ +export async function runCompaction( + store: CompactorStore, + embedder: CompactorEmbedder, + config: CompactionConfig, + scopes?: string[], + logger?: CompactorLogger, +): Promise { + const cutoff = Date.now() - config.minAgeDays * 24 * 60 * 60 * 1000; + + const entries = await store.fetchForCompaction( + cutoff, + scopes, + config.maxMemoriesToScan, + ); + + if (entries.length === 0) { + return { + scanned: 0, + clustersFound: 0, + memoriesDeleted: 0, + memoriesCreated: 0, + dryRun: config.dryRun, + }; + } + + // Filter out entries without vectors (shouldn't happen but be safe) + const valid = entries.filter((e) => e.vector && e.vector.length > 0); + + const plans = buildClusters( + valid, + config.similarityThreshold, + config.minClusterSize, + ); + + if (config.dryRun) { + logger?.info( + `memory-compactor [dry-run]: scanned=${valid.length} clusters=${plans.length}`, + ); + return { + scanned: valid.length, + clustersFound: plans.length, + memoriesDeleted: 0, + memoriesCreated: 0, + dryRun: true, + }; + } + + let memoriesDeleted = 0; + let memoriesCreated = 0; + + for (const plan of plans) { + const members = plan.memberIndices.map((i) => valid[i]); + + try { + // Embed the merged text + const vector = await embedder.embedPassage(plan.merged.text); + + // Store merged entry + await store.store({ + text: plan.merged.text, + vector, + importance: plan.merged.importance, + category: plan.merged.category, + scope: plan.merged.scope, + metadata: plan.merged.metadata, + }); + memoriesCreated++; + + // Delete source entries + for (const m of members) { + const deleted = await store.delete(m.id); + if (deleted) memoriesDeleted++; + } + } catch (err) { + logger?.warn( + `memory-compactor: failed to merge cluster of ${members.length}: ${String(err)}`, + ); + } + } + + logger?.info( + `memory-compactor: scanned=${valid.length} clusters=${plans.length} ` + + `deleted=${memoriesDeleted} created=${memoriesCreated}`, + ); + + return { + scanned: valid.length, + clustersFound: plans.length, + memoriesDeleted, + memoriesCreated, + dryRun: false, + }; +} + +// ============================================================================ +// Cooldown helper +// ============================================================================ + +/** + * Check whether enough time has passed since the last compaction run. + * Uses a simple JSON file at `stateFile` to persist the last-run timestamp. + */ +export async function shouldRunCompaction( + stateFile: string, + cooldownHours: number, +): Promise { + try { + const { readFile } = await import("node:fs/promises"); + const raw = await readFile(stateFile, "utf8"); + const state = JSON.parse(raw) as { lastRunAt?: number }; + if (typeof state.lastRunAt === "number") { + const elapsed = Date.now() - state.lastRunAt; + return elapsed >= cooldownHours * 60 * 60 * 1000; + } + } catch { + // File doesn't exist or is malformed — treat as never run + } + return true; +} + +export async function recordCompactionRun(stateFile: string): Promise { + const { writeFile, mkdir } = await import("node:fs/promises"); + const { dirname } = await import("node:path"); + await mkdir(dirname(stateFile), { recursive: true }); + await writeFile(stateFile, JSON.stringify({ lastRunAt: Date.now() }), "utf8"); +} diff --git a/src/preference-slots.ts b/src/preference-slots.ts new file mode 100644 index 00000000..300c5d57 --- /dev/null +++ b/src/preference-slots.ts @@ -0,0 +1,76 @@ +const ROLE_PREFIX_RE = /^\[(用户|助手)\]\s*/gm; +const PREFERENCE_SPLIT_RE = /(?:、|,|，|\/|以及|及|与|和| and | & )/iu; +const PREFERENCE_CLAUSE_STOP_RE = /(?:因为|所以|但是|不过|if |when |because |but )/iu; +const BRAND_ITEM_PREFERENCE_PATTERNS = [ + /(?:^|[\s，,。；;！!？?])(?:我|用户)?(?:很|更|还)?(?:喜欢|爱吃|偏爱|常吃|想吃)(?:吃|喝|用|买)?(?[\p{Script=Han}A-Za-z0-9&·'\-]{1,24})的(?[\p{Script=Han}A-Za-z0-9&·'\-\s、,，和及与/]{1,80})/u, + /\b(?:i|user)?\s*(?:really\s+|still\s+|also\s+)?(?:like|love|prefer|enjoy)\s+(?[a-z0-9'&\-\s]{1,80})\s+from\s+(?[a-z0-9'&\-\s]{1,40})/iu, +] as const; + +export interface ParsedBrandItemPreference { + brand: string; + items: string[]; + aggregate: boolean; +} + +export interface AtomicBrandItemPreferenceSlot { + type: "brand-item"; + brand: string; + item: string; +} + +function normalizePreferenceText(value: string): string { + return value + .replace(ROLE_PREFIX_RE, "") + .replace(/\s+/g, " ") + .trim(); +} + +export function normalizePreferenceToken(value: string): string { + return normalizePreferenceText(value) + .replace(/^[“"'`‘’]+|[”"'`‘’。！!？?，,；;:：]+$/gu, "") + .replace(/\b(?:the|a|an)\s+/giu, "") + .replace(/\s+/g, "") + .toLowerCase(); +} + +function splitPreferenceItems(rawItems: string): string[] { + const trimmed = rawItems.split(PREFERENCE_CLAUSE_STOP_RE)[0] || rawItems; + return trimmed + .split(PREFERENCE_SPLIT_RE) + .map((item) => normalizePreferenceToken(item)) + .filter((item) => item.length > 0); +} + +export function parseBrandItemPreference(text: string): ParsedBrandItemPreference | null { + const normalizedText = normalizePreferenceText(text); + + for (const pattern of BRAND_ITEM_PREFERENCE_PATTERNS) { + const match = normalizedText.match(pattern); + if (!match?.groups) continue; + + const brand = normalizePreferenceToken(match.groups.brand || ""); + const items = splitPreferenceItems(match.groups.items || ""); + if (!brand || items.length === 0) continue; + + return { + brand, + items, + aggregate: items.length > 1, + }; + } + + return null; +} + +export function inferAtomicBrandItemPreferenceSlot(text: string): AtomicBrandItemPreferenceSlot | null { + const parsed = parseBrandItemPreference(text); + if (!parsed || parsed.aggregate || parsed.items.length !== 1) { + return null; + } + + return { + type: "brand-item", + brand: parsed.brand, + item: parsed.items[0], + }; +} diff --git a/src/retrieval-stats.ts b/src/retrieval-stats.ts new file mode 100644 index 00000000..60994040 --- /dev/null +++ b/src/retrieval-stats.ts @@ -0,0 +1,152 @@ +/** + * Retrieval Statistics — Aggregate query metrics + * + * Collects per-query traces and produces aggregate statistics + * for monitoring retrieval quality and performance. + */ + +import type { RetrievalTrace } from "./retrieval-trace.js"; + +// ============================================================================ +// Types +// ============================================================================ + +export interface AggregateStats { + /** Total number of queries recorded */ + totalQueries: number; + /** Number of queries that returned zero results */ + zeroResultQueries: number; + /** Average latency across all queries (ms) */ + avgLatencyMs: number; + /** 95th percentile latency (ms) */ + p95LatencyMs: number; + /** Average number of results returned */ + avgResultCount: number; + /** Number of queries where reranking was applied */ + rerankUsed: number; + /** Number of queries where noise filter removed results */ + noiseFiltered: number; + /** Query counts broken down by source */ + queriesBySource: Record; + /** Stages that drop the most entries across all queries */ + topDropStages: { name: string; totalDropped: number }[]; +} + +// ============================================================================ +// RetrievalStatsCollector +// ============================================================================ + +interface QueryRecord { + trace: RetrievalTrace; + source: string; +} + +export class RetrievalStatsCollector { + private _records: QueryRecord[] = []; + private readonly _maxRecords: number; + + constructor(maxRecords = 1000) { + this._maxRecords = maxRecords; + } + + /** + * Record a completed query trace. + * @param trace - The finalized retrieval trace + * @param source - Query source identifier (e.g. "manual", "auto-recall") + */ + recordQuery(trace: RetrievalTrace, source: string): void { + this._records.push({ trace, source }); + // Evict oldest if over capacity + if (this._records.length > this._maxRecords) { + this._records.shift(); + } + } + + /** + * Compute aggregate statistics from all recorded queries. + */ + getStats(): AggregateStats { + const n = this._records.length; + if (n === 0) { + return { + totalQueries: 0, + zeroResultQueries: 0, + avgLatencyMs: 0, + p95LatencyMs: 0, + avgResultCount: 0, + rerankUsed: 0, + noiseFiltered: 0, + queriesBySource: {}, + topDropStages: [], + }; + } + + let totalLatency = 0; + let totalResults = 0; + let zeroResultQueries = 0; + let rerankUsed = 0; + let noiseFiltered = 0; + const latencies: number[] = []; + const queriesBySource: Record = {}; + const dropsByStage: Record = {}; + + for (const { trace, source } of this._records) { + totalLatency += trace.totalMs; + totalResults += trace.finalCount; + latencies.push(trace.totalMs); + + if (trace.finalCount === 0) { + zeroResultQueries++; + } + + queriesBySource[source] = (queriesBySource[source] || 0) + 1; + + for (const stage of trace.stages) { + const dropped = stage.inputCount - stage.outputCount; + if (dropped > 0) { + dropsByStage[stage.name] = (dropsByStage[stage.name] || 0) + dropped; + } + if (stage.name === "rerank") { + rerankUsed++; + } + if (stage.name === "noise_filter" && dropped > 0) { + noiseFiltered++; + } + } + } + + // Sort latencies for percentile calculation + latencies.sort((a, b) => a - b); + const p95Index = Math.min(Math.ceil(n * 0.95) - 1, n - 1); + + // Top drop stages sorted by total dropped descending + const topDropStages = Object.entries(dropsByStage) + .map(([name, totalDropped]) => ({ name, totalDropped })) + .sort((a, b) => b.totalDropped - a.totalDropped) + .slice(0, 5); + + return { + totalQueries: n, + zeroResultQueries, + avgLatencyMs: Math.round(totalLatency / n), + p95LatencyMs: latencies[p95Index], + avgResultCount: Math.round((totalResults / n) * 10) / 10, + rerankUsed, + noiseFiltered, + queriesBySource, + topDropStages, + }; + } + + /** + * Reset all collected statistics. + */ + reset(): void { + this._records = []; + } + + /** Number of recorded queries. */ + get count(): number { + return this._records.length; + } +} diff --git a/src/retrieval-trace.ts b/src/retrieval-trace.ts new file mode 100644 index 00000000..a17af36e --- /dev/null +++ b/src/retrieval-trace.ts @@ -0,0 +1,173 @@ +/** + * Retrieval Trace — Observable pipeline diagnostics + * + * Tracks entry IDs through each retrieval stage, computes drops, + * score ranges, and timing. Zero overhead when not used. + */ + +// ============================================================================ +// Types +// ============================================================================ + +export interface RetrievalStageResult { + /** Stage name, e.g. "vector_search", "bm25_search", "rrf_fusion" */ + name: string; + /** Number of entries entering this stage */ + inputCount: number; + /** Number of entries surviving this stage */ + outputCount: number; + /** IDs that were present in input but not in output */ + droppedIds: string[]; + /** [min, max] score range of surviving entries, null if no scores */ + scoreRange: [number, number] | null; + /** Wall-clock duration of this stage in milliseconds */ + durationMs: number; +} + +export interface RetrievalTrace { + /** The original search query */ + query: string; + /** Retrieval mode used */ + mode: "hybrid" | "vector" | "bm25"; + /** Timestamp when retrieval started (epoch ms) */ + startedAt: number; + /** Per-stage results in pipeline order */ + stages: RetrievalStageResult[]; + /** Number of results after all stages */ + finalCount: number; + /** Total wall-clock time in milliseconds */ + totalMs: number; +} + +// ============================================================================ +// TraceCollector +// ============================================================================ + +interface PendingStage { + name: string; + inputIds: Set; + startTime: number; +} + +export class TraceCollector { + private readonly _startTime: number; + private readonly _stages: RetrievalStageResult[] = []; + private _pending: PendingStage | null = null; + + constructor() { + this._startTime = Date.now(); + } + + /** + * Begin tracking a pipeline stage. + * @param name - Stage identifier (e.g. "vector_search") + * @param entryIds - IDs of entries entering this stage + */ + startStage(name: string, entryIds: string[]): void { + // Auto-close any unclosed previous stage (defensive) + if (this._pending) { + this.endStage([...this._pending.inputIds]); + } + this._pending = { + name, + inputIds: new Set(entryIds), + startTime: Date.now(), + }; + } + + /** + * End the current stage. + * @param survivingIds - IDs of entries that survived this stage + * @param scores - Optional scores for surviving entries (parallel to survivingIds) + */ + endStage(survivingIds: string[], scores?: number[]): void { + if (!this._pending) return; + + const { name, inputIds, startTime } = this._pending; + const survivingSet = new Set(survivingIds); + + const droppedIds: string[] = []; + for (const id of inputIds) { + if (!survivingSet.has(id)) { + droppedIds.push(id); + } + } + + let scoreRange: [number, number] | null = null; + if (scores && scores.length > 0) { + let min = Infinity; + let max = -Infinity; + for (const s of scores) { + if (s < min) min = s; + if (s > max) max = s; + } + scoreRange = [min, max]; + } + + this._stages.push({ + name, + inputCount: inputIds.size, + outputCount: survivingIds.length, + droppedIds, + scoreRange, + durationMs: Date.now() - startTime, + }); + + this._pending = null; + } + + /** + * Finalize the trace and produce the complete RetrievalTrace object. + */ + finalize(query: string, mode: string): RetrievalTrace { + // Auto-close any unclosed stage + if (this._pending) { + this.endStage([...this._pending.inputIds]); + } + + const lastStage = this._stages[this._stages.length - 1]; + return { + query, + mode: mode as "hybrid" | "vector" | "bm25", + startedAt: this._startTime, + stages: this._stages, + finalCount: lastStage ? lastStage.outputCount : 0, + totalMs: Date.now() - this._startTime, + }; + } + + /** + * Produce a human-readable summary of the trace. + */ + summarize(): string { + const lines: string[] = []; + lines.push(`Retrieval trace (${this._stages.length} stages):`); + for (const stage of this._stages) { + const dropped = stage.inputCount - stage.outputCount; + const scoreStr = stage.scoreRange + ? ` scores=[${stage.scoreRange[0].toFixed(3)}, ${stage.scoreRange[1].toFixed(3)}]` + : ""; + lines.push( + ` ${stage.name}: ${stage.inputCount} -> ${stage.outputCount} (-${dropped}) ${stage.durationMs}ms${scoreStr}`, + ); + if (stage.droppedIds.length > 0 && stage.droppedIds.length <= 5) { + lines.push(` dropped: ${stage.droppedIds.join(", ")}`); + } else if (stage.droppedIds.length > 5) { + lines.push( + ` dropped: ${stage.droppedIds.slice(0, 5).join(", ")} (+${stage.droppedIds.length - 5} more)`, + ); + } + } + const lastStage = this._stages[this._stages.length - 1]; + const totalMs = Date.now() - this._startTime; + lines.push( + ` total: ${totalMs}ms, final count: ${lastStage ? lastStage.outputCount : 0}`, + ); + return lines.join("\n"); + } + + /** Access collected stages (read-only). */ + get stages(): readonly RetrievalStageResult[] { + return this._stages; + } +} diff --git a/src/session-compressor.ts b/src/session-compressor.ts new file mode 100644 index 00000000..769904ce --- /dev/null +++ b/src/session-compressor.ts @@ -0,0 +1,331 @@ +/** + * Session Compressor + * + * Scores and compresses conversation texts before memory extraction. + * Prioritizes high-signal content (tool calls, corrections, decisions) over + * low-signal content (greetings, acknowledgments) so that the fixed extraction + * budget captures the most important parts of a conversation. + */ + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export interface ScoredText { + /** Original index in the texts array */ + index: number; + /** The text content */ + text: string; + /** Score from 0.0 (noise) to 1.0 (high value) */ + score: number; + /** Human-readable reason for the score */ + reason: string; +} + +export interface CompressResult { + /** Selected texts in chronological order */ + texts: string[]; + /** Detailed scoring for all input texts */ + scored: ScoredText[]; + /** Number of texts dropped */ + dropped: number; + /** Total chars in output */ + totalChars: number; +} + +// --------------------------------------------------------------------------- +// Indicator patterns +// --------------------------------------------------------------------------- + +const TOOL_CALL_INDICATORS = [ + /\btool_use\b/i, + /\btool_result\b/i, + /\bfunction_call\b/i, + /\b(memory_store|memory_recall|memory_forget|memory_update)\b/i, + // Removed over-broad patterns: fenced code blocks and "$ " matched normal pasted code +]; + +const CORRECTION_INDICATORS = [ + /^no[,.\s]/i, + /\bactually\b/i, + /\binstead\b/i, + /\bwrong\b/i, + /\bcorrect(ion)?\b/i, + /\bfix\b/i, + /不对/, + /应该是/, + /應該是/, + /错了/, + /錯了/, + /改成/, + /不是.*而是/, +]; + +const DECISION_INDICATORS = [ + /\blet'?s go with\b/i, + /\bconfirmed?\b/i, + /\bapproved?\b/i, + /\bdecided?\b/i, + /\bwe'?ll use\b/i, + /\bgoing forward\b/i, + /\bfrom now on\b/i, + /\bagreed\b/i, + /决定/, + /決定/, + /确认/, + /確認/, + /选择了/, + /選擇了/, + /就这样/, + /就這樣/, +]; + +const ACKNOWLEDGMENT_PATTERNS = [ + /^(ok|okay|k|sure|fine|thanks|thank you|thx|ty|got it|understood|cool|nice|great|good|perfect|awesome|alright|yep|yup|yeah|right)\s*[.!]?$/i, + /^好的?\s*[。！]?$/, + /^嗯\s*[。]?$/, + /^收到\s*[。！]?$/, + /^了解\s*[。！]?$/, + /^明白\s*[。！]?$/, + /^谢谢\s*[。！]?$/, + /^感谢\s*[。！]?$/, + /^👍\s*$/, +]; + +// --------------------------------------------------------------------------- +// Scoring +// --------------------------------------------------------------------------- + +/** + * Score a single text segment by its information density. + */ +export function scoreText(text: string, index: number): ScoredText { + const trimmed = text.trim(); + + // Empty / whitespace-only + if (trimmed.length === 0) { + return { index, text, score: 0.0, reason: "empty" }; + } + + // Tool call indicators → highest value + if (TOOL_CALL_INDICATORS.some((p) => p.test(trimmed))) { + return { index, text, score: 1.0, reason: "tool_call" }; + } + + // Corrections → very high value (user correcting agent = strong signal) + if (CORRECTION_INDICATORS.some((p) => p.test(trimmed))) { + return { index, text, score: 0.95, reason: "correction" }; + } + + // Decisions / confirmations → high value + if (DECISION_INDICATORS.some((p) => p.test(trimmed))) { + return { index, text, score: 0.85, reason: "decision" }; + } + + // Acknowledgments → very low value + if (ACKNOWLEDGMENT_PATTERNS.some((p) => p.test(trimmed))) { + return { index, text, score: 0.1, reason: "acknowledgment" }; + } + + // Substantive content vs short questions + // CJK characters carry ~2-3x more meaning per character, so use a lower + // threshold (same approach as adaptive-retrieval.ts). + const hasCJK = /[\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af]/.test(trimmed); + const substantiveMinLength = hasCJK ? 30 : 80; + if (trimmed.length > substantiveMinLength) { + // Check for boilerplate (XML tags, system messages) + if (/^<[a-z-]+>/.test(trimmed) && /<\/[a-z-]+>\s*$/.test(trimmed)) { + return { index, text, score: 0.3, reason: "system_xml" }; + } + return { index, text, score: 0.7, reason: "substantive" }; + } + + // Short questions + if (trimmed.includes("?") || trimmed.includes("\uff1f")) { + return { index, text, score: 0.5, reason: "short_question" }; + } + + // Short but not a question and not an acknowledgment + return { index, text, score: 0.4, reason: "short_statement" }; +} + +// --------------------------------------------------------------------------- +// Compression +// --------------------------------------------------------------------------- + +/** Default minimum texts to keep even if all score low */ +const DEFAULT_MIN_TEXTS = 3; + +/** + * Compress an array of text segments to fit within a character budget. + * + * Strategy: + * 1. Score all texts + * 2. Always include first and last text (session boundaries) + * 3. Sort remaining by score descending + * 4. Greedily select until budget exhausted + * 5. Handle paired texts (tool call + result: indices i, i+1) + * 6. Re-sort selected by original index + * 7. If all texts score < threshold, keep at least minTexts + */ +export function compressTexts( + texts: string[], + maxChars: number, + options: { minTexts?: number; minScoreToKeep?: number } = {}, +): CompressResult { + const minTexts = options.minTexts ?? DEFAULT_MIN_TEXTS; + const minScoreToKeep = options.minScoreToKeep ?? 0.3; + + if (texts.length === 0) { + return { texts: [], scored: [], dropped: 0, totalChars: 0 }; + } + + // Score everything + const scored = texts.map((t, i) => scoreText(t, i)); + + // Total chars of all texts + const allChars = texts.reduce((sum, t) => sum + t.length, 0); + + // If already within budget, return all + if (allChars <= maxChars) { + return { + texts: [...texts], + scored, + dropped: 0, + totalChars: allChars, + }; + } + + // Build selected set starting with first and last + const selectedIndices = new Set(); + let usedChars = 0; + + const addIndex = (idx: number): boolean => { + if (selectedIndices.has(idx) || idx < 0 || idx >= texts.length) return false; + const len = texts[idx].length; + if (usedChars + len > maxChars) { + // Hard cap: even the first/last text cannot exceed budget + return false; + } + selectedIndices.add(idx); + usedChars += len; + return true; + }; + + // Always keep first and last + addIndex(0); + if (texts.length > 1) { + addIndex(texts.length - 1); + } + + // Build candidate list excluding first/last, sorted by score desc (stable by index asc on tie) + const candidates = scored + .filter((s) => s.index !== 0 && s.index !== texts.length - 1) + .sort((a, b) => b.score - a.score || a.index - b.index); + + // Identify paired indices (tool call at i → result at i+1). + // Only pair from a tool_call line, NOT from tool_result — a result line + // should not pull in the next unrelated line as its "partner". + const pairedWith = new Map(); + for (const s of scored) { + if ( + s.reason === "tool_call" && + s.index + 1 < texts.length && + !pairedWith.has(s.index) && // not already claimed + !pairedWith.has(s.index + 1) // partner not already claimed + ) { + pairedWith.set(s.index, s.index + 1); + pairedWith.set(s.index + 1, s.index); + } + } + + // Greedily add candidates + for (const candidate of candidates) { + if (usedChars >= maxChars) break; + + const added = addIndex(candidate.index); + if (added) { + // If this is part of a pair, try to add the partner + const partner = pairedWith.get(candidate.index); + if (partner !== undefined) { + addIndex(partner); + } + } + } + + // All-low-score fallback: if everything scored below threshold, ensure + // we keep at least minTexts (the last N by original order) + const allLow = scored.every((s) => s.score < minScoreToKeep); + if (allLow && selectedIndices.size < Math.min(minTexts, texts.length)) { + // Add from the end (most recent = most relevant for low-value sessions) + for (let i = texts.length - 1; i >= 0 && selectedIndices.size < Math.min(minTexts, texts.length); i--) { + addIndex(i); + } + } + + // Re-sort selected by original index to preserve chronological order + const sortedIndices = [...selectedIndices].sort((a, b) => a - b); + const resultTexts = sortedIndices.map((i) => texts[i]); + const totalChars = resultTexts.reduce((sum, t) => sum + t.length, 0); + + return { + texts: resultTexts, + scored, + dropped: texts.length - sortedIndices.length, + totalChars, + }; +} + +// --------------------------------------------------------------------------- +// Conversation Value Estimation (for Feature 7: Adaptive Throttling) +// --------------------------------------------------------------------------- + +/** + * Estimate the overall value of a conversation for memory extraction. + * Returns a number between 0.0 and 1.0. + * + * Used by the adaptive extraction throttle to skip low-value conversations. + */ +export function estimateConversationValue(texts: string[]): number { + if (texts.length === 0) return 0; + + let value = 0; + + const joined = texts.join(" "); + + // Has explicit memory intent? (e.g. "remember this", "记住") +0.5 + // These should NEVER be skipped by the low-value gate. + const MEMORY_INTENT = /\b(remember|recall|don'?t forget|note that|keep in mind)\b/i; + const MEMORY_INTENT_CJK = /(记住|記住|别忘|不要忘|记一下|記一下)/; + if (MEMORY_INTENT.test(joined) || MEMORY_INTENT_CJK.test(joined)) { + value += 0.5; + } + + // Has tool calls? +0.4 + if (TOOL_CALL_INDICATORS.some((p) => p.test(joined))) { + value += 0.4; + } + + // Has corrections or decisions? +0.3 + const hasCorrectionOrDecision = + CORRECTION_INDICATORS.some((p) => p.test(joined)) || + DECISION_INDICATORS.some((p) => p.test(joined)); + if (hasCorrectionOrDecision) { + value += 0.3; + } + + // Total substantive text > 200 chars? +0.2 + const substantiveChars = texts + .filter((t) => t.trim().length > 20) // skip very short lines + .reduce((sum, t) => sum + t.length, 0); + if (substantiveChars > 200) { + value += 0.2; + } + + // Has multi-turn exchanges (>6 texts)? +0.1 + if (texts.length > 6) { + value += 0.1; + } + + return Math.min(value, 1.0); +} diff --git a/src/workspace-boundary.ts b/src/workspace-boundary.ts new file mode 100644 index 00000000..06897ab9 --- /dev/null +++ b/src/workspace-boundary.ts @@ -0,0 +1,154 @@ +import { + classifyIdentityAndAddressingMemory, +} from "./identity-addressing.js"; +import { parseSmartMetadata } from "./smart-metadata.js"; + +export interface UserMdExclusiveConfig { + enabled?: boolean; + routeProfile?: boolean; + routeCanonicalName?: boolean; + routeCanonicalAddressing?: boolean; + filterRecall?: boolean; +} + +export interface WorkspaceBoundaryConfig { + userMdExclusive?: UserMdExclusiveConfig; +} + +export interface ResolvedUserMdExclusiveConfig { + enabled: boolean; + routeProfile: boolean; + routeCanonicalName: boolean; + routeCanonicalAddressing: boolean; + filterRecall: boolean; +} + +type UserMdExclusiveSlot = "profile" | "name" | "addressing"; + +type BoundaryEntryLike = { + text: string; + metadata?: string; + category?: "preference" | "fact" | "decision" | "entity" | "other" | "reflection"; + importance?: number; + timestamp?: number; +}; + +const PROFILE_HINT_PATTERNS = [ + /^User profile:/im, + /^##\s*(?:Background|Profile|Context)$/im, + /(?:^|\n)-\s*(?:Timezone|Pronouns?|Role|Language|Working style|Collaboration style)\s*:/i, + /(?:我的时区是|我的代词是|我是|我的身份是|my timezone is|my pronouns are|i am)\b/iu, + /(?:时区|代词|协作方式|工作方式|语言偏好)/u, +]; + +export function resolveUserMdExclusiveConfig( + workspaceBoundary?: WorkspaceBoundaryConfig | null, +): ResolvedUserMdExclusiveConfig { + const raw = workspaceBoundary?.userMdExclusive; + const enabled = raw?.enabled === true; + return { + enabled, + routeProfile: enabled && raw?.routeProfile !== false, + routeCanonicalName: enabled && raw?.routeCanonicalName !== false, + routeCanonicalAddressing: enabled && raw?.routeCanonicalAddressing !== false, + filterRecall: enabled && raw?.filterRecall !== false, + }; +} + +export function shouldFilterUserMdExclusiveRecall( + workspaceBoundary?: WorkspaceBoundaryConfig | null, +): boolean { + return resolveUserMdExclusiveConfig(workspaceBoundary).filterRecall; +} + +export function isUserMdExclusiveMemory( + params: { + memoryCategory?: string; + factKey?: string; + text?: string; + abstract?: string; + overview?: string; + content?: string; + }, + workspaceBoundary?: WorkspaceBoundaryConfig | null, +): boolean { + const config = resolveUserMdExclusiveConfig(workspaceBoundary); + if (!config.enabled) return false; + + const slots = new Set(); + if (params.memoryCategory === "profile") { + slots.add("profile"); + } + + const semantics = classifyIdentityAndAddressingMemory({ + factKey: params.factKey, + text: params.text, + abstract: params.abstract, + overview: params.overview, + content: params.content, + }); + + if (semantics.slots.has("name")) { + slots.add("name"); + } + if (semantics.slots.has("addressing")) { + slots.add("addressing"); + } + + const probe = [ + params.text, + params.abstract, + params.overview, + params.content, + ] + .filter((value): value is string => typeof value === "string" && value.trim().length > 0) + .map((value) => value.trim()) + .join("\n"); + + if (probe && PROFILE_HINT_PATTERNS.some((pattern) => pattern.test(probe))) { + slots.add("profile"); + } + + if (config.routeProfile && slots.has("profile")) { + return true; + } + + if (config.routeCanonicalName && slots.has("name")) { + return true; + } + + if (config.routeCanonicalAddressing && slots.has("addressing")) { + return true; + } + + return false; +} + +export function isUserMdExclusiveEntry( + entry: BoundaryEntryLike, + workspaceBoundary?: WorkspaceBoundaryConfig | null, +): boolean { + const meta = parseSmartMetadata(entry.metadata, entry); + return isUserMdExclusiveMemory( + { + memoryCategory: meta.memory_category, + factKey: meta.fact_key, + text: entry.text, + abstract: meta.l0_abstract, + overview: meta.l1_overview, + content: meta.l2_content, + }, + workspaceBoundary, + ); +} + +export function filterUserMdExclusiveRecallResults( + results: T[], + workspaceBoundary?: WorkspaceBoundaryConfig | null, +): T[] { + if (!shouldFilterUserMdExclusiveRecall(workspaceBoundary)) { + return results; + } + + return results.filter((result) => !isUserMdExclusiveEntry(result.entry, workspaceBoundary)); +} diff --git a/test/batch-dedup.test.mjs b/test/batch-dedup.test.mjs new file mode 100644 index 00000000..83d85e83 --- /dev/null +++ b/test/batch-dedup.test.mjs @@ -0,0 +1,196 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import jitiFactory from "jiti"; + +const jiti = jitiFactory(import.meta.url, { interopDefault: true }); + +const { batchDedup, createExtractionCostStats } = jiti( + "../src/batch-dedup.ts", +); + +// ============================================================================ +// Helpers +// ============================================================================ + +/** + * Create a normalized unit vector with slight variation. + * seed controls the angle: same seed = same direction = high cosine similarity. + */ +function makeVector(seed, dim = 128) { + const vec = new Array(dim).fill(0); + for (let i = 0; i < dim; i++) { + vec[i] = Math.sin(seed * (i + 1)) + Math.cos(seed * (i + 2)); + } + // Normalize + const norm = Math.sqrt(vec.reduce((sum, v) => sum + v * v, 0)); + if (norm > 0) { + for (let i = 0; i < dim; i++) vec[i] /= norm; + } + return vec; +} + +/** + * Create a vector very similar to a base vector (add small noise). + */ +function makeSimilarVector(base, noise = 0.01) { + const vec = base.map((v) => v + (Math.random() - 0.5) * noise); + const norm = Math.sqrt(vec.reduce((sum, v) => sum + v * v, 0)); + if (norm > 0) { + for (let i = 0; i < vec.length; i++) vec[i] /= norm; + } + return vec; +} + +// ============================================================================ +// batchDedup tests +// ============================================================================ + +describe("batchDedup", () => { + it("returns all indices when no duplicates", () => { + const v1 = makeVector(1.0); + const v2 = makeVector(5.0); + const v3 = makeVector(10.0); + + const result = batchDedup( + ["abstract A", "abstract B", "abstract C"], + [v1, v2, v3], + 0.85, + ); + + assert.equal(result.inputCount, 3); + assert.equal(result.outputCount, 3); + assert.deepEqual(result.survivingIndices, [0, 1, 2]); + assert.deepEqual(result.duplicateIndices, []); + }); + + it("marks similar candidates as duplicates", () => { + const v1 = makeVector(1.0); + const v2 = makeSimilarVector(v1, 0.001); // Very similar to v1 + const v3 = makeVector(10.0); // Very different + + const result = batchDedup( + ["similar abstract 1", "similar abstract 2", "different abstract"], + [v1, v2, v3], + 0.85, + ); + + assert.equal(result.inputCount, 3); + assert.equal(result.outputCount, 2); + assert.ok(result.survivingIndices.includes(0)); + assert.ok(result.survivingIndices.includes(2)); + assert.ok(result.duplicateIndices.includes(1)); + }); + + it("keeps first of duplicate pair", () => { + const v1 = makeVector(1.0); + const v2 = makeSimilarVector(v1, 0.0001); // Nearly identical + + const result = batchDedup( + ["abstract A", "abstract A (duplicate)"], + [v1, v2], + 0.85, + ); + + assert.equal(result.outputCount, 1); + assert.deepEqual(result.survivingIndices, [0]); + assert.deepEqual(result.duplicateIndices, [1]); + }); + + it("handles single candidate", () => { + const result = batchDedup( + ["only abstract"], + [makeVector(1.0)], + 0.85, + ); + + assert.equal(result.inputCount, 1); + assert.equal(result.outputCount, 1); + assert.deepEqual(result.survivingIndices, [0]); + assert.deepEqual(result.duplicateIndices, []); + }); + + it("handles empty input", () => { + const result = batchDedup([], [], 0.85); + + assert.equal(result.inputCount, 0); + assert.equal(result.outputCount, 0); + assert.deepEqual(result.survivingIndices, []); + assert.deepEqual(result.duplicateIndices, []); + }); + + it("respects threshold: low threshold drops more", () => { + const v1 = makeVector(1.0); + const v2 = makeVector(1.3); // Somewhat similar + const v3 = makeVector(10.0); // Very different + + const strictResult = batchDedup( + ["a", "b", "c"], + [v1, v2, v3], + 0.5, // Very low threshold - more aggressive dedup + ); + + const lenientResult = batchDedup( + ["a", "b", "c"], + [v1, v2, v3], + 0.99, // Very high threshold - almost no dedup + ); + + // Strict should drop more or equal candidates + assert.ok(strictResult.outputCount <= lenientResult.outputCount); + }); + + it("handles empty/missing vectors gracefully", () => { + const v1 = makeVector(1.0); + + const result = batchDedup( + ["abstract A", "abstract B", "abstract C"], + [v1, [], v1], // Second vector is empty + 0.85, + ); + + // Should not crash; candidates with empty vectors survive + assert.ok(result.outputCount >= 1); + }); + + it("deduplicates multiple similar pairs correctly", () => { + const v1 = makeVector(1.0); + const v1dup = makeSimilarVector(v1, 0.0001); + const v2 = makeVector(10.0); + const v2dup = makeSimilarVector(v2, 0.0001); + + const result = batchDedup( + ["topic A", "topic A copy", "topic B", "topic B copy"], + [v1, v1dup, v2, v2dup], + 0.85, + ); + + assert.equal(result.inputCount, 4); + assert.equal(result.outputCount, 2); + assert.ok(result.survivingIndices.includes(0)); + assert.ok(result.survivingIndices.includes(2)); + }); +}); + +// ============================================================================ +// ExtractionCostStats tests +// ============================================================================ + +describe("ExtractionCostStats", () => { + it("creates fresh stats with zero values", () => { + const stats = createExtractionCostStats(); + assert.equal(stats.batchDeduped, 0); + assert.equal(stats.durationMs, 0); + assert.equal(stats.llmCalls, 0); + }); + + it("tracks batch dedup count", () => { + const stats = createExtractionCostStats(); + stats.batchDeduped = 3; + stats.durationMs = 1500; + stats.llmCalls = 2; + + assert.equal(stats.batchDeduped, 3); + assert.equal(stats.durationMs, 1500); + assert.equal(stats.llmCalls, 2); + }); +}); diff --git a/test/cjk-recursion-regression.test.mjs b/test/cjk-recursion-regression.test.mjs new file mode 100644 index 00000000..247e7c14 --- /dev/null +++ b/test/cjk-recursion-regression.test.mjs @@ -0,0 +1,338 @@ +import assert from "node:assert/strict"; +import http from "node:http"; + +import jitiFactory from "jiti"; + +const jiti = jitiFactory(import.meta.url, { interopDefault: true }); +const { Embedder } = jiti("../src/embedder.ts"); +const { smartChunk } = jiti("../src/chunker.ts"); + +function generateCJKText(charCount) { + const chars = "中文字符测试数据内容关键词信息处理系统计算机软件硬件网络数据库服务器客户端浏览器应用程序编程语言算法数据结构人工智能机器学习深度学习神经网络。".split(""); + let text = ""; + for (let i = 0; i < charCount; i++) text += chars[i % chars.length]; + return text; +} + +function createJsonServer(handler) { + const server = http.createServer(async (req, res) => { + if (req.url !== "/v1/embeddings" || req.method !== "POST") { + res.writeHead(404); + res.end("not found"); + return; + } + + let body = ""; + req.on("data", (chunk) => { + body += chunk; + }); + req.on("end", async () => { + try { + await handler(JSON.parse(body || "{}"), req, res); + } catch (error) { + res.writeHead(500, { "content-type": "application/json" }); + res.end(JSON.stringify({ error: { message: String(error?.message || error), code: "test_handler_error" } })); + } + }); + }); + return server; +} + +async function withServer(handler, fn) { + const server = createJsonServer(handler); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + const port = typeof address === "object" && address ? address.port : 0; + const baseURL = `http://127.0.0.1:${port}/v1`; + try { + await fn({ baseURL }); + } finally { + await new Promise((resolve) => server.close(resolve)); + } +} + +async function testSingleChunkFallbackTerminates() { + console.log("Test 1: single-chunk fallback terminates instead of looping"); + + let callCount = 0; + await withServer((payload, _req, res) => { + callCount++; + const input = Array.isArray(payload.input) ? payload.input[0] : payload.input; + if (typeof input === "string" && input.length > 100) { + res.writeHead(400, { "content-type": "application/json" }); + res.end(JSON.stringify({ error: { message: "Input length exceeds maximum tokens (max 8192)", code: "context_length_exceeded" } })); + return; + } + + const dims = 1024; + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ data: [{ embedding: Array.from({ length: dims }, () => 1), index: 0 }] })); + }, async ({ baseURL }) => { + const embedder = new Embedder({ + provider: "openai-compatible", + apiKey: "test-key", + model: "mxbai-embed-large", + baseURL, + dimensions: 1024, + }); + + await assert.rejects( + () => embedder.embedPassage(generateCJKText(3000)), + (error) => { + assert.match(error.message, /Failed to embed: input too large for model context after 3 retries/i); + assert(callCount < 20, `Expected bounded retries, got ${callCount}`); + return true; + } + ); + }); + + console.log(` API calls before termination: ${callCount}`); + console.log(" PASSED\n"); +} + +async function testDepthLimitTermination() { + console.log("Test 2: depth limit terminates repeated forced reductions"); + + await withServer((_payload, _req, res) => { + res.writeHead(400, { "content-type": "application/json" }); + res.end(JSON.stringify({ error: { message: "Input length exceeds maximum tokens (max 8192)", code: "context_length_exceeded" } })); + }, async ({ baseURL }) => { + const embedder = new Embedder({ + provider: "openai-compatible", + apiKey: "test-key", + model: "mxbai-embed-large", + baseURL, + dimensions: 1024, + }); + + await assert.rejects( + () => embedder.embedPassage(generateCJKText(220)), + (error) => { + assert.match(error.message, /Failed to embed: input too large for model context after 3 retries|chunking couldn't reduce input size enough/i); + return true; + } + ); + }); + + console.log(" PASSED\n"); +} + +async function testCjkAwareChunkSizing() { + console.log("Test 3: CJK-aware chunk sizing produces more chunks than Latin text for same model budget"); + const cjkText = generateCJKText(5000); + const latinText = "english text sentence. ".repeat(220); + const cjkResult = smartChunk(cjkText, "mxbai-embed-large"); + const latinResult = smartChunk(latinText, "mxbai-embed-large"); + + assert(cjkResult.chunkCount > 1, "Expected multiple chunks for long CJK text"); + assert(cjkResult.chunks[0].length < latinResult.chunks[0].length, "Expected smaller CJK chunks than Latin chunks"); + console.log(` CJK first chunk: ${cjkResult.chunks[0].length} chars`); + console.log(` Latin first chunk: ${latinResult.chunks[0].length} chars`); + console.log(" PASSED\n"); +} + +async function testChunkErrorSurfaced() { + console.log("Test 4: chunkError is surfaced instead of generic context_length_exceeded wrapper"); + + await withServer((payload, _req, res) => { + const input = Array.isArray(payload.input) ? payload.input[0] : payload.input; + if (typeof input === "string" && input.length > 1500) { + res.writeHead(400, { "content-type": "application/json" }); + res.end(JSON.stringify({ error: { message: "Input length exceeds maximum tokens (max 8192)", code: "context_length_exceeded" } })); + return; + } + + res.writeHead(400, { "content-type": "application/json" }); + res.end(JSON.stringify({ error: { message: "chunk child failed with synthetic downstream error", code: "synthetic_chunk_failure" } })); + }, async ({ baseURL }) => { + const embedder = new Embedder({ + provider: "openai-compatible", + apiKey: "test-key", + model: "mxbai-embed-large", + baseURL, + dimensions: 1024, + }); + + await assert.rejects( + () => embedder.embedPassage(generateCJKText(5000)), + (error) => { + assert.match(error.message, /synthetic_chunk_failure|synthetic downstream error|chunk child failed/i); + assert.doesNotMatch(error.message, /context_length_exceeded/i); + return true; + } + ); + }); + + console.log(" PASSED\n"); +} + +async function testSmallContextChunking() { + console.log("Test 5: small-context model no longer keeps a 1000-char hard floor"); + const text = generateCJKText(2000); + const result = smartChunk(text, "all-MiniLM-L6-v2"); + assert(result.chunkCount > 1, "Expected multiple chunks for small-context CJK text"); + const maxChunkLen = Math.max(...result.chunks.map((c) => c.length)); + assert(maxChunkLen <= 200, `Expected chunk size <= 200 chars after clamp, got ${maxChunkLen}`); + console.log(` Largest chunk: ${maxChunkLen} chars`); + console.log(" PASSED\n"); +} + +async function testTimeoutAbortPropagation() { + console.log("Test 6: timeout abort propagates to underlying request path"); + + await withServer(async (_payload, req, res) => { + await new Promise((resolve) => setTimeout(resolve, 11_000)); + if (req.aborted || req.destroyed) { + return; + } + const dims = 1024; + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ data: [{ embedding: Array.from({ length: dims }, () => 0), index: 0 }] })); + }, async ({ baseURL }) => { + const embedder = new Embedder({ + provider: "openai-compatible", + apiKey: "test-key", + model: "mxbai-embed-large", + baseURL, + dimensions: 1024, + }); + + await assert.rejects( + () => embedder.embedPassage("short timeout probe"), + (error) => { + assert.match(error.message, /aborted|abort|timed out|fetch failed/i); + return true; + } + ); + }); + + console.log(" PASSED\n"); +} + +async function testBatchEmbeddingStillWorks() { + console.log("Test 7: batch embedding still works without withTimeout wrapper"); + + await withServer((_payload, _req, res) => { + const dims = 1024; + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ + data: [0, 1, 2].map((index) => ({ embedding: Array.from({ length: dims }, () => index), index })), + })); + }, async ({ baseURL }) => { + const embedder = new Embedder({ + provider: "openai-compatible", + apiKey: "test-key", + model: "mxbai-embed-large", + baseURL, + dimensions: 1024, + }); + + const embeddings = await embedder.embedBatchPassage(["a", "b", "c"]); + assert.equal(embeddings.length, 3); + assert.equal(embeddings[0].length, 1024); + assert.equal(embeddings[2][0], 2); + }); + + console.log(" PASSED\n"); +} + +async function testOllamaAbortWithNativeFetch() { + console.log("Test 8: Ollama native fetch respects external AbortSignal (PR354 fix regression)"); + + // Author's analysis: the previous test used withServer() on a random port but hardcoded + // http://127.0.0.1:11434/v1 for the Embedder — so the request always hit "connection refused" + // immediately and never touched the slow handler. This test fixes that by: + // 1. Binding the mock server directly to 127.0.0.1:11434 (so isOllamaProvider() is true) + // 2. Delaying the response by 5 seconds + // 3. Passing an external AbortSignal that fires after 2 seconds + // 4. Asserting total time ≈ 2s (proving abort interrupted the slow request) + + const SLOW_DELAY_MS = 5_000; + const ABORT_AFTER_MS = 2_000; + const DIMS = 1024; + + const server = http.createServer((req, res) => { + if (req.url === "/v1/embeddings" && req.method === "POST") { + const timer = setTimeout(() => { + if (res.writableEnded) return; // already aborted + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ + data: [{ embedding: Array.from({ length: DIMS }, () => 0.1), index: 0 }] + })); + }, SLOW_DELAY_MS); + req.on("aborted", () => clearTimeout(timer)); + return; + } + res.writeHead(404); + res.end("not found"); + }); + + // Bind directly to 127.0.0.1:11434 so isOllamaProvider() returns true + await new Promise((resolve) => server.listen(11434, "127.0.0.1", resolve)); + + try { + const embedder = new Embedder({ + provider: "openai-compatible", + apiKey: "test-key", + model: "mxbai-embed-large", + baseURL: "http://127.0.0.1:11434/v1", + dimensions: DIMS, + }); + + assert.equal( + embedder.isOllamaProvider ? embedder.isOllamaProvider() : false, + true, + "isOllamaProvider should return true for 127.0.0.1:11434" + ); + + const start = Date.now(); + const controller = new AbortController(); + const abortTimer = setTimeout(() => controller.abort(), ABORT_AFTER_MS); + + let errorCaught; + try { + // Pass external AbortSignal — should interrupt the 5-second slow response at ~2s + await embedder.embedPassage("abort test probe", controller.signal); + } catch (e) { + errorCaught = e; + } + + clearTimeout(abortTimer); + const elapsed = Date.now() - start; + + assert.ok(errorCaught, "embedPassage should throw (abort or timeout)"); + const msg = errorCaught instanceof Error ? errorCaught.message : String(errorCaught); + assert.ok( + /timed out|abort|ollama|ECONNREFUSED/i.test(msg), + `Expected abort/timeout error, got: ${msg}` + ); + + // If abort works: elapsed ≈ 2000ms. If abort fails: elapsed ≈ 5000ms. + assert.ok( + elapsed < SLOW_DELAY_MS * 0.75, + `Expected abort ~${ABORT_AFTER_MS}ms, got ${elapsed}ms — abort did NOT interrupt slow request` + ); + + console.log(` PASSED (aborted in ${elapsed}ms < ${SLOW_DELAY_MS}ms threshold)\n`); + } finally { + await new Promise((resolve) => server.close(resolve)); + } +} + +async function run() { + console.log("Running regression tests for PR #238...\n"); + await testSingleChunkFallbackTerminates(); + await testDepthLimitTermination(); + await testCjkAwareChunkSizing(); + await testChunkErrorSurfaced(); + await testSmallContextChunking(); + await testTimeoutAbortPropagation(); + await testBatchEmbeddingStillWorks(); + await testOllamaAbortWithNativeFetch(); + console.log("All regression tests passed!"); +} + +run().catch((err) => { + console.error("Test failed:", err); + process.exit(1); +}); diff --git a/test/clawteam-scope.test.mjs b/test/clawteam-scope.test.mjs new file mode 100644 index 00000000..14759394 --- /dev/null +++ b/test/clawteam-scope.test.mjs @@ -0,0 +1,128 @@ +import { describe, it, beforeEach } from "node:test"; +import assert from "node:assert/strict"; +import jitiFactory from "jiti"; + +const jiti = jitiFactory(import.meta.url, { interopDefault: true }); +const { MemoryScopeManager, _resetLegacyFallbackWarningState } = jiti("../src/scopes.ts"); +const { parseClawteamScopes, applyClawteamScopes } = jiti("../src/clawteam-scope.ts"); + +describe("ClawTeam Scope Integration", () => { + let manager; + + beforeEach(() => { + manager = new MemoryScopeManager({ default: "global", agentAccess: {} }); + _resetLegacyFallbackWarningState(); + }); + + // ── parseClawteamScopes ────────────────────────────────────────────── + + describe("parseClawteamScopes", () => { + it("parses comma-separated scope names", () => { + assert.deepStrictEqual( + parseClawteamScopes("custom:team-a,custom:team-b"), + ["custom:team-a", "custom:team-b"], + ); + }); + + it("trims whitespace around scope names", () => { + assert.deepStrictEqual( + parseClawteamScopes(" custom:team-a , custom:team-b "), + ["custom:team-a", "custom:team-b"], + ); + }); + + it("returns empty array for undefined", () => { + assert.deepStrictEqual(parseClawteamScopes(undefined), []); + }); + + it("returns empty array for empty string", () => { + assert.deepStrictEqual(parseClawteamScopes(""), []); + }); + + it("filters out empty segments from trailing commas", () => { + assert.deepStrictEqual( + parseClawteamScopes("custom:team-a,,, "), + ["custom:team-a"], + ); + }); + + it("handles single scope without commas", () => { + assert.deepStrictEqual( + parseClawteamScopes("custom:team-demo"), + ["custom:team-demo"], + ); + }); + }); + + // ── applyClawteamScopes ────────────────────────────────────────────── + + describe("applyClawteamScopes", () => { + it("registers scope definitions for unknown scopes", () => { + assert.strictEqual(manager.getScopeDefinition("custom:team-x"), undefined); + + applyClawteamScopes(manager, ["custom:team-x"]); + + const def = manager.getScopeDefinition("custom:team-x"); + assert.notStrictEqual(def, undefined); + assert.match(def.description, /ClawTeam shared scope/); + }); + + it("does not overwrite existing scope definitions", () => { + manager.addScopeDefinition("custom:team-x", { description: "My custom def" }); + + applyClawteamScopes(manager, ["custom:team-x"]); + + assert.strictEqual(manager.getScopeDefinition("custom:team-x").description, "My custom def"); + }); + + it("extends getAccessibleScopes for a normal agent", () => { + applyClawteamScopes(manager, ["custom:team-demo"]); + + const scopes = manager.getAccessibleScopes("agent-1"); + assert.ok(scopes.includes("custom:team-demo"), "should include team scope"); + }); + + it("preserves original agent scopes after extension", () => { + applyClawteamScopes(manager, ["custom:team-demo"]); + + const scopes = manager.getAccessibleScopes("main"); + assert.ok(scopes.includes("global"), "should still have global"); + assert.ok(scopes.includes("agent:main"), "should still have agent:main"); + assert.ok(scopes.includes("reflection:agent:main"), "should still have reflection scope"); + }); + + it("does not duplicate scopes already in the base list", () => { + // global is always in the base list + applyClawteamScopes(manager, ["global"]); + + const scopes = manager.getAccessibleScopes("main"); + const globalCount = scopes.filter(s => s === "global").length; + assert.strictEqual(globalCount, 1, "global should appear exactly once"); + }); + + it("supports multiple team scopes", () => { + applyClawteamScopes(manager, ["custom:team-a", "custom:team-b"]); + + const scopes = manager.getAccessibleScopes("agent-1"); + assert.ok(scopes.includes("custom:team-a")); + assert.ok(scopes.includes("custom:team-b")); + }); + + it("no-ops when given empty scopes array", () => { + const before = manager.getAccessibleScopes("main"); + applyClawteamScopes(manager, []); + const after = manager.getAccessibleScopes("main"); + assert.deepStrictEqual(before, after); + }); + }); + + // ── Baseline (no ClawTeam) ─────────────────────────────────────────── + + describe("without applyClawteamScopes", () => { + it("agent does not have team scopes by default", () => { + const scopes = manager.getAccessibleScopes("main"); + assert.ok(!scopes.includes("custom:team-demo"), "should NOT include team scope"); + assert.deepStrictEqual(scopes, ["global", "agent:main", "reflection:agent:main"]); + }); + }); +}); diff --git a/test/cli-oauth-login.test.mjs b/test/cli-oauth-login.test.mjs new file mode 100644 index 00000000..1ae0e75e --- /dev/null +++ b/test/cli-oauth-login.test.mjs @@ -0,0 +1,577 @@ +import assert from "node:assert/strict"; +import { afterEach, beforeEach, describe, it } from "node:test"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import http from "node:http"; +import { Command } from "commander"; +import jitiFactory from "jiti"; + +const jiti = jitiFactory(import.meta.url, { interopDefault: true }); +const { createMemoryCLI } = jiti("../cli.ts"); + +const ENV_KEYS = [ + "MEMORY_PRO_OAUTH_AUTHORIZE_URL", + "MEMORY_PRO_OAUTH_TOKEN_URL", + "MEMORY_PRO_OAUTH_REDIRECT_URI", + "MEMORY_PRO_OAUTH_CLIENT_ID", + "OPENCLAW_HOME", +]; + +function encodeSegment(value) { + return Buffer.from(JSON.stringify(value)).toString("base64url"); +} + +function makeJwt(accountId) { + return [ + encodeSegment({ alg: "none", typ: "JWT" }), + encodeSegment({ + exp: Math.floor((Date.now() + 3_600_000) / 1000), + "https://api.openai.com/auth": { chatgpt_account_id: accountId }, + }), + "signature", + ].join("."); +} + +function getBackupPath(oauthPath) { + const parsed = path.parse(oauthPath); + const fileName = parsed.ext + ? `${parsed.name}.llm-backup${parsed.ext}` + : `${parsed.base}.llm-backup.json`; + return path.join(parsed.dir, fileName); +} + +describe("memory-pro auth", () => { + let tempDir; + let server; + let originalEnv; + let originalCwd; + + beforeEach(() => { + tempDir = mkdtempSync(path.join(tmpdir(), "memory-cli-oauth-")); + originalEnv = Object.fromEntries(ENV_KEYS.map((key) => [key, process.env[key]])); + originalCwd = process.cwd(); + }); + + afterEach(async () => { + process.chdir(originalCwd); + for (const key of ENV_KEYS) { + if (originalEnv[key] === undefined) { + delete process.env[key]; + } else { + process.env[key] = originalEnv[key]; + } + } + if (server) { + await new Promise((resolve) => server.close(resolve)); + server = null; + } + rmSync(tempDir, { recursive: true, force: true }); + }); + + it("round-trips a dedicated llm api-key config through OAuth login/logout", async () => { + const authCode = "test-auth-code"; + const accountId = "acct_cli_123"; + const redirectPort = 18765; + let tokenRequests = 0; + + server = http.createServer(async (req, res) => { + if (req.method !== "POST" || req.url !== "/oauth/token") { + res.writeHead(404).end(); + return; + } + + let body = ""; + for await (const chunk of req) body += chunk; + const params = new URLSearchParams(body); + tokenRequests += 1; + + assert.equal(params.get("grant_type"), "authorization_code"); + assert.equal(params.get("code"), authCode); + + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ + access_token: makeJwt(accountId), + refresh_token: "refresh-cli-token", + expires_in: 3600, + })); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const tokenPort = server.address().port; + + process.env.MEMORY_PRO_OAUTH_AUTHORIZE_URL = `http://127.0.0.1:${tokenPort}/oauth/authorize`; + process.env.MEMORY_PRO_OAUTH_TOKEN_URL = `http://127.0.0.1:${tokenPort}/oauth/token`; + process.env.MEMORY_PRO_OAUTH_REDIRECT_URI = `http://localhost:${redirectPort}/auth/callback`; + process.env.MEMORY_PRO_OAUTH_CLIENT_ID = "test-client-id"; + + const configPath = path.join(tempDir, "openclaw.json"); + const oauthPath = path.join(tempDir, ".memory-lancedb-pro", "oauth.json"); + const backupPath = getBackupPath(oauthPath); + const originalLlmConfig = { + auth: "api-key", + apiKey: "old-llm-key", + model: "gpt-4o-mini", + baseURL: "https://api.openai.com/v1", + timeoutMs: 45000, + }; + writeFileSync(configPath, JSON.stringify({ + plugins: { + entries: { + "memory-lancedb-pro": { + enabled: true, + config: { + embedding: { + provider: "openai-compatible", + apiKey: "embed-key", + }, + llm: originalLlmConfig, + }, + }, + }, + }, + }, null, 2)); + + let capturedAuthorizeUrl = ""; + const program = new Command(); + program.exitOverride(); + createMemoryCLI({ + store: {} , + retriever: {}, + scopeManager: {}, + migrator: {}, + pluginId: "memory-lancedb-pro", + pluginConfig: { + llm: { + model: "openai/gpt-5.4", + }, + }, + oauthTestHooks: { + authorizeUrl: async (url) => { + capturedAuthorizeUrl = url; + const parsed = new URL(url); + const state = parsed.searchParams.get("state"); + setTimeout(() => { + const callback = new URL(process.env.MEMORY_PRO_OAUTH_REDIRECT_URI); + callback.searchParams.set("code", authCode); + callback.searchParams.set("state", state || ""); + http.get(callback); + }, 25); + }, + }, + })({ program }); + + const logs = []; + const originalLog = console.log; + console.log = (...args) => logs.push(args.join(" ")); + try { + await program.parseAsync([ + "node", + "openclaw", + "memory-pro", + "auth", + "login", + "--config", + configPath, + "--provider", + "openai-codex", + "--oauth-path", + oauthPath, + "--model", + "openai/gpt-5.4", + "--no-browser", + ]); + } finally { + console.log = originalLog; + } + + assert.equal(tokenRequests, 1); + assert.ok(capturedAuthorizeUrl.includes("client_id=test-client-id")); + assert.ok(readFileSync(oauthPath, "utf8").includes(accountId)); + + const updatedConfig = JSON.parse(readFileSync(configPath, "utf8")); + const pluginConfig = updatedConfig.plugins.entries["memory-lancedb-pro"].config; + assert.equal(pluginConfig.llm.auth, "oauth"); + assert.equal(pluginConfig.llm.oauthProvider, "openai-codex"); + assert.equal(pluginConfig.llm.oauthPath, oauthPath); + assert.equal(pluginConfig.llm.model, "gpt-5.4"); + assert.equal(pluginConfig.llm.timeoutMs, 45000); + assert.equal(Object.prototype.hasOwnProperty.call(pluginConfig.llm, "apiKey"), false); + assert.equal(Object.prototype.hasOwnProperty.call(pluginConfig.llm, "baseURL"), false); + + const backup = JSON.parse(readFileSync(backupPath, "utf8")); + assert.equal(backup.hadLlmConfig, true); + assert.deepEqual(backup.llm, originalLlmConfig); + + const output = logs.join("\n"); + assert.match(output, /Provider: OpenAI Codex \(openai-codex,/); + assert.match(output, /Authorization URL:/); + assert.match(output, /OAuth login completed/); + assert.match(output, /Updated memory-lancedb-pro config: llm.auth=oauth, llm.oauthProvider=openai-codex/); + + const logoutProgram = new Command(); + logoutProgram.exitOverride(); + createMemoryCLI({ + store: {}, + retriever: {}, + scopeManager: {}, + migrator: {}, + pluginId: "memory-lancedb-pro", + })({ program: logoutProgram }); + + const logoutLogs = []; + console.log = (...args) => logoutLogs.push(args.join(" ")); + try { + await logoutProgram.parseAsync([ + "node", + "openclaw", + "memory-pro", + "auth", + "logout", + "--config", + configPath, + ]); + } finally { + console.log = originalLog; + } + + assert.equal(existsSync(oauthPath), false); + assert.equal(existsSync(backupPath), false); + + const restoredConfig = JSON.parse(readFileSync(configPath, "utf8")); + const restoredPluginConfig = restoredConfig.plugins.entries["memory-lancedb-pro"].config; + assert.deepEqual(restoredPluginConfig.llm, originalLlmConfig); + + const logoutOutput = logoutLogs.join("\n"); + assert.match(logoutOutput, /Updated memory-lancedb-pro config: llm.auth=api-key/); + }); + + it("supports interactive provider selection when --provider is omitted", async () => { + const authCode = "test-auth-code"; + const accountId = "acct_cli_prompt_123"; + const redirectPort = 18766; + + server = http.createServer(async (req, res) => { + if (req.method !== "POST" || req.url !== "/oauth/token") { + res.writeHead(404).end(); + return; + } + + let body = ""; + for await (const chunk of req) body += chunk; + const params = new URLSearchParams(body); + + assert.equal(params.get("grant_type"), "authorization_code"); + assert.equal(params.get("code"), authCode); + + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ + access_token: makeJwt(accountId), + refresh_token: "refresh-cli-token", + expires_in: 3600, + })); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const tokenPort = server.address().port; + + process.env.MEMORY_PRO_OAUTH_AUTHORIZE_URL = `http://127.0.0.1:${tokenPort}/oauth/authorize`; + process.env.MEMORY_PRO_OAUTH_TOKEN_URL = `http://127.0.0.1:${tokenPort}/oauth/token`; + process.env.MEMORY_PRO_OAUTH_REDIRECT_URI = `http://localhost:${redirectPort}/auth/callback`; + process.env.MEMORY_PRO_OAUTH_CLIENT_ID = "test-client-id"; + + const configPath = path.join(tempDir, "openclaw.json"); + const oauthPath = path.join(tempDir, ".memory-lancedb-pro", "oauth.json"); + writeFileSync(configPath, JSON.stringify({ + plugins: { + entries: { + "memory-lancedb-pro": { + enabled: true, + config: { + embedding: { + provider: "openai-compatible", + apiKey: "embed-key", + }, + }, + }, + }, + }, + }, null, 2)); + + const selectedProviders = []; + const program = new Command(); + program.exitOverride(); + createMemoryCLI({ + store: {} , + retriever: {}, + scopeManager: {}, + migrator: {}, + pluginId: "memory-lancedb-pro", + oauthTestHooks: { + chooseProvider: async (providers, currentProviderId) => { + selectedProviders.push(currentProviderId); + selectedProviders.push(...providers.map((provider) => provider.id)); + return "openai-codex"; + }, + authorizeUrl: async (url) => { + const parsed = new URL(url); + const state = parsed.searchParams.get("state"); + setTimeout(() => { + const callback = new URL(process.env.MEMORY_PRO_OAUTH_REDIRECT_URI); + callback.searchParams.set("code", authCode); + callback.searchParams.set("state", state || ""); + http.get(callback); + }, 25); + }, + }, + })({ program }); + + const logs = []; + const originalLog = console.log; + console.log = (...args) => logs.push(args.join(" ")); + try { + await program.parseAsync([ + "node", + "openclaw", + "memory-pro", + "auth", + "login", + "--config", + configPath, + "--oauth-path", + oauthPath, + "--model", + "openai/gpt-5.4", + "--no-browser", + ]); + } finally { + console.log = originalLog; + } + + assert.deepEqual(selectedProviders, ["openai-codex", "openai-codex"]); + + const updatedConfig = JSON.parse(readFileSync(configPath, "utf8")); + const pluginConfig = updatedConfig.plugins.entries["memory-lancedb-pro"].config; + assert.equal(pluginConfig.llm.oauthProvider, "openai-codex"); + + const output = logs.join("\n"); + assert.match(output, /Provider: OpenAI Codex \(openai-codex, prompt\)/); + }); + + it("defaults the OAuth file to the plugin-scoped path under OPENCLAW_HOME", async () => { + const authCode = "test-auth-code"; + const accountId = "acct_cli_default_path_123"; + const redirectPort = 18767; + + server = http.createServer(async (req, res) => { + if (req.method !== "POST" || req.url !== "/oauth/token") { + res.writeHead(404).end(); + return; + } + + let body = ""; + for await (const chunk of req) body += chunk; + const params = new URLSearchParams(body); + + assert.equal(params.get("grant_type"), "authorization_code"); + assert.equal(params.get("code"), authCode); + + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ + access_token: makeJwt(accountId), + refresh_token: "refresh-cli-token", + expires_in: 3600, + })); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const tokenPort = server.address().port; + + process.env.MEMORY_PRO_OAUTH_AUTHORIZE_URL = `http://127.0.0.1:${tokenPort}/oauth/authorize`; + process.env.MEMORY_PRO_OAUTH_TOKEN_URL = `http://127.0.0.1:${tokenPort}/oauth/token`; + process.env.MEMORY_PRO_OAUTH_REDIRECT_URI = `http://localhost:${redirectPort}/auth/callback`; + process.env.MEMORY_PRO_OAUTH_CLIENT_ID = "test-client-id"; + process.env.OPENCLAW_HOME = path.join(tempDir, "openclaw-home"); + + const configPath = path.join(tempDir, "openclaw.json"); + const oauthPath = path.join(process.env.OPENCLAW_HOME, ".memory-lancedb-pro", "oauth.json"); + const backupPath = getBackupPath(oauthPath); + writeFileSync(configPath, JSON.stringify({ + plugins: { + entries: { + "memory-lancedb-pro": { + enabled: true, + config: { + embedding: { + provider: "openai-compatible", + apiKey: "embed-key", + }, + }, + }, + }, + }, + }, null, 2)); + + const program = new Command(); + program.exitOverride(); + createMemoryCLI({ + store: {}, + retriever: {}, + scopeManager: {}, + migrator: {}, + pluginId: "memory-lancedb-pro", + oauthTestHooks: { + authorizeUrl: async (url) => { + const parsed = new URL(url); + const state = parsed.searchParams.get("state"); + setTimeout(() => { + const callback = new URL(process.env.MEMORY_PRO_OAUTH_REDIRECT_URI); + callback.searchParams.set("code", authCode); + callback.searchParams.set("state", state || ""); + http.get(callback); + }, 25); + }, + }, + })({ program }); + + await program.parseAsync([ + "node", + "openclaw", + "memory-pro", + "auth", + "login", + "--config", + configPath, + "--provider", + "openai-codex", + "--model", + "openai/gpt-5.4", + "--no-browser", + ]); + + assert.equal(existsSync(oauthPath), true); + assert.equal(existsSync(backupPath), true); + + const updatedConfig = JSON.parse(readFileSync(configPath, "utf8")); + const pluginConfig = updatedConfig.plugins.entries["memory-lancedb-pro"].config; + assert.equal(pluginConfig.llm.oauthPath, oauthPath); + }); + + it("resolves stored relative oauthPath against the config location during logout", async () => { + const workspaceDir = path.join(tempDir, "workspace"); + const otherDir = path.join(tempDir, "other"); + mkdirSync(workspaceDir, { recursive: true }); + mkdirSync(otherDir, { recursive: true }); + + const configPath = path.join(workspaceDir, "openclaw.json"); + const storedOauthPath = ".memory-lancedb-pro/oauth.json"; + const actualOauthPath = path.join(workspaceDir, ".memory-lancedb-pro", "oauth.json"); + mkdirSync(path.dirname(actualOauthPath), { recursive: true }); + writeFileSync(actualOauthPath, JSON.stringify({ access_token: "token" }), "utf8"); + writeFileSync(configPath, JSON.stringify({ + plugins: { + entries: { + "memory-lancedb-pro": { + enabled: true, + config: { + llm: { + auth: "oauth", + oauthPath: storedOauthPath, + baseURL: "https://chatgpt-proxy.example/v1", + }, + }, + }, + }, + }, + }, null, 2)); + + process.chdir(otherDir); + + const program = new Command(); + program.exitOverride(); + createMemoryCLI({ + store: {}, + retriever: {}, + scopeManager: {}, + migrator: {}, + pluginId: "memory-lancedb-pro", + })({ program }); + + const logs = []; + const originalLog = console.log; + console.log = (...args) => logs.push(args.join(" ")); + try { + await program.parseAsync([ + "node", + "openclaw", + "memory-pro", + "auth", + "logout", + "--config", + configPath, + ]); + } finally { + console.log = originalLog; + } + + assert.equal(existsSync(actualOauthPath), false); + + const updatedConfig = JSON.parse(readFileSync(configPath, "utf8")); + const pluginConfig = updatedConfig.plugins.entries["memory-lancedb-pro"].config; + assert.equal(pluginConfig.llm.baseURL, "https://chatgpt-proxy.example/v1"); + assert.equal(Object.prototype.hasOwnProperty.call(pluginConfig.llm, "oauthPath"), false); + assert.equal(Object.prototype.hasOwnProperty.call(pluginConfig.llm, "oauthProvider"), false); + assert.equal(Object.prototype.hasOwnProperty.call(pluginConfig.llm, "auth"), false); + + const output = logs.join("\n"); + assert.match(output, new RegExp(`Deleted OAuth file: ${actualOauthPath.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}`)); + }); + + it("removes llm config on logout when only OAuth-generated fields remain and no backup exists", async () => { + const workspaceDir = path.join(tempDir, "workspace"); + mkdirSync(workspaceDir, { recursive: true }); + + const configPath = path.join(workspaceDir, "openclaw.json"); + const oauthPath = path.join(workspaceDir, ".memory-lancedb-pro", "oauth.json"); + mkdirSync(path.dirname(oauthPath), { recursive: true }); + writeFileSync(oauthPath, JSON.stringify({ access_token: "token" }), "utf8"); + writeFileSync(configPath, JSON.stringify({ + plugins: { + entries: { + "memory-lancedb-pro": { + enabled: true, + config: { + llm: { + auth: "oauth", + oauthProvider: "openai-codex", + oauthPath, + model: "gpt-5.4", + }, + }, + }, + }, + }, + }, null, 2)); + + const program = new Command(); + program.exitOverride(); + createMemoryCLI({ + store: {}, + retriever: {}, + scopeManager: {}, + migrator: {}, + pluginId: "memory-lancedb-pro", + })({ program }); + + await program.parseAsync([ + "node", + "openclaw", + "memory-pro", + "auth", + "logout", + "--config", + configPath, + ]); + + const updatedConfig = JSON.parse(readFileSync(configPath, "utf8")); + const pluginConfig = updatedConfig.plugins.entries["memory-lancedb-pro"].config; + assert.equal(Object.prototype.hasOwnProperty.call(pluginConfig, "llm"), false); + }); +}); diff --git a/test/cross-process-lock.test.mjs b/test/cross-process-lock.test.mjs new file mode 100644 index 00000000..9370a954 --- /dev/null +++ b/test/cross-process-lock.test.mjs @@ -0,0 +1,119 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync, existsSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import jitiFactory from "jiti"; + +const jiti = jitiFactory(import.meta.url, { interopDefault: true }); +const { MemoryStore } = jiti("../src/store.ts"); + +function makeStore() { + const dir = mkdtempSync(join(tmpdir(), "memory-lancedb-pro-lock-")); + const store = new MemoryStore({ dbPath: dir, vectorDim: 3 }); + return { store, dir }; +} + +function makeEntry(i) { + return { + text: `memory-${i}`, + vector: [0.1 * i, 0.2 * i, 0.3 * i], + category: "fact", + scope: "global", + importance: 0.5, + metadata: "{}", + }; +} + +describe("Cross-process file lock", () => { + it("creates .memory-write.lock file on first write", async () => { + const { store, dir } = makeStore(); + try { + await store.store(makeEntry(1)); + assert.ok(existsSync(join(dir, ".memory-write.lock")), "lock file should exist"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("sequential writes succeed without conflict", async () => { + const { store, dir } = makeStore(); + try { + const e1 = await store.store(makeEntry(1)); + const e2 = await store.store(makeEntry(2)); + assert.ok(e1.id !== e2.id, "entries should have different IDs"); + + const all = await store.list(undefined, undefined, 20, 0); + assert.strictEqual(all.length, 2); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("concurrent writes do not lose data", async () => { + const { store, dir } = makeStore(); + const count = 4; + try { + // Fire 4 concurrent stores (realistic ClawTeam swarm size) + const results = await Promise.all( + Array.from({ length: count }, (_, i) => store.store(makeEntry(i + 1))), + ); + + assert.strictEqual(results.length, count, "all store calls should resolve"); + + const ids = new Set(results.map(r => r.id)); + assert.strictEqual(ids.size, count, "all entries should have unique IDs"); + + const all = await store.list(undefined, undefined, 100, 0); + assert.strictEqual(all.length, count, "all entries should be retrievable"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("concurrent updates do not corrupt data", async () => { + const { store, dir } = makeStore(); + try { + // Seed entries + const entries = await Promise.all( + Array.from({ length: 4 }, (_, i) => store.store(makeEntry(i + 1))), + ); + + // Concurrently update all of them + const updated = await Promise.all( + entries.map((e, i) => + store.update(e.id, { text: `updated-${i}`, importance: 0.9 }), + ), + ); + + assert.strictEqual(updated.filter(Boolean).length, 4, "all updates should succeed"); + + // Verify data integrity + for (let i = 0; i < 4; i++) { + const fetched = await store.getById(entries[i].id); + assert.ok(fetched, `entry ${i} should exist`); + assert.strictEqual(fetched.text, `updated-${i}`); + assert.strictEqual(fetched.importance, 0.9); + } + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("lock is released after each operation", async () => { + const { store, dir } = makeStore(); + try { + await store.store(makeEntry(1)); + // If lock were stuck, this second store would hang/fail + await store.store(makeEntry(2)); + await store.delete((await store.list(undefined, undefined, 1, 0))[0].id); + // Still works after delete + await store.store(makeEntry(3)); + + const all = await store.list(undefined, undefined, 20, 0); + assert.strictEqual(all.length, 2, "should have 2 entries after store+store+delete+store"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/test/embedder-ollama-abort.test.mjs b/test/embedder-ollama-abort.test.mjs new file mode 100644 index 00000000..73a55f2d --- /dev/null +++ b/test/embedder-ollama-abort.test.mjs @@ -0,0 +1,99 @@ +import assert from "node:assert/strict"; +import http from "node:http"; +import { test } from "node:test"; + +import jitiFactory from "jiti"; + +const jiti = jitiFactory(import.meta.url, { interopDefault: true }); +const { Embedder } = jiti("../src/embedder.ts"); + +/** + * Test: Ollama native fetch correctly aborts a slow HTTP request. + * + * Root cause (Issue #361 / PR #383): + * OpenAI SDK's HTTP client does not reliably abort Ollama TCP connections + * when AbortController.abort() fires in Node.js, causing stalled sockets + * that hang until the gateway-level timeout. + * + * Fix: For Ollama endpoints (localhost:11434), use Node.js native fetch + * instead of the OpenAI SDK. Native fetch properly closes TCP on abort. + * + * This test verifies the fix by: + * 1. Mocking a slow Ollama server on 127.0.0.1:11434 (5s delay) + * 2. Calling embedPassage with an AbortSignal that fires after 2s + * 3. Asserting total time ≈ 2s (not 5s) — proving abort interrupted the request + * + * Note: The mock server is bound to 127.0.0.1:11434 (not a random port) so that + * isOllamaProvider() returns true and the native fetch path is exercised. + */ +test("Ollama embedWithNativeFetch aborts slow request within expected time", async () => { + const SLOW_DELAY_MS = 5_000; + const ABORT_AFTER_MS = 2_000; + const DIMS = 1024; + + const server = http.createServer((req, res) => { + if (req.url === "/v1/embeddings" && req.method === "POST") { + const timer = setTimeout(() => { + if (res.writableEnded) return; // already aborted + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ + data: [{ embedding: Array.from({ length: DIMS }, () => 0.1), index: 0 }] + })); + }, SLOW_DELAY_MS); + req.on("aborted", () => clearTimeout(timer)); + return; + } + res.writeHead(404); + res.end("not found"); + }); + + // Bind to 127.0.0.1:11434 so isOllamaProvider() returns true → native fetch path + await new Promise((resolve) => server.listen(11434, "127.0.0.1", resolve)); + + try { + const embedder = new Embedder({ + provider: "openai-compatible", + apiKey: "test-key", + model: "mxbai-embed-large", + baseURL: "http://127.0.0.1:11434/v1", + dimensions: DIMS, + }); + + assert.ok( + embedder.isOllamaProvider(), + "isOllamaProvider() should return true for http://127.0.0.1:11434", + ); + + const start = Date.now(); + const controller = new AbortController(); + const abortTimer = setTimeout(() => controller.abort(), ABORT_AFTER_MS); + + let errorCaught; + try { + await embedder.embedPassage("abort test probe", controller.signal); + assert.fail("embedPassage should have thrown"); + } catch (e) { + errorCaught = e; + } + + clearTimeout(abortTimer); + const elapsed = Date.now() - start; + + assert.ok(errorCaught, "embedPassage should have thrown (abort or timeout)"); + const msg = errorCaught instanceof Error ? errorCaught.message : String(errorCaught); + assert.ok( + /timed out|abort|ollama/i.test(msg), + `Expected abort/timeout/Ollama error, got: ${msg}`, + ); + + // Elapsed time must be close to ABORT_AFTER_MS, NOT SLOW_DELAY_MS. + // If abort worked: elapsed ≈ 2000ms. + // If abort failed: elapsed ≈ 5000ms (waited for slow response). + assert.ok( + elapsed < SLOW_DELAY_MS * 0.75, + `Expected abort ~${ABORT_AFTER_MS}ms, got ${elapsed}ms — abort did NOT interrupt slow request`, + ); + } finally { + await new Promise((resolve) => server.close(resolve)); + } +}); diff --git a/test/governance-metadata.test.mjs b/test/governance-metadata.test.mjs new file mode 100644 index 00000000..085dd9ce --- /dev/null +++ b/test/governance-metadata.test.mjs @@ -0,0 +1,72 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import jitiFactory from "jiti"; + +const jiti = jitiFactory(import.meta.url, { interopDefault: true }); +const { + parseSmartMetadata, + buildSmartMetadata, +} = jiti("../src/smart-metadata.ts"); + +describe("governance metadata compatibility", () => { + it("fills governance defaults for legacy metadata", () => { + const meta = parseSmartMetadata(undefined, { + text: "legacy memory", + category: "fact", + importance: 0.7, + timestamp: 1710000000000, + }); + + assert.equal(meta.state, "confirmed"); + assert.equal(meta.source, "legacy"); + assert.equal(meta.memory_layer, "working"); + assert.equal(meta.injected_count, 0); + assert.equal(meta.bad_recall_count, 0); + assert.equal(meta.suppressed_until_turn, 0); + }); + + it("maps session-summary records to archived/reflection defaults", () => { + const meta = parseSmartMetadata( + JSON.stringify({ type: "session-summary", l0_abstract: "summary" }), + { + text: "summary", + category: "other", + }, + ); + + assert.equal(meta.source, "session-summary"); + assert.equal(meta.state, "archived"); + assert.equal(meta.memory_layer, "reflection"); + }); + + it("buildSmartMetadata preserves and updates governance fields", () => { + const original = { + text: "captured note", + category: "other", + timestamp: 1710000000000, + metadata: JSON.stringify({ + state: "pending", + source: "auto-capture", + memory_layer: "working", + injected_count: 2, + bad_recall_count: 1, + }), + }; + + const patched = buildSmartMetadata(original, { + state: "confirmed", + source: "manual", + memory_layer: "durable", + injected_count: 3, + bad_recall_count: 0, + last_confirmed_use_at: 1710000001234, + }); + + assert.equal(patched.state, "confirmed"); + assert.equal(patched.source, "manual"); + assert.equal(patched.memory_layer, "durable"); + assert.equal(patched.injected_count, 3); + assert.equal(patched.bad_recall_count, 0); + assert.equal(patched.last_confirmed_use_at, 1710000001234); + }); +}); diff --git a/test/intent-analyzer.test.mjs b/test/intent-analyzer.test.mjs new file mode 100644 index 00000000..8d25876f --- /dev/null +++ b/test/intent-analyzer.test.mjs @@ -0,0 +1,209 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { analyzeIntent, applyCategoryBoost, formatAtDepth } from "../src/intent-analyzer.ts"; + +describe("analyzeIntent", () => { + it("detects preference intent (English)", () => { + const result = analyzeIntent("What is my preferred coding style?"); + assert.equal(result.label, "preference"); + assert.equal(result.confidence, "high"); + assert.equal(result.depth, "l0"); + assert.ok(result.categories.includes("preference")); + }); + + it("detects preference intent (Chinese)", () => { + const result = analyzeIntent("我的代码风格偏好是什么？"); + assert.equal(result.label, "preference"); + assert.equal(result.confidence, "high"); + }); + + it("detects decision intent", () => { + const result = analyzeIntent("Why did we choose PostgreSQL over MySQL?"); + assert.equal(result.label, "decision"); + assert.equal(result.confidence, "high"); + assert.equal(result.depth, "l1"); + assert.ok(result.categories.includes("decision")); + }); + + it("detects decision intent (Chinese)", () => { + const result = analyzeIntent("当时决定用哪个方案？"); + assert.equal(result.label, "decision"); + assert.equal(result.confidence, "high"); + }); + + it("detects entity intent", () => { + const result = analyzeIntent("Who is the project lead for auth service?"); + assert.equal(result.label, "entity"); + assert.equal(result.confidence, "high"); + assert.ok(result.categories.includes("entity")); + }); + + it("detects entity intent (Chinese)", () => { + const result = analyzeIntent("谁是这个项目的负责人？"); + assert.equal(result.label, "entity"); + assert.equal(result.confidence, "high"); + }); + + it("does NOT misclassify tool/component queries as entity", () => { + // These should match fact, not entity (Codex review finding #4) + const tool = analyzeIntent("How do I install the tool?"); + assert.notEqual(tool.label, "entity"); + const component = analyzeIntent("How does this component work?"); + assert.notEqual(component.label, "entity"); + }); + + it("detects event intent and routes to entity+decision categories", () => { + const result = analyzeIntent("What happened during last week's deploy?"); + assert.equal(result.label, "event"); + assert.equal(result.confidence, "high"); + assert.equal(result.depth, "full"); + // event is not a stored category — should route to entity + decision + assert.ok(result.categories.includes("entity")); + assert.ok(result.categories.includes("decision")); + assert.ok(!result.categories.includes("event")); + }); + + it("detects event intent (Chinese)", () => { + const result = analyzeIntent("最近发生了什么？"); + assert.equal(result.label, "event"); + assert.equal(result.confidence, "high"); + assert.ok(!result.categories.includes("event")); + }); + + it("detects fact intent", () => { + const result = analyzeIntent("How does the authentication API work?"); + assert.equal(result.label, "fact"); + assert.equal(result.confidence, "high"); + assert.equal(result.depth, "l1"); + }); + + it("detects fact intent (Chinese)", () => { + const result = analyzeIntent("这个接口怎么配置？"); + assert.equal(result.label, "fact"); + assert.equal(result.confidence, "high"); + }); + + it("returns broad signal for ambiguous queries", () => { + const result = analyzeIntent("write a function to sort arrays"); + assert.equal(result.label, "broad"); + assert.equal(result.confidence, "low"); + assert.deepEqual(result.categories, []); + assert.equal(result.depth, "l0"); + }); + + it("returns empty signal for empty input", () => { + const result = analyzeIntent(""); + assert.equal(result.label, "empty"); + assert.equal(result.confidence, "low"); + }); +}); + +describe("applyCategoryBoost", () => { + const mockResults = [ + { entry: { category: "fact" }, score: 0.8 }, + { entry: { category: "preference" }, score: 0.75 }, + { entry: { category: "entity" }, score: 0.7 }, + ]; + + it("boosts matching categories and re-sorts", () => { + const intent = { + categories: ["preference"], + depth: "l0", + confidence: "high", + label: "preference", + }; + const boosted = applyCategoryBoost(mockResults, intent); + // preference entry (0.75 * 1.15 = 0.8625) should now rank first + assert.equal(boosted[0].entry.category, "preference"); + assert.ok(boosted[0].score > 0.75); + }); + + it("returns results unchanged for low confidence", () => { + const intent = { + categories: [], + depth: "l0", + confidence: "low", + label: "broad", + }; + const result = applyCategoryBoost(mockResults, intent); + assert.equal(result[0].entry.category, "fact"); // original order preserved + }); + + it("caps boosted scores at 1.0", () => { + const highScoreResults = [ + { entry: { category: "preference" }, score: 0.95 }, + ]; + const intent = { + categories: ["preference"], + depth: "l0", + confidence: "high", + label: "preference", + }; + const boosted = applyCategoryBoost(highScoreResults, intent); + assert.ok(boosted[0].score <= 1.0); + }); +}); + +describe("formatAtDepth", () => { + const entry = { + text: "User prefers TypeScript over JavaScript for all new projects. This was decided after the migration incident in Q3 where type errors caused a production outage.", + category: "preference", + scope: "global", + }; + + it("l0: returns compact one-line summary", () => { + const line = formatAtDepth(entry, "l0", 0.85, 0); + assert.ok(line.length < entry.text.length + 30); // shorter than full + assert.ok(line.includes("[preference]")); + assert.ok(line.includes("85%")); + assert.ok(!line.includes("global")); // l0 omits scope + }); + + it("l1: returns medium detail with scope", () => { + const line = formatAtDepth(entry, "l1", 0.72, 1); + assert.ok(line.includes("[preference:global]")); + assert.ok(line.includes("72%")); + }); + + it("full: returns complete text", () => { + const line = formatAtDepth(entry, "full", 0.9, 0); + assert.ok(line.includes(entry.text)); + assert.ok(line.includes("[preference:global]")); + }); + + it("includes BM25 and rerank source tags", () => { + const line = formatAtDepth(entry, "full", 0.8, 0, { bm25Hit: true, reranked: true }); + assert.ok(line.includes("vector+BM25")); + assert.ok(line.includes("+reranked")); + }); + + it("handles short text without truncation", () => { + const short = { text: "Use tabs.", category: "preference", scope: "global" }; + const l0 = formatAtDepth(short, "l0", 0.9, 0); + assert.ok(l0.includes("Use tabs.")); + }); + + it("splits CJK sentences correctly at l0 depth", () => { + const cjk = { + text: "第一句结束。第二句开始，这里有更多内容需要处理。", + category: "fact", + scope: "global", + }; + const l0 = formatAtDepth(cjk, "l0", 0.8, 0); + // Should stop at first 。 not include second sentence + assert.ok(l0.includes("第一句结束。")); + assert.ok(!l0.includes("第二句开始")); + }); + + it("applies sanitize function when provided", () => { + const malicious = { + text: ' normal text', + category: "fact", + scope: "global", + }; + const sanitize = (t) => t.replace(/<[^>]*>/g, "").trim(); + const line = formatAtDepth(malicious, "full", 0.8, 0, { sanitize }); + assert.ok(!line.includes("