diff --git a/README.md b/README.md index 6923582e..ae5611aa 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,8 @@ Read these in order: 5. [`platform/docs/ARCHITECTURE.md`](platform/docs/ARCHITECTURE.md) 6. [`platform/docs/PRODUCTION_READINESS_CHECKLIST.md`](platform/docs/PRODUCTION_READINESS_CHECKLIST.md) 7. [`platform/docs/DEVFLOW.md`](platform/docs/DEVFLOW.md) +8. [`platform/docs/PLUGIN_INVENTORY.md`](platform/docs/PLUGIN_INVENTORY.md) +9. [`platform/docs/DEGRADATION.md`](platform/docs/DEGRADATION.md) ## Repository map diff --git a/platform/configs/openclaw/75-strongclaw-hypermemory.example.json5 b/platform/configs/openclaw/75-strongclaw-hypermemory.example.json5 index a08c76a2..3f3b6b65 100644 --- a/platform/configs/openclaw/75-strongclaw-hypermemory.example.json5 +++ b/platform/configs/openclaw/75-strongclaw-hypermemory.example.json5 @@ -6,6 +6,8 @@ "enabled": true, "config": { "command": [ + "__PYTHON_EXECUTABLE__", + "-m", "clawops" ], "configPath": "__HYPERMEMORY_SQLITE_CONFIG_PATH__", diff --git a/platform/configs/openclaw/77-hypermemory.example.json5 b/platform/configs/openclaw/77-hypermemory.example.json5 index 0abc4fa7..e005d6f9 100644 --- a/platform/configs/openclaw/77-hypermemory.example.json5 +++ b/platform/configs/openclaw/77-hypermemory.example.json5 @@ -13,6 +13,8 @@ "enabled": true, "config": { "command": [ + "__PYTHON_EXECUTABLE__", + "-m", "clawops" ], "configPath": "__HYPERMEMORY_CONFIG_PATH__", diff --git a/platform/docs/BACKUP_AND_RECOVERY.md b/platform/docs/BACKUP_AND_RECOVERY.md index 77e9f70c..4f5d7cbd 100644 --- a/platform/docs/BACKUP_AND_RECOVERY.md +++ b/platform/docs/BACKUP_AND_RECOVERY.md @@ -15,6 +15,19 @@ - `clawops recovery prune-retention` - `clawops recovery rotate-secrets` +## Scheduled maintenance + +StrongClaw host service activation now installs a daily maintenance schedule at `04:00` local time: + +- systemd: `openclaw-maintenance.timer` -> `openclaw-maintenance.service` +- launchd: `ai.openclaw.maintenance` + +The scheduled command is: + +- `clawops recovery --home-dir prune-retention` + +This maintenance path is idempotent and retention-only. It prunes expired StrongClaw recovery artifacts and does not mutate upstream OpenClaw internals. + ## Development-mode repo-local compose state If you keep compose state under `platform/compose/state` during development, use the explicit dev wrappers instead of relying on implicit leftover mounts: diff --git a/platform/docs/CI_AND_SECURITY.md b/platform/docs/CI_AND_SECURITY.md index d1e88d40..d75be2bd 100644 --- a/platform/docs/CI_AND_SECURITY.md +++ b/platform/docs/CI_AND_SECURITY.md @@ -93,3 +93,5 @@ hypermemory Qdrant checks against the official pinned Qdrant GHCR image instead compile-checks the repo, runs targeted devflow tests, and validates `clawops devflow plan --goal "contract smoke"` without live ACP providers. - Operators can verify published provenance with GitHub's attestation tooling after a tagged release lands. + +Canonical plugin support status lives in [Plugin Inventory](./PLUGIN_INVENTORY.md). diff --git a/platform/docs/DEGRADATION.md b/platform/docs/DEGRADATION.md new file mode 100644 index 00000000..fd29ccea --- /dev/null +++ b/platform/docs/DEGRADATION.md @@ -0,0 +1,37 @@ +# Degradation Contract + +This document defines how dependency health maps to StrongClaw runtime impact. + +## Sidecar Dependency Matrix + +| Dependency | Used by | Required for `clawops ops sidecars up` success | Impact when unavailable | +| --- | --- | --- | --- | +| Postgres | runtime metadata/session state | Yes | **fatal**: sidecar bring-up fails | +| LiteLLM | loopback model routing | Yes | **fatal**: sidecar bring-up fails | +| Qdrant | dense/sparse retrieval (`openclaw-qmd`, `hypermemory`) | Required when the active profile uses QMD or hypermemory retrieval | **degraded**: retrieval lanes depending on Qdrant are unavailable | +| Neo4j | graph expansion for `hypermemory` context | Required when the active profile uses hypermemory graph expansion | **degraded**: graph expansion is unavailable | +| OTel Collector | telemetry export | No | **observational**: tracing/metrics export unavailable, runtime behavior unaffected | + +## Operator Output Contract + +- `clawops ops status` reports dependency health under `readiness.required` and `readiness.optional`. +- Each readiness entry includes: + - `service` + - `required` + - `impact` (`fatal`, `degraded`, or `observational`) + - `reason` + - `ready` + - expected vs observed state/health fields +- `ok` and `readiness.requiredReady` are `true` only when every required dependency is ready. + +## Plugin Startup Contract + +- `strongclaw-hypermemory` runs startup preflight (`clawops hypermemory verify --json`) before serving memory tools. +- If preflight fails, tool responses return a disabled/unavailable payload instead of silent fallback. +- Existing configs that only define `timeoutMs` remain valid; startup/tool timeout split is optional via `startupTimeoutMs` and `toolTimeoutMs`. + +## Related Docs + +- [Plugin Inventory](./PLUGIN_INVENTORY.md) +- [Observability](./OBSERVABILITY.md) +- [Hypermemory](./HYPERMEMORY.md) diff --git a/platform/docs/HYPERMEMORY.md b/platform/docs/HYPERMEMORY.md index ba4ae0c2..68bf2643 100644 --- a/platform/docs/HYPERMEMORY.md +++ b/platform/docs/HYPERMEMORY.md @@ -8,6 +8,11 @@ The built-in OpenClaw fallback remains available as `openclaw-default`, and the explicit built-ins-plus-QMD fallback remains available as `openclaw-qmd`. +Canonical support/degradation references: + +- [Plugin Inventory](./PLUGIN_INVENTORY.md) +- [Degradation Contract](./DEGRADATION.md) + ## Design goals - preserve OpenClaw-compatible `memory_search` and `memory_get` diff --git a/platform/docs/OBSERVABILITY.md b/platform/docs/OBSERVABILITY.md index f180e7cc..4a8163ae 100644 --- a/platform/docs/OBSERVABILITY.md +++ b/platform/docs/OBSERVABILITY.md @@ -32,3 +32,29 @@ Collector-side redaction is mandatory before broader trace export. Do not assume - vector sync runs and sync failures Those signals reuse the shared ClawOps telemetry path instead of adding a separate exporter or collector. + +## Runtime Bring-Up Events + +`clawops ops` now emits structured readiness events: + +- `clawops.ops.sidecars.wait.start` +- `clawops.ops.sidecars.wait.ready` +- `clawops.ops.sidecars.wait.timeout` +- `clawops.ops.sidecars.ready` +- `clawops.ops.sidecars.status` + +Wait-timeout events include `service`, `target`, `observed`, and `timeout_seconds` so failed dependencies can be diagnosed without replaying the command interactively. + +## Host Service Activation Events + +`clawops services install --activate` now emits: + +- `clawops.services.activate` with `service_manager=launchd` and `step` values: + - `sidecars_bootstrap` + - `gateway_bootstrap` + - `maintenance_bootstrap` +- `clawops.services.activate` with `service_manager=systemd` and `step` values: + - `daemon_reload` + - `enable_now` (`unit` included per activation) + +These events are intentionally coarse-grained to avoid high-volume per-poll logging. diff --git a/platform/docs/PLUGIN_INVENTORY.md b/platform/docs/PLUGIN_INVENTORY.md new file mode 100644 index 00000000..c8b0ae2d --- /dev/null +++ b/platform/docs/PLUGIN_INVENTORY.md @@ -0,0 +1,22 @@ +# Plugin Inventory + +This page is the canonical StrongClaw plugin support matrix. + +## Shipped Plugins + +| Plugin | Purpose | Default status | Build/runtime expectation | CI coverage | Support level | +| --- | --- | --- | --- | --- | --- | +| `strongclaw-hypermemory` | StrongClaw-owned memory plugin that proxies `clawops hypermemory` and preserves OpenClaw memory tool names. | Enabled by default in the `hypermemory` profile; opt-in via standalone overlay `75-strongclaw-hypermemory.example.json5`. | Requires a valid rendered `configPath` plus a callable StrongClaw command (`python -m clawops` in managed overlays). Startup preflight runs `clawops hypermemory verify --json` before serving tools. | `tests/suites/integration/clawops/hypermemory/test_plugin.py` and `platform/plugins/strongclaw-hypermemory/test/openclaw-host-functional.mjs` in CI. | **Supported (StrongClaw-owned)** | +| `memory-lancedb-pro` (vendored) | Vendored upstream memory plugin for migration bridge and compatibility use cases. | Not default; enabled through `memory-lancedb-pro` profile. | Vendored bundle under `platform/plugins/memory-lancedb-pro`, rendered via managed profile overlays. | `tests/suites/unit/ci/test_memory_plugin_verification.py` and `.github/workflows/memory-plugin-verification.yml`. | **Supported (vendored bridge path)** | + +## Support Policy + +- Support labels in this table must match real CI evidence and workflow coverage. +- New plugin entries must update this table and `platform/docs/CI_AND_SECURITY.md` in the same change. +- If a plugin is experimental or unavailable by default, that status must be explicit in this table and in any setup docs that mention it. + +## Related Docs + +- [Hypermemory](./HYPERMEMORY.md) +- [CI and Security](./CI_AND_SECURITY.md) +- [Degradation Contract](./DEGRADATION.md) diff --git a/platform/launchd/ai.openclaw.maintenance.plist.template b/platform/launchd/ai.openclaw.maintenance.plist.template new file mode 100644 index 00000000..5d44c5e8 --- /dev/null +++ b/platform/launchd/ai.openclaw.maintenance.plist.template @@ -0,0 +1,55 @@ + + + + + Label + ai.openclaw.maintenance + ProgramArguments + + __PYTHON_EXECUTABLE__ + -m + clawops + recovery + --home-dir + __HOME_DIR__ + prune-retention + + WorkingDirectory + __REPO_ROOT__ + EnvironmentVariables + + HOME + __HOME_DIR__ + XDG_CONFIG_HOME + __HOME_DIR__/.config + OPENCLAW_HOME + __OPENCLAW_HOME__ + OPENCLAW_STATE_DIR + __STATE_DIR__ + OPENCLAW_CONFIG_PATH + __OPENCLAW_CONFIG_PATH__ + OPENCLAW_CONFIG + __OPENCLAW_CONFIG__ + OPENCLAW_PROFILE + __OPENCLAW_PROFILE__ + STRONGCLAW_RUNTIME_ROOT + __STRONGCLAW_RUNTIME_ROOT__ + PATH + __HOME_DIR__/.config/varlock/bin:__HOME_DIR__/.local/bin:/usr/local/bin:/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin +__LAUNCHD_EXTRA_ENV__ + + StartCalendarInterval + + Hour + 4 + Minute + 0 + + StandardOutPath + __STATE_DIR__/logs/launchd-maintenance.out.log + StandardErrorPath + __STATE_DIR__/logs/launchd-maintenance.err.log + KeepAlive + + + diff --git a/platform/plugins/strongclaw-hypermemory/index.js b/platform/plugins/strongclaw-hypermemory/index.js index 999083b4..05f04358 100644 --- a/platform/plugins/strongclaw-hypermemory/index.js +++ b/platform/plugins/strongclaw-hypermemory/index.js @@ -1,4 +1,5 @@ import { spawn } from "node:child_process"; +import { existsSync } from "node:fs"; const DEFAULT_COMMAND = ["clawops"]; const DEFAULT_TIMEOUT_MS = 20000; @@ -119,6 +120,9 @@ function resolvePluginConfig(rawConfig) { "strongclaw-hypermemory requires plugins.entries.strongclaw-hypermemory.config.configPath", ); } + if (!existsSync(configPath)) { + throw new Error(`strongclaw-hypermemory configPath does not exist: ${configPath}`); + } const command = Array.isArray(input.command) && input.command.length > 0 ? input.command.filter((entry) => typeof entry === "string" && entry.trim()).map((entry) => entry.trim()) @@ -127,8 +131,18 @@ function resolvePluginConfig(rawConfig) { throw new Error("strongclaw-hypermemory command must contain at least one executable"); } const timeoutMs = readNumberParam(input, "timeoutMs"); + const startupTimeoutMs = readNumberParam(input, "startupTimeoutMs"); + const toolTimeoutMs = readNumberParam(input, "toolTimeoutMs"); const recallMaxResults = readNumberParam(input, "recallMaxResults"); const captureMinMessages = readNumberParam(input, "captureMinMessages"); + const resolvedTimeoutMs = + timeoutMs && timeoutMs >= 1000 ? Math.min(120000, Math.trunc(timeoutMs)) : DEFAULT_TIMEOUT_MS; + const resolvedStartupTimeoutMs = + startupTimeoutMs && startupTimeoutMs >= 1000 + ? Math.min(120000, Math.trunc(startupTimeoutMs)) + : resolvedTimeoutMs; + const resolvedToolTimeoutMs = + toolTimeoutMs && toolTimeoutMs >= 1000 ? Math.min(120000, Math.trunc(toolTimeoutMs)) : resolvedTimeoutMs; return { command, configPath, @@ -139,12 +153,17 @@ function resolvePluginConfig(rawConfig) { recallMaxResults && recallMaxResults >= 1 ? Math.min(10, Math.trunc(recallMaxResults)) : 3, captureMinMessages: captureMinMessages && captureMinMessages >= 1 ? Math.min(20, Math.trunc(captureMinMessages)) : 4, - timeoutMs: - timeoutMs && timeoutMs >= 1000 ? Math.min(120000, Math.trunc(timeoutMs)) : DEFAULT_TIMEOUT_MS, + timeoutMs: resolvedTimeoutMs, + startupTimeoutMs: resolvedStartupTimeoutMs, + toolTimeoutMs: resolvedToolTimeoutMs, }; } -async function runClawopsCommand(pluginConfig, args, { captureJson = true } = {}) { +async function runClawopsCommand( + pluginConfig, + args, + { captureJson = true, timeoutMs = pluginConfig.toolTimeoutMs, phase = "tool" } = {}, +) { const [command, ...commandArgs] = pluginConfig.command; const fullArgs = [...commandArgs, "hypermemory", "--config", pluginConfig.configPath, ...args]; return await new Promise((resolve, reject) => { @@ -158,7 +177,7 @@ async function runClawopsCommand(pluginConfig, args, { captureJson = true } = {} const timer = setTimeout(() => { timedOut = true; child.kill("SIGKILL"); - }, pluginConfig.timeoutMs); + }, timeoutMs); if (captureJson) { child.stdout.on("data", (chunk) => { stdout += chunk.toString(); @@ -167,15 +186,22 @@ async function runClawopsCommand(pluginConfig, args, { captureJson = true } = {} stderr += chunk.toString(); }); } - child.on("error", reject); + child.on("error", (error) => { + clearTimeout(timer); + reject( + new Error( + `failed to start strongclaw-hypermemory command (${command}): ${error instanceof Error ? error.message : String(error)}`, + ), + ); + }); child.on("close", (code) => { clearTimeout(timer); if (timedOut) { - reject(new Error(`clawops hypermemory timed out after ${pluginConfig.timeoutMs}ms`)); + reject(new Error(`clawops hypermemory ${phase} timed out after ${timeoutMs}ms`)); return; } if (code !== 0) { - reject(new Error(stderr.trim() || `clawops hypermemory exited with code ${code}`)); + reject(new Error(stderr.trim() || `clawops hypermemory ${phase} exited with code ${code}`)); return; } if (!captureJson) { @@ -195,6 +221,36 @@ async function runClawopsCommand(pluginConfig, args, { captureJson = true } = {} }); } +function createStartupGate(pluginConfig) { + let startupPromise; + async function ensureReady() { + if (startupPromise) { + return startupPromise; + } + startupPromise = runClawopsCommand(pluginConfig, ["status", "--json"], { + timeoutMs: pluginConfig.startupTimeoutMs, + phase: "startup preflight", + }).then((payload) => { + if (payload && typeof payload === "object" && payload.ok === false) { + const reason = + typeof payload.message === "string" && payload.message.trim() + ? payload.message.trim() + : "status returned ok=false"; + throw new Error(`strongclaw-hypermemory startup preflight failed: ${reason}`); + } + }); + return startupPromise; + } + function start(logger) { + void ensureReady().catch((error) => { + if (logger && typeof logger.warn === "function") { + logger.warn(`strongclaw-hypermemory startup preflight failed: ${String(error)}`); + } + }); + } + return { ensureReady, start }; +} + function buildDisabledSearchResult(error) { return { results: [], @@ -216,8 +272,14 @@ function formatRecallContext(results) { return lines.join("\n"); } -function fireAndForget(pluginConfig, args) { - runClawopsCommand(pluginConfig, args).catch(() => {}); +function fireAndForget(pluginConfig, args, { ensureReady } = {}) { + const runPromise = + typeof ensureReady === "function" + ? Promise.resolve() + .then(() => ensureReady()) + .then(() => runClawopsCommand(pluginConfig, args)) + : runClawopsCommand(pluginConfig, args); + runPromise.catch(() => {}); } function extractSessionMessages(event) { @@ -300,14 +362,14 @@ function splitFeedbackIds(entries, responseText) { return { confirmed, badRecall }; } -function registerMemoryCli(program, pluginConfig) { +function registerMemoryCli(program, runClawops) { const memory = program.command("memory").description("Use StrongClaw hypermemory."); memory .command("status") .description("Show StrongClaw hypermemory status.") .option("--json", "Print JSON.") .action(async (opts) => { - await runClawopsCommand(pluginConfig, ["status", ...(opts.json ? ["--json"] : [])], { + await runClawops(["status", ...(opts.json ? ["--json"] : [])], { captureJson: false, }); }); @@ -317,7 +379,7 @@ function registerMemoryCli(program, pluginConfig) { .description("Rebuild the StrongClaw hypermemory index.") .option("--json", "Print JSON.") .action(async (opts) => { - await runClawopsCommand(pluginConfig, ["index", ...(opts.json ? ["--json"] : [])], { + await runClawops(["index", ...(opts.json ? ["--json"] : [])], { captureJson: false, }); }); @@ -370,7 +432,7 @@ function registerMemoryCli(program, pluginConfig) { if (opts.json) { args.push("--json"); } - await runClawopsCommand(pluginConfig, args, { captureJson: false }); + await runClawops(args, { captureJson: false }); }); memory @@ -390,7 +452,7 @@ function registerMemoryCli(program, pluginConfig) { if (opts.json) { args.push("--json"); } - await runClawopsCommand(pluginConfig, args, { captureJson: false }); + await runClawops(args, { captureJson: false }); }); memory @@ -416,7 +478,7 @@ function registerMemoryCli(program, pluginConfig) { if (opts.json) { args.push("--json"); } - await runClawopsCommand(pluginConfig, args, { captureJson: false }); + await runClawops(args, { captureJson: false }); }); memory @@ -435,7 +497,7 @@ function registerMemoryCli(program, pluginConfig) { if (opts.json) { args.push("--json"); } - await runClawopsCommand(pluginConfig, args, { captureJson: false }); + await runClawops(args, { captureJson: false }); }); memory @@ -445,7 +507,7 @@ function registerMemoryCli(program, pluginConfig) { .option("--json", "Print JSON.") .action(async (opts) => { const args = ["reflect", "--mode", String(opts.mode ?? "safe"), ...(opts.json ? ["--json"] : [])]; - await runClawopsCommand(pluginConfig, args, { + await runClawops(args, { captureJson: false, }); }); @@ -475,7 +537,7 @@ function registerMemoryCli(program, pluginConfig) { if (opts.json) { args.push("--json"); } - await runClawopsCommand(pluginConfig, args, { captureJson: false }); + await runClawops(args, { captureJson: false }); }); memory @@ -495,7 +557,7 @@ function registerMemoryCli(program, pluginConfig) { if (opts.json) { args.push("--json"); } - await runClawopsCommand(pluginConfig, args, { captureJson: false }); + await runClawops(args, { captureJson: false }); }); } @@ -520,6 +582,8 @@ const strongclawHypermemoryPlugin = { captureMinMessages: { type: "number", minimum: 1, maximum: 20 }, recallMaxResults: { type: "number", minimum: 1, maximum: 10 }, timeoutMs: { type: "number", minimum: 1000, maximum: 120000 }, + startupTimeoutMs: { type: "number", minimum: 1000, maximum: 120000 }, + toolTimeoutMs: { type: "number", minimum: 1000, maximum: 120000 }, }, required: ["configPath"], }, @@ -527,6 +591,13 @@ const strongclawHypermemoryPlugin = { const pluginConfig = resolvePluginConfig(api.pluginConfig); const sessionFeedback = new Map(); const sessionKeyFor = (event) => String(event?.sessionId ?? event?.conversationId ?? "default"); + const startupGate = createStartupGate(pluginConfig); + const ensureReady = startupGate.ensureReady; + startupGate.start(api.logger); + const runClawops = async (args, options) => { + await ensureReady(); + return runClawopsCommand(pluginConfig, args, options); + }; api.registerTool( { @@ -574,10 +645,14 @@ const strongclawHypermemoryPlugin = { if (params?.explain === true) { args.push("--explain"); } - const payload = await runClawopsCommand(pluginConfig, args); + const payload = await runClawops(args); const injectedIds = collectInjectedItemIds(payload?.results); if (injectedIds.length > 0) { - fireAndForget(pluginConfig, ["access", "--json", "--item-ids", JSON.stringify(injectedIds)]); + fireAndForget( + pluginConfig, + ["access", "--json", "--item-ids", JSON.stringify(injectedIds)], + { ensureReady }, + ); } return jsonResult(payload); } catch (error) { @@ -607,7 +682,7 @@ const strongclawHypermemoryPlugin = { args.push("--lines", String(lines)); } try { - const payload = await runClawopsCommand(pluginConfig, args); + const payload = await runClawops(args); return jsonResult(payload); } catch (error) { return jsonResult({ path, text: "", disabled: true, error: String(error) }); @@ -640,7 +715,7 @@ const strongclawHypermemoryPlugin = { if (scope) { args.push("--scope", scope); } - return jsonResult(await runClawopsCommand(pluginConfig, args)); + return jsonResult(await runClawops(args)); } catch (error) { return jsonResult({ ok: false, error: String(error) }); } @@ -673,7 +748,7 @@ const strongclawHypermemoryPlugin = { if (params?.all === true) { args.push("--all"); } - return jsonResult(await runClawopsCommand(pluginConfig, args)); + return jsonResult(await runClawops(args)); } catch (error) { return jsonResult({ ok: false, error: String(error) }); } @@ -695,7 +770,7 @@ const strongclawHypermemoryPlugin = { if (mode) { args.push("--mode", mode); } - return jsonResult(await runClawopsCommand(pluginConfig, args)); + return jsonResult(await runClawops(args)); } catch (error) { return jsonResult({ ok: false, error: String(error) }); } @@ -728,7 +803,7 @@ const strongclawHypermemoryPlugin = { if (params?.hardDelete === true) { args.push("--hard-delete"); } - return jsonResult(await runClawopsCommand(pluginConfig, args)); + return jsonResult(await runClawops(args)); } catch (error) { return jsonResult({ ok: false, error: String(error) }); } @@ -754,7 +829,7 @@ const strongclawHypermemoryPlugin = { if (scope) { args.push("--scope", scope); } - return jsonResult(await runClawopsCommand(pluginConfig, args)); + return jsonResult(await runClawops(args)); } catch (error) { return jsonResult({ ok: false, error: String(error) }); } @@ -764,7 +839,7 @@ const strongclawHypermemoryPlugin = { ); api.registerCli(({ program }) => { - registerMemoryCli(program, pluginConfig); + registerMemoryCli(program, runClawops); }, { commands: ["memory"] }); if (pluginConfig.autoRecall) { @@ -774,7 +849,7 @@ const strongclawHypermemoryPlugin = { return; } try { - const payload = await runClawopsCommand(pluginConfig, [ + const payload = await runClawops([ "search", "--json", "--query", @@ -795,7 +870,7 @@ const strongclawHypermemoryPlugin = { "--json", "--item-ids", JSON.stringify(injectedIds), - ]); + ], { ensureReady }); } return { prependContext: formatRecallContext(results.slice(0, pluginConfig.recallMaxResults)), @@ -827,7 +902,7 @@ const strongclawHypermemoryPlugin = { "--json", "--item-ids", JSON.stringify(feedback.confirmed), - ]); + ], { ensureReady }); } if (feedback.badRecall.length > 0) { fireAndForget(pluginConfig, [ @@ -835,7 +910,7 @@ const strongclawHypermemoryPlugin = { "--json", "--item-ids", JSON.stringify(feedback.badRecall), - ]); + ], { ensureReady }); } }); } @@ -847,7 +922,7 @@ const strongclawHypermemoryPlugin = { if (messages.length < pluginConfig.captureMinMessages) { return; } - await runClawopsCommand(pluginConfig, [ + await runClawops([ "capture", "--json", "--messages", @@ -862,7 +937,7 @@ const strongclawHypermemoryPlugin = { if (pluginConfig.autoReflect) { const runReflect = async (hookName) => { try { - await runClawopsCommand(pluginConfig, ["reflect", "--json"]); + await runClawops(["reflect", "--json"]); } catch (error) { api.logger.warn(`strongclaw-hypermemory ${hookName} reflect failed: ${String(error)}`); } @@ -876,7 +951,7 @@ const strongclawHypermemoryPlugin = { } api.on("session_end", async () => { - fireAndForget(pluginConfig, ["flush-metadata", "--json"]); + fireAndForget(pluginConfig, ["flush-metadata", "--json"], { ensureReady }); }); }, }; diff --git a/platform/plugins/strongclaw-hypermemory/openclaw.plugin.json b/platform/plugins/strongclaw-hypermemory/openclaw.plugin.json index 669ea761..92e747ef 100644 --- a/platform/plugins/strongclaw-hypermemory/openclaw.plugin.json +++ b/platform/plugins/strongclaw-hypermemory/openclaw.plugin.json @@ -38,6 +38,16 @@ "type": "number", "minimum": 1000, "maximum": 120000 + }, + "startupTimeoutMs": { + "type": "number", + "minimum": 1000, + "maximum": 120000 + }, + "toolTimeoutMs": { + "type": "number", + "minimum": 1000, + "maximum": 120000 } }, "required": [ diff --git a/platform/plugins/strongclaw-hypermemory/test/openclaw-host-functional.mjs b/platform/plugins/strongclaw-hypermemory/test/openclaw-host-functional.mjs index 416ac4e5..9f57a5d5 100644 --- a/platform/plugins/strongclaw-hypermemory/test/openclaw-host-functional.mjs +++ b/platform/plugins/strongclaw-hypermemory/test/openclaw-host-functional.mjs @@ -116,12 +116,69 @@ async function main() { try { writeHypermemoryConfig(workspaceDir, memoryConfigPath); + assert.throws(() => { + const missingConfigStub = createPluginApiStub({ + configPath: path.join(workspaceDir, "missing-config.yaml"), + }); + strongclawHypermemoryPlugin.register(missingConfigStub.api); + }, /configPath does not exist/); + + const badCommandStub = createPluginApiStub({ + configPath: memoryConfigPath, + command: ["command-does-not-exist-strongclaw-hypermemory"], + autoRecall: false, + autoReflect: false, + timeoutMs: 5_000, + startupTimeoutMs: 5_000, + toolTimeoutMs: 5_000, + }); + strongclawHypermemoryPlugin.register(badCommandStub.api); + const badSearchTool = badCommandStub.tools.get("memory_search"); + assert.ok(badSearchTool); + const badSearchResult = await badSearchTool.execute("bad-tool", { + query: "anything", + }); + assert.equal(badSearchResult.details.disabled, true); + assert.match(String(badSearchResult.details.error), /failed to start/); + + const timeoutCommandPath = path.join(runDir, "slow-command.mjs"); + writeFileSync( + timeoutCommandPath, + [ + "#!/usr/bin/env node", + "setTimeout(() => {", + " process.stdout.write(\"{}\\n\");", + " process.exit(0);", + "}, 1500);", + ].join("\n"), + { encoding: "utf8", mode: 0o755 }, + ); + const timeoutStub = createPluginApiStub({ + configPath: memoryConfigPath, + command: [timeoutCommandPath], + autoRecall: false, + autoReflect: false, + timeoutMs: 20_000, + startupTimeoutMs: 1_000, + toolTimeoutMs: 1_000, + }); + strongclawHypermemoryPlugin.register(timeoutStub.api); + const timeoutSearchTool = timeoutStub.tools.get("memory_search"); + assert.ok(timeoutSearchTool); + const timeoutSearchResult = await timeoutSearchTool.execute("timeout-tool", { + query: "anything", + }); + assert.equal(timeoutSearchResult.details.disabled, true); + assert.match(String(timeoutSearchResult.details.error), /startup preflight timed out/i); + const stub = createPluginApiStub({ configPath: memoryConfigPath, command: ["uv", "run", "--project", repoRoot, "python", "-m", "clawops"], autoRecall: false, autoReflect: false, timeoutMs: 20_000, + startupTimeoutMs: 20_000, + toolTimeoutMs: 20_000, }); strongclawHypermemoryPlugin.register(stub.api); @@ -136,6 +193,9 @@ async function main() { commands[0].subcommands.map((command) => command.name), ["status", "index", "search", "get", "store", "update", "reflect", "forget", "list-facts"], ); + const indexCommand = commands[0].subcommands.find((command) => command.name === "index"); + assert.ok(indexCommand?.action); + await indexCommand.action({ json: true }); const memorySearch = stub.tools.get("memory_search"); assert.ok(memorySearch); diff --git a/platform/systemd/openclaw-gateway.service b/platform/systemd/openclaw-gateway.service index a8cf3612..4785dbfc 100644 --- a/platform/systemd/openclaw-gateway.service +++ b/platform/systemd/openclaw-gateway.service @@ -1,7 +1,7 @@ [Unit] Description=OpenClaw Gateway -After=network-online.target -Wants=network-online.target +After=network-online.target openclaw-sidecars.service +Wants=network-online.target openclaw-sidecars.service [Service] Type=simple diff --git a/platform/systemd/openclaw-maintenance.service b/platform/systemd/openclaw-maintenance.service new file mode 100644 index 00000000..3319e9b2 --- /dev/null +++ b/platform/systemd/openclaw-maintenance.service @@ -0,0 +1,16 @@ +[Unit] +Description=StrongClaw maintenance tasks +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=__REPO_ROOT__ +Environment=OPENCLAW_STATE_DIR=__STATE_DIR__ +Environment=OPENCLAW_HOME=__OPENCLAW_HOME__ +Environment=OPENCLAW_CONFIG_PATH=__OPENCLAW_CONFIG_PATH__ +Environment=OPENCLAW_CONFIG=__OPENCLAW_CONFIG__ +Environment=OPENCLAW_PROFILE=__OPENCLAW_PROFILE__ +Environment=STRONGCLAW_RUNTIME_ROOT=__STRONGCLAW_RUNTIME_ROOT__ +Environment=PATH=%h/.config/varlock/bin:%h/.local/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin +ExecStart=__PYTHON_EXECUTABLE__ -m clawops recovery --home-dir __HOME_DIR__ prune-retention diff --git a/platform/systemd/openclaw-maintenance.timer b/platform/systemd/openclaw-maintenance.timer new file mode 100644 index 00000000..399c0d37 --- /dev/null +++ b/platform/systemd/openclaw-maintenance.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Run StrongClaw maintenance daily + +[Timer] +OnCalendar=*-*-* 04:00:00 +Persistent=true +Unit=openclaw-maintenance.service + +[Install] +WantedBy=timers.target diff --git a/src/clawops/approval_dispatch.py b/src/clawops/approval_dispatch.py new file mode 100644 index 00000000..493f03dc --- /dev/null +++ b/src/clawops/approval_dispatch.py @@ -0,0 +1,123 @@ +"""Reviewer packet dispatch for approval-gated operations.""" + +from __future__ import annotations + +import dataclasses +import json +import pathlib + +from clawops.common import write_json +from clawops.op_journal import Operation, OperationJournal + +REVIEW_PACKET_VERSION = 1 +LOCAL_DISPATCH_CHANNEL = "local_file" + + +@dataclasses.dataclass(frozen=True, slots=True) +class ApprovalDispatchOutcome: + """Result of dispatching a reviewer packet for one operation.""" + + operation: Operation + artifact_path: pathlib.Path + dispatched: bool + channel: str + error: str | None = None + + def to_dict(self) -> dict[str, object]: + """Serialize the outcome for workflow/wrapper payloads.""" + payload: dict[str, object] = { + "dispatched": self.dispatched, + "channel": self.channel, + "artifactPath": self.artifact_path.as_posix(), + } + if self.error is not None: + payload["error"] = self.error + return payload + + +def _decode_json(value: str | None, *, field_name: str) -> object: + """Decode one persisted JSON string while preserving malformed payloads.""" + if value is None: + return None + try: + return json.loads(value) + except json.JSONDecodeError: + return {"_decodeError": f"invalid JSON in {field_name}", "raw": value} + + +def _default_artifact_path(journal: OperationJournal, *, op_id: str) -> pathlib.Path: + """Return the default on-disk reviewer packet path for one operation.""" + return journal.db_path.parent / "reviews" / f"{op_id}.json" + + +def build_review_packet(operation: Operation) -> dict[str, object]: + """Build a stable reviewer packet for one pending operation.""" + return { + "version": REVIEW_PACKET_VERSION, + "opId": operation.op_id, + "status": operation.status, + "scope": operation.scope, + "kind": operation.kind, + "trustZone": operation.trust_zone, + "normalizedTarget": operation.normalized_target, + "createdAtMs": operation.created_at_ms, + "updatedAtMs": operation.updated_at_ms, + "approvalRequired": bool(operation.approval_required), + "review": { + "mode": operation.review_mode, + "target": operation.review_target, + "status": operation.review_status, + "payload": _decode_json( + operation.review_payload_json, field_name="review_payload_json" + ), + }, + "policy": { + "decision": operation.policy_decision, + "detail": _decode_json( + operation.policy_decision_json, field_name="policy_decision_json" + ), + }, + "executionContract": _decode_json( + operation.execution_contract_json, + field_name="execution_contract_json", + ), + "inputs": _decode_json(operation.inputs_json, field_name="inputs_json"), + } + + +def dispatch_pending_approval( + *, + journal: OperationJournal, + operation: Operation, +) -> ApprovalDispatchOutcome: + """Write and register one durable reviewer packet for a pending operation.""" + if operation.status != "pending_approval": + raise ValueError("dispatch requires a pending_approval operation") + artifact_path = ( + pathlib.Path(operation.review_artifact_path).expanduser().resolve() + if operation.review_artifact_path + else _default_artifact_path(journal, op_id=operation.op_id) + ) + payload = build_review_packet(operation) + try: + write_json(artifact_path, payload) + updated = journal.transition( + operation.op_id, + "pending_approval", + review_artifact_path=artifact_path.as_posix(), + review_status=operation.review_status or "pending", + ) + except Exception as exc: # pragma: no cover - exercised through failure handling tests + return ApprovalDispatchOutcome( + operation=operation, + artifact_path=artifact_path, + dispatched=False, + channel=LOCAL_DISPATCH_CHANNEL, + error=str(exc), + ) + return ApprovalDispatchOutcome( + operation=updated, + artifact_path=artifact_path, + dispatched=True, + channel=LOCAL_DISPATCH_CHANNEL, + ) diff --git a/src/clawops/assets/platform/configs/openclaw/75-strongclaw-hypermemory.example.json5 b/src/clawops/assets/platform/configs/openclaw/75-strongclaw-hypermemory.example.json5 index a08c76a2..3f3b6b65 100644 --- a/src/clawops/assets/platform/configs/openclaw/75-strongclaw-hypermemory.example.json5 +++ b/src/clawops/assets/platform/configs/openclaw/75-strongclaw-hypermemory.example.json5 @@ -6,6 +6,8 @@ "enabled": true, "config": { "command": [ + "__PYTHON_EXECUTABLE__", + "-m", "clawops" ], "configPath": "__HYPERMEMORY_SQLITE_CONFIG_PATH__", diff --git a/src/clawops/assets/platform/configs/openclaw/77-hypermemory.example.json5 b/src/clawops/assets/platform/configs/openclaw/77-hypermemory.example.json5 index 0abc4fa7..e005d6f9 100644 --- a/src/clawops/assets/platform/configs/openclaw/77-hypermemory.example.json5 +++ b/src/clawops/assets/platform/configs/openclaw/77-hypermemory.example.json5 @@ -13,6 +13,8 @@ "enabled": true, "config": { "command": [ + "__PYTHON_EXECUTABLE__", + "-m", "clawops" ], "configPath": "__HYPERMEMORY_CONFIG_PATH__", diff --git a/src/clawops/assets/platform/docs/BACKUP_AND_RECOVERY.md b/src/clawops/assets/platform/docs/BACKUP_AND_RECOVERY.md index 77e9f70c..4f5d7cbd 100644 --- a/src/clawops/assets/platform/docs/BACKUP_AND_RECOVERY.md +++ b/src/clawops/assets/platform/docs/BACKUP_AND_RECOVERY.md @@ -15,6 +15,19 @@ - `clawops recovery prune-retention` - `clawops recovery rotate-secrets` +## Scheduled maintenance + +StrongClaw host service activation now installs a daily maintenance schedule at `04:00` local time: + +- systemd: `openclaw-maintenance.timer` -> `openclaw-maintenance.service` +- launchd: `ai.openclaw.maintenance` + +The scheduled command is: + +- `clawops recovery --home-dir prune-retention` + +This maintenance path is idempotent and retention-only. It prunes expired StrongClaw recovery artifacts and does not mutate upstream OpenClaw internals. + ## Development-mode repo-local compose state If you keep compose state under `platform/compose/state` during development, use the explicit dev wrappers instead of relying on implicit leftover mounts: diff --git a/src/clawops/assets/platform/docs/CI_AND_SECURITY.md b/src/clawops/assets/platform/docs/CI_AND_SECURITY.md index d1e88d40..d75be2bd 100644 --- a/src/clawops/assets/platform/docs/CI_AND_SECURITY.md +++ b/src/clawops/assets/platform/docs/CI_AND_SECURITY.md @@ -93,3 +93,5 @@ hypermemory Qdrant checks against the official pinned Qdrant GHCR image instead compile-checks the repo, runs targeted devflow tests, and validates `clawops devflow plan --goal "contract smoke"` without live ACP providers. - Operators can verify published provenance with GitHub's attestation tooling after a tagged release lands. + +Canonical plugin support status lives in [Plugin Inventory](./PLUGIN_INVENTORY.md). diff --git a/src/clawops/assets/platform/docs/DEGRADATION.md b/src/clawops/assets/platform/docs/DEGRADATION.md new file mode 100644 index 00000000..fd29ccea --- /dev/null +++ b/src/clawops/assets/platform/docs/DEGRADATION.md @@ -0,0 +1,37 @@ +# Degradation Contract + +This document defines how dependency health maps to StrongClaw runtime impact. + +## Sidecar Dependency Matrix + +| Dependency | Used by | Required for `clawops ops sidecars up` success | Impact when unavailable | +| --- | --- | --- | --- | +| Postgres | runtime metadata/session state | Yes | **fatal**: sidecar bring-up fails | +| LiteLLM | loopback model routing | Yes | **fatal**: sidecar bring-up fails | +| Qdrant | dense/sparse retrieval (`openclaw-qmd`, `hypermemory`) | Required when the active profile uses QMD or hypermemory retrieval | **degraded**: retrieval lanes depending on Qdrant are unavailable | +| Neo4j | graph expansion for `hypermemory` context | Required when the active profile uses hypermemory graph expansion | **degraded**: graph expansion is unavailable | +| OTel Collector | telemetry export | No | **observational**: tracing/metrics export unavailable, runtime behavior unaffected | + +## Operator Output Contract + +- `clawops ops status` reports dependency health under `readiness.required` and `readiness.optional`. +- Each readiness entry includes: + - `service` + - `required` + - `impact` (`fatal`, `degraded`, or `observational`) + - `reason` + - `ready` + - expected vs observed state/health fields +- `ok` and `readiness.requiredReady` are `true` only when every required dependency is ready. + +## Plugin Startup Contract + +- `strongclaw-hypermemory` runs startup preflight (`clawops hypermemory verify --json`) before serving memory tools. +- If preflight fails, tool responses return a disabled/unavailable payload instead of silent fallback. +- Existing configs that only define `timeoutMs` remain valid; startup/tool timeout split is optional via `startupTimeoutMs` and `toolTimeoutMs`. + +## Related Docs + +- [Plugin Inventory](./PLUGIN_INVENTORY.md) +- [Observability](./OBSERVABILITY.md) +- [Hypermemory](./HYPERMEMORY.md) diff --git a/src/clawops/assets/platform/docs/HYPERMEMORY.md b/src/clawops/assets/platform/docs/HYPERMEMORY.md index ba4ae0c2..68bf2643 100644 --- a/src/clawops/assets/platform/docs/HYPERMEMORY.md +++ b/src/clawops/assets/platform/docs/HYPERMEMORY.md @@ -8,6 +8,11 @@ The built-in OpenClaw fallback remains available as `openclaw-default`, and the explicit built-ins-plus-QMD fallback remains available as `openclaw-qmd`. +Canonical support/degradation references: + +- [Plugin Inventory](./PLUGIN_INVENTORY.md) +- [Degradation Contract](./DEGRADATION.md) + ## Design goals - preserve OpenClaw-compatible `memory_search` and `memory_get` diff --git a/src/clawops/assets/platform/docs/OBSERVABILITY.md b/src/clawops/assets/platform/docs/OBSERVABILITY.md index f180e7cc..4a8163ae 100644 --- a/src/clawops/assets/platform/docs/OBSERVABILITY.md +++ b/src/clawops/assets/platform/docs/OBSERVABILITY.md @@ -32,3 +32,29 @@ Collector-side redaction is mandatory before broader trace export. Do not assume - vector sync runs and sync failures Those signals reuse the shared ClawOps telemetry path instead of adding a separate exporter or collector. + +## Runtime Bring-Up Events + +`clawops ops` now emits structured readiness events: + +- `clawops.ops.sidecars.wait.start` +- `clawops.ops.sidecars.wait.ready` +- `clawops.ops.sidecars.wait.timeout` +- `clawops.ops.sidecars.ready` +- `clawops.ops.sidecars.status` + +Wait-timeout events include `service`, `target`, `observed`, and `timeout_seconds` so failed dependencies can be diagnosed without replaying the command interactively. + +## Host Service Activation Events + +`clawops services install --activate` now emits: + +- `clawops.services.activate` with `service_manager=launchd` and `step` values: + - `sidecars_bootstrap` + - `gateway_bootstrap` + - `maintenance_bootstrap` +- `clawops.services.activate` with `service_manager=systemd` and `step` values: + - `daemon_reload` + - `enable_now` (`unit` included per activation) + +These events are intentionally coarse-grained to avoid high-volume per-poll logging. diff --git a/src/clawops/assets/platform/docs/PLUGIN_INVENTORY.md b/src/clawops/assets/platform/docs/PLUGIN_INVENTORY.md new file mode 100644 index 00000000..c8b0ae2d --- /dev/null +++ b/src/clawops/assets/platform/docs/PLUGIN_INVENTORY.md @@ -0,0 +1,22 @@ +# Plugin Inventory + +This page is the canonical StrongClaw plugin support matrix. + +## Shipped Plugins + +| Plugin | Purpose | Default status | Build/runtime expectation | CI coverage | Support level | +| --- | --- | --- | --- | --- | --- | +| `strongclaw-hypermemory` | StrongClaw-owned memory plugin that proxies `clawops hypermemory` and preserves OpenClaw memory tool names. | Enabled by default in the `hypermemory` profile; opt-in via standalone overlay `75-strongclaw-hypermemory.example.json5`. | Requires a valid rendered `configPath` plus a callable StrongClaw command (`python -m clawops` in managed overlays). Startup preflight runs `clawops hypermemory verify --json` before serving tools. | `tests/suites/integration/clawops/hypermemory/test_plugin.py` and `platform/plugins/strongclaw-hypermemory/test/openclaw-host-functional.mjs` in CI. | **Supported (StrongClaw-owned)** | +| `memory-lancedb-pro` (vendored) | Vendored upstream memory plugin for migration bridge and compatibility use cases. | Not default; enabled through `memory-lancedb-pro` profile. | Vendored bundle under `platform/plugins/memory-lancedb-pro`, rendered via managed profile overlays. | `tests/suites/unit/ci/test_memory_plugin_verification.py` and `.github/workflows/memory-plugin-verification.yml`. | **Supported (vendored bridge path)** | + +## Support Policy + +- Support labels in this table must match real CI evidence and workflow coverage. +- New plugin entries must update this table and `platform/docs/CI_AND_SECURITY.md` in the same change. +- If a plugin is experimental or unavailable by default, that status must be explicit in this table and in any setup docs that mention it. + +## Related Docs + +- [Hypermemory](./HYPERMEMORY.md) +- [CI and Security](./CI_AND_SECURITY.md) +- [Degradation Contract](./DEGRADATION.md) diff --git a/src/clawops/assets/platform/launchd/ai.openclaw.maintenance.plist.template b/src/clawops/assets/platform/launchd/ai.openclaw.maintenance.plist.template new file mode 100644 index 00000000..5d44c5e8 --- /dev/null +++ b/src/clawops/assets/platform/launchd/ai.openclaw.maintenance.plist.template @@ -0,0 +1,55 @@ + + + + + Label + ai.openclaw.maintenance + ProgramArguments + + __PYTHON_EXECUTABLE__ + -m + clawops + recovery + --home-dir + __HOME_DIR__ + prune-retention + + WorkingDirectory + __REPO_ROOT__ + EnvironmentVariables + + HOME + __HOME_DIR__ + XDG_CONFIG_HOME + __HOME_DIR__/.config + OPENCLAW_HOME + __OPENCLAW_HOME__ + OPENCLAW_STATE_DIR + __STATE_DIR__ + OPENCLAW_CONFIG_PATH + __OPENCLAW_CONFIG_PATH__ + OPENCLAW_CONFIG + __OPENCLAW_CONFIG__ + OPENCLAW_PROFILE + __OPENCLAW_PROFILE__ + STRONGCLAW_RUNTIME_ROOT + __STRONGCLAW_RUNTIME_ROOT__ + PATH + __HOME_DIR__/.config/varlock/bin:__HOME_DIR__/.local/bin:/usr/local/bin:/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin +__LAUNCHD_EXTRA_ENV__ + + StartCalendarInterval + + Hour + 4 + Minute + 0 + + StandardOutPath + __STATE_DIR__/logs/launchd-maintenance.out.log + StandardErrorPath + __STATE_DIR__/logs/launchd-maintenance.err.log + KeepAlive + + + diff --git a/src/clawops/assets/platform/plugins/strongclaw-hypermemory/index.js b/src/clawops/assets/platform/plugins/strongclaw-hypermemory/index.js index 999083b4..05f04358 100644 --- a/src/clawops/assets/platform/plugins/strongclaw-hypermemory/index.js +++ b/src/clawops/assets/platform/plugins/strongclaw-hypermemory/index.js @@ -1,4 +1,5 @@ import { spawn } from "node:child_process"; +import { existsSync } from "node:fs"; const DEFAULT_COMMAND = ["clawops"]; const DEFAULT_TIMEOUT_MS = 20000; @@ -119,6 +120,9 @@ function resolvePluginConfig(rawConfig) { "strongclaw-hypermemory requires plugins.entries.strongclaw-hypermemory.config.configPath", ); } + if (!existsSync(configPath)) { + throw new Error(`strongclaw-hypermemory configPath does not exist: ${configPath}`); + } const command = Array.isArray(input.command) && input.command.length > 0 ? input.command.filter((entry) => typeof entry === "string" && entry.trim()).map((entry) => entry.trim()) @@ -127,8 +131,18 @@ function resolvePluginConfig(rawConfig) { throw new Error("strongclaw-hypermemory command must contain at least one executable"); } const timeoutMs = readNumberParam(input, "timeoutMs"); + const startupTimeoutMs = readNumberParam(input, "startupTimeoutMs"); + const toolTimeoutMs = readNumberParam(input, "toolTimeoutMs"); const recallMaxResults = readNumberParam(input, "recallMaxResults"); const captureMinMessages = readNumberParam(input, "captureMinMessages"); + const resolvedTimeoutMs = + timeoutMs && timeoutMs >= 1000 ? Math.min(120000, Math.trunc(timeoutMs)) : DEFAULT_TIMEOUT_MS; + const resolvedStartupTimeoutMs = + startupTimeoutMs && startupTimeoutMs >= 1000 + ? Math.min(120000, Math.trunc(startupTimeoutMs)) + : resolvedTimeoutMs; + const resolvedToolTimeoutMs = + toolTimeoutMs && toolTimeoutMs >= 1000 ? Math.min(120000, Math.trunc(toolTimeoutMs)) : resolvedTimeoutMs; return { command, configPath, @@ -139,12 +153,17 @@ function resolvePluginConfig(rawConfig) { recallMaxResults && recallMaxResults >= 1 ? Math.min(10, Math.trunc(recallMaxResults)) : 3, captureMinMessages: captureMinMessages && captureMinMessages >= 1 ? Math.min(20, Math.trunc(captureMinMessages)) : 4, - timeoutMs: - timeoutMs && timeoutMs >= 1000 ? Math.min(120000, Math.trunc(timeoutMs)) : DEFAULT_TIMEOUT_MS, + timeoutMs: resolvedTimeoutMs, + startupTimeoutMs: resolvedStartupTimeoutMs, + toolTimeoutMs: resolvedToolTimeoutMs, }; } -async function runClawopsCommand(pluginConfig, args, { captureJson = true } = {}) { +async function runClawopsCommand( + pluginConfig, + args, + { captureJson = true, timeoutMs = pluginConfig.toolTimeoutMs, phase = "tool" } = {}, +) { const [command, ...commandArgs] = pluginConfig.command; const fullArgs = [...commandArgs, "hypermemory", "--config", pluginConfig.configPath, ...args]; return await new Promise((resolve, reject) => { @@ -158,7 +177,7 @@ async function runClawopsCommand(pluginConfig, args, { captureJson = true } = {} const timer = setTimeout(() => { timedOut = true; child.kill("SIGKILL"); - }, pluginConfig.timeoutMs); + }, timeoutMs); if (captureJson) { child.stdout.on("data", (chunk) => { stdout += chunk.toString(); @@ -167,15 +186,22 @@ async function runClawopsCommand(pluginConfig, args, { captureJson = true } = {} stderr += chunk.toString(); }); } - child.on("error", reject); + child.on("error", (error) => { + clearTimeout(timer); + reject( + new Error( + `failed to start strongclaw-hypermemory command (${command}): ${error instanceof Error ? error.message : String(error)}`, + ), + ); + }); child.on("close", (code) => { clearTimeout(timer); if (timedOut) { - reject(new Error(`clawops hypermemory timed out after ${pluginConfig.timeoutMs}ms`)); + reject(new Error(`clawops hypermemory ${phase} timed out after ${timeoutMs}ms`)); return; } if (code !== 0) { - reject(new Error(stderr.trim() || `clawops hypermemory exited with code ${code}`)); + reject(new Error(stderr.trim() || `clawops hypermemory ${phase} exited with code ${code}`)); return; } if (!captureJson) { @@ -195,6 +221,36 @@ async function runClawopsCommand(pluginConfig, args, { captureJson = true } = {} }); } +function createStartupGate(pluginConfig) { + let startupPromise; + async function ensureReady() { + if (startupPromise) { + return startupPromise; + } + startupPromise = runClawopsCommand(pluginConfig, ["status", "--json"], { + timeoutMs: pluginConfig.startupTimeoutMs, + phase: "startup preflight", + }).then((payload) => { + if (payload && typeof payload === "object" && payload.ok === false) { + const reason = + typeof payload.message === "string" && payload.message.trim() + ? payload.message.trim() + : "status returned ok=false"; + throw new Error(`strongclaw-hypermemory startup preflight failed: ${reason}`); + } + }); + return startupPromise; + } + function start(logger) { + void ensureReady().catch((error) => { + if (logger && typeof logger.warn === "function") { + logger.warn(`strongclaw-hypermemory startup preflight failed: ${String(error)}`); + } + }); + } + return { ensureReady, start }; +} + function buildDisabledSearchResult(error) { return { results: [], @@ -216,8 +272,14 @@ function formatRecallContext(results) { return lines.join("\n"); } -function fireAndForget(pluginConfig, args) { - runClawopsCommand(pluginConfig, args).catch(() => {}); +function fireAndForget(pluginConfig, args, { ensureReady } = {}) { + const runPromise = + typeof ensureReady === "function" + ? Promise.resolve() + .then(() => ensureReady()) + .then(() => runClawopsCommand(pluginConfig, args)) + : runClawopsCommand(pluginConfig, args); + runPromise.catch(() => {}); } function extractSessionMessages(event) { @@ -300,14 +362,14 @@ function splitFeedbackIds(entries, responseText) { return { confirmed, badRecall }; } -function registerMemoryCli(program, pluginConfig) { +function registerMemoryCli(program, runClawops) { const memory = program.command("memory").description("Use StrongClaw hypermemory."); memory .command("status") .description("Show StrongClaw hypermemory status.") .option("--json", "Print JSON.") .action(async (opts) => { - await runClawopsCommand(pluginConfig, ["status", ...(opts.json ? ["--json"] : [])], { + await runClawops(["status", ...(opts.json ? ["--json"] : [])], { captureJson: false, }); }); @@ -317,7 +379,7 @@ function registerMemoryCli(program, pluginConfig) { .description("Rebuild the StrongClaw hypermemory index.") .option("--json", "Print JSON.") .action(async (opts) => { - await runClawopsCommand(pluginConfig, ["index", ...(opts.json ? ["--json"] : [])], { + await runClawops(["index", ...(opts.json ? ["--json"] : [])], { captureJson: false, }); }); @@ -370,7 +432,7 @@ function registerMemoryCli(program, pluginConfig) { if (opts.json) { args.push("--json"); } - await runClawopsCommand(pluginConfig, args, { captureJson: false }); + await runClawops(args, { captureJson: false }); }); memory @@ -390,7 +452,7 @@ function registerMemoryCli(program, pluginConfig) { if (opts.json) { args.push("--json"); } - await runClawopsCommand(pluginConfig, args, { captureJson: false }); + await runClawops(args, { captureJson: false }); }); memory @@ -416,7 +478,7 @@ function registerMemoryCli(program, pluginConfig) { if (opts.json) { args.push("--json"); } - await runClawopsCommand(pluginConfig, args, { captureJson: false }); + await runClawops(args, { captureJson: false }); }); memory @@ -435,7 +497,7 @@ function registerMemoryCli(program, pluginConfig) { if (opts.json) { args.push("--json"); } - await runClawopsCommand(pluginConfig, args, { captureJson: false }); + await runClawops(args, { captureJson: false }); }); memory @@ -445,7 +507,7 @@ function registerMemoryCli(program, pluginConfig) { .option("--json", "Print JSON.") .action(async (opts) => { const args = ["reflect", "--mode", String(opts.mode ?? "safe"), ...(opts.json ? ["--json"] : [])]; - await runClawopsCommand(pluginConfig, args, { + await runClawops(args, { captureJson: false, }); }); @@ -475,7 +537,7 @@ function registerMemoryCli(program, pluginConfig) { if (opts.json) { args.push("--json"); } - await runClawopsCommand(pluginConfig, args, { captureJson: false }); + await runClawops(args, { captureJson: false }); }); memory @@ -495,7 +557,7 @@ function registerMemoryCli(program, pluginConfig) { if (opts.json) { args.push("--json"); } - await runClawopsCommand(pluginConfig, args, { captureJson: false }); + await runClawops(args, { captureJson: false }); }); } @@ -520,6 +582,8 @@ const strongclawHypermemoryPlugin = { captureMinMessages: { type: "number", minimum: 1, maximum: 20 }, recallMaxResults: { type: "number", minimum: 1, maximum: 10 }, timeoutMs: { type: "number", minimum: 1000, maximum: 120000 }, + startupTimeoutMs: { type: "number", minimum: 1000, maximum: 120000 }, + toolTimeoutMs: { type: "number", minimum: 1000, maximum: 120000 }, }, required: ["configPath"], }, @@ -527,6 +591,13 @@ const strongclawHypermemoryPlugin = { const pluginConfig = resolvePluginConfig(api.pluginConfig); const sessionFeedback = new Map(); const sessionKeyFor = (event) => String(event?.sessionId ?? event?.conversationId ?? "default"); + const startupGate = createStartupGate(pluginConfig); + const ensureReady = startupGate.ensureReady; + startupGate.start(api.logger); + const runClawops = async (args, options) => { + await ensureReady(); + return runClawopsCommand(pluginConfig, args, options); + }; api.registerTool( { @@ -574,10 +645,14 @@ const strongclawHypermemoryPlugin = { if (params?.explain === true) { args.push("--explain"); } - const payload = await runClawopsCommand(pluginConfig, args); + const payload = await runClawops(args); const injectedIds = collectInjectedItemIds(payload?.results); if (injectedIds.length > 0) { - fireAndForget(pluginConfig, ["access", "--json", "--item-ids", JSON.stringify(injectedIds)]); + fireAndForget( + pluginConfig, + ["access", "--json", "--item-ids", JSON.stringify(injectedIds)], + { ensureReady }, + ); } return jsonResult(payload); } catch (error) { @@ -607,7 +682,7 @@ const strongclawHypermemoryPlugin = { args.push("--lines", String(lines)); } try { - const payload = await runClawopsCommand(pluginConfig, args); + const payload = await runClawops(args); return jsonResult(payload); } catch (error) { return jsonResult({ path, text: "", disabled: true, error: String(error) }); @@ -640,7 +715,7 @@ const strongclawHypermemoryPlugin = { if (scope) { args.push("--scope", scope); } - return jsonResult(await runClawopsCommand(pluginConfig, args)); + return jsonResult(await runClawops(args)); } catch (error) { return jsonResult({ ok: false, error: String(error) }); } @@ -673,7 +748,7 @@ const strongclawHypermemoryPlugin = { if (params?.all === true) { args.push("--all"); } - return jsonResult(await runClawopsCommand(pluginConfig, args)); + return jsonResult(await runClawops(args)); } catch (error) { return jsonResult({ ok: false, error: String(error) }); } @@ -695,7 +770,7 @@ const strongclawHypermemoryPlugin = { if (mode) { args.push("--mode", mode); } - return jsonResult(await runClawopsCommand(pluginConfig, args)); + return jsonResult(await runClawops(args)); } catch (error) { return jsonResult({ ok: false, error: String(error) }); } @@ -728,7 +803,7 @@ const strongclawHypermemoryPlugin = { if (params?.hardDelete === true) { args.push("--hard-delete"); } - return jsonResult(await runClawopsCommand(pluginConfig, args)); + return jsonResult(await runClawops(args)); } catch (error) { return jsonResult({ ok: false, error: String(error) }); } @@ -754,7 +829,7 @@ const strongclawHypermemoryPlugin = { if (scope) { args.push("--scope", scope); } - return jsonResult(await runClawopsCommand(pluginConfig, args)); + return jsonResult(await runClawops(args)); } catch (error) { return jsonResult({ ok: false, error: String(error) }); } @@ -764,7 +839,7 @@ const strongclawHypermemoryPlugin = { ); api.registerCli(({ program }) => { - registerMemoryCli(program, pluginConfig); + registerMemoryCli(program, runClawops); }, { commands: ["memory"] }); if (pluginConfig.autoRecall) { @@ -774,7 +849,7 @@ const strongclawHypermemoryPlugin = { return; } try { - const payload = await runClawopsCommand(pluginConfig, [ + const payload = await runClawops([ "search", "--json", "--query", @@ -795,7 +870,7 @@ const strongclawHypermemoryPlugin = { "--json", "--item-ids", JSON.stringify(injectedIds), - ]); + ], { ensureReady }); } return { prependContext: formatRecallContext(results.slice(0, pluginConfig.recallMaxResults)), @@ -827,7 +902,7 @@ const strongclawHypermemoryPlugin = { "--json", "--item-ids", JSON.stringify(feedback.confirmed), - ]); + ], { ensureReady }); } if (feedback.badRecall.length > 0) { fireAndForget(pluginConfig, [ @@ -835,7 +910,7 @@ const strongclawHypermemoryPlugin = { "--json", "--item-ids", JSON.stringify(feedback.badRecall), - ]); + ], { ensureReady }); } }); } @@ -847,7 +922,7 @@ const strongclawHypermemoryPlugin = { if (messages.length < pluginConfig.captureMinMessages) { return; } - await runClawopsCommand(pluginConfig, [ + await runClawops([ "capture", "--json", "--messages", @@ -862,7 +937,7 @@ const strongclawHypermemoryPlugin = { if (pluginConfig.autoReflect) { const runReflect = async (hookName) => { try { - await runClawopsCommand(pluginConfig, ["reflect", "--json"]); + await runClawops(["reflect", "--json"]); } catch (error) { api.logger.warn(`strongclaw-hypermemory ${hookName} reflect failed: ${String(error)}`); } @@ -876,7 +951,7 @@ const strongclawHypermemoryPlugin = { } api.on("session_end", async () => { - fireAndForget(pluginConfig, ["flush-metadata", "--json"]); + fireAndForget(pluginConfig, ["flush-metadata", "--json"], { ensureReady }); }); }, }; diff --git a/src/clawops/assets/platform/plugins/strongclaw-hypermemory/openclaw.plugin.json b/src/clawops/assets/platform/plugins/strongclaw-hypermemory/openclaw.plugin.json index 669ea761..92e747ef 100644 --- a/src/clawops/assets/platform/plugins/strongclaw-hypermemory/openclaw.plugin.json +++ b/src/clawops/assets/platform/plugins/strongclaw-hypermemory/openclaw.plugin.json @@ -38,6 +38,16 @@ "type": "number", "minimum": 1000, "maximum": 120000 + }, + "startupTimeoutMs": { + "type": "number", + "minimum": 1000, + "maximum": 120000 + }, + "toolTimeoutMs": { + "type": "number", + "minimum": 1000, + "maximum": 120000 } }, "required": [ diff --git a/src/clawops/assets/platform/plugins/strongclaw-hypermemory/test/openclaw-host-functional.mjs b/src/clawops/assets/platform/plugins/strongclaw-hypermemory/test/openclaw-host-functional.mjs index 416ac4e5..9f57a5d5 100644 --- a/src/clawops/assets/platform/plugins/strongclaw-hypermemory/test/openclaw-host-functional.mjs +++ b/src/clawops/assets/platform/plugins/strongclaw-hypermemory/test/openclaw-host-functional.mjs @@ -116,12 +116,69 @@ async function main() { try { writeHypermemoryConfig(workspaceDir, memoryConfigPath); + assert.throws(() => { + const missingConfigStub = createPluginApiStub({ + configPath: path.join(workspaceDir, "missing-config.yaml"), + }); + strongclawHypermemoryPlugin.register(missingConfigStub.api); + }, /configPath does not exist/); + + const badCommandStub = createPluginApiStub({ + configPath: memoryConfigPath, + command: ["command-does-not-exist-strongclaw-hypermemory"], + autoRecall: false, + autoReflect: false, + timeoutMs: 5_000, + startupTimeoutMs: 5_000, + toolTimeoutMs: 5_000, + }); + strongclawHypermemoryPlugin.register(badCommandStub.api); + const badSearchTool = badCommandStub.tools.get("memory_search"); + assert.ok(badSearchTool); + const badSearchResult = await badSearchTool.execute("bad-tool", { + query: "anything", + }); + assert.equal(badSearchResult.details.disabled, true); + assert.match(String(badSearchResult.details.error), /failed to start/); + + const timeoutCommandPath = path.join(runDir, "slow-command.mjs"); + writeFileSync( + timeoutCommandPath, + [ + "#!/usr/bin/env node", + "setTimeout(() => {", + " process.stdout.write(\"{}\\n\");", + " process.exit(0);", + "}, 1500);", + ].join("\n"), + { encoding: "utf8", mode: 0o755 }, + ); + const timeoutStub = createPluginApiStub({ + configPath: memoryConfigPath, + command: [timeoutCommandPath], + autoRecall: false, + autoReflect: false, + timeoutMs: 20_000, + startupTimeoutMs: 1_000, + toolTimeoutMs: 1_000, + }); + strongclawHypermemoryPlugin.register(timeoutStub.api); + const timeoutSearchTool = timeoutStub.tools.get("memory_search"); + assert.ok(timeoutSearchTool); + const timeoutSearchResult = await timeoutSearchTool.execute("timeout-tool", { + query: "anything", + }); + assert.equal(timeoutSearchResult.details.disabled, true); + assert.match(String(timeoutSearchResult.details.error), /startup preflight timed out/i); + const stub = createPluginApiStub({ configPath: memoryConfigPath, command: ["uv", "run", "--project", repoRoot, "python", "-m", "clawops"], autoRecall: false, autoReflect: false, timeoutMs: 20_000, + startupTimeoutMs: 20_000, + toolTimeoutMs: 20_000, }); strongclawHypermemoryPlugin.register(stub.api); @@ -136,6 +193,9 @@ async function main() { commands[0].subcommands.map((command) => command.name), ["status", "index", "search", "get", "store", "update", "reflect", "forget", "list-facts"], ); + const indexCommand = commands[0].subcommands.find((command) => command.name === "index"); + assert.ok(indexCommand?.action); + await indexCommand.action({ json: true }); const memorySearch = stub.tools.get("memory_search"); assert.ok(memorySearch); diff --git a/src/clawops/assets/platform/systemd/openclaw-gateway.service b/src/clawops/assets/platform/systemd/openclaw-gateway.service index a8cf3612..4785dbfc 100644 --- a/src/clawops/assets/platform/systemd/openclaw-gateway.service +++ b/src/clawops/assets/platform/systemd/openclaw-gateway.service @@ -1,7 +1,7 @@ [Unit] Description=OpenClaw Gateway -After=network-online.target -Wants=network-online.target +After=network-online.target openclaw-sidecars.service +Wants=network-online.target openclaw-sidecars.service [Service] Type=simple diff --git a/src/clawops/assets/platform/systemd/openclaw-maintenance.service b/src/clawops/assets/platform/systemd/openclaw-maintenance.service new file mode 100644 index 00000000..3319e9b2 --- /dev/null +++ b/src/clawops/assets/platform/systemd/openclaw-maintenance.service @@ -0,0 +1,16 @@ +[Unit] +Description=StrongClaw maintenance tasks +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=__REPO_ROOT__ +Environment=OPENCLAW_STATE_DIR=__STATE_DIR__ +Environment=OPENCLAW_HOME=__OPENCLAW_HOME__ +Environment=OPENCLAW_CONFIG_PATH=__OPENCLAW_CONFIG_PATH__ +Environment=OPENCLAW_CONFIG=__OPENCLAW_CONFIG__ +Environment=OPENCLAW_PROFILE=__OPENCLAW_PROFILE__ +Environment=STRONGCLAW_RUNTIME_ROOT=__STRONGCLAW_RUNTIME_ROOT__ +Environment=PATH=%h/.config/varlock/bin:%h/.local/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin +ExecStart=__PYTHON_EXECUTABLE__ -m clawops recovery --home-dir __HOME_DIR__ prune-retention diff --git a/src/clawops/assets/platform/systemd/openclaw-maintenance.timer b/src/clawops/assets/platform/systemd/openclaw-maintenance.timer new file mode 100644 index 00000000..399c0d37 --- /dev/null +++ b/src/clawops/assets/platform/systemd/openclaw-maintenance.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Run StrongClaw maintenance daily + +[Timer] +OnCalendar=*-*-* 04:00:00 +Persistent=true +Unit=openclaw-maintenance.service + +[Install] +WantedBy=timers.target diff --git a/src/clawops/config_cli.py b/src/clawops/config_cli.py index 9fb012ce..714bc2ae 100644 --- a/src/clawops/config_cli.py +++ b/src/clawops/config_cli.py @@ -3,13 +3,18 @@ from __future__ import annotations import argparse -import dataclasses import json import pathlib from collections.abc import Mapping from clawops.cli_roots import add_asset_root_argument, resolve_asset_root_argument from clawops.common import write_json +from clawops.memory_profiles import ( + MANAGED_MEMORY_PROFILE_IDS, + MEMORY_PROFILES, + MemoryProfileSpec, + require_memory_profile, +) from clawops.openclaw_config import ( DEFAULT_OPENCLAW_CONFIG_OUTPUT, materialize_runtime_memory_configs, @@ -20,53 +25,9 @@ from clawops.strongclaw_runtime import resolve_home_dir -@dataclasses.dataclass(frozen=True, slots=True) -class MemoryProfileSpec: - """StrongClaw-managed OpenClaw memory profile.""" - - profile_id: str - render_profile: str - description: str - installs_qmd: bool = False - installs_lossless_claw: bool = False - installs_memory_pro: bool = False - - -MEMORY_PROFILES: dict[str, MemoryProfileSpec] = { - "hypermemory": MemoryProfileSpec( - profile_id="hypermemory", - render_profile="hypermemory", - description="Default StrongClaw profile: lossless-claw + strongclaw-hypermemory.", - installs_lossless_claw=True, - ), - "openclaw-default": MemoryProfileSpec( - profile_id="openclaw-default", - render_profile="openclaw-default", - description="Built-in OpenClaw defaults: legacy context engine + memory-core.", - ), - "openclaw-qmd": MemoryProfileSpec( - profile_id="openclaw-qmd", - render_profile="openclaw-qmd", - description="Built-in OpenClaw defaults plus the experimental QMD memory backend.", - installs_qmd=True, - ), - "memory-lancedb-pro": MemoryProfileSpec( - profile_id="memory-lancedb-pro", - render_profile="memory-lancedb-pro", - description="Vendored memory-lancedb-pro with Ollama-backed smart extraction.", - installs_qmd=True, - installs_memory_pro=True, - ), -} - - def _memory_profile(profile_id: str) -> MemoryProfileSpec: """Resolve one supported StrongClaw memory profile.""" - try: - return MEMORY_PROFILES[profile_id] - except KeyError as exc: - available = ", ".join(sorted(MEMORY_PROFILES)) - raise ValueError(f"unknown memory profile: {profile_id} (choose from {available})") from exc + return require_memory_profile(profile_id) def _print_payload(payload: Mapping[str, object], *, as_json: bool) -> None: @@ -123,7 +84,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: memory_parser = subparsers.add_parser("memory", help="Manage StrongClaw memory profiles.") memory_parser.add_argument( "--set-profile", - choices=sorted(MEMORY_PROFILES), + choices=sorted(MANAGED_MEMORY_PROFILE_IDS), help="Install assets as needed and render the selected memory profile.", ) memory_parser.add_argument( @@ -162,6 +123,7 @@ def main(argv: list[str] | None = None) -> int: "description": profile.description, } for profile in MEMORY_PROFILES.values() + if profile.managed ] } _print_payload(list_payload, as_json=bool(args.json)) diff --git a/src/clawops/context/codebase/service.py b/src/clawops/context/codebase/service.py index a7480ffc..0e484793 100644 --- a/src/clawops/context/codebase/service.py +++ b/src/clawops/context/codebase/service.py @@ -1086,6 +1086,8 @@ class IndexStats: indexed_files: int skipped_files: int deleted_files: int + deleted_graph_nodes: int = 0 + deleted_graph_edges: int = 0 def to_dict(self) -> dict[str, int]: """Serialize the stats for CLI/reporting surfaces.""" @@ -1094,6 +1096,8 @@ def to_dict(self) -> dict[str, int]: "indexed_files": self.indexed_files, "skipped_files": self.skipped_files, "deleted_files": self.deleted_files, + "deleted_graph_nodes": self.deleted_graph_nodes, + "deleted_graph_edges": self.deleted_graph_edges, } @@ -1175,6 +1179,14 @@ class GraphNode: kind: str +@dataclasses.dataclass(frozen=True, slots=True) +class GraphCleanupStats: + """Graph cleanup summary from one backend upsert.""" + + deleted_nodes: int = 0 + deleted_edges: int = 0 + + @dataclasses.dataclass(frozen=True, slots=True) class GraphConfig: """Graph expansion configuration.""" @@ -1247,7 +1259,7 @@ def upsert( nodes: Sequence[GraphNode], edges: Sequence[EdgeRecord], snapshot_id: str, - ) -> None: + ) -> GraphCleanupStats: """Upsert graph nodes and edges.""" ... @@ -1281,7 +1293,7 @@ def upsert( nodes: Sequence[GraphNode], edges: Sequence[EdgeRecord], snapshot_id: str, - ) -> None: + ) -> GraphCleanupStats: """Upsert graph data into SQLite.""" del nodes, snapshot_id with self._conn_factory() as conn: @@ -1299,6 +1311,7 @@ def upsert( ], ) conn.commit() + return GraphCleanupStats() def neighbors( self, @@ -1416,10 +1429,10 @@ def upsert( nodes: Sequence[GraphNode], edges: Sequence[EdgeRecord], snapshot_id: str, - ) -> None: + ) -> GraphCleanupStats: """Upsert graph nodes and edges into Neo4j.""" if not nodes: - return + return GraphCleanupStats() self._ensure_constraints() node_payload = [ { @@ -1466,14 +1479,40 @@ def upsert( """, parameters={"edges": edge_payload}, ) - self._run_query( + stale_edge_records = self._run_query( + """ + MATCH ()-[rel:CODE_EDGE]->() + WHERE rel.snapshot_id <> $snapshot_id + WITH collect(rel) AS stale_edges + FOREACH (edge IN stale_edges | DELETE edge) + RETURN size(stale_edges) AS deleted_edges + """, + parameters={"snapshot_id": snapshot_id}, + ) + stale_node_records = self._run_query( """ MATCH (n:CodeNode) WHERE n.snapshot_id <> $snapshot_id - DETACH DELETE n + WITH collect(n) AS stale_nodes + FOREACH (node IN stale_nodes | DETACH DELETE node) + RETURN size(stale_nodes) AS deleted_nodes """, parameters={"snapshot_id": snapshot_id}, ) + deleted_edges_raw = ( + stale_edge_records[0].get("deleted_edges", 0) if stale_edge_records else 0 + ) + deleted_nodes_raw = ( + stale_node_records[0].get("deleted_nodes", 0) if stale_node_records else 0 + ) + return GraphCleanupStats( + deleted_nodes=( + int(deleted_nodes_raw) if isinstance(deleted_nodes_raw, (int, float)) else 0 + ), + deleted_edges=( + int(deleted_edges_raw) if isinstance(deleted_edges_raw, (int, float)) else 0 + ), + ) def neighbors( self, @@ -2376,6 +2415,7 @@ def index_with_stats(self) -> IndexStats: skipped_files=skipped_files, deleted_files=len(stale_paths), ) + graph_cleanup = GraphCleanupStats() elapsed_ms = int((time.perf_counter() - started_at) * 1000) snapshot_id = self.index_snapshot_id(conn=conn) conn.executemany( @@ -2396,11 +2436,16 @@ def index_with_stats(self) -> IndexStats: features = self._runtime_features() if "graph" in features.backend_modes: graph_backend = self._active_graph_backend() - graph_backend.upsert( + graph_cleanup = graph_backend.upsert( nodes=self._load_graph_nodes(conn), edges=self._load_graph_edges(conn), snapshot_id=snapshot_id, ) + stats = dataclasses.replace( + stats, + deleted_graph_nodes=graph_cleanup.deleted_nodes, + deleted_graph_edges=graph_cleanup.deleted_edges, + ) observation: dict[str, bool | int | str] = { "repo": self.repo.as_posix(), @@ -2410,6 +2455,8 @@ def index_with_stats(self) -> IndexStats: "indexed_files": stats.indexed_files, "skipped_files": stats.skipped_files, "deleted_files": stats.deleted_files, + "deleted_graph_nodes": stats.deleted_graph_nodes, + "deleted_graph_edges": stats.deleted_graph_edges, "elapsed_ms": elapsed_ms, "hybrid_sync_pending": self._hybrid_requested() and not self._hybrid_state_ready(), } diff --git a/src/clawops/memory_profiles.py b/src/clawops/memory_profiles.py new file mode 100644 index 00000000..3e03002e --- /dev/null +++ b/src/clawops/memory_profiles.py @@ -0,0 +1,79 @@ +"""Shared StrongClaw memory profile registry.""" + +from __future__ import annotations + +import dataclasses + + +@dataclasses.dataclass(frozen=True, slots=True) +class MemoryProfileSpec: + """StrongClaw-managed OpenClaw memory profile.""" + + profile_id: str + render_profile: str + description: str + installs_qmd: bool = False + installs_lossless_claw: bool = False + installs_memory_pro: bool = False + enables_hypermemory_backend: bool = False + managed: bool = True + + +MEMORY_PROFILES: dict[str, MemoryProfileSpec] = { + "hypermemory": MemoryProfileSpec( + profile_id="hypermemory", + render_profile="hypermemory", + description="Default StrongClaw profile: lossless-claw + strongclaw-hypermemory.", + installs_lossless_claw=True, + enables_hypermemory_backend=True, + ), + "openclaw-default": MemoryProfileSpec( + profile_id="openclaw-default", + render_profile="openclaw-default", + description="Built-in OpenClaw defaults: legacy context engine + memory-core.", + ), + "openclaw-qmd": MemoryProfileSpec( + profile_id="openclaw-qmd", + render_profile="openclaw-qmd", + description="Built-in OpenClaw defaults plus the experimental QMD memory backend.", + installs_qmd=True, + ), + "memory-lancedb-pro": MemoryProfileSpec( + profile_id="memory-lancedb-pro", + render_profile="memory-lancedb-pro", + description="Vendored memory-lancedb-pro with Ollama-backed smart extraction.", + installs_qmd=True, + installs_memory_pro=True, + ), + "acp": MemoryProfileSpec( + profile_id="acp", + render_profile="acp", + description="Legacy OpenClaw built-ins plus ACP worker agents.", + installs_qmd=True, + managed=False, + ), + "browser-lab": MemoryProfileSpec( + profile_id="browser-lab", + render_profile="browser-lab", + description="Legacy OpenClaw built-ins plus browser-lab integration.", + managed=False, + ), +} + +MANAGED_MEMORY_PROFILE_IDS: tuple[str, ...] = tuple( + profile_id for profile_id, profile in MEMORY_PROFILES.items() if profile.managed +) + + +def resolve_memory_profile(profile_id: str) -> MemoryProfileSpec | None: + """Resolve one profile from the shared registry.""" + return MEMORY_PROFILES.get(profile_id) + + +def require_memory_profile(profile_id: str) -> MemoryProfileSpec: + """Resolve one managed profile and raise when unknown or unmanaged.""" + profile = resolve_memory_profile(profile_id) + if profile is None or not profile.managed: + available = ", ".join(sorted(MANAGED_MEMORY_PROFILE_IDS)) + raise ValueError(f"unknown memory profile: {profile_id} (choose from {available})") + return profile diff --git a/src/clawops/openclaw_config.py b/src/clawops/openclaw_config.py index 972f8ef7..6041ec32 100644 --- a/src/clawops/openclaw_config.py +++ b/src/clawops/openclaw_config.py @@ -14,6 +14,7 @@ from clawops.common import load_overlay, write_json from clawops.json_merge import merge_documents from clawops.runtime_assets import resolve_asset_path, resolve_runtime_layout +from clawops.strongclaw_runtime import managed_python REPO_ROOT_PLACEHOLDER = "__REPO_ROOT__" HOME_PLACEHOLDER = "__HOME__" @@ -32,6 +33,7 @@ HYPERMEMORY_WORKSPACE_ROOT_PLACEHOLDER = "__HYPERMEMORY_WORKSPACE_ROOT__" HYPERMEMORY_CONFIG_PATH_PLACEHOLDER = "__HYPERMEMORY_CONFIG_PATH__" HYPERMEMORY_SQLITE_CONFIG_PATH_PLACEHOLDER = "__HYPERMEMORY_SQLITE_CONFIG_PATH__" +PYTHON_EXECUTABLE_PLACEHOLDER = "__PYTHON_EXECUTABLE__" OPENCLAW_CONFIG_DIR = pathlib.Path("platform/configs/openclaw") DEFAULT_PROFILE_NAME = "hypermemory" DEFAULT_OPENCLAW_CONFIG_OUTPUT: pathlib.Path | None = None @@ -161,6 +163,7 @@ def build_placeholder_map( HYPERMEMORY_SQLITE_CONFIG_PATH_PLACEHOLDER: ( layout.hypermemory_sqlite_config_path.as_posix() ), + PYTHON_EXECUTABLE_PLACEHOLDER: managed_python(repo_root).as_posix(), } if lossless_claw_plugin_path is not None: replacements[LOSSLESS_CLAW_PLUGIN_PATH_PLACEHOLDER] = ( diff --git a/src/clawops/strongclaw_ops.py b/src/clawops/strongclaw_ops.py index 49a66383..ac205e42 100644 --- a/src/clawops/strongclaw_ops.py +++ b/src/clawops/strongclaw_ops.py @@ -15,12 +15,15 @@ from typing import cast from clawops.cli_roots import add_asset_root_argument, resolve_asset_root_argument +from clawops.observability import emit_structured_log from clawops.runtime_assets import resolve_asset_path, resolve_runtime_layout from clawops.strongclaw_compose import compose_project_name, resolve_compose_file from clawops.strongclaw_runtime import ( CommandError, ensure_docker_backend_ready, load_env_assignments, + rendered_openclaw_uses_hypermemory, + rendered_openclaw_uses_qmd, resolve_openclaw_config_path, resolve_openclaw_state_dir, resolve_repo_local_compose_state_dir, @@ -46,6 +49,9 @@ COMPOSE_STATUS_TIMEOUT_SECONDS = 30 POSTGRES_HEALTH_TIMEOUT_SECONDS = 180 LITELLM_BOOTSTRAP_TIMEOUT_SECONDS = 1800 +LITELLM_HEALTH_TIMEOUT_SECONDS = 180 +QDRANT_HEALTH_TIMEOUT_SECONDS = 180 +NEO4J_HEALTH_TIMEOUT_SECONDS = 180 COMPOSE_POLL_INTERVAL_SECONDS = 2.0 @@ -79,6 +85,19 @@ class _ComposeServiceStatus: health: str | None = None +@dataclasses.dataclass(frozen=True, slots=True) +class _SidecarReadinessTarget: + """Readiness contract for one sidecar dependency.""" + + service_name: str + required: bool + impact: str + reason: str + state: str = "running" + health: str | None = "healthy" + timeout_seconds: int = 180 + + def _compose_state_dir(repo_root: pathlib.Path, *, repo_local_state: bool) -> pathlib.Path: """Return the effective compose-state directory.""" if repo_local_state: @@ -269,21 +288,49 @@ def _wait_for_compose_service( timeout_seconds: int, ) -> None: """Wait for one compose service to reach the requested state.""" + started_at = time.monotonic() + target = state if health is None else f"{state}/{health}" + emit_structured_log( + "clawops.ops.sidecars.wait.start", + { + "service": service_name, + "target": target, + "timeout_seconds": timeout_seconds, + }, + ) deadline = time.monotonic() + timeout_seconds last_status: _ComposeServiceStatus | None = None while True: last_status = _compose_service_statuses(execution).get(service_name) if _service_matches(last_status, state=state, health=health): + emit_structured_log( + "clawops.ops.sidecars.wait.ready", + { + "service": service_name, + "target": target, + "state": None if last_status is None else last_status.state, + "health": None if last_status is None else last_status.health, + "elapsed_ms": int((time.monotonic() - started_at) * 1000), + }, + ) return if time.monotonic() >= deadline: break time.sleep(COMPOSE_POLL_INTERVAL_SECONDS) - target = state if health is None else f"{state}/{health}" observed = ( "service not listed in compose status" if last_status is None else f"state={last_status.state!r}, health={last_status.health or 'n/a'!r}" ) + emit_structured_log( + "clawops.ops.sidecars.wait.timeout", + { + "service": service_name, + "target": target, + "observed": observed, + "timeout_seconds": timeout_seconds, + }, + ) raise CommandError( f"timed out waiting for compose service '{service_name}' to reach {target}; " f"last observed {observed}." @@ -309,10 +356,117 @@ def _run_litellm_schema_bootstrap(execution: _ComposeExecution) -> int: ) +def _resolve_profile_dependency_flags(config_path: pathlib.Path) -> dict[str, object]: + """Resolve profile-dependent sidecar flags from one rendered OpenClaw config.""" + try: + uses_qmd = rendered_openclaw_uses_qmd(config_path) + uses_hypermemory = rendered_openclaw_uses_hypermemory(config_path) + return { + "configPath": config_path.as_posix(), + "source": "rendered-config", + "usesQmd": uses_qmd, + "usesHypermemory": uses_hypermemory, + } + except Exception as exc: + # NOTE: missing/invalid config should keep startup checks conservative. + return { + "configPath": config_path.as_posix(), + "source": "fallback-conservative", + "usesQmd": True, + "usesHypermemory": True, + "resolutionError": str(exc), + } + + +def _sidecar_readiness_targets( + profile_flags: Mapping[str, object], +) -> tuple[_SidecarReadinessTarget, ...]: + """Return the readiness contract for the active profile.""" + uses_qmd = bool(profile_flags.get("usesQmd")) + uses_hypermemory = bool(profile_flags.get("usesHypermemory")) + qdrant_required = uses_qmd or uses_hypermemory + neo4j_required = uses_hypermemory + return ( + _SidecarReadinessTarget( + service_name=POSTGRES_SERVICE_NAME, + required=True, + impact="fatal", + reason="runtime metadata and session state storage", + timeout_seconds=POSTGRES_HEALTH_TIMEOUT_SECONDS, + ), + _SidecarReadinessTarget( + service_name=LITELLM_SERVICE_NAME, + required=True, + impact="fatal", + reason="loopback model routing boundary", + timeout_seconds=LITELLM_HEALTH_TIMEOUT_SECONDS, + ), + _SidecarReadinessTarget( + service_name="qdrant", + required=qdrant_required, + impact="degraded", + reason="dense/sparse retrieval lanes for qmd or hypermemory profiles", + timeout_seconds=QDRANT_HEALTH_TIMEOUT_SECONDS, + ), + _SidecarReadinessTarget( + service_name="neo4j", + required=neo4j_required, + impact="degraded", + reason="graph-backed context expansion for the hypermemory profile", + timeout_seconds=NEO4J_HEALTH_TIMEOUT_SECONDS, + ), + _SidecarReadinessTarget( + service_name="otel-collector", + required=False, + impact="observational", + reason="runtime telemetry export", + health=None, + timeout_seconds=60, + ), + ) + + +def _compose_rows(statuses: Mapping[str, _ComposeServiceStatus]) -> list[dict[str, object]]: + """Return stable compose rows from structured statuses.""" + return [ + { + "Service": status.name, + "State": status.state, + "Health": status.health, + } + for status in sorted(statuses.values(), key=lambda item: item.name) + ] + + +def _readiness_entries( + statuses: Mapping[str, _ComposeServiceStatus], + targets: Sequence[_SidecarReadinessTarget], +) -> list[dict[str, object]]: + """Build a structured readiness report from compose statuses.""" + entries: list[dict[str, object]] = [] + for target in targets: + status = statuses.get(target.service_name) + ready = _service_matches(status, state=target.state, health=target.health) + entries.append( + { + "service": target.service_name, + "required": target.required, + "impact": target.impact, + "reason": target.reason, + "ready": ready, + "targetState": target.state, + "targetHealth": target.health, + "observedState": None if status is None else status.state, + "observedHealth": None if status is None else status.health, + } + ) + return entries + + def gateway_start(repo_root: pathlib.Path) -> int: """Run the OpenClaw gateway under Varlock when available.""" command = wrap_command_with_varlock(repo_root, ["openclaw", "gateway"]) - return run_command_inherited(command, cwd=repo_root, timeout_seconds=1800) + return run_command_inherited(command, cwd=repo_root, timeout_seconds=None) def sidecars_up(repo_root: pathlib.Path, *, repo_local_state: bool) -> int: @@ -322,6 +476,9 @@ def sidecars_up(repo_root: pathlib.Path, *, repo_local_state: bool) -> int: compose_name=SIDECARS_COMPOSE_NAME, repo_local_state=repo_local_state, ) + config_path = pathlib.Path(execution.env["OPENCLAW_CONFIG"]).expanduser().resolve() + profile_flags = _resolve_profile_dependency_flags(config_path) + targets = _sidecar_readiness_targets(profile_flags) postgres_exit = _run_compose_command_with_context( execution, arguments=("up", "-d", POSTGRES_SERVICE_NAME), @@ -352,14 +509,41 @@ def sidecars_up(repo_root: pathlib.Path, *, repo_local_state: bool) -> int: for service_name in SIDECAR_RUNTIME_SERVICE_NAMES if service_name != LITELLM_SERVICE_NAME ) - return _run_compose_command_with_context( + sidecars_exit = _run_compose_command_with_context( execution, arguments=("up", "-d", *runtime_services), ) - return _run_compose_command_with_context( - execution, - arguments=("up", "-d", *SIDECAR_RUNTIME_SERVICE_NAMES), + else: + sidecars_exit = _run_compose_command_with_context( + execution, + arguments=("up", "-d", *SIDECAR_RUNTIME_SERVICE_NAMES), + ) + if sidecars_exit != 0: + return sidecars_exit + for target in targets: + if not target.required: + continue + _wait_for_compose_service( + execution, + service_name=target.service_name, + state=target.state, + health=target.health, + timeout_seconds=target.timeout_seconds, + ) + readiness_entries = _readiness_entries(_compose_service_statuses(execution), targets) + required_ready = all( + bool(entry["ready"]) for entry in readiness_entries if bool(entry["required"]) ) + emit_structured_log( + "clawops.ops.sidecars.ready", + { + "required_ready": required_ready, + "required_count": sum(1 for entry in readiness_entries if bool(entry["required"])), + "optional_count": sum(1 for entry in readiness_entries if not bool(entry["required"])), + "profile_source": str(profile_flags.get("source", "")), + }, + ) + return 0 def sidecars_down(repo_root: pathlib.Path, *, repo_local_state: bool) -> int: @@ -394,23 +578,91 @@ def browser_lab_down(repo_root: pathlib.Path, *, repo_local_state: bool) -> int: def status(repo_root: pathlib.Path, *, repo_local_state: bool) -> dict[str, object]: """Return the current sidecar compose status.""" - compose_name = "docker-compose.aux-stack.yaml" - env = _compose_env(repo_root, repo_local_state=repo_local_state, compose_name=compose_name) - compose_path = _compose_path(repo_root, compose_name) - compose_cwd = resolve_asset_path("platform/compose", repo_root=repo_root) - compose_result = run_command( - ["docker", "compose", "-f", str(compose_path), "ps", "--format", "json"], - cwd=compose_cwd, - env=env, - timeout_seconds=30, + try: + execution = _compose_execution( + repo_root, + compose_name=SIDECARS_COMPOSE_NAME, + repo_local_state=repo_local_state, + ) + except CommandError as exc: + detail = str(exc) + emit_structured_log( + "clawops.ops.sidecars.status", + { + "ok": False, + "error": detail, + }, + ) + return { + "ok": False, + "composeStateDir": "", + "openclawConfig": "", + "profile": {}, + "services": {}, + "readiness": { + "requiredReady": False, + "required": [], + "optional": [], + }, + "compose": detail, + } + config_path = pathlib.Path(execution.env["OPENCLAW_CONFIG"]).expanduser().resolve() + profile_flags = _resolve_profile_dependency_flags(config_path) + targets = _sidecar_readiness_targets(profile_flags) + try: + statuses = _compose_service_statuses(execution) + except CommandError as exc: + detail = str(exc) + emit_structured_log( + "clawops.ops.sidecars.status", + { + "ok": False, + "error": detail, + }, + ) + return { + "ok": False, + "composeStateDir": execution.env["STRONGCLAW_COMPOSE_STATE_DIR"], + "openclawConfig": execution.env["OPENCLAW_CONFIG"], + "profile": profile_flags, + "services": {}, + "readiness": { + "requiredReady": False, + "required": [], + "optional": [], + }, + "compose": detail, + } + entries = _readiness_entries(statuses, targets) + required_entries = [entry for entry in entries if bool(entry["required"])] + optional_entries = [entry for entry in entries if not bool(entry["required"])] + required_ready = all(bool(entry["ready"]) for entry in required_entries) + services = { + status.name: {"state": status.state, "health": status.health} + for status in statuses.values() + } + compose_rows = _compose_rows(statuses) + emit_structured_log( + "clawops.ops.sidecars.status", + { + "ok": required_ready, + "required_ready": required_ready, + "required_count": len(required_entries), + "optional_count": len(optional_entries), + }, ) return { - "ok": compose_result.ok, - "composeStateDir": env["STRONGCLAW_COMPOSE_STATE_DIR"], - "openclawConfig": env["OPENCLAW_CONFIG"], - "compose": ( - compose_result.stdout.strip() if compose_result.ok else compose_result.stderr.strip() - ), + "ok": required_ready, + "composeStateDir": execution.env["STRONGCLAW_COMPOSE_STATE_DIR"], + "openclawConfig": execution.env["OPENCLAW_CONFIG"], + "profile": profile_flags, + "services": services, + "readiness": { + "requiredReady": required_ready, + "required": required_entries, + "optional": optional_entries, + }, + "compose": json.dumps(compose_rows, separators=(",", ":")), } diff --git a/src/clawops/strongclaw_runtime.py b/src/clawops/strongclaw_runtime.py index 1f0fac4c..afc0f109 100644 --- a/src/clawops/strongclaw_runtime.py +++ b/src/clawops/strongclaw_runtime.py @@ -33,6 +33,7 @@ strongclaw_varlock_dir, strongclaw_workspace_dir, ) +from clawops.memory_profiles import MemoryProfileSpec, resolve_memory_profile from clawops.platform_compat import ( DEFAULT_ACPX_VERSION, DEFAULT_MANAGED_PROJECT_PYTHON_VERSION, @@ -151,24 +152,33 @@ def resolve_profile(profile_name: str | None = None) -> str: return configured or os.environ.get("STRONGCLAW_DEFAULT_PROFILE", DEFAULT_PROFILE_NAME) +def _resolved_profile_spec(profile_name: str | None = None) -> MemoryProfileSpec | None: + """Resolve the active profile from the shared registry when available.""" + return resolve_memory_profile(resolve_profile(profile_name)) + + def profile_requires_qmd(profile_name: str | None = None) -> bool: """Return whether the profile requires the QMD asset.""" - return resolve_profile(profile_name) in {"openclaw-qmd", "memory-lancedb-pro", "acp"} + spec = _resolved_profile_spec(profile_name) + return bool(spec is not None and spec.installs_qmd) def profile_requires_lossless_claw(profile_name: str | None = None) -> bool: """Return whether the profile requires the lossless-claw plugin.""" - return resolve_profile(profile_name) == "hypermemory" + spec = _resolved_profile_spec(profile_name) + return bool(spec is not None and spec.installs_lossless_claw) def profile_requires_hypermemory_backend(profile_name: str | None = None) -> bool: """Return whether the profile enables strongclaw-hypermemory.""" - return resolve_profile(profile_name) == "hypermemory" + spec = _resolved_profile_spec(profile_name) + return bool(spec is not None and spec.enables_hypermemory_backend) def profile_requires_memory_pro_plugin(profile_name: str | None = None) -> bool: """Return whether the profile requires the vendored memory-pro plugin.""" - return resolve_profile(profile_name) == "memory-lancedb-pro" + spec = _resolved_profile_spec(profile_name) + return bool(spec is not None and spec.installs_memory_pro) def profile_bootstrap_capabilities(profile_name: str | None = None) -> tuple[str, ...]: @@ -287,14 +297,27 @@ def run_command_inherited( *, cwd: pathlib.Path | None = None, env: Mapping[str, str] | None = None, - timeout_seconds: int = 1800, + timeout_seconds: int | None = 1800, ) -> int: """Run a subprocess with inherited stdio.""" + argv = [str(part) for part in command] + cwd_value = None if cwd is None else str(cwd) + env_value = None if env is None else dict(env) + if timeout_seconds is None: + completed = subprocess.run( + argv, + check=False, + cwd=cwd_value, + env=env_value, + ) + return int(completed.returncode) + if timeout_seconds <= 0: + raise ValueError("timeout_seconds must be positive when provided") completed = subprocess.run( - [str(part) for part in command], + argv, check=False, - cwd=None if cwd is None else str(cwd), - env=None if env is None else dict(env), + cwd=cwd_value, + env=env_value, timeout=timeout_seconds, ) return int(completed.returncode) diff --git a/src/clawops/strongclaw_services.py b/src/clawops/strongclaw_services.py index 4a2853f1..4e5ba554 100644 --- a/src/clawops/strongclaw_services.py +++ b/src/clawops/strongclaw_services.py @@ -12,6 +12,7 @@ from clawops.cli_roots import add_asset_root_argument, resolve_asset_root_argument from clawops.common import load_text, write_text +from clawops.observability import emit_structured_log from clawops.platform_compat import detect_host_platform, resolve_service_manager from clawops.runtime_assets import RuntimeLayout, resolve_runtime_layout from clawops.strongclaw_runtime import ( @@ -24,11 +25,13 @@ ) LAUNCHD_ACTIVATE_LABELS: Final[tuple[str, ...]] = ( - "ai.openclaw.gateway", "ai.openclaw.sidecars", + "ai.openclaw.gateway", + "ai.openclaw.maintenance", ) LAUNCHD_GATEWAY_LABEL: Final[str] = "ai.openclaw.gateway" LAUNCHD_SIDECARS_LABEL: Final[str] = "ai.openclaw.sidecars" +LAUNCHD_MAINTENANCE_LABEL: Final[str] = "ai.openclaw.maintenance" LAUNCHD_GATEWAY_TIMEOUT_ENV_VAR: Final[str] = "STRONGCLAW_LAUNCHD_GATEWAY_TIMEOUT_SECONDS" LAUNCHD_SIDECARS_TIMEOUT_ENV_VAR: Final[str] = "STRONGCLAW_LAUNCHD_SIDECARS_TIMEOUT_SECONDS" LAUNCHD_PASSTHROUGH_ENV_VARS: Final[tuple[str, ...]] = ( @@ -44,6 +47,7 @@ SYSTEMD_ACTIVATE_UNITS: Final[tuple[str, ...]] = ( "openclaw-sidecars.service", "openclaw-gateway.service", + "openclaw-maintenance.timer", ) @@ -124,19 +128,20 @@ def render_service_files( resolved_state_dir.mkdir(parents=True, exist_ok=True) (resolved_state_dir / "logs").mkdir(parents=True, exist_ok=True) manager = service_manager or resolve_service_manager(detect_host_platform()) + template_paths: list[pathlib.Path] if manager == "launchd": template_dir = resolved_repo_root / "platform" / "launchd" output_dir = launchd_dir() - pattern = "*.template" + template_paths = sorted(template_dir.glob("*.template")) elif manager == "systemd": template_dir = resolved_repo_root / "platform" / "systemd" output_dir = systemd_dir() - pattern = "*.service" + template_paths = sorted([*template_dir.glob("*.service"), *template_dir.glob("*.timer")]) else: raise ValueError(f"unsupported service manager: {manager}") output_dir.mkdir(parents=True, exist_ok=True) rendered_files: list[str] = [] - for template_path in sorted(template_dir.glob(pattern)): + for template_path in template_paths: if not template_path.is_file(): continue output_name = ( @@ -289,11 +294,14 @@ def activate_services( ) gateway_plist = output_dir / f"{LAUNCHD_GATEWAY_LABEL}.plist" sidecars_plist = output_dir / f"{LAUNCHD_SIDECARS_LABEL}.plist" - _activate_launchd_service(domain, LAUNCHD_GATEWAY_LABEL, gateway_plist) - _wait_for_launchd_service( - LAUNCHD_GATEWAY_LABEL, - persistent=True, - timeout_seconds=gateway_timeout_seconds, + maintenance_plist = output_dir / f"{LAUNCHD_MAINTENANCE_LABEL}.plist" + emit_structured_log( + "clawops.services.activate", + { + "service_manager": "launchd", + "step": "sidecars_bootstrap", + "label": LAUNCHD_SIDECARS_LABEL, + }, ) _activate_launchd_oneshot_service( domain, @@ -301,10 +309,40 @@ def activate_services( sidecars_plist, timeout_seconds=sidecars_timeout_seconds, ) + emit_structured_log( + "clawops.services.activate", + { + "service_manager": "launchd", + "step": "gateway_bootstrap", + "label": LAUNCHD_GATEWAY_LABEL, + }, + ) + _activate_launchd_service(domain, LAUNCHD_GATEWAY_LABEL, gateway_plist) + _wait_for_launchd_service( + LAUNCHD_GATEWAY_LABEL, + persistent=True, + timeout_seconds=gateway_timeout_seconds, + ) + emit_structured_log( + "clawops.services.activate", + { + "service_manager": "launchd", + "step": "maintenance_bootstrap", + "label": LAUNCHD_MAINTENANCE_LABEL, + }, + ) + _activate_launchd_service(domain, LAUNCHD_MAINTENANCE_LABEL, maintenance_plist) return { **render_payload, "activated": list(LAUNCHD_ACTIVATE_LABELS), } + emit_structured_log( + "clawops.services.activate", + { + "service_manager": "systemd", + "step": "daemon_reload", + }, + ) reload_result = run_command(["systemctl", "--user", "daemon-reload"], timeout_seconds=30) if not reload_result.ok: detail = ( @@ -314,6 +352,14 @@ def activate_services( ) raise RuntimeError(detail) for unit in SYSTEMD_ACTIVATE_UNITS: + emit_structured_log( + "clawops.services.activate", + { + "service_manager": "systemd", + "step": "enable_now", + "unit": unit, + }, + ) enable_result = run_command( ["systemctl", "--user", "enable", "--now", unit], timeout_seconds=60, diff --git a/src/clawops/workflow_runner.py b/src/clawops/workflow_runner.py index ad22fa78..7668cca3 100644 --- a/src/clawops/workflow_runner.py +++ b/src/clawops/workflow_runner.py @@ -12,6 +12,7 @@ from clawops.acp_runner import SessionSpec, run_session from clawops.acpx_adapter import AcpxPermissionMode from clawops.app_paths import scoped_state_dir +from clawops.approval_dispatch import dispatch_pending_approval from clawops.common import canonical_json, load_json, load_yaml, write_json, write_text from clawops.context.contracts import validate_context_provider, validate_context_scale from clawops.context.registry import create_context_service @@ -658,7 +659,7 @@ def _worker_dispatch_step(self, step: Mapping[str, object]) -> StepResult: details["journal_db"] = str(journal_db) contract_json = canonical_json(task.to_contract()) if task.approval_required: - journal.transition( + pending = journal.transition( op.op_id, "pending_approval", policy_decision="require_approval", @@ -666,6 +667,16 @@ def _worker_dispatch_step(self, step: Mapping[str, object]) -> StepResult: execution_contract_json=contract_json, approval_required=True, ) + dispatch_outcome = dispatch_pending_approval(journal=journal, operation=pending) + details["dispatch"] = dispatch_outcome.to_dict() + details["review_artifact_path"] = dispatch_outcome.artifact_path.as_posix() + if dispatch_outcome.error is not None: + return self._store_step_result( + step_name=str(step["name"]), + ok=False, + message=f"approval dispatch failed for {op.op_id}: {dispatch_outcome.error}", + details=details, + ) approved_by = step.get("approved_by") if not isinstance(approved_by, str) or not approved_by.strip(): return self._store_step_result( diff --git a/src/clawops/wrappers/base.py b/src/clawops/wrappers/base.py index 48d75f63..38693e70 100644 --- a/src/clawops/wrappers/base.py +++ b/src/clawops/wrappers/base.py @@ -15,6 +15,7 @@ import requests from clawops import __version__ +from clawops.approval_dispatch import ApprovalDispatchOutcome, dispatch_pending_approval from clawops.common import canonical_json from clawops.observability import emit_structured_log, observed_span from clawops.op_journal import Operation, OperationJournal @@ -890,6 +891,17 @@ def prepare_operation( decision_payload: Mapping[str, Any], ) -> PreparedOperation: """Prepare an operation and transition it into a pre-execution state.""" + + def _attach_dispatch_payload( + result_payload: dict[str, Any] | None, + dispatch_outcome: ApprovalDispatchOutcome | None, + ) -> dict[str, Any] | None: + if result_payload is None or dispatch_outcome is None: + return result_payload + result_payload["review_artifact_path"] = dispatch_outcome.artifact_path.as_posix() + result_payload["dispatch"] = dispatch_outcome.to_dict() + return result_payload + op = ctx.journal.begin( scope=scope, kind=kind, @@ -898,6 +910,7 @@ def prepare_operation( inputs=payload, ) if op.status != "proposed": + dispatch_outcome: ApprovalDispatchOutcome | None = None decision = decision_from_operation(op) if op.status in {"pending_approval", "approved", "running"}: op, decision = ensure_execution_contract( @@ -905,7 +918,13 @@ def prepare_operation( op=op, decision_payload=decision_payload, ) - result = replay_result_from_operation(op, decision=decision, dry_run=ctx.dry_run) + if op.status == "pending_approval": + dispatch_outcome = dispatch_pending_approval(journal=ctx.journal, operation=op) + op = dispatch_outcome.operation + result = _attach_dispatch_payload( + replay_result_from_operation(op, decision=decision, dry_run=ctx.dry_run), + dispatch_outcome, + ) return PreparedOperation( op, decision, result=result, should_execute=op.status == "approved" and not ctx.dry_run ) @@ -943,11 +962,16 @@ def prepare_operation( review_status="pending", review_payload_json=(None if not review_payload else canonical_json(review_payload)), ) + dispatch_outcome = dispatch_pending_approval(journal=ctx.journal, operation=updated) + updated = dispatch_outcome.operation return PreparedOperation( updated, decision, - result=result_from_operation( - updated, decision=decision, ok=True, accepted=True, executed=False + result=_attach_dispatch_payload( + result_from_operation( + updated, decision=decision, ok=True, accepted=True, executed=False + ), + dispatch_outcome, ), ) contract = _build_execution_contract(op, decision) diff --git a/tests/suites/contracts/repo/test_docs_parity.py b/tests/suites/contracts/repo/test_docs_parity.py index a7dfb8e0..be78dff7 100644 --- a/tests/suites/contracts/repo/test_docs_parity.py +++ b/tests/suites/contracts/repo/test_docs_parity.py @@ -200,3 +200,24 @@ def test_operator_docs_no_longer_surface_root_shell_entrypoints() -> None: for markdown_file in _official_markdown_files(REPO_ROOT): text = markdown_file.read_text(encoding="utf-8") assert "./scripts/" not in text + + +def test_operator_docs_surface_plugin_inventory_and_degradation_contract() -> None: + readme = (REPO_ROOT / "README.md").read_text(encoding="utf-8") + inventory = (REPO_ROOT / "platform/docs/PLUGIN_INVENTORY.md").read_text(encoding="utf-8") + degradation = (REPO_ROOT / "platform/docs/DEGRADATION.md").read_text(encoding="utf-8") + hypermemory = (REPO_ROOT / "platform/docs/HYPERMEMORY.md").read_text(encoding="utf-8") + ci_doc = (REPO_ROOT / "platform/docs/CI_AND_SECURITY.md").read_text(encoding="utf-8") + + assert "platform/docs/PLUGIN_INVENTORY.md" in readme + assert "platform/docs/DEGRADATION.md" in readme + assert "`strongclaw-hypermemory`" in inventory + assert "`memory-lancedb-pro`" in inventory + assert "Support level" in inventory + assert "clawops ops status" in degradation + assert "fatal" in degradation + assert "degraded" in degradation + assert "observational" in degradation + assert "[Plugin Inventory](./PLUGIN_INVENTORY.md)" in hypermemory + assert "[Degradation Contract](./DEGRADATION.md)" in hypermemory + assert "[Plugin Inventory](./PLUGIN_INVENTORY.md)" in ci_doc diff --git a/tests/suites/contracts/repo/test_scripts_migration_surfaces.py b/tests/suites/contracts/repo/test_scripts_migration_surfaces.py index cc6de6b9..3cabc21a 100644 --- a/tests/suites/contracts/repo/test_scripts_migration_surfaces.py +++ b/tests/suites/contracts/repo/test_scripts_migration_surfaces.py @@ -52,6 +52,12 @@ def test_service_templates_call_repo_venv_python() -> None: browserlab = (REPO_ROOT / "platform/systemd/openclaw-browserlab.service").read_text( encoding="utf-8" ) + maintenance_service = (REPO_ROOT / "platform/systemd/openclaw-maintenance.service").read_text( + encoding="utf-8" + ) + maintenance_timer = (REPO_ROOT / "platform/systemd/openclaw-maintenance.timer").read_text( + encoding="utf-8" + ) sidecars = (REPO_ROOT / "platform/systemd/openclaw-sidecars.service").read_text( encoding="utf-8" ) @@ -64,13 +70,22 @@ def test_service_templates_call_repo_venv_python() -> None: launchd_browserlab = ( REPO_ROOT / "platform/launchd/ai.openclaw.browserlab.plist.template" ).read_text(encoding="utf-8") + launchd_maintenance = ( + REPO_ROOT / "platform/launchd/ai.openclaw.maintenance.plist.template" + ).read_text(encoding="utf-8") assert "scripts/ops/" not in gateway assert "scripts/ops/" not in sidecars assert "__PYTHON_EXECUTABLE__ -m clawops" in gateway assert "__PYTHON_EXECUTABLE__ -m clawops" in browserlab + assert "__PYTHON_EXECUTABLE__ -m clawops" in maintenance_service assert "__PYTHON_EXECUTABLE__ -m clawops" in sidecars + assert "openclaw-sidecars.service" in gateway + assert "Unit=openclaw-maintenance.service" in maintenance_timer assert "__PYTHON_EXECUTABLE__" in launchd_gateway + assert "__PYTHON_EXECUTABLE__" in launchd_maintenance + assert "recovery" in launchd_maintenance + assert "prune-retention" in launchd_maintenance assert ( "Environment=PATH=%h/.config/varlock/bin:%h/.local/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin" in gateway diff --git a/tests/suites/integration/clawops/hypermemory/test_plugin.py b/tests/suites/integration/clawops/hypermemory/test_plugin.py index 3318f336..843cd503 100644 --- a/tests/suites/integration/clawops/hypermemory/test_plugin.py +++ b/tests/suites/integration/clawops/hypermemory/test_plugin.py @@ -17,6 +17,8 @@ def test_hypermemory_plugin_manifest_and_package_metadata() -> None: assert "configPath" in manifest["configSchema"]["properties"] assert "autoCapture" in manifest["configSchema"]["properties"] assert "captureMinMessages" in manifest["configSchema"]["properties"] + assert "startupTimeoutMs" in manifest["configSchema"]["properties"] + assert "toolTimeoutMs" in manifest["configSchema"]["properties"] assert package["openclaw"]["extensions"] == ["./index.js"] assert package["scripts"]["test:openclaw-host"] == "node test/openclaw-host-functional.mjs" assert (plugin_root / "test" / "openclaw-host-functional.mjs").exists() @@ -40,3 +42,5 @@ def test_hypermemory_plugin_uses_compatible_tool_names() -> None: assert 'name: "memory_list_facts"' in plugin_source assert "autoCapture" in plugin_source assert "record-injection" in plugin_source + assert "createStartupGate" in plugin_source + assert "startup preflight" in plugin_source diff --git a/tests/suites/integration/clawops/test_workflow_runner_descriptors.py b/tests/suites/integration/clawops/test_workflow_runner_descriptors.py index faa6161f..8f2c9e7a 100644 --- a/tests/suites/integration/clawops/test_workflow_runner_descriptors.py +++ b/tests/suites/integration/clawops/test_workflow_runner_descriptors.py @@ -5,7 +5,7 @@ import pathlib from clawops.common import write_yaml -from clawops.typed_values import as_string +from clawops.typed_values import as_mapping, as_string from clawops.workflow_runner import WorkflowRunner from tests.plugins.infrastructure.context import TestContext from tests.utils.helpers.cli import write_fake_acpx, write_status_script @@ -134,6 +134,69 @@ def test_workflow_runner_worker_dispatch_and_poll_support_non_git_workspace( assert results[1].message == "succeeded" +def test_workflow_runner_worker_dispatch_writes_review_packet_when_approval_is_required( + tmp_path: pathlib.Path, + test_context: TestContext, +) -> None: + test_context.env.apply_profile( + "workflow_state", + overrides={"STRONGCLAW_STATE_DIR": tmp_path / "state"}, + ) + project, workspace, config = build_context_project(tmp_path) + (workspace / "main.py").write_text("def run_task():\n return 'ok'\n", encoding="utf-8") + write_yaml(config, {"index": {"db_path": ".clawops/context.sqlite"}}) + + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + write_fake_acpx(bin_dir) + write_status_script(bin_dir, "codex", stdout_text="Logged in using ChatGPT") + test_context.env.prepend_path(bin_dir) + + runner = WorkflowRunner( + { + "steps": [ + { + "name": "dispatch", + "kind": "worker_dispatch", + "journal_db": str(tmp_path / "workflow.sqlite"), + "task": { + "project": {"root": str(project)}, + "workspace": {"kind": "local_dir", "path": str(workspace)}, + "lane": "feature-a", + "role": "developer", + "backend": "codex", + "prompt": "Implement feature A", + "operation_kind": "implement", + "required_auth_mode": "subscription", + "approval_required": True, + "context": { + "provider": "codebase", + "scale": "small", + "config": str(config), + "query": "run_task", + }, + }, + } + ] + } + ) + + results = runner.run() + + assert len(results) == 1 + assert results[0].ok is False + assert "approval required before dispatch" in results[0].message + review_artifact_path = pathlib.Path( + as_string( + results[0].details["review_artifact_path"], + path="results[0].details.review_artifact_path", + ) + ) + dispatch = as_mapping(results[0].details["dispatch"], path="results[0].details.dispatch") + assert dispatch["dispatched"] is True + assert review_artifact_path.exists() + + def test_workflow_runner_approval_and_artifact_gates( tmp_path: pathlib.Path, ) -> None: diff --git a/tests/suites/unit/clawops/context/test_codebase_graph.py b/tests/suites/unit/clawops/context/test_codebase_graph.py index e03e65fd..3a6fdb14 100644 --- a/tests/suites/unit/clawops/context/test_codebase_graph.py +++ b/tests/suites/unit/clawops/context/test_codebase_graph.py @@ -8,7 +8,9 @@ from clawops.common import write_yaml from clawops.context.codebase.service import ( + EdgeRecord, GraphConfig, + GraphNode, Neo4jGraphBackend, normalize_neo4j_driver_url, service_from_config, @@ -187,3 +189,57 @@ def test_neo4j_neighbors_reject_invalid_depth() -> None: depth=0, limit=4, ) + + +def test_neo4j_upsert_prunes_stale_edges_and_reports_cleanup_counts( + test_context: TestContext, +) -> None: + backend = Neo4jGraphBackend(GraphConfig()) + seen_queries: list[str] = [] + + def _fake_run_query( + query: str, + *, + parameters: dict[str, object] | None = None, + ) -> list[dict[str, object]]: + del parameters + seen_queries.append(query) + if "deleted_edges" in query: + return [{"deleted_edges": 2}] + if "deleted_nodes" in query: + return [{"deleted_nodes": 1}] + return [] + + test_context.patch.patch_object(backend, "_run_query", new=_fake_run_query) + + cleanup = backend.upsert( + nodes=[ + GraphNode( + node_id="symbol:provider.rotate_token", + path="provider.py", + language="python", + kind="symbol", + ), + GraphNode( + node_id="symbol:consumer.dispatch", + path="consumer.py", + language="python", + kind="symbol", + ), + ], + edges=[ + EdgeRecord( + src_id="symbol:consumer.dispatch", + dst_id="symbol:provider.rotate_token", + edge_type="CALLS", + path="consumer.py", + weight=1, + ) + ], + snapshot_id="snapshot-2", + ) + + assert cleanup.deleted_edges == 2 + assert cleanup.deleted_nodes == 1 + assert any("MATCH ()-[rel:CODE_EDGE]->()" in query for query in seen_queries) + assert any("rel.snapshot_id <> $snapshot_id" in query for query in seen_queries) diff --git a/tests/suites/unit/clawops/test_approval_dispatch.py b/tests/suites/unit/clawops/test_approval_dispatch.py new file mode 100644 index 00000000..e38a6a1b --- /dev/null +++ b/tests/suites/unit/clawops/test_approval_dispatch.py @@ -0,0 +1,82 @@ +"""Unit tests for approval reviewer packet dispatch.""" + +from __future__ import annotations + +import json +import pathlib + +from clawops.approval_dispatch import dispatch_pending_approval +from clawops.op_journal import Operation, OperationJournal +from tests.plugins.infrastructure.context import TestContext +from tests.utils.helpers.journal import create_journal + + +def _seed_pending_operation(journal: OperationJournal, *, root: pathlib.Path) -> Operation: + op = journal.begin( + scope="session-1", + kind="workflow-dispatch", + trust_zone="automation", + normalized_target=root.as_posix(), + inputs={"task": "ship"}, + ) + return journal.transition( + op.op_id, + "pending_approval", + policy_decision="require_approval", + policy_decision_json=json.dumps( + {"decision": "require_approval", "reason": "manual review"} + ), + execution_contract_version=1, + execution_contract_json=json.dumps({"version": 1, "kind": "workflow-dispatch"}), + approval_required=True, + review_mode="manual", + review_status="pending", + review_payload_json=json.dumps({"checklist": ["confirm release notes"]}), + ) + + +def test_dispatch_pending_approval_writes_packet_and_updates_journal( + tmp_path: pathlib.Path, +) -> None: + journal = create_journal(tmp_path / "workflow.sqlite") + pending = _seed_pending_operation(journal, root=tmp_path) + + outcome = dispatch_pending_approval(journal=journal, operation=pending) + + assert outcome.dispatched is True + assert outcome.error is None + assert outcome.artifact_path.exists() + packet = json.loads(outcome.artifact_path.read_text(encoding="utf-8")) + assert packet["opId"] == pending.op_id + assert packet["review"]["status"] == "pending" + assert packet["policy"]["decision"] == "require_approval" + persisted = journal.get(pending.op_id) + assert persisted.review_artifact_path == outcome.artifact_path.as_posix() + assert persisted.status == "pending_approval" + + +def test_dispatch_pending_approval_surfaces_local_write_failures( + tmp_path: pathlib.Path, + test_context: TestContext, +) -> None: + journal = create_journal(tmp_path / "workflow.sqlite") + pending = _seed_pending_operation(journal, root=tmp_path) + + def _raise_write_failure(_path: pathlib.Path, _value: object, *, indent: int = 2) -> None: + del indent + raise OSError("disk full") + + import clawops.approval_dispatch as approval_dispatch + + test_context.patch.patch_object( + approval_dispatch, + "write_json", + new=_raise_write_failure, + ) + + outcome = dispatch_pending_approval(journal=journal, operation=pending) + + assert outcome.dispatched is False + assert outcome.error is not None + assert "disk full" in outcome.error + assert journal.get(pending.op_id).review_artifact_path is None diff --git a/tests/suites/unit/clawops/test_openclaw_config.py b/tests/suites/unit/clawops/test_openclaw_config.py index f58ad0a8..f5f7e428 100644 --- a/tests/suites/unit/clawops/test_openclaw_config.py +++ b/tests/suites/unit/clawops/test_openclaw_config.py @@ -23,6 +23,7 @@ render_openclaw_profile, render_qmd_overlay, ) +from clawops.strongclaw_runtime import managed_python from tests.plugins.infrastructure.context import TestContext from tests.utils.helpers.repo import REPO_ROOT @@ -226,6 +227,7 @@ def test_hypermemory_overlay_template_renders_repo_local_paths() -> None: plugin_config = rendered["plugins"]["entries"]["strongclaw-hypermemory"]["config"] assert rendered["plugins"]["slots"]["memory"] == "strongclaw-hypermemory" + assert plugin_config["command"] == [managed_python(repo_root).as_posix(), "-m", "clawops"] assert plugin_config["configPath"] == f"{memory_config_dir.as_posix()}/hypermemory.sqlite.yaml" assert rendered["plugins"]["load"]["paths"] == [ f"{repo_root.as_posix()}/platform/plugins/strongclaw-hypermemory" @@ -271,6 +273,7 @@ def test_render_hypermemory_profile_merges_baseline_and_plugin_slots() -> None: } plugin_config = rendered["plugins"]["entries"]["strongclaw-hypermemory"]["config"] assert plugin_config["configPath"] == f"{memory_config_dir.as_posix()}/hypermemory.yaml" + assert plugin_config["command"] == [managed_python(repo_root).as_posix(), "-m", "clawops"] assert plugin_config["autoRecall"] is True assert plugin_config["autoReflect"] is False diff --git a/tests/suites/unit/clawops/test_strongclaw_ops.py b/tests/suites/unit/clawops/test_strongclaw_ops.py index 58ed2dc9..a4d6ebb8 100644 --- a/tests/suites/unit/clawops/test_strongclaw_ops.py +++ b/tests/suites/unit/clawops/test_strongclaw_ops.py @@ -2,7 +2,6 @@ from __future__ import annotations -import json import pathlib import re from collections.abc import Callable, Mapping, Sequence @@ -11,7 +10,6 @@ import pytest from clawops import strongclaw_ops -from clawops.strongclaw_runtime import ExecResult from tests.plugins.infrastructure.context import TestContext from tests.utils.helpers.repo import REPO_ROOT @@ -48,22 +46,6 @@ def __call__( ) -def _exec_result( - *argv: str, - stdout: str = "", - stderr: str = "", - returncode: int = 0, -) -> ExecResult: - """Build a typed subprocess result for StrongClaw runtime helpers.""" - return ExecResult( - argv=argv, - returncode=returncode, - stdout=stdout, - stderr=stderr, - duration_ms=1, - ) - - def _fixed_compose_env( _repo_root: pathlib.Path, *, @@ -72,7 +54,11 @@ def _fixed_compose_env( ) -> dict[str, str]: """Return a minimal compose environment for command-sequencing tests.""" del repo_local_state, compose_name - return {"PATH": "/usr/bin"} + return { + "PATH": "/usr/bin", + "OPENCLAW_CONFIG": "/tmp/openclaw.json", + "STRONGCLAW_COMPOSE_STATE_DIR": "/tmp/compose", + } def _identity_varlock_command( @@ -83,11 +69,6 @@ def _identity_varlock_command( return [str(part) for part in command] -def _compose_ps_output(*entries: dict[str, object]) -> str: - """Render Compose `ps --format json` output in the newline-delimited form.""" - return "\n".join(json.dumps(entry) for entry in entries) - - def test_compose_state_dir_defaults_to_openclaw_state_root( monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path ) -> None: @@ -358,13 +339,8 @@ def test_sidecars_up_bootstraps_litellm_before_starting_runtime_services( """Sidecars startup should bootstrap LiteLLM before starting the runtime service.""" compose_path = tmp_path / "compose.yaml" compose_path.write_text("services: {}\n", encoding="utf-8") - compose_status_payloads = iter( - ( - _compose_ps_output({"Service": "postgres", "State": "running", "Health": "healthy"}), - _compose_ps_output({"Service": "postgres", "State": "running", "Health": "healthy"}), - ) - ) inherited_calls: list[tuple[str, ...]] = [] + waited_services: list[tuple[str, str, str | None, int]] = [] def _compose_path_override(_repo_root: pathlib.Path, _compose_name: str) -> pathlib.Path: return compose_path @@ -374,18 +350,70 @@ def _compose_path_override(_repo_root: pathlib.Path, _compose_name: str) -> path monkeypatch.setattr(strongclaw_ops, "_compose_env", _fixed_compose_env) monkeypatch.setattr(strongclaw_ops, "wrap_command_with_varlock", _identity_varlock_command) - def fake_run_command( - command: list[str], + def _profile_flags(_path: pathlib.Path) -> dict[str, bool | str]: + return { + "usesQmd": True, + "usesHypermemory": True, + "source": "test", + } + + monkeypatch.setattr( + strongclaw_ops, + "_resolve_profile_dependency_flags", + _profile_flags, + ) + + compose_statuses = iter( + ( + { + "postgres": cast(Any, strongclaw_ops)._ComposeServiceStatus( + name="postgres", + state="running", + health="healthy", + ) + }, + { + "postgres": cast(Any, strongclaw_ops)._ComposeServiceStatus( + name="postgres", + state="running", + health="healthy", + ), + "litellm": cast(Any, strongclaw_ops)._ComposeServiceStatus( + name="litellm", + state="running", + health="healthy", + ), + "qdrant": cast(Any, strongclaw_ops)._ComposeServiceStatus( + name="qdrant", + state="running", + health="healthy", + ), + "neo4j": cast(Any, strongclaw_ops)._ComposeServiceStatus( + name="neo4j", + state="running", + health="healthy", + ), + "otel-collector": cast(Any, strongclaw_ops)._ComposeServiceStatus( + name="otel-collector", + state="running", + health=None, + ), + }, + ) + ) + + def _compose_service_statuses(_execution: object) -> dict[str, object]: + return cast(dict[str, object], next(compose_statuses)) + + def _wait_for_compose_service( + _execution: object, *, - cwd: pathlib.Path | None = None, - env: dict[str, str] | None = None, - timeout_seconds: int = 30, - capture_output: bool = True, - input_text: str | None = None, - check: bool = False, - ) -> ExecResult: - del cwd, env, timeout_seconds, capture_output, input_text, check - return _exec_result(*command, stdout=next(compose_status_payloads)) + service_name: str, + state: str, + health: str | None = None, + timeout_seconds: int, + ) -> None: + waited_services.append((service_name, state, health, timeout_seconds)) def fake_run_command_inherited( command: list[str], @@ -398,10 +426,18 @@ def fake_run_command_inherited( inherited_calls.append(tuple(command)) return 0 - monkeypatch.setattr(strongclaw_ops, "run_command", fake_run_command) + monkeypatch.setattr(strongclaw_ops, "_compose_service_statuses", _compose_service_statuses) + monkeypatch.setattr(strongclaw_ops, "_wait_for_compose_service", _wait_for_compose_service) monkeypatch.setattr(strongclaw_ops, "run_command_inherited", fake_run_command_inherited) assert strongclaw_ops.sidecars_up(REPO_ROOT, repo_local_state=False) == 0 + assert [service for service, _, _, _ in waited_services] == [ + "postgres", + "postgres", + "litellm", + "qdrant", + "neo4j", + ] assert inherited_calls == [ ("docker", "compose", "-f", str(compose_path), "up", "-d", "postgres"), ( @@ -449,16 +485,8 @@ def test_sidecars_up_skips_bootstrap_when_litellm_is_already_healthy( """Healthy LiteLLM runtimes should not be re-bootstrapped on idempotent startup.""" compose_path = tmp_path / "compose.yaml" compose_path.write_text("services: {}\n", encoding="utf-8") - compose_status_payloads = iter( - ( - _compose_ps_output({"Service": "postgres", "State": "running", "Health": "healthy"}), - _compose_ps_output( - {"Service": "postgres", "State": "running", "Health": "healthy"}, - {"Service": "litellm", "State": "running", "Health": "healthy"}, - ), - ) - ) inherited_calls: list[tuple[str, ...]] = [] + waited_services: list[str] = [] def _compose_path_override(_repo_root: pathlib.Path, _compose_name: str) -> pathlib.Path: return compose_path @@ -468,18 +496,76 @@ def _compose_path_override(_repo_root: pathlib.Path, _compose_name: str) -> path monkeypatch.setattr(strongclaw_ops, "_compose_env", _fixed_compose_env) monkeypatch.setattr(strongclaw_ops, "wrap_command_with_varlock", _identity_varlock_command) - def fake_run_command( - command: list[str], + def _profile_flags(_path: pathlib.Path) -> dict[str, bool | str]: + return { + "usesQmd": True, + "usesHypermemory": True, + "source": "test", + } + + monkeypatch.setattr( + strongclaw_ops, + "_resolve_profile_dependency_flags", + _profile_flags, + ) + + compose_statuses = iter( + ( + { + "postgres": cast(Any, strongclaw_ops)._ComposeServiceStatus( + name="postgres", + state="running", + health="healthy", + ), + "litellm": cast(Any, strongclaw_ops)._ComposeServiceStatus( + name="litellm", + state="running", + health="healthy", + ), + }, + { + "postgres": cast(Any, strongclaw_ops)._ComposeServiceStatus( + name="postgres", + state="running", + health="healthy", + ), + "litellm": cast(Any, strongclaw_ops)._ComposeServiceStatus( + name="litellm", + state="running", + health="healthy", + ), + "qdrant": cast(Any, strongclaw_ops)._ComposeServiceStatus( + name="qdrant", + state="running", + health="healthy", + ), + "neo4j": cast(Any, strongclaw_ops)._ComposeServiceStatus( + name="neo4j", + state="running", + health="healthy", + ), + "otel-collector": cast(Any, strongclaw_ops)._ComposeServiceStatus( + name="otel-collector", + state="running", + health=None, + ), + }, + ) + ) + + def _compose_service_statuses(_execution: object) -> dict[str, object]: + return cast(dict[str, object], next(compose_statuses)) + + def _wait_for_compose_service( + _execution: object, *, - cwd: pathlib.Path | None = None, - env: dict[str, str] | None = None, - timeout_seconds: int = 30, - capture_output: bool = True, - input_text: str | None = None, - check: bool = False, - ) -> ExecResult: - del cwd, env, timeout_seconds, capture_output, input_text, check - return _exec_result(*command, stdout=next(compose_status_payloads)) + service_name: str, + state: str, + health: str | None = None, + timeout_seconds: int, + ) -> None: + del state, health, timeout_seconds + waited_services.append(service_name) def fake_run_command_inherited( command: list[str], @@ -492,10 +578,12 @@ def fake_run_command_inherited( inherited_calls.append(tuple(command)) return 0 - monkeypatch.setattr(strongclaw_ops, "run_command", fake_run_command) + monkeypatch.setattr(strongclaw_ops, "_compose_service_statuses", _compose_service_statuses) + monkeypatch.setattr(strongclaw_ops, "_wait_for_compose_service", _wait_for_compose_service) monkeypatch.setattr(strongclaw_ops, "run_command_inherited", fake_run_command_inherited) assert strongclaw_ops.sidecars_up(REPO_ROOT, repo_local_state=False) == 0 + assert waited_services == ["postgres", "postgres", "litellm", "qdrant", "neo4j"] assert inherited_calls == [ ("docker", "compose", "-f", str(compose_path), "up", "-d", "postgres"), ( @@ -520,42 +608,46 @@ def test_sidecars_up_fails_when_postgres_never_turns_healthy( compose_path = tmp_path / "compose.yaml" compose_path.write_text("services: {}\n", encoding="utf-8") inherited_calls: list[tuple[str, ...]] = [] - monotonic_values = iter((0.0, 2.0)) def _compose_path_override(_repo_root: pathlib.Path, _compose_name: str) -> pathlib.Path: return compose_path - def _monotonic() -> float: - return next(monotonic_values) - - def _sleep(_seconds: float) -> None: - return None - monkeypatch.setattr(strongclaw_ops, "ensure_docker_backend_ready", lambda: None) monkeypatch.setattr(strongclaw_ops, "_compose_path", _compose_path_override) monkeypatch.setattr(strongclaw_ops, "_compose_env", _fixed_compose_env) monkeypatch.setattr(strongclaw_ops, "wrap_command_with_varlock", _identity_varlock_command) - monkeypatch.setattr(strongclaw_ops, "POSTGRES_HEALTH_TIMEOUT_SECONDS", 1) - monkeypatch.setattr(strongclaw_ops.time, "monotonic", _monotonic) - monkeypatch.setattr(strongclaw_ops.time, "sleep", _sleep) - def fake_run_command( - command: list[str], + def _profile_flags(_path: pathlib.Path) -> dict[str, bool | str]: + return { + "usesQmd": True, + "usesHypermemory": True, + "source": "test", + } + + monkeypatch.setattr( + strongclaw_ops, + "_resolve_profile_dependency_flags", + _profile_flags, + ) + + def _wait_for_compose_service( + _execution: object, *, - cwd: pathlib.Path | None = None, - env: dict[str, str] | None = None, - timeout_seconds: int = 30, - capture_output: bool = True, - input_text: str | None = None, - check: bool = False, - ) -> ExecResult: - del cwd, env, timeout_seconds, capture_output, input_text, check - return _exec_result( - *command, - stdout=_compose_ps_output( - {"Service": "postgres", "State": "running", "Health": "starting"} - ), - ) + service_name: str, + state: str, + health: str | None = None, + timeout_seconds: int, + ) -> None: + del state, health, timeout_seconds + if service_name == "postgres": + raise strongclaw_ops.CommandError("timed out waiting for compose service 'postgres'") + + monkeypatch.setattr(strongclaw_ops, "_wait_for_compose_service", _wait_for_compose_service) + + def _empty_compose_statuses(_execution: object) -> dict[str, object]: + return {} + + monkeypatch.setattr(strongclaw_ops, "_compose_service_statuses", _empty_compose_statuses) def fake_run_command_inherited( command: list[str], @@ -568,7 +660,6 @@ def fake_run_command_inherited( inherited_calls.append(tuple(command)) return 0 - monkeypatch.setattr(strongclaw_ops, "run_command", fake_run_command) monkeypatch.setattr(strongclaw_ops, "run_command_inherited", fake_run_command_inherited) with pytest.raises( @@ -579,3 +670,125 @@ def fake_run_command_inherited( assert inherited_calls == [ ("docker", "compose", "-f", str(compose_path), "up", "-d", "postgres") ] + + +def test_wait_for_compose_service_emits_start_and_ready_events( + monkeypatch: pytest.MonkeyPatch, +) -> None: + observed_events: list[tuple[str, dict[str, object]]] = [] + execution = cast(Any, strongclaw_ops)._ComposeExecution( + repo_root=REPO_ROOT, + compose_path=pathlib.Path("/tmp/compose.yaml"), + cwd=pathlib.Path("/tmp"), + env={}, + ) + + def _compose_service_statuses_ready(_execution: object) -> dict[str, object]: + return { + "postgres": cast(Any, strongclaw_ops)._ComposeServiceStatus( + name="postgres", + state="running", + health="healthy", + ) + } + + def _emit_structured_log(event: str, payload: object) -> None: + observed_events.append((str(event), cast(dict[str, object], payload))) + + monkeypatch.setattr( + strongclaw_ops, "_compose_service_statuses", _compose_service_statuses_ready + ) + monkeypatch.setattr(strongclaw_ops, "emit_structured_log", _emit_structured_log) + + cast(Any, strongclaw_ops)._wait_for_compose_service( + execution, + service_name="postgres", + state="running", + health="healthy", + timeout_seconds=30, + ) + + assert observed_events[0][0] == "clawops.ops.sidecars.wait.start" + assert observed_events[0][1]["service"] == "postgres" + assert observed_events[1][0] == "clawops.ops.sidecars.wait.ready" + assert observed_events[1][1]["service"] == "postgres" + + +def test_gateway_start_uses_unbounded_subprocess_timeout( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: dict[str, object] = {} + + monkeypatch.setattr(strongclaw_ops, "wrap_command_with_varlock", _identity_varlock_command) + + def _run_command_inherited( + command: list[str], + *, + cwd: pathlib.Path | None = None, + env: dict[str, str] | None = None, + timeout_seconds: int | None = 1800, + ) -> int: + del env + captured["command"] = command + captured["cwd"] = cwd + captured["timeout_seconds"] = timeout_seconds + return 0 + + monkeypatch.setattr(strongclaw_ops, "run_command_inherited", _run_command_inherited) + + assert strongclaw_ops.gateway_start(REPO_ROOT) == 0 + assert captured["command"] == ["openclaw", "gateway"] + assert captured["cwd"] == REPO_ROOT + assert captured["timeout_seconds"] is None + + +def test_status_returns_structured_readiness_with_impact_classification( + monkeypatch: pytest.MonkeyPatch, +) -> None: + execution = cast(Any, strongclaw_ops)._ComposeExecution( + repo_root=REPO_ROOT, + compose_path=pathlib.Path("/tmp/compose.yaml"), + cwd=pathlib.Path("/tmp"), + env={ + "OPENCLAW_CONFIG": "/tmp/openclaw.json", + "STRONGCLAW_COMPOSE_STATE_DIR": "/tmp/compose", + }, + ) + + def _compose_execution_stub(*_args: object, **_kwargs: object) -> object: + return execution + + def _profile_flags(_path: pathlib.Path) -> dict[str, bool | str]: + return { + "usesQmd": False, + "usesHypermemory": False, + "source": "test", + } + + def _compose_service_statuses(_execution: object) -> dict[str, object]: + return { + "postgres": cast(Any, strongclaw_ops)._ComposeServiceStatus( + name="postgres", + state="running", + health="healthy", + ), + "litellm": cast(Any, strongclaw_ops)._ComposeServiceStatus( + name="litellm", + state="running", + health="healthy", + ), + } + + monkeypatch.setattr(strongclaw_ops, "_compose_execution", _compose_execution_stub) + monkeypatch.setattr(strongclaw_ops, "_resolve_profile_dependency_flags", _profile_flags) + monkeypatch.setattr(strongclaw_ops, "_compose_service_statuses", _compose_service_statuses) + + payload = strongclaw_ops.status(REPO_ROOT, repo_local_state=False) + + assert payload["ok"] is True + readiness = cast(dict[str, object], payload["readiness"]) + assert readiness["requiredReady"] is True + optional = cast(list[dict[str, object]], readiness["optional"]) + qdrant = next(entry for entry in optional if entry["service"] == "qdrant") + assert qdrant["impact"] == "degraded" + assert qdrant["ready"] is False diff --git a/tests/suites/unit/clawops/test_strongclaw_runtime.py b/tests/suites/unit/clawops/test_strongclaw_runtime.py index 76a90162..41db2401 100644 --- a/tests/suites/unit/clawops/test_strongclaw_runtime.py +++ b/tests/suites/unit/clawops/test_strongclaw_runtime.py @@ -1,12 +1,15 @@ from __future__ import annotations import pathlib +import subprocess import sys +from typing import Any, cast import pytest import clawops.strongclaw_runtime as runtime from clawops.app_paths import strongclaw_varlock_dir +from clawops.memory_profiles import MANAGED_MEMORY_PROFILE_IDS, MEMORY_PROFILES from clawops.strongclaw_runtime import CommandError, ExecResult, write_env_assignments from tests.plugins.infrastructure.context import TestContext from tests.utils.helpers.repo import REPO_ROOT @@ -224,3 +227,66 @@ def _load_env_assignments(_: pathlib.Path) -> dict[str, str]: resolved = runtime.resolve_openclaw_state_dir(REPO_ROOT, home_dir=tmp_path / "home") assert resolved == runtime_root / ".openclaw" + + +def test_run_command_inherited_omits_timeout_when_none( + test_context: TestContext, +) -> None: + recorded_kwargs: dict[str, Any] = {} + + def _fake_subprocess_run(*args: object, **kwargs: object) -> object: + del args + recorded_kwargs.update(cast(dict[str, Any], kwargs)) + return subprocess.CompletedProcess[str](args=["echo"], returncode=0) + + test_context.patch.patch_object(runtime.subprocess, "run", new=_fake_subprocess_run) + + exit_code = runtime.run_command_inherited(["echo", "ready"], timeout_seconds=None) + + assert exit_code == 0 + assert "timeout" not in recorded_kwargs + + +def test_run_command_inherited_passes_numeric_timeout( + test_context: TestContext, +) -> None: + recorded_kwargs: dict[str, Any] = {} + + def _fake_subprocess_run(*args: object, **kwargs: object) -> object: + del args + recorded_kwargs.update(cast(dict[str, Any], kwargs)) + return subprocess.CompletedProcess[str](args=["echo"], returncode=0) + + test_context.patch.patch_object(runtime.subprocess, "run", new=_fake_subprocess_run) + + exit_code = runtime.run_command_inherited(["echo", "ready"], timeout_seconds=45) + + assert exit_code == 0 + assert recorded_kwargs["timeout"] == 45 + + +def test_profile_requirement_helpers_track_managed_registry_flags() -> None: + for profile_id in MANAGED_MEMORY_PROFILE_IDS: + profile = MEMORY_PROFILES[profile_id] + assert runtime.profile_requires_qmd(profile_id) is profile.installs_qmd + assert runtime.profile_requires_lossless_claw(profile_id) is profile.installs_lossless_claw + assert ( + runtime.profile_requires_hypermemory_backend(profile_id) + is profile.enables_hypermemory_backend + ) + assert runtime.profile_requires_memory_pro_plugin(profile_id) is profile.installs_memory_pro + + +def test_profile_requirement_helpers_return_false_for_unknown_profile() -> None: + unknown = "unknown-profile-name" + + assert runtime.profile_requires_qmd(unknown) is False + assert runtime.profile_requires_lossless_claw(unknown) is False + assert runtime.profile_requires_hypermemory_backend(unknown) is False + assert runtime.profile_requires_memory_pro_plugin(unknown) is False + + +def test_profile_requirement_helpers_include_non_managed_runtime_profiles() -> None: + assert runtime.profile_requires_qmd("acp") is True + assert runtime.profile_requires_lossless_claw("acp") is False + assert runtime.profile_requires_hypermemory_backend("acp") is False diff --git a/tests/suites/unit/clawops/test_strongclaw_services.py b/tests/suites/unit/clawops/test_strongclaw_services.py index e5b1669f..66fba8eb 100644 --- a/tests/suites/unit/clawops/test_strongclaw_services.py +++ b/tests/suites/unit/clawops/test_strongclaw_services.py @@ -263,6 +263,20 @@ def test_render_service_files_include_isolated_runtime_env( assert f"Environment=STRONGCLAW_RUNTIME_ROOT={runtime_root}" in rendered_gateway +def test_render_service_files_includes_maintenance_timer_and_service( + test_context: TestContext, + tmp_path: pathlib.Path, +) -> None: + output_dir = tmp_path / "systemd" + test_context.patch.patch_object(strongclaw_services, "systemd_dir", new=lambda: output_dir) + + payload = strongclaw_services.render_service_files(REPO_ROOT, service_manager="systemd") + + assert payload["serviceManager"] == "systemd" + assert (output_dir / "openclaw-maintenance.service").exists() + assert (output_dir / "openclaw-maintenance.timer").exists() + + def test_render_service_files_omits_launchd_passthrough_env_when_unset( test_context: TestContext, tmp_path: pathlib.Path ) -> None: @@ -357,12 +371,13 @@ def fake_wait(label: str, *, persistent: bool, timeout_seconds: int) -> None: assert activated["activated"] == list(strongclaw_services.LAUNCHD_ACTIVATE_LABELS) assert calls == [ - ("activate", "gui/501:ai.openclaw.gateway"), - ("wait", "ai.openclaw.gateway:True"), ("activate", "gui/501:ai.openclaw.sidecars"), ("wait", "ai.openclaw.sidecars:False"), ("activate", "gui/501:ai.openclaw.sidecars"), ("wait", "ai.openclaw.sidecars:False"), + ("activate", "gui/501:ai.openclaw.gateway"), + ("wait", "ai.openclaw.gateway:True"), + ("activate", "gui/501:ai.openclaw.maintenance"), ] @@ -438,8 +453,82 @@ def fake_wait(label: str, *, persistent: bool, timeout_seconds: int) -> None: assert activated["activated"] == list(strongclaw_services.LAUNCHD_ACTIVATE_LABELS) assert waits == [ - (strongclaw_services.LAUNCHD_GATEWAY_LABEL, True, 45), (strongclaw_services.LAUNCHD_SIDECARS_LABEL, False, 2700), + (strongclaw_services.LAUNCHD_GATEWAY_LABEL, True, 45), + ] + + +def test_activate_services_enables_systemd_units_in_declared_order( + test_context: TestContext, + tmp_path: pathlib.Path, +) -> None: + payload: dict[str, object] = { + "ok": True, + "serviceManager": "systemd", + "outputDir": str(tmp_path / "systemd"), + "stateDir": str(tmp_path / "state"), + "renderedFiles": [], + } + observed_commands: list[tuple[str, ...]] = [] + observed_events: list[tuple[str, dict[str, object]]] = [] + + def _render_service_files(*args: object, **kwargs: object) -> dict[str, object]: + del args, kwargs + return payload + + def _run_command(command: list[str], *, timeout_seconds: int = 30) -> ExecResult: + del timeout_seconds + observed_commands.append(tuple(command)) + return ExecResult( + argv=tuple(command), + returncode=0, + stdout="", + stderr="", + duration_ms=1, + ) + + test_context.patch.patch_object( + strongclaw_services, + "render_service_files", + new=_render_service_files, + ) + test_context.patch.patch_object( + strongclaw_services, + "ensure_docker_backend_ready", + new=lambda: None, + ) + test_context.patch.patch_object( + strongclaw_services, + "run_command", + new=_run_command, + ) + + def _emit_structured_log(event: str, payload: object) -> None: + observed_events.append((str(event), cast(dict[str, object], payload))) + + test_context.patch.patch_object( + strongclaw_services, + "emit_structured_log", + new=_emit_structured_log, + ) + + payload_out = strongclaw_services.activate_services(tmp_path, service_manager="systemd") + + assert payload_out["activated"] == list(strongclaw_services.SYSTEMD_ACTIVATE_UNITS) + assert observed_commands == [ + ("systemctl", "--user", "daemon-reload"), + ("systemctl", "--user", "enable", "--now", "openclaw-sidecars.service"), + ("systemctl", "--user", "enable", "--now", "openclaw-gateway.service"), + ("systemctl", "--user", "enable", "--now", "openclaw-maintenance.timer"), + ] + assert observed_events[0] == ( + "clawops.services.activate", + {"service_manager": "systemd", "step": "daemon_reload"}, + ) + assert [event[1].get("unit") for event in observed_events[1:]] == [ + "openclaw-sidecars.service", + "openclaw-gateway.service", + "openclaw-maintenance.timer", ] diff --git a/tests/suites/unit/clawops/wrappers/test_wrapper_lifecycle.py b/tests/suites/unit/clawops/wrappers/test_wrapper_lifecycle.py index 66a88cd6..1ee606df 100644 --- a/tests/suites/unit/clawops/wrappers/test_wrapper_lifecycle.py +++ b/tests/suites/unit/clawops/wrappers/test_wrapper_lifecycle.py @@ -38,6 +38,9 @@ def test_wrapper_replays_pending_approval_without_side_effect( assert first["ok"] is True assert first["accepted"] is True assert first["executed"] is False + assert isinstance(first.get("review_artifact_path"), str) + dispatch = as_mapping(first["dispatch"], path="first.dispatch") + assert dispatch["dispatched"] is True assert second == first assert calls == [] @@ -49,6 +52,8 @@ def test_wrapper_replays_pending_approval_without_side_effect( assert persisted.review_mode == "manual" assert persisted.review_status == "pending" assert persisted.review_payload_json is not None + assert persisted.review_artifact_path == first["review_artifact_path"] + assert pathlib.Path(str(first["review_artifact_path"])).exists() @pytest.mark.parametrize("spec", SPECS, ids=[spec.name for spec in SPECS])