From a9acd1a9bc3ad44945f4ac767e20cea773082561 Mon Sep 17 00:00:00 2001 From: nutstrut Date: Mon, 18 May 2026 15:27:03 -0500 Subject: [PATCH] Harden verifier gate failure handling and fix imports --- README.md | 20 ++++++- metaclaw/api_server.py | 58 +++++++++++++++++++ metaclaw/cli.py | 2 +- metaclaw/config.py | 16 +++++ metaclaw/config_store.py | 16 +++++ metaclaw/launcher.py | 4 +- metaclaw/setup_wizard.py | 2 +- metaclaw/skill_manager.py | 20 +++++-- metaclaw/verification/__init__.py | 2 + metaclaw/verification/base.py | 27 +++++++++ metaclaw/verification/factory.py | 29 ++++++++++ metaclaw/verification/local_verifier.py | 18 ++++++ metaclaw/verification/null_verifier.py | 8 +++ metaclaw/verification/remote_http_verifier.py | 15 +++++ metaclaw/verification/settlement_witness.py | 5 ++ tests/test_verification.py | 30 ++++++++++ 16 files changed, 261 insertions(+), 11 deletions(-) create mode 100644 metaclaw/verification/__init__.py create mode 100644 metaclaw/verification/base.py create mode 100644 metaclaw/verification/factory.py create mode 100644 metaclaw/verification/local_verifier.py create mode 100644 metaclaw/verification/null_verifier.py create mode 100644 metaclaw/verification/remote_http_verifier.py create mode 100644 metaclaw/verification/settlement_witness.py create mode 100644 tests/test_verification.py diff --git a/README.md b/README.md index 91e3f25a..9e479f37 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,7 @@ Configure once with `metaclaw setup`, then `metaclaw start` brings up the proxy, | Mode | Default | What it does | |------|---------|--------------| | `skills_only` | | Proxy your LLM API. Skills injected and auto-summarized after each session. No GPU/Tinker required. | +| `verified_skills` | | Same as `skills_only`, but skill promotion is gated by a pluggable verifier (local by default, no network required). | | `rl` | | Skills + RL training (GRPO). Trains immediately when a batch is full. Optional OPD for teacher distillation. | | `auto` | ✅ | Skills + RL + smart scheduler. RL weight updates only run during sleep/idle/meeting windows. | @@ -285,7 +286,7 @@ When you start MetaClaw, the command waits until the local proxy becomes healthy Full config reference (click to expand) ```yaml -mode: auto # "auto" | "rl" | "skills_only" +mode: auto # "auto" | "rl" | "skills_only" | "verified_skills" claw_type: openclaw # "openclaw" | "copaw" | "ironclaw" | "picoclaw" | "zeroclaw" | "nanoclaw" | "nemoclaw" | "hermes" | "none" llm: @@ -574,3 +575,20 @@ MetaClaw builds on top of the following open-source projects: ## 📄 License This project is licensed under the [MIT License](LICENSE). + + +### Verified skills (optional) + +`verified_skills` adds a verifier gate before a generated skill is persisted. By default, use: + +```yaml +mode: verified_skills +verification: + enabled: true + verifier: local + require_pass_for_promotion: true + allow_indeterminate_promotion: false + redact_inputs: true +``` + +Remote verifiers are optional (`remote_http` and `settlement_witness`). SettlementWitness is supported as an adapter only and is **not required**. diff --git a/metaclaw/api_server.py b/metaclaw/api_server.py index 0d3da241..430b4873 100644 --- a/metaclaw/api_server.py +++ b/metaclaw/api_server.py @@ -25,6 +25,7 @@ import re import threading import time +from datetime import datetime from itertools import count from typing import Any, Optional @@ -34,6 +35,8 @@ from openai import OpenAI from .config import MetaClawConfig +from .verification import build_verifier_safe +from .verification.base import VerificationResult, VerificationVerdict from .data_formatter import ConversationSample from .memory.scope import base_scope, derive_memory_scope from .prm_scorer import PRMScorer @@ -516,6 +519,7 @@ def __init__( self.prm_scorer = prm_scorer self.skill_evolver = skill_evolver self.memory_manager = memory_manager + self._verifier = build_verifier_safe(config, logger=logger) # Optional LastRequestTracker for scheduler idle detection self._last_request_tracker = last_request_tracker @@ -2141,9 +2145,63 @@ async def _evolve_skills_for_session(self, turns: list[dict]): added = 0 for skill in new_skills: category = skill.get("category", "general") + if self.config.mode == "verified_skills": + task_id, spec, output, context = self._build_verification_packet(skill) + vr = self._verifier.verify(task_id=task_id, spec=spec, output=output, context=context) + skill.setdefault("metadata", {})["promotion_source"] = "verified_skills" + skill["metadata"]["verification"] = self._verification_metadata(vr) + if ( + vr.verifier_id == "null" + and vr.reason_code == "VERIFIER_CONFIG_ERROR" + and self.config.verification.require_pass_for_promotion + and not self.config.verification.allow_fallback_promotion_on_config_error + ): + vr = VerificationResult( + verdict=VerificationVerdict.INDETERMINATE, + verifier_id="null", + reason_code="VERIFIER_CONFIG_ERROR", + message="Verifier configuration failed.", + ) + if self.config.verification.require_pass_for_promotion and vr.verdict != VerificationVerdict.PASS: + bucket = "pending_review" if vr.verdict == VerificationVerdict.INDETERMINATE and not self.config.verification.allow_indeterminate_promotion else "rejected" + self._audit_skill_decision(bucket, skill, vr) + continue added += self.skill_manager.add_skills([skill], category=category) logger.info("[SkillEvolver] session analysis added %d new skills", added) + + def _build_verification_packet(self, skill: dict) -> tuple[str, dict, dict, dict]: + from datetime import datetime + ts = datetime.utcnow().replace(microsecond=0).isoformat() + "Z" + task_id = f"skill-promotion-{skill.get('name','unknown')}-{ts}" + spec = {"verification_type": "skill_promotion", "skill_name": skill.get("name", ""), "criteria": ["non_empty", "has_evaluation"]} + output = { + "skill_name": skill.get("name", ""), + "skill_content": skill.get("content", ""), + "evaluation_criteria": ["generated_from_session"], + "evaluation_result": {"passed": True, "criteria_satisfied": ["generated_from_session"]}, + } + existing_contents = [] + if self.skill_manager: + for group in self.skill_manager.skills.values(): + if isinstance(group, list): + existing_contents.extend([x.get("content", "") for x in group if isinstance(x, dict)]) + elif isinstance(group, dict): + for arr in group.values(): + existing_contents.extend([x.get("content", "") for x in arr if isinstance(x, dict)]) + context = {"source": "skills_only_session_analysis", "existing_skill_contents": existing_contents, "redacted": True} + return task_id, spec, output, context + + def _verification_metadata(self, vr: VerificationResult) -> dict: + return {"verified": vr.verdict == VerificationVerdict.PASS and vr.verifier_id != "null", "verifier_id": vr.verifier_id, "verdict": vr.verdict.value, "confidence": vr.confidence, "reason_code": vr.reason_code, "receipt_id": vr.receipt_id, "verifier_kid": vr.verifier_kid} + + def _audit_skill_decision(self, bucket: str, skill: dict, vr: VerificationResult) -> None: + base = os.path.join(self.config.skills_dir, bucket) + os.makedirs(base, exist_ok=True) + path = os.path.join(base, f"{skill.get('name','unknown')}.json") + with open(path, "w", encoding="utf-8") as f: + json.dump({"skill_name": skill.get("name"), "timestamp": datetime.utcnow().isoformat() + "Z", "promotion_source": self.config.mode, "verifier_id": vr.verifier_id, "verdict": vr.verdict.value, "reason_code": vr.reason_code, "message": vr.message, "receipt_id": vr.receipt_id}, f) + # ------------------------------------------------------------------ # # Skill injection # # ------------------------------------------------------------------ # diff --git a/metaclaw/cli.py b/metaclaw/cli.py index 6e12f701..fd7dc12a 100644 --- a/metaclaw/cli.py +++ b/metaclaw/cli.py @@ -69,7 +69,7 @@ def setup(): @metaclaw.command() @click.option( "--mode", - type=click.Choice(["auto", "skills_only", "rl"]), + type=click.Choice(["auto", "skills_only", "verified_skills", "rl"]), default=None, help="Override operating mode for this session.", ) diff --git a/metaclaw/config.py b/metaclaw/config.py index 588d3a92..b66f6901 100644 --- a/metaclaw/config.py +++ b/metaclaw/config.py @@ -5,6 +5,19 @@ """ from dataclasses import dataclass, field +@dataclass +class VerificationConfig: + enabled: bool = False + verifier: str = "local" + require_pass_for_promotion: bool = True + allow_indeterminate_promotion: bool = False + rollback_threshold: int = 3 + store_receipts: bool = True + redact_inputs: bool = True + endpoint: str = "" + agent_id: str = "" + timeout_seconds: int = 10 + allow_fallback_promotion_on_config_error: bool = False @dataclass @@ -130,6 +143,7 @@ class MetaClawConfig: # "auto" — v0.3: RL + scheduler (trains during idle/sleep windows) # "rl" — v0.2: RL without scheduler (trains immediately on full batch) # "skills_only" — proxy + skill injection only (no Tinker, no RL) + # "verified_skills" — like skills_only, but gate promotion via verifier mode: str = "auto" # When True (RL/auto mode only), the trainer does NOT run its own # collection loop. Instead it waits for ``metaclaw train-step`` CLI @@ -176,3 +190,5 @@ class MetaClawConfig: # WeChat (official openclaw-weixin plugin, auto-installed) # # ------------------------------------------------------------------ # wechat_enabled: bool = False + + verification: VerificationConfig = field(default_factory=VerificationConfig) diff --git a/metaclaw/config_store.py b/metaclaw/config_store.py index 448a1e2f..21d17de7 100644 --- a/metaclaw/config_store.py +++ b/metaclaw/config_store.py @@ -79,6 +79,19 @@ "wechat": { "enabled": False, }, + "verification": { + "enabled": False, + "verifier": "local", + "require_pass_for_promotion": True, + "allow_indeterminate_promotion": False, + "rollback_threshold": 3, + "store_receipts": True, + "redact_inputs": True, + "endpoint": "", + "agent_id": "", + "timeout_seconds": 10, + "allow_fallback_promotion_on_config_error": False, + }, } @@ -185,6 +198,7 @@ def to_metaclaw_config(self) -> MetaClawConfig: sched = data.get("scheduler", {}) sched_cal = sched.get("calendar", {}) wx = data.get("wechat", {}) + verification = data.get("verification", {}) mode = data.get("mode", "auto") rl_enabled = mode in ("rl", "auto") or bool(rl.get("enabled", False)) @@ -236,6 +250,7 @@ def to_metaclaw_config(self) -> MetaClawConfig: except Exception: pass + from .config import VerificationConfig return MetaClawConfig( # Mode mode=mode, @@ -329,6 +344,7 @@ def describe(self) -> str: rl = data.get("rl", {}) memory = data.get("memory", {}) wx = data.get("wechat", {}) + verification = data.get("verification", {}) mode = data.get("mode", "auto") lines = [ f"mode: {mode}", diff --git a/metaclaw/launcher.py b/metaclaw/launcher.py index 7d5a2199..47c85d3e 100644 --- a/metaclaw/launcher.py +++ b/metaclaw/launcher.py @@ -63,7 +63,7 @@ async def start(self): if mode == "auto": cfg.scheduler_enabled = True await self._start_rl(cfg) - elif mode == "skills_only": + elif mode in ("skills_only", "verified_skills"): await self._start_skills_only(cfg) else: await self._start_rl(cfg) @@ -341,7 +341,7 @@ def _setup_evolver_env(self, cfg): pre-existing OPENAI_* env vars (force-assign). In other modes the existing env vars win (setdefault), preserving previous behaviour. """ - force = cfg.mode == "skills_only" + force = cfg.mode in ("skills_only", "verified_skills") _set = (lambda k, v: os.environ.__setitem__(k, v)) if force else os.environ.setdefault if cfg.evolver_api_base: _set("OPENAI_BASE_URL", cfg.evolver_api_base) diff --git a/metaclaw/setup_wizard.py b/metaclaw/setup_wizard.py index 995ab6ab..27ebe75c 100644 --- a/metaclaw/setup_wizard.py +++ b/metaclaw/setup_wizard.py @@ -128,7 +128,7 @@ def run(self): current_mode = existing.get("mode", "auto") mode = _prompt_choice( "Operating mode", - ["auto", "skills_only", "rl"], + ["auto", "skills_only", "verified_skills", "rl"], default=current_mode, ) diff --git a/metaclaw/skill_manager.py b/metaclaw/skill_manager.py index 490e4866..e185e492 100644 --- a/metaclaw/skill_manager.py +++ b/metaclaw/skill_manager.py @@ -29,6 +29,7 @@ import glob import logging import os +import json from typing import Any, Dict, List, Optional logger = logging.getLogger(__name__) @@ -127,12 +128,15 @@ def _parse_skill_md(path: str) -> Optional[Dict[str, Any]]: logger.warning("[SkillManager] skipping %s — missing name or description", path) return None - return { - "name": name, - "description": description, - "category": category, - "content": body, - } + result = {"name": name, "description": description, "category": category, "content": body} + meta_path = os.path.join(os.path.dirname(path), "metadata.json") + if os.path.exists(meta_path): + try: + with open(meta_path, encoding="utf-8") as mf: + result["metadata"] = json.load(mf) + except Exception: + pass + return result # ------------------------------------------------------------------ # @@ -495,6 +499,10 @@ def _write_skill_md(self, skill: dict) -> None: with open(path, "w", encoding="utf-8") as f: f.write(text) logger.info("[SkillManager] wrote skill file: %s", path) + metadata = skill.get("metadata") + if isinstance(metadata, dict): + with open(os.path.join(skill_dir, "metadata.json"), "w", encoding="utf-8") as mf: + json.dump(metadata, mf, ensure_ascii=False, indent=2) except OSError as e: logger.warning("[SkillManager] could not write %s: %s", path, e) diff --git a/metaclaw/verification/__init__.py b/metaclaw/verification/__init__.py new file mode 100644 index 00000000..bf8499ae --- /dev/null +++ b/metaclaw/verification/__init__.py @@ -0,0 +1,2 @@ +from .base import VerificationResult, VerificationVerdict, VerifierInterface +from .factory import build_verifier, build_verifier_safe diff --git a/metaclaw/verification/base.py b/metaclaw/verification/base.py new file mode 100644 index 00000000..aed2feaf --- /dev/null +++ b/metaclaw/verification/base.py @@ -0,0 +1,27 @@ +from __future__ import annotations +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, Dict, Optional + +class VerificationVerdict(str, Enum): + PASS = "PASS" + FAIL = "FAIL" + INDETERMINATE = "INDETERMINATE" + +@dataclass +class VerificationResult: + verdict: VerificationVerdict + confidence: float = 0.0 + verifier_id: str = "unknown" + reason_code: Optional[str] = None + message: Optional[str] = None + receipt_id: Optional[str] = None + verifier_kid: Optional[str] = None + receipt: Optional[Dict[str, Any]] = None + evidence: Dict[str, Any] = field(default_factory=dict) + +class VerifierInterface(ABC): + @abstractmethod + def verify(self, *, task_id: str, spec: Dict[str, Any], output: Dict[str, Any], context: Optional[Dict[str, Any]] = None) -> VerificationResult: + raise NotImplementedError diff --git a/metaclaw/verification/factory.py b/metaclaw/verification/factory.py new file mode 100644 index 00000000..0b86bee2 --- /dev/null +++ b/metaclaw/verification/factory.py @@ -0,0 +1,29 @@ +import logging +from .local_verifier import LocalVerifier +from .null_verifier import NullVerifier +from .remote_http_verifier import RemoteHTTPVerifier +from .settlement_witness import SettlementWitnessVerifier + + +def build_verifier(config): + vc = config.verification + if not vc.enabled: + return NullVerifier() + if vc.verifier == "local": + return LocalVerifier() + if vc.verifier == "remote_http": + return RemoteHTTPVerifier(vc) + if vc.verifier == "settlement_witness": + return SettlementWitnessVerifier(vc) + if vc.verifier == "null": + return NullVerifier() + raise ValueError(f"Unknown verifier: {vc.verifier}") + + +def build_verifier_safe(config, logger=None): + logger = logger or logging.getLogger(__name__) + try: + return build_verifier(config) + except Exception as exc: + logger.warning("Verifier configuration failed; falling back to NullVerifier: %s", exc) + return NullVerifier(reason_code="VERIFIER_CONFIG_ERROR") diff --git a/metaclaw/verification/local_verifier.py b/metaclaw/verification/local_verifier.py new file mode 100644 index 00000000..c4d53c1a --- /dev/null +++ b/metaclaw/verification/local_verifier.py @@ -0,0 +1,18 @@ +from .base import VerificationResult, VerificationVerdict, VerifierInterface + +class LocalVerifier(VerifierInterface): + def verify(self, *, task_id, spec, output, context=None): + context = context or {} + skill_name = (output.get("skill_name") or "").strip() + content = (output.get("skill_content") or "").strip() + if not skill_name or not content: + return VerificationResult(VerificationVerdict.FAIL, verifier_id="local", reason_code="MALFORMED_SKILL") + if not output.get("evaluation_criteria") or output.get("evaluation_result") is None: + return VerificationResult(VerificationVerdict.FAIL, verifier_id="local", reason_code="MISSING_EVAL") + existing_contents = context.get("existing_skill_contents") or [] + if content in existing_contents: + return VerificationResult(VerificationVerdict.FAIL, verifier_id="local", reason_code="DUPLICATE_SKILL_CONTENT") + satisfied = (output.get("evaluation_result") or {}).get("criteria_satisfied") or [] + if not satisfied: + return VerificationResult(VerificationVerdict.INDETERMINATE, verifier_id="local", reason_code="INSUFFICIENT_IMPROVEMENT_EVIDENCE") + return VerificationResult(VerificationVerdict.PASS, verifier_id="local", confidence=0.72, reason_code="LOCAL_CHECKS_PASSED") diff --git a/metaclaw/verification/null_verifier.py b/metaclaw/verification/null_verifier.py new file mode 100644 index 00000000..8518f9b1 --- /dev/null +++ b/metaclaw/verification/null_verifier.py @@ -0,0 +1,8 @@ +from .base import VerificationResult, VerificationVerdict, VerifierInterface + +class NullVerifier(VerifierInterface): + def __init__(self, reason_code: str | None = None): + self.reason_code = reason_code + + def verify(self, *, task_id, spec, output, context=None): + return VerificationResult(verdict=VerificationVerdict.PASS, verifier_id="null", reason_code=self.reason_code) diff --git a/metaclaw/verification/remote_http_verifier.py b/metaclaw/verification/remote_http_verifier.py new file mode 100644 index 00000000..2507a7a4 --- /dev/null +++ b/metaclaw/verification/remote_http_verifier.py @@ -0,0 +1,15 @@ +from __future__ import annotations +import requests +from .base import VerificationResult, VerificationVerdict, VerifierInterface + +class RemoteHTTPVerifier(VerifierInterface): + def __init__(self, config): + self.endpoint = config.endpoint + self.timeout = config.timeout_seconds + def verify(self, *, task_id, spec, output, context=None): + try: + r = requests.post(self.endpoint, json={"task_id": task_id, "spec": spec, "output": output, "context": context or {}}, timeout=self.timeout) + data = r.json() + return VerificationResult(VerificationVerdict(data.get("verdict", "INDETERMINATE")), confidence=float(data.get("confidence", 0.0)), verifier_id=data.get("verifier_id", "remote_http"), reason_code=data.get("reason_code"), message=data.get("message"), receipt_id=data.get("receipt_id"), verifier_kid=data.get("verifier_kid"), receipt=data.get("receipt"), evidence=data.get("evidence") or {}) + except Exception: + return VerificationResult(VerificationVerdict.INDETERMINATE, verifier_id="remote_http", reason_code="VERIFIER_UNAVAILABLE", message="Remote verifier unavailable or timed out.") diff --git a/metaclaw/verification/settlement_witness.py b/metaclaw/verification/settlement_witness.py new file mode 100644 index 00000000..af037fc7 --- /dev/null +++ b/metaclaw/verification/settlement_witness.py @@ -0,0 +1,5 @@ +from .remote_http_verifier import RemoteHTTPVerifier + +class SettlementWitnessVerifier(RemoteHTTPVerifier): + def verify(self, *, task_id, spec, output, context=None): + return super().verify(task_id=task_id, spec=spec, output=output, context=context) diff --git a/tests/test_verification.py b/tests/test_verification.py new file mode 100644 index 00000000..b3a5841d --- /dev/null +++ b/tests/test_verification.py @@ -0,0 +1,30 @@ +from metaclaw.config import MetaClawConfig, VerificationConfig +from metaclaw.verification.base import VerificationVerdict +from metaclaw.verification.factory import build_verifier, build_verifier_safe +from metaclaw.verification.local_verifier import LocalVerifier +from metaclaw.verification.null_verifier import NullVerifier + + +def test_null_verifier_pass(): + r = NullVerifier().verify(task_id='t', spec={}, output={}) + assert r.verdict == VerificationVerdict.PASS + + +def test_local_verifier_cases(): + v = LocalVerifier() + fail = v.verify(task_id='t', spec={}, output={'skill_name':'a','skill_content':'','evaluation_criteria':[1],'evaluation_result':{}}, context={}) + assert fail.verdict == VerificationVerdict.FAIL + dup = v.verify(task_id='t', spec={}, output={'skill_name':'a','skill_content':'x','evaluation_criteria':[1],'evaluation_result':{'criteria_satisfied':['a']}}, context={'existing_skill_contents':['x']}) + assert dup.verdict == VerificationVerdict.FAIL + ind = v.verify(task_id='t', spec={}, output={'skill_name':'a','skill_content':'x','evaluation_criteria':[1],'evaluation_result':{}}, context={}) + assert ind.verdict == VerificationVerdict.INDETERMINATE + ok = v.verify(task_id='t', spec={}, output={'skill_name':'a','skill_content':'x','evaluation_criteria':[1],'evaluation_result':{'criteria_satisfied':['a']}}, context={}) + assert ok.verdict == VerificationVerdict.PASS + + +def test_factory_selection_and_fallback(): + cfg = MetaClawConfig(mode='verified_skills', verification=VerificationConfig(enabled=True, verifier='local')) + assert build_verifier(cfg).__class__.__name__ == 'LocalVerifier' + cfg.verification.verifier = 'unknown' + fv = build_verifier_safe(cfg) + assert isinstance(fv, NullVerifier)