From d46c998a44be348feba9b63d3fbd160dd6b49b52 Mon Sep 17 00:00:00 2001 From: JTInventory Date: Sun, 28 Jun 2026 22:52:57 +0000 Subject: [PATCH 1/2] feat(cognee): add local lookup manifest checker --- bin/fm-cognee-lookup.sh | 85 ++++++++++ bin/fm-cognee-manifest-check.sh | 275 ++++++++++++++++++++++++++++++++ docs/scripts.md | 2 + tests/fm-cognee-lookup.test.sh | 188 ++++++++++++++++++++++ 4 files changed, 550 insertions(+) create mode 100755 bin/fm-cognee-lookup.sh create mode 100755 bin/fm-cognee-manifest-check.sh create mode 100755 tests/fm-cognee-lookup.test.sh diff --git a/bin/fm-cognee-lookup.sh b/bin/fm-cognee-lookup.sh new file mode 100755 index 00000000..a984445d --- /dev/null +++ b/bin/fm-cognee-lookup.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# Local dry-run wrapper for future Cognee lookup integration. +# +# This script deliberately does not call Cognee. It accepts a local answer +# fixture, treats it as an untrusted hint, and asks the local manifest checker to +# prove whether any cited source can be reopened and checksum-verified. +set -eu + +usage() { + cat >&2 <<'USAGE' +usage: fm-cognee-lookup.sh --dry-run --query [--manifest --answer-file ] + +No live mode exists yet. Without --dry-run this command fails closed before any +network, environment, MCP, or config access can happen. +USAGE +} + +die() { + echo "error: $*" >&2 + exit 1 +} + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DRY_RUN=false +QUERY= +MANIFEST= +ANSWER_FILE= + +while [ $# -gt 0 ]; do + case "$1" in + --dry-run) + DRY_RUN=true + shift + ;; + --query) + QUERY=${2:-} + [ -n "$QUERY" ] || die "--query requires text" + shift 2 + ;; + --manifest) + MANIFEST=${2:-} + [ -n "$MANIFEST" ] || die "--manifest requires a path" + shift 2 + ;; + --answer-file) + ANSWER_FILE=${2:-} + [ -n "$ANSWER_FILE" ] || die "--answer-file requires a path" + shift 2 + ;; + --help|-h) + usage + exit 0 + ;; + *) + usage + exit 1 + ;; + esac +done + +if ! "$DRY_RUN"; then + echo "label=blocked_missing_proof reason=live_cognee_lookup_not_implemented external_action_authorized=false" >&2 + exit 2 +fi + +[ -n "$QUERY" ] || die "--query is required in dry-run mode" + +query_hash=$(printf '%s' "$QUERY" | sha256sum | awk '{print $1}') +query_bytes=$(printf '%s' "$QUERY" | wc -c | tr -d ' ') + +echo "mode=dry-run" +echo "query_sha256=$query_hash" +echo "query_bytes=$query_bytes" +echo "cognee_answer_status=hint_only" +echo "external_action_authorized=false" + +if [ -z "$MANIFEST" ] && [ -z "$ANSWER_FILE" ]; then + echo "label=hint_only reason=no_manifest_or_answer_fixture external_action_authorized=false" + exit 0 +fi + +[ -n "$MANIFEST" ] || die "--manifest is required when --answer-file is used" +[ -n "$ANSWER_FILE" ] || die "--answer-file is required when --manifest is used" + +"$SCRIPT_DIR/fm-cognee-manifest-check.sh" --manifest "$MANIFEST" --answer-file "$ANSWER_FILE" diff --git a/bin/fm-cognee-manifest-check.sh b/bin/fm-cognee-manifest-check.sh new file mode 100755 index 00000000..15735726 --- /dev/null +++ b/bin/fm-cognee-manifest-check.sh @@ -0,0 +1,275 @@ +#!/usr/bin/env bash +# Validate local Cognee manifest rows and verify answer references against them. +# +# This is intentionally local-only. It never calls Cognee, never mutates config, +# and never treats a generated answer as proof without reopening the local source. +set -eu + +usage() { + cat >&2 <<'USAGE' +usage: fm-cognee-manifest-check.sh --manifest [--validate | --answer-file ] + +The manifest must be TSV with the row fields from the Cognee import policy. +Answer verification looks for SOURCE_ID=, SOURCE_PATH=, and SEED_FILE= labels. +USAGE +} + +die() { + echo "error: $*" >&2 + exit 1 +} + +MANIFEST= +ANSWER_FILE= +VALIDATE=false + +while [ $# -gt 0 ]; do + case "$1" in + --manifest) + MANIFEST=${2:-} + [ -n "$MANIFEST" ] || die "--manifest requires a path" + shift 2 + ;; + --answer-file) + ANSWER_FILE=${2:-} + [ -n "$ANSWER_FILE" ] || die "--answer-file requires a path" + shift 2 + ;; + --validate) + VALIDATE=true + shift + ;; + --help|-h) + usage + exit 0 + ;; + *) + usage + exit 1 + ;; + esac +done + +[ -n "$MANIFEST" ] || { usage; exit 1; } +[ -f "$MANIFEST" ] || die "manifest not found: $MANIFEST" +if [ -n "$ANSWER_FILE" ]; then + [ -f "$ANSWER_FILE" ] || die "answer file not found: $ANSWER_FILE" +fi +if ! "$VALIDATE" && [ -z "$ANSWER_FILE" ]; then + usage + exit 1 +fi + +required_fields='row_id source_group source_path source_truth_pointer source_kind recommended_tier decision_status redaction_status redaction_notes sensitivity_label stale_risk supersession_check source_size_bytes source_mtime_utc source_sha256 estimated_words estimated_tokens estimated_cost_formula import_text_prefix raw_readback_status verification_status' +allowed_redaction=' passed redacted summarized path_index_only ' +allowed_stale=' low medium high live_external unknown ' +allowed_raw=' not_attempted passed failed_404 not_trusted not_applicable ' +allowed_verification=' not_imported verified_local_source hint_only failed stale ' + +declare -A IDX +IFS=$'\t' read -r -a HEADER < "$MANIFEST" || die "manifest is empty" +for i in "${!HEADER[@]}"; do + IDX["${HEADER[$i]}"]=$i +done + +for field in $required_fields; do + [ -n "${IDX[$field]+x}" ] || die "manifest missing required field: $field" +done + +field_value() { + local name=$1 idx=${IDX[$1]} + printf '%s' "${COLS[$idx]:-}" +} + +sha256_file() { + sha256sum "$1" | awk '{print $1}' +} + +row_matches_ref() { + local kind=$1 value=$2 + local row_id source_path import_text cognee_source_id cognee_data_id cognee_chunk_ids summary_path seed_name + row_id=$(field_value row_id) + source_path=$(field_value source_path) + import_text=$(field_value import_text_prefix) + cognee_source_id= + cognee_data_id= + cognee_chunk_ids= + summary_path= + [ -n "${IDX[cognee_source_id]+x}" ] && cognee_source_id=$(field_value cognee_source_id) + [ -n "${IDX[cognee_data_id]+x}" ] && cognee_data_id=$(field_value cognee_data_id) + [ -n "${IDX[cognee_chunk_ids]+x}" ] && cognee_chunk_ids=$(field_value cognee_chunk_ids) + [ -n "${IDX[summary_path]+x}" ] && summary_path=$(field_value summary_path) + seed_name=$(basename "$source_path") + + case "$kind" in + SOURCE_ID) + [ "$value" = "$row_id" ] && return 0 + [ -n "$cognee_source_id" ] && [ "$value" = "$cognee_source_id" ] && return 0 + [ -n "$cognee_data_id" ] && [ "$value" = "$cognee_data_id" ] && return 0 + case " $cognee_chunk_ids " in *" $value "*) return 0 ;; esac + case "$import_text" in *"SOURCE_ID=$value"*|*"SOURCE_ID: $value"*) return 0 ;; esac + ;; + SOURCE_PATH) + [ "$value" = "$source_path" ] && return 0 + [ -n "$summary_path" ] && [ "$value" = "$summary_path" ] && return 0 + ;; + SEED_FILE) + [ "$value" = "$seed_name" ] && return 0 + case "$import_text" in *"SEED_FILE=$value"*|*"SEED_FILE: $value"*) return 0 ;; esac + ;; + esac + return 1 +} + +validate_current_row() { + local field value source_path source_path_lc expected actual actual_size redaction stale raw verification size + + for field in $required_fields; do + value=$(field_value "$field") + [ -n "$value" ] || { + echo "label=blocked_missing_proof reason=missing_$field external_action_authorized=false" + return 1 + } + done + + source_path=$(field_value source_path) + source_path_lc=$(printf '%s' "$source_path" | tr '[:upper:]' '[:lower:]') + case "$source_path_lc" in + *secret*|*token*|*api_key*|*password*|*credential*|*auth*|*bearer*|*cookie*|*private_key*|*.env*|*session*|*oauth*|*signed*) + echo "label=blocked_missing_proof reason=path_risk_scan_failed external_action_authorized=false" + return 1 + ;; + esac + + case "$source_path" in + /*) ;; + *) + echo "label=blocked_missing_proof reason=source_path_not_absolute source_path=$source_path external_action_authorized=false" + return 1 + ;; + esac + [ -r "$source_path" ] || { + echo "label=blocked_missing_proof reason=source_unreadable source_path=$source_path external_action_authorized=false" + return 1 + } + + redaction=$(field_value redaction_status) + case "$allowed_redaction" in + *" $redaction "*) ;; + *) + echo "label=blocked_missing_proof reason=redaction_not_passed redaction_status=$redaction source_path=$source_path external_action_authorized=false" + return 1 + ;; + esac + + stale=$(field_value stale_risk) + case "$allowed_stale" in + *" $stale "*) ;; + *) + echo "label=blocked_missing_proof reason=invalid_stale_risk stale_risk=$stale source_path=$source_path external_action_authorized=false" + return 1 + ;; + esac + + raw=$(field_value raw_readback_status) + case "$allowed_raw" in + *" $raw "*) ;; + *) + echo "label=blocked_missing_proof reason=invalid_raw_readback_status raw_readback_status=$raw source_path=$source_path external_action_authorized=false" + return 1 + ;; + esac + + verification=$(field_value verification_status) + case "$allowed_verification" in + *" $verification "*) ;; + *) + echo "label=blocked_missing_proof reason=invalid_verification_status verification_status=$verification source_path=$source_path external_action_authorized=false" + return 1 + ;; + esac + + size=$(field_value source_size_bytes) + actual_size=$(wc -c < "$source_path" | tr -d ' ') + [ "$size" = "$actual_size" ] || { + echo "label=blocked_missing_proof reason=size_mismatch source_path=$source_path external_action_authorized=false" + return 1 + } + + expected=$(field_value source_sha256) + actual=$(sha256_file "$source_path") + [ "$expected" = "$actual" ] || { + echo "label=blocked_missing_proof reason=checksum_mismatch source_path=$source_path external_action_authorized=false" + return 1 + } + + return 0 +} + +sanitize_ref() { + # Strip wrapping punctuation without printing answer text. + sed -E 's/^[`"'\''[:space:]]+//; s/[`"'\'',.;:)[:space:]]+$//' +} + +REFS=$(mktemp "${TMPDIR:-/tmp}/fm-cognee-refs.XXXXXX") +trap 'rm -f "$REFS"' EXIT + +if [ -n "$ANSWER_FILE" ]; then + { + grep -Eoh 'SOURCE_ID[[:space:]]*[:=][[:space:]]*[^[:space:],;)]+' "$ANSWER_FILE" 2>/dev/null \ + | sed -E 's/^SOURCE_ID[[:space:]]*[:=][[:space:]]*//' | sanitize_ref | awk 'NF {print "SOURCE_ID\t"$0}' + grep -Eoh 'SOURCE_PATH[[:space:]]*[:=][[:space:]]*[^[:space:],;)]+' "$ANSWER_FILE" 2>/dev/null \ + | sed -E 's/^SOURCE_PATH[[:space:]]*[:=][[:space:]]*//' | sanitize_ref | awk 'NF {print "SOURCE_PATH\t"$0}' + grep -Eoh 'SEED_FILE[[:space:]]*[:=][[:space:]]*[^[:space:],;)]+' "$ANSWER_FILE" 2>/dev/null \ + | sed -E 's/^SEED_FILE[[:space:]]*[:=][[:space:]]*//' | sanitize_ref | awk 'NF {print "SEED_FILE\t"$0}' + } | sort -u > "$REFS" +fi + +if "$VALIDATE"; then + ok=true + while IFS=$'\t' read -r -a COLS || [ "${#COLS[@]}" -gt 0 ]; do + [ "${#COLS[@]}" -eq 0 ] && continue + validate_current_row || ok=false + done < <(tail -n +2 "$MANIFEST") + "$ok" || exit 1 + echo "manifest_status=valid external_action_authorized=false" +fi + +if [ -n "$ANSWER_FILE" ]; then + if [ ! -s "$REFS" ]; then + echo "label=blocked_missing_proof reason=no_usable_citations external_action_authorized=false" + exit 3 + fi + + found=false + failed=false + while IFS=$'\t' read -r -a COLS || [ "${#COLS[@]}" -gt 0 ]; do + [ "${#COLS[@]}" -eq 0 ] && continue + while IFS=$'\t' read -r ref_kind ref_value; do + if row_matches_ref "$ref_kind" "$ref_value"; then + if validate_current_row >/dev/null; then + found=true + row_id=$(field_value row_id) + source_path=$(field_value source_path) + stale=$(field_value stale_risk) + verification=$(field_value verification_status) + label=verified_local_source + case "$stale" in high|live_external) label=stale_warning ;; esac + echo "label=$label row_id=$row_id matched_ref=$ref_kind source_path=$source_path stale_risk=$stale manifest_verification_status=$verification external_action_authorized=false" + else + failed=true + validate_current_row || true + fi + break + fi + done < "$REFS" + done < <(tail -n +2 "$MANIFEST") + + if ! "$found"; then + if "$failed"; then + exit 1 + fi + echo "label=blocked_missing_proof reason=manifest_reference_not_found external_action_authorized=false" + exit 3 + fi +fi diff --git a/docs/scripts.md b/docs/scripts.md index 929258e2..722b53cf 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -18,6 +18,8 @@ Each file also starts with a short header comment. | `fm-route.sh` | Classify a task into a deterministic route profile, harness, model, effort, reason, override, and risk flags without changing spawn behavior | | `fm-merge-local.sh` | Fast-forward a `local-only` project's local default branch after approval | | `fm-review-diff.sh` | Review a crewmate branch against the authoritative base, with optional `--stat` output | +| `fm-cognee-lookup.sh` | Local-only dry-run wrapper for future Cognee lookup; treats answer fixtures as hints and delegates source proof to the manifest checker | +| `fm-cognee-manifest-check.sh` | Validate TSV Cognee manifest rows and verify `SOURCE_ID`, `SOURCE_PATH`, or `SEED_FILE` answer references against reopened local files | | `fm-marker-lib.sh` | Shared from-firstmate request marker and detector sourced by `fm-send.sh`, `fm-brief.sh`, and tests | | `fm-watch-arm.sh` | Verified per-home watcher re-arm; reports `started`, `healthy`, or `FAILED`; `--restart` relaunches only this home's watcher | | `fm-watch-session.sh` | Home-scoped durable active watcher runner with `--start`, `--status`, `--stop`, `--foreground`, and `--tmux` helpers | diff --git a/tests/fm-cognee-lookup.test.sh b/tests/fm-cognee-lookup.test.sh new file mode 100755 index 00000000..ee993e3a --- /dev/null +++ b/tests/fm-cognee-lookup.test.sh @@ -0,0 +1,188 @@ +#!/usr/bin/env bash +# Behavior tests for local-only Cognee lookup and manifest verification. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +TMP_ROOT=$(fm_test_tmproot fm-cognee-lookup-tests) + +write_manifest() { + local manifest=$1 row_id=$2 source_path=$3 source_sha=$4 redaction=${5:-passed} stale=${6:-low} + local size mtime + size=$(wc -c < "$source_path" | tr -d ' ') + mtime=$(date -u -r "$source_path" '+%Y-%m-%dT%H:%M:%SZ') + { + printf '%s\t' row_id source_group source_path source_truth_pointer source_kind recommended_tier decision_status redaction_status redaction_notes sensitivity_label stale_risk supersession_check source_size_bytes source_mtime_utc source_sha256 estimated_words estimated_tokens estimated_cost_formula import_text_prefix raw_readback_status verification_status cognee_source_id + printf '\n' + printf '%s\t' "$row_id" firstmate_reports "$source_path" "$source_path" report direct_import allowed "$redaction" "test scan" internal "$stale" checked_current "$size" "$mtime" "$source_sha" 12 16 "words * 1.3" "SOURCE_ID=$row_id SOURCE_PATH=$source_path SEED_FILE=$(basename "$source_path")" not_trusted verified_local_source "$row_id" + printf '\n' + } > "$manifest" +} + +test_dry_run_blocks_live_mode() { + local out code + out=$("$ROOT/bin/fm-cognee-lookup.sh" --query "find cognee proof" 2>&1) + code=$? + expect_code 2 "$code" "live lookup fails closed" + assert_contains "$out" "reason=live_cognee_lookup_not_implemented" "live mode states why it is blocked" + pass "cognee lookup has no live/API mode" +} + +test_dry_run_verifies_local_source_without_echoing_answer() { + local dir source manifest answer out code sha + dir="$TMP_ROOT/verified" + mkdir -p "$dir" + source="$dir/source.md" + manifest="$dir/manifest.tsv" + answer="$dir/answer.txt" + printf 'local source truth\n' > "$source" + sha=$(sha256sum "$source" | awk '{print $1}') + write_manifest "$manifest" fm-cognee-001 "$source" "$sha" + printf 'Generated answer with SOURCE_ID=fm-cognee-001 and secret-like body DO_NOT_PRINT_ME.\n' > "$answer" + + out=$("$ROOT/bin/fm-cognee-lookup.sh" --dry-run --query "which report matters" --manifest "$manifest" --answer-file "$answer") + code=$? + expect_code 0 "$code" "verified dry-run exits successfully" + assert_contains "$out" "cognee_answer_status=hint_only" "raw Cognee answer remains hint-only" + assert_contains "$out" "label=verified_local_source" "manifest and source checksum produce verified label" + assert_contains "$out" "external_action_authorized=false" "verified lookup still cannot authorize action" + assert_not_contains "$out" "DO_NOT_PRINT_ME" "wrapper does not echo answer body" + assert_not_contains "$out" "which report matters" "wrapper does not echo query text" + pass "dry-run verifies local source without echoing raw query or answer" +} + +test_missing_citation_blocks_proof() { + local dir source manifest answer out code sha + dir="$TMP_ROOT/missing-citation" + mkdir -p "$dir" + source="$dir/source.md" + manifest="$dir/manifest.tsv" + answer="$dir/answer.txt" + printf 'local source truth\n' > "$source" + sha=$(sha256sum "$source" | awk '{print $1}') + write_manifest "$manifest" fm-cognee-002 "$source" "$sha" + printf 'Generated answer with no source labels.\n' > "$answer" + + out=$("$ROOT/bin/fm-cognee-lookup.sh" --dry-run --query "lookup" --manifest "$manifest" --answer-file "$answer") + code=$? + expect_code 3 "$code" "missing citation exits as blocked" + assert_contains "$out" "label=blocked_missing_proof" "missing citation cannot verify" + assert_contains "$out" "reason=no_usable_citations" "missing citation reason is explicit" + pass "missing Cognee citations fail closed" +} + +test_source_path_and_seed_file_references_verify() { + local dir source manifest answer out code sha + dir="$TMP_ROOT/path-seed" + mkdir -p "$dir" + source="$dir/source.md" + manifest="$dir/manifest.tsv" + answer="$dir/answer.txt" + printf 'local source truth\n' > "$source" + sha=$(sha256sum "$source" | awk '{print $1}') + write_manifest "$manifest" fm-cognee-002b "$source" "$sha" + printf 'Generated answer cites SOURCE_PATH=%s.\n' "$source" > "$answer" + + out=$("$ROOT/bin/fm-cognee-lookup.sh" --dry-run --query "lookup" --manifest "$manifest" --answer-file "$answer") + code=$? + expect_code 0 "$code" "source path dry-run exits successfully" + assert_contains "$out" "label=verified_local_source" "SOURCE_PATH references can verify" + assert_contains "$out" "matched_ref=SOURCE_PATH" "source path reference is recognized" + + printf 'Generated answer cites SEED_FILE=%s.\n' "$(basename "$source")" > "$answer" + out=$("$ROOT/bin/fm-cognee-lookup.sh" --dry-run --query "lookup" --manifest "$manifest" --answer-file "$answer") + code=$? + expect_code 0 "$code" "seed file dry-run exits successfully" + assert_contains "$out" "label=verified_local_source" "SEED_FILE references can verify" + assert_contains "$out" "matched_ref=SEED_FILE" "seed file reference is recognized" + pass "SOURCE_PATH and SEED_FILE references can verify local sources" +} + +test_checksum_mismatch_blocks_proof() { + local dir source manifest answer out code sha + dir="$TMP_ROOT/checksum" + mkdir -p "$dir" + source="$dir/source.md" + manifest="$dir/manifest.tsv" + answer="$dir/answer.txt" + printf 'local source truth\n' > "$source" + sha=$(printf 'wrong' | sha256sum | awk '{print $1}') + write_manifest "$manifest" fm-cognee-003 "$source" "$sha" + printf 'SOURCE_ID=fm-cognee-003\n' > "$answer" + + out=$("$ROOT/bin/fm-cognee-lookup.sh" --dry-run --query "lookup" --manifest "$manifest" --answer-file "$answer") + code=$? + expect_code 1 "$code" "checksum mismatch exits as invalid" + assert_contains "$out" "reason=checksum_mismatch" "checksum mismatch is reported" + assert_not_contains "$out" "label=verified_local_source" "bad checksum cannot verify" + pass "checksum mismatch blocks proof" +} + +test_redaction_not_checked_blocks_proof() { + local dir source manifest answer out code sha + dir="$TMP_ROOT/redaction" + mkdir -p "$dir" + source="$dir/source.md" + manifest="$dir/manifest.tsv" + answer="$dir/answer.txt" + printf 'local source truth\n' > "$source" + sha=$(sha256sum "$source" | awk '{print $1}') + write_manifest "$manifest" fm-cognee-004 "$source" "$sha" not_checked + printf 'SOURCE_ID=fm-cognee-004\n' > "$answer" + + out=$("$ROOT/bin/fm-cognee-lookup.sh" --dry-run --query "lookup" --manifest "$manifest" --answer-file "$answer") + code=$? + expect_code 1 "$code" "unchecked redaction exits as invalid" + assert_contains "$out" "reason=redaction_not_passed" "redaction status blocks local verification" + pass "unchecked redaction blocks proof" +} + +test_secret_risk_path_blocks_without_echoing_path() { + local dir source manifest answer out code sha + dir="$TMP_ROOT/secret-token-source" + mkdir -p "$dir" + source="$dir/source.md" + manifest="$dir/manifest.tsv" + answer="$dir/answer.txt" + printf 'local source truth\n' > "$source" + sha=$(sha256sum "$source" | awk '{print $1}') + write_manifest "$manifest" fm-cognee-004b "$source" "$sha" + printf 'SOURCE_ID=fm-cognee-004b\n' > "$answer" + + out=$("$ROOT/bin/fm-cognee-lookup.sh" --dry-run --query "lookup" --manifest "$manifest" --answer-file "$answer") + code=$? + expect_code 1 "$code" "secret-looking path exits as invalid" + assert_contains "$out" "reason=path_risk_scan_failed" "path risk scan blocks local verification" + assert_not_contains "$out" "$dir" "risky source path is not echoed" + pass "secret-risk paths block without echoing the risky path" +} + +test_high_stale_risk_warns_after_local_verification() { + local dir source manifest answer out code sha + dir="$TMP_ROOT/stale" + mkdir -p "$dir" + source="$dir/source.md" + manifest="$dir/manifest.tsv" + answer="$dir/answer.txt" + printf 'historical source truth\n' > "$source" + sha=$(sha256sum "$source" | awk '{print $1}') + write_manifest "$manifest" fm-cognee-005 "$source" "$sha" passed high + printf 'SOURCE_ID=fm-cognee-005\n' > "$answer" + + out=$("$ROOT/bin/fm-cognee-lookup.sh" --dry-run --query "lookup" --manifest "$manifest" --answer-file "$answer") + code=$? + expect_code 0 "$code" "stale warning exits successfully" + assert_contains "$out" "label=stale_warning" "high stale risk is not plain proof" + assert_contains "$out" "stale_risk=high" "stale risk is surfaced" + pass "high stale risk remains visible after local verification" +} + +test_dry_run_blocks_live_mode +test_dry_run_verifies_local_source_without_echoing_answer +test_missing_citation_blocks_proof +test_source_path_and_seed_file_references_verify +test_checksum_mismatch_blocks_proof +test_redaction_not_checked_blocks_proof +test_secret_risk_path_blocks_without_echoing_path +test_high_stale_risk_warns_after_local_verification From 0159209011bdf80f2af7f7f97e400d1f39538428 Mon Sep 17 00:00:00 2001 From: JTInventory Date: Sun, 28 Jun 2026 22:49:08 +0000 Subject: [PATCH 2/2] feat(cognee): verify local source hints --- bin/fm-cognee-verify-source.sh | 352 ++++++++++++++++++++++++++ tests/fm-cognee-source-verify.test.sh | 211 +++++++++++++++ 2 files changed, 563 insertions(+) create mode 100755 bin/fm-cognee-verify-source.sh create mode 100644 tests/fm-cognee-source-verify.test.sh diff --git a/bin/fm-cognee-verify-source.sh b/bin/fm-cognee-verify-source.sh new file mode 100755 index 00000000..36872b43 --- /dev/null +++ b/bin/fm-cognee-verify-source.sh @@ -0,0 +1,352 @@ +#!/usr/bin/env bash +# Parse Cognee hint text and verify references against a local manifest. +# +# This is intentionally local-only: it reads a saved answer fixture and a JSONL +# manifest, reopens the referenced local source file, and never calls Cognee. +# Usage: fm-cognee-verify-source.sh --manifest --answer +set -eu + +usage() { + echo "usage: fm-cognee-verify-source.sh --manifest --answer " >&2 +} + +MANIFEST= +ANSWER= +while [ $# -gt 0 ]; do + case "$1" in + --manifest) + [ $# -ge 2 ] || { usage; exit 64; } + MANIFEST=$2 + shift 2 + ;; + --answer) + [ $# -ge 2 ] || { usage; exit 64; } + ANSWER=$2 + shift 2 + ;; + --help|-h) + usage + exit 0 + ;; + *) + usage + exit 64 + ;; + esac +done + +[ -n "$MANIFEST" ] || { usage; exit 64; } +[ -n "$ANSWER" ] || { usage; exit 64; } + +python3 - "$MANIFEST" "$ANSWER" <<'PY' +import datetime as dt +import hashlib +import json +import re +import sys +from pathlib import Path + + +manifest_path = Path(sys.argv[1]) +answer_path = Path(sys.argv[2]) + + +UUID_RE = re.compile( + r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}\b" +) +LABEL_RE = re.compile( + r"\b(SOURCE_ID|SOURCE_PATH|SEED_FILE|DATA_ID|DATA_UUID|CHUNK_ID|CHUNK_UUID)\s*[:=]\s*" + r"(?:\"([^\"]*)\"|'([^']*)'|([^\s,;\]\)]+))" +) + + +def _json(status, outcome, *, row=None, parsed=None, local=None, errors=None, warnings=None): + parsed = parsed or {} + row = row or {} + local = local or {} + errors = errors or [] + warnings = warnings or [] + now = dt.datetime.now(dt.timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + result = { + "schema_version": "cognee_source_verification.v1", + "event_type": "source_verification", + "ts_utc": now, + "operation": { + "operation_name": "local_source_verify", + "mutates_remote": False, + }, + "source_reference": { + "source_ids": sorted(parsed.get("source_ids", [])), + "source_paths": sorted(parsed.get("source_paths", [])), + "seed_files": sorted(parsed.get("seed_files", [])), + "data_ids": sorted(parsed.get("data_ids", [])), + "chunk_ids": sorted(parsed.get("chunk_ids", [])), + "uuid_mentions": sorted(parsed.get("uuid_mentions", [])), + "malformed_uuid_count": parsed.get("malformed_uuid_count", 0), + }, + "manifest": { + "manifest_path": str(manifest_path), + "manifest_row_found": bool(row), + "manifest_checksum_algorithm": "sha256" if _manifest_checksum(row) else None, + "manifest_checksum_match": local.get("checksum_match"), + "redaction_status": row.get("redaction_status"), + "stale_risk": row.get("stale_risk"), + "source_family": row.get("source_family"), + }, + "local_file": { + "local_file_opened": local.get("opened", False), + "local_file_readable": local.get("readable", False), + "local_file_size_bytes": local.get("size_bytes"), + "local_file_mtime_utc": local.get("mtime_utc"), + }, + "verification_result": { + "status": status, + "outcome": outcome, + "errors": errors, + "warnings": warnings, + }, + "policy": { + "cognee_is_source_of_truth": False, + "action_authorized": False, + }, + } + print(json.dumps(result, sort_keys=True)) + + +def _load_manifest(): + rows = [] + with manifest_path.open("r", encoding="utf-8") as handle: + for line_no, line in enumerate(handle, 1): + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except json.JSONDecodeError as exc: + raise ValueError(f"manifest line {line_no} is not JSON: {exc}") from exc + row["_line_no"] = line_no + rows.append(row) + return rows + + +def _parse_answer(text): + labels = { + "source_ids": set(), + "source_paths": set(), + "seed_files": set(), + "data_ids": set(), + "chunk_ids": set(), + "uuid_mentions": set(), + } + malformed = 0 + for match in LABEL_RE.finditer(text): + label = match.group(1) + cleaned = next(group for group in match.groups()[1:] if group is not None).strip() + if label == "SOURCE_ID": + labels["source_ids"].add(cleaned) + elif label == "SOURCE_PATH": + labels["source_paths"].add(cleaned) + elif label == "SEED_FILE": + labels["seed_files"].add(cleaned) + elif label in ("DATA_ID", "DATA_UUID"): + if UUID_RE.fullmatch(cleaned): + labels["data_ids"].add(cleaned.lower()) + else: + malformed += 1 + elif label in ("CHUNK_ID", "CHUNK_UUID"): + if UUID_RE.fullmatch(cleaned): + labels["chunk_ids"].add(cleaned.lower()) + else: + malformed += 1 + + valid_labelled_uuids = labels["data_ids"] | labels["chunk_ids"] + + for value in UUID_RE.findall(text): + lower = value.lower() + if lower not in valid_labelled_uuids: + labels["uuid_mentions"].add(lower) + + labels["malformed_uuid_count"] = malformed + return labels + + +def _field_set(row, field): + value = row.get(field) + if value is None: + return set() + if isinstance(value, list): + return {str(item) for item in value} + return {str(value)} + + +def _lower_field_set(row, field): + return {item.lower() for item in _field_set(row, field)} + + +def _source_path(row): + return str(row.get("source_path") or row.get("path") or "") + + +def _seed_file(row): + return str(row.get("seed_file") or "") + + +def _manifest_checksum(row): + return row.get("checksum_sha256") or row.get("sha256") or row.get("checksum") + + +def _resolve_path(row): + raw = _source_path(row) + if not raw: + return None + path = Path(raw) + if path.is_absolute(): + return path + return (manifest_path.parent / path).resolve() + + +def _mtime_utc(path): + return dt.datetime.fromtimestamp(path.stat().st_mtime, dt.timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def _sha256(path): + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _find_row(rows, parsed): + source_ids = parsed["source_ids"] + if source_ids: + for row in rows: + if str(row.get("source_id")) in source_ids: + return row + return None + + source_paths = parsed["source_paths"] + seed_files = parsed["seed_files"] + data_ids = parsed["data_ids"] | parsed["uuid_mentions"] + chunk_ids = parsed["chunk_ids"] | parsed["uuid_mentions"] + for row in rows: + if source_paths and (_source_path(row) in source_paths or str(_resolve_path(row)) in source_paths): + return row + if seed_files and _seed_file(row) in seed_files: + return row + if data_ids and data_ids & _lower_field_set(row, "data_ids"): + return row + if chunk_ids and chunk_ids & _lower_field_set(row, "chunk_ids"): + return row + return None + + +def _verify(): + if not manifest_path.is_file(): + _json("failed_closed", "failed_closed_manifest_unreadable", errors=[f"manifest not found: {manifest_path}"]) + return 2 + if not answer_path.is_file(): + _json("failed_closed", "failed_closed_answer_unreadable", errors=[f"answer not found: {answer_path}"]) + return 2 + + try: + rows = _load_manifest() + except Exception as exc: + _json("failed_closed", "failed_closed_manifest_unreadable", errors=[str(exc)]) + return 2 + + try: + answer_text = answer_path.read_text(encoding="utf-8") + except Exception as exc: + _json("failed_closed", "failed_closed_answer_unreadable", errors=[str(exc)]) + return 2 + + parsed = _parse_answer(answer_text) + has_reference = any(parsed[key] for key in ("source_ids", "source_paths", "seed_files", "data_ids", "chunk_ids", "uuid_mentions")) + if not has_reference: + _json("failed_closed", "failed_closed_missing_reference", parsed=parsed, errors=["no parseable source reference"]) + return 2 + + row = _find_row(rows, parsed) + if not row: + _json("failed_closed", "hint_only_manifest_miss", parsed=parsed, errors=["no matching manifest row"]) + return 2 + + errors = [] + warnings = [] + source_path = _resolve_path(row) + local = {"opened": False, "readable": False, "checksum_match": None} + + row_source_id = {str(row.get("source_id"))} + if parsed["source_ids"] - row_source_id: + errors.append("SOURCE_ID does not match manifest row") + row_raw_path = _source_path(row) + row_resolved_path = str(source_path) if source_path else "" + if parsed["source_paths"] - {row_raw_path, row_resolved_path}: + errors.append("SOURCE_PATH does not match manifest row") + if parsed["seed_files"] - {_seed_file(row)}: + errors.append("SEED_FILE does not match manifest row") + + row_data_ids = _lower_field_set(row, "data_ids") + row_chunk_ids = _lower_field_set(row, "chunk_ids") + if parsed["data_ids"] - row_data_ids: + errors.append("DATA_ID does not match manifest row") + if parsed["chunk_ids"] - row_chunk_ids: + errors.append("CHUNK_ID does not match manifest row") + unknown_uuid_mentions = parsed["uuid_mentions"] - row_data_ids - row_chunk_ids + if unknown_uuid_mentions: + errors.append("UUID mention does not match manifest row") + + if not source_path or not source_path.is_file(): + errors.append("local source file is missing") + else: + try: + local["opened"] = True + local["readable"] = True + local["size_bytes"] = source_path.stat().st_size + local["mtime_utc"] = _mtime_utc(source_path) + expected_size = row.get("size_bytes") + if expected_size is not None and int(expected_size) != local["size_bytes"]: + errors.append("local source size does not match manifest") + expected_checksum = _manifest_checksum(row) + if expected_checksum: + local["checksum_match"] = _sha256(source_path).lower() == str(expected_checksum).lower() + if not local["checksum_match"]: + errors.append("local source checksum does not match manifest") + except Exception as exc: + local["readable"] = False + errors.append(f"local source file could not be read: {exc}") + + stale_risk = str(row.get("stale_risk") or "").lower() + if stale_risk in {"high", "critical"}: + warnings.append(f"stale_risk={stale_risk}") + + raw_status = str(row.get("raw_readback_status") or row.get("raw_status") or "ok").lower() + raw_blocked = raw_status not in {"", "ok", "passed", "available", "readable", "200"} + if raw_blocked: + errors.append(f"raw_readback_status={raw_status}") + + if errors: + if any("checksum" in error for error in errors): + outcome = "failed_closed_checksum_mismatch" + elif any("raw_readback_status" in error for error in errors): + outcome = "failed_closed_raw_durability" + elif any("DATA_ID" in error or "CHUNK_ID" in error or "UUID" in error for error in errors): + outcome = "failed_closed_identifier_mismatch" + elif any("SOURCE_PATH" in error for error in errors): + outcome = "failed_closed_path_mismatch" + elif any("SEED_FILE" in error for error in errors): + outcome = "failed_closed_seed_mismatch" + elif any("local source" in error for error in errors): + outcome = "failed_closed_missing_proof" + else: + outcome = "failed_closed_missing_proof" + _json("failed_closed", outcome, row=row, parsed=parsed, local=local, errors=errors, warnings=warnings) + return 2 + + _json("verified", "verified_local_source", row=row, parsed=parsed, local=local, warnings=warnings) + return 0 + + +sys.exit(_verify()) +PY diff --git a/tests/fm-cognee-source-verify.test.sh b/tests/fm-cognee-source-verify.test.sh new file mode 100644 index 00000000..7c58112f --- /dev/null +++ b/tests/fm-cognee-source-verify.test.sh @@ -0,0 +1,211 @@ +#!/usr/bin/env bash +# Behavior tests for local Cognee source parsing and verification. +# +# These fixtures are saved/redacted local files only. They prove Cognee text is +# only a hint until a manifest row and readable local source file agree. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +VERIFY="$ROOT/bin/fm-cognee-verify-source.sh" +TMP_ROOT=$(fm_test_tmproot fm-cognee-source-verify) + +sha256_file() { + sha256sum "$1" | awk '{print $1}' +} + +write_manifest() { + local manifest=$1 source=$2 checksum=$3 + cat > "$manifest" < "$source" + write_manifest "$manifest" "$source" "$(sha256_file "$source")" + cat > "$answer" < "$source" + write_manifest "$manifest" "$source" "$(sha256_file "$source")" + cat > "$answer" < "$source" + write_manifest "$manifest" "$source" "$(sha256_file "$source")" + printf '\377' > "$answer" + + set +e + out=$("$VERIFY" --manifest "$manifest" --answer "$answer" 2>&1) + code=$? + set -e + expect_code 2 "$code" "invalid UTF-8 answer" + [ "$(printf '%s' "$out" | jq -r '.verification_result.outcome')" = "failed_closed_answer_unreadable" ] || fail "invalid UTF-8 outcome" + assert_not_contains "$out" "Traceback" "invalid UTF-8 must not print traceback" +} + +test_unknown_source_id_fails_closed() { + local case_dir source manifest answer out code + case_dir="$TMP_ROOT/unknown" + mkdir -p "$case_dir" + source="$case_dir/redacted-source.md" + manifest="$case_dir/manifest.jsonl" + answer="$case_dir/answer.txt" + printf 'Redacted report fixture.\n' > "$source" + write_manifest "$manifest" "$source" "$(sha256_file "$source")" + printf 'SOURCE_ID=seed-missing\n' > "$answer" + + set +e + out=$("$VERIFY" --manifest "$manifest" --answer "$answer") + code=$? + set -e + expect_code 2 "$code" "unknown source id" + [ "$(printf '%s' "$out" | jq -r '.verification_result.status')" = "failed_closed" ] || fail "unknown id must fail closed" + [ "$(printf '%s' "$out" | jq -r '.verification_result.outcome')" = "hint_only_manifest_miss" ] || fail "unknown id outcome" +} + +test_checksum_mismatch_fails_closed() { + local case_dir source manifest answer out code + case_dir="$TMP_ROOT/checksum" + mkdir -p "$case_dir" + source="$case_dir/redacted-source.md" + manifest="$case_dir/manifest.jsonl" + answer="$case_dir/answer.txt" + printf 'Redacted report fixture.\n' > "$source" + write_manifest "$manifest" "$source" "0000000000000000000000000000000000000000000000000000000000000000" + printf 'SOURCE_ID=seed-07\nSOURCE_PATH=%s\nSEED_FILE=seed/redacted-07.md\n' "$source" > "$answer" + + set +e + out=$("$VERIFY" --manifest "$manifest" --answer "$answer") + code=$? + set -e + expect_code 2 "$code" "checksum mismatch" + [ "$(printf '%s' "$out" | jq -r '.verification_result.outcome')" = "failed_closed_checksum_mismatch" ] || fail "checksum mismatch outcome" + [ "$(printf '%s' "$out" | jq -r '.manifest.manifest_checksum_match')" = "false" ] || fail "checksum match must be false" +} + +test_extra_source_id_fails_closed() { + local case_dir source manifest answer out code + case_dir="$TMP_ROOT/extra-source-id" + mkdir -p "$case_dir" + source="$case_dir/redacted-source.md" + manifest="$case_dir/manifest.jsonl" + answer="$case_dir/answer.txt" + printf 'Redacted report fixture.\n' > "$source" + write_manifest "$manifest" "$source" "$(sha256_file "$source")" + cat > "$answer" < "$source" + write_manifest "$manifest" "$source" "$(sha256_file "$source")" + printf 'SOURCE_ID=seed-raw-404\nSOURCE_PATH=%s\nSEED_FILE=seed/raw-404.md\n' "$source" > "$answer" + + set +e + out=$("$VERIFY" --manifest "$manifest" --answer "$answer") + code=$? + set -e + expect_code 2 "$code" "raw 404 durability blocker" + [ "$(printf '%s' "$out" | jq -r '.verification_result.outcome')" = "failed_closed_raw_durability" ] || fail "raw 404 outcome" + [ "$(printf '%s' "$out" | jq -r '.local_file.local_file_opened')" = "true" ] || fail "local file still reopens for diagnostics" +} + +test_malformed_uuid_is_ignored_but_unknown_well_formed_uuid_fails() { + local case_dir source manifest answer out code + case_dir="$TMP_ROOT/uuid" + mkdir -p "$case_dir" + source="$case_dir/redacted-source.md" + manifest="$case_dir/manifest.jsonl" + answer="$case_dir/answer.txt" + printf 'Redacted report fixture.\n' > "$source" + write_manifest "$manifest" "$source" "$(sha256_file "$source")" + cat > "$answer" <