test: golden-envelope snapshot tests + million-row scale test (#6)

CHANGELOG.md

@@ -6,6 +6,18 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
 
+### Testing
+
+- **Golden-envelope snapshot tests** (`anomalyx/tests/golden.rs`). Run the actual
+  binary and pin its byte-exact stdout for `schema`, `describe`, and a
+  representative `scan` envelope against committed goldens — so any accidental
+  contract drift (renamed field, changed dense-row layout, shifted
+  `config_version`, recalibrated confidence) fails CI as a visible diff.
+  Regenerate intentional changes with `BLESS=1`.
+- **Million-row scale test** (`ax-validate`): a 1,000,000-row scan must be
+  byte-identical across runs *and* recover exactly the injected outliers —
+  determinism and correctness verified at scale, not just on toy inputs.
+
 ## [0.8.0] - 2026-06-01
 
 ### Changed

crates/anomalyx/tests/golden.rs

@@ -0,0 +1,70 @@
+//! Golden-envelope snapshot tests — the contract's tripwire.
+//!
+//! These run the actual `anomalyx` binary and compare its stdout, byte-for-byte,
+//! against committed golden files. They pin the *exact* wire output of the three
+//! contract surfaces — `schema`, `describe`, and a representative `scan` envelope
+//! — so any accidental drift (a renamed field, a changed dense-row layout, a
+//! shifted `config_version`, a recalibrated confidence) fails CI as a visible
+//! diff rather than slipping out in a release.
+//!
+//! Intentional changes are expected to update the goldens: regenerate with
+//! `BLESS=1 cargo test -p anomalyx --test golden`.
+
+use std::io::Write;
+use std::process::{Command, Stdio};
+
+/// Runs the built binary with `args`, feeding `stdin`, returning stdout as text.
+fn run(args: &[&str], stdin: &[u8]) -> String {
+    let mut child = Command::new(env!("CARGO_BIN_EXE_anomalyx"))
+        .args(args)
+        .stdin(Stdio::piped())
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .spawn()
+        .expect("spawn anomalyx");
+    child
+        .stdin
+        .take()
+        .expect("stdin")
+        .write_all(stdin)
+        .expect("write stdin");
+    let out = child.wait_with_output().expect("wait");
+    String::from_utf8(out.stdout).expect("utf-8 stdout")
+}
+
+/// Compares `actual` to the committed golden `name`; `BLESS=1` rewrites it.
+fn check_golden(name: &str, actual: &str) {
+    let path = format!("{}/tests/golden/{name}", env!("CARGO_MANIFEST_DIR"));
+    if std::env::var_os("BLESS").is_some() {
+        std::fs::write(&path, actual).expect("write golden");
+        return;
+    }
+    let expected = std::fs::read_to_string(&path)
+        .unwrap_or_else(|_| panic!("missing golden {path}; regenerate with BLESS=1"));
+    assert!(
+        actual == expected,
+        "golden mismatch for {name}.\nIf this change is intentional, regenerate with \
+         `BLESS=1 cargo test -p anomalyx --test golden`."
+    );
+}
+
+/// A fixed corpus exercising column roles (`id` → identifier, skipped) and the
+/// point detector (`reading` → one clear outlier). Piped via stdin so the
+/// envelope's `source` is the stable `"-"`.
+const SCAN_INPUT: &[u8] =
+    b"id,reading\n1,10\n2,11\n3,9\n4,10\n5,12\n6,8\n7,11\n8,10\n9,9\n10,1000\n";
+
+#[test]
+fn golden_schema() {
+    check_golden("schema.json", &run(&["schema"], b""));
+}
+
+#[test]
+fn golden_describe() {
+    check_golden("describe.json", &run(&["describe"], b""));
+}
+
+#[test]
+fn golden_scan_basic() {
+    check_golden("scan_basic.json", &run(&["scan"], SCAN_INPUT));
+}

crates/anomalyx/tests/golden/describe.json

@@ -0,0 +1,103 @@
+{
+  "protocol": "anomalyx/tq1",
+  "tool": "anomalyx",
+  "summary": "Contract-first anomaly detection over arbitrary corpora.",
+  "commands": {
+    "describe": "Emit this protocol metadata.",
+    "schema": "Emit the JSON Schema of `scan` output.",
+    "scan": "Normalize input (file or stdin) and emit a tq1 anomaly envelope.",
+    "explain": "Resolve a finding handle to its underlying evidence."
+  },
+  "input_formats": [
+    "parquet",
+    "arrow",
+    "evtx",
+    "pcap",
+    "xlsx",
+    "sqlite",
+    "avro",
+    "orc",
+    "otlp",
+    "cloudtrail",
+    "eve",
+    "journal",
+    "osquery",
+    "ndjson",
+    "zeek",
+    "logfmt",
+    "accesslog",
+    "syslog",
+    "cef",
+    "leef",
+    "auditd",
+    "dns",
+    "prometheus",
+    "xml",
+    "json",
+    "yaml",
+    "toml",
+    "ini",
+    "netflow",
+    "vpcflow",
+    "tsv",
+    "csv"
+  ],
+  "anomaly_classes": [
+    "point",
+    "contextual",
+    "collective",
+    "distributional",
+    "structural",
+    "multivariate",
+    "cadence"
+  ],
+  "detectors": [
+    "point.modz",
+    "struct.schema",
+    "dist.ks",
+    "dist.psi",
+    "dist.chi2",
+    "mv.mahalanobis",
+    "ctx.seasonal",
+    "coll.cusum",
+    "cad.regularity"
+  ],
+  "finding_columns": [
+    "detector",
+    "class",
+    "handle",
+    "confidence",
+    "severity",
+    "score",
+    "reason"
+  ],
+  "exit_codes": {
+    "clean": 0,
+    "anomalies": 1,
+    "error": 2
+  },
+  "config": {
+    "point_threshold": 3.5,
+    "point_min_n": 8,
+    "column_roles": true,
+    "point_fdr_q": null,
+    "dist_alpha": 0.05,
+    "psi_threshold": 0.2,
+    "psi_bins": 10,
+    "dist_min_n": 20,
+    "struct_null_rate": 0.5,
+    "mv_alpha": 0.001,
+    "mv_min_n": 20,
+    "mv_ridge": 1e-9,
+    "ctx_period": 0,
+    "ctx_threshold": 3.5,
+    "ctx_min_per_phase": 4,
+    "coll_min_n": 20,
+    "coll_threshold": 5.0,
+    "cadence_column": null,
+    "cad_max_cv": 0.05,
+    "cad_min_n": 20
+  },
+  "config_version": "anomalyx-cfg/8;pt=3.5000;ptn=8;cr=true;pfdr=;da=0.0500;psi=0.2000;psib=10;dmn=20;snr=0.5000;mva=0.00100;mvn=20;mvr=1e-9;cxp=0;cxt=3.5000;cxm=4;cln=20;clt=5.0000;cdc=;cdcv=0.0500;cdn=20",
+  "determinism": "Same input + same config_version yields byte-identical output."
+}

crates/anomalyx/tests/golden/scan_basic.json

@@ -0,0 +1 @@
+{"protocol":"anomalyx/tq1","config_version":"anomalyx-cfg/8;pt=3.5000;ptn=8;cr=true;pfdr=;da=0.0500;psi=0.2000;psib=10;dmn=20;snr=0.5000;mva=0.00100;mvn=20;mvr=1e-9;cxp=0;cxt=3.5000;cxm=4;cln=20;clt=5.0000;cdc=;cdcv=0.0500;cdn=20","source":"-","format":"csv","rows_scanned":10,"dict":["point.modz","point","cell:reading:9","critical","reading = 1000.000000: modified z-score 450.395 exceeds 3.500 (center=10.000000, scale=1.482600)"],"columns":["detector","class","handle","confidence","severity","score","reason"],"rows":[[0,1,2,1.0,3,450.3945770942938,4]],"absent":[{"detector":"dist.ks","reason":"no baseline provided; distributional drift requires --baseline"},{"detector":"dist.psi","reason":"no baseline provided; distributional drift requires --baseline"},{"detector":"dist.chi2","reason":"no baseline provided; distributional drift requires --baseline"},{"detector":"mv.mahalanobis","reason":"fewer than 20 complete numeric rows to estimate a covariance"},{"detector":"ctx.seasonal","reason":"contextual detection needs a declared period ≥ 2 (pass --period N)"},{"detector":"coll.cusum","reason":"no numeric column with ≥ 20 values and non-zero variance"},{"detector":"cad.regularity","reason":"cadence detection requires a time column (pass --cadence <COL>)"}],"roles":[{"column":"id","role":"identifier"},{"column":"reading","role":"measurement"}],"summary":{"total":1,"max_severity":"critical","by_class":[{"class":"point","count":1},{"class":"contextual","count":0},{"class":"collective","count":0},{"class":"distributional","count":0},{"class":"structural","count":0},{"class":"multivariate","count":0},{"class":"cadence","count":0}]},"exit":1}