diff --git a/CHANGELOG.md b/CHANGELOG.md index 241024b..ffa8493 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,18 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Testing + +- **Golden-envelope snapshot tests** (`anomalyx/tests/golden.rs`). Run the actual + binary and pin its byte-exact stdout for `schema`, `describe`, and a + representative `scan` envelope against committed goldens — so any accidental + contract drift (renamed field, changed dense-row layout, shifted + `config_version`, recalibrated confidence) fails CI as a visible diff. + Regenerate intentional changes with `BLESS=1`. +- **Million-row scale test** (`ax-validate`): a 1,000,000-row scan must be + byte-identical across runs *and* recover exactly the injected outliers — + determinism and correctness verified at scale, not just on toy inputs. + ## [0.8.0] - 2026-06-01 ### Changed diff --git a/crates/anomalyx/tests/golden.rs b/crates/anomalyx/tests/golden.rs new file mode 100644 index 0000000..7e00188 --- /dev/null +++ b/crates/anomalyx/tests/golden.rs @@ -0,0 +1,70 @@ +//! Golden-envelope snapshot tests — the contract's tripwire. +//! +//! These run the actual `anomalyx` binary and compare its stdout, byte-for-byte, +//! against committed golden files. They pin the *exact* wire output of the three +//! contract surfaces — `schema`, `describe`, and a representative `scan` envelope +//! — so any accidental drift (a renamed field, a changed dense-row layout, a +//! shifted `config_version`, a recalibrated confidence) fails CI as a visible +//! diff rather than slipping out in a release. +//! +//! Intentional changes are expected to update the goldens: regenerate with +//! `BLESS=1 cargo test -p anomalyx --test golden`. + +use std::io::Write; +use std::process::{Command, Stdio}; + +/// Runs the built binary with `args`, feeding `stdin`, returning stdout as text. +fn run(args: &[&str], stdin: &[u8]) -> String { + let mut child = Command::new(env!("CARGO_BIN_EXE_anomalyx")) + .args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("spawn anomalyx"); + child + .stdin + .take() + .expect("stdin") + .write_all(stdin) + .expect("write stdin"); + let out = child.wait_with_output().expect("wait"); + String::from_utf8(out.stdout).expect("utf-8 stdout") +} + +/// Compares `actual` to the committed golden `name`; `BLESS=1` rewrites it. +fn check_golden(name: &str, actual: &str) { + let path = format!("{}/tests/golden/{name}", env!("CARGO_MANIFEST_DIR")); + if std::env::var_os("BLESS").is_some() { + std::fs::write(&path, actual).expect("write golden"); + return; + } + let expected = std::fs::read_to_string(&path) + .unwrap_or_else(|_| panic!("missing golden {path}; regenerate with BLESS=1")); + assert!( + actual == expected, + "golden mismatch for {name}.\nIf this change is intentional, regenerate with \ + `BLESS=1 cargo test -p anomalyx --test golden`." + ); +} + +/// A fixed corpus exercising column roles (`id` → identifier, skipped) and the +/// point detector (`reading` → one clear outlier). Piped via stdin so the +/// envelope's `source` is the stable `"-"`. +const SCAN_INPUT: &[u8] = + b"id,reading\n1,10\n2,11\n3,9\n4,10\n5,12\n6,8\n7,11\n8,10\n9,9\n10,1000\n"; + +#[test] +fn golden_schema() { + check_golden("schema.json", &run(&["schema"], b"")); +} + +#[test] +fn golden_describe() { + check_golden("describe.json", &run(&["describe"], b"")); +} + +#[test] +fn golden_scan_basic() { + check_golden("scan_basic.json", &run(&["scan"], SCAN_INPUT)); +} diff --git a/crates/anomalyx/tests/golden/describe.json b/crates/anomalyx/tests/golden/describe.json new file mode 100644 index 0000000..ac48f8c --- /dev/null +++ b/crates/anomalyx/tests/golden/describe.json @@ -0,0 +1,103 @@ +{ + "protocol": "anomalyx/tq1", + "tool": "anomalyx", + "summary": "Contract-first anomaly detection over arbitrary corpora.", + "commands": { + "describe": "Emit this protocol metadata.", + "schema": "Emit the JSON Schema of `scan` output.", + "scan": "Normalize input (file or stdin) and emit a tq1 anomaly envelope.", + "explain": "Resolve a finding handle to its underlying evidence." + }, + "input_formats": [ + "parquet", + "arrow", + "evtx", + "pcap", + "xlsx", + "sqlite", + "avro", + "orc", + "otlp", + "cloudtrail", + "eve", + "journal", + "osquery", + "ndjson", + "zeek", + "logfmt", + "accesslog", + "syslog", + "cef", + "leef", + "auditd", + "dns", + "prometheus", + "xml", + "json", + "yaml", + "toml", + "ini", + "netflow", + "vpcflow", + "tsv", + "csv" + ], + "anomaly_classes": [ + "point", + "contextual", + "collective", + "distributional", + "structural", + "multivariate", + "cadence" + ], + "detectors": [ + "point.modz", + "struct.schema", + "dist.ks", + "dist.psi", + "dist.chi2", + "mv.mahalanobis", + "ctx.seasonal", + "coll.cusum", + "cad.regularity" + ], + "finding_columns": [ + "detector", + "class", + "handle", + "confidence", + "severity", + "score", + "reason" + ], + "exit_codes": { + "clean": 0, + "anomalies": 1, + "error": 2 + }, + "config": { + "point_threshold": 3.5, + "point_min_n": 8, + "column_roles": true, + "point_fdr_q": null, + "dist_alpha": 0.05, + "psi_threshold": 0.2, + "psi_bins": 10, + "dist_min_n": 20, + "struct_null_rate": 0.5, + "mv_alpha": 0.001, + "mv_min_n": 20, + "mv_ridge": 1e-9, + "ctx_period": 0, + "ctx_threshold": 3.5, + "ctx_min_per_phase": 4, + "coll_min_n": 20, + "coll_threshold": 5.0, + "cadence_column": null, + "cad_max_cv": 0.05, + "cad_min_n": 20 + }, + "config_version": "anomalyx-cfg/8;pt=3.5000;ptn=8;cr=true;pfdr=;da=0.0500;psi=0.2000;psib=10;dmn=20;snr=0.5000;mva=0.00100;mvn=20;mvr=1e-9;cxp=0;cxt=3.5000;cxm=4;cln=20;clt=5.0000;cdc=;cdcv=0.0500;cdn=20", + "determinism": "Same input + same config_version yields byte-identical output." +} diff --git a/crates/anomalyx/tests/golden/scan_basic.json b/crates/anomalyx/tests/golden/scan_basic.json new file mode 100644 index 0000000..a0ad173 --- /dev/null +++ b/crates/anomalyx/tests/golden/scan_basic.json @@ -0,0 +1 @@ +{"protocol":"anomalyx/tq1","config_version":"anomalyx-cfg/8;pt=3.5000;ptn=8;cr=true;pfdr=;da=0.0500;psi=0.2000;psib=10;dmn=20;snr=0.5000;mva=0.00100;mvn=20;mvr=1e-9;cxp=0;cxt=3.5000;cxm=4;cln=20;clt=5.0000;cdc=;cdcv=0.0500;cdn=20","source":"-","format":"csv","rows_scanned":10,"dict":["point.modz","point","cell:reading:9","critical","reading = 1000.000000: modified z-score 450.395 exceeds 3.500 (center=10.000000, scale=1.482600)"],"columns":["detector","class","handle","confidence","severity","score","reason"],"rows":[[0,1,2,1.0,3,450.3945770942938,4]],"absent":[{"detector":"dist.ks","reason":"no baseline provided; distributional drift requires --baseline"},{"detector":"dist.psi","reason":"no baseline provided; distributional drift requires --baseline"},{"detector":"dist.chi2","reason":"no baseline provided; distributional drift requires --baseline"},{"detector":"mv.mahalanobis","reason":"fewer than 20 complete numeric rows to estimate a covariance"},{"detector":"ctx.seasonal","reason":"contextual detection needs a declared period ≥ 2 (pass --period N)"},{"detector":"coll.cusum","reason":"no numeric column with ≥ 20 values and non-zero variance"},{"detector":"cad.regularity","reason":"cadence detection requires a time column (pass --cadence