From 995f8acf66e5be50226ff1048da37a350be1d09c Mon Sep 17 00:00:00 2001 From: zuub-don Date: Mon, 1 Jun 2026 11:55:26 -0700 Subject: [PATCH] feat: unified cross-detector confidence calibration (0.8.0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Confidence was computed three incompatible ways — 1−p (dist/mv), logistic-over-threshold (point/ctx/coll/psi), linear (cadence) — so a 0.9 meant different things per detector and severity couldn't rank across detectors. Now every detector routes through one shared ax_detect::calibrate module: confidence = logistic(STEEPNESS · relative excess past the firing threshold) from_exceedance(stat, thr) for "fires above" (modz, CUSUM shift, PSI, null-rate); from_undercut(stat, thr) for "fires below" (p-value < alpha, CV < max, BH cutoff). 0.5 at the threshold, →1 as it gets more extreme; a finding "2× past threshold" earns the same confidence on any detector. Removed the duplicated/divergent shift_confidence, psi_confidence, robustz::confidence helpers. Recalibrates every published confidence and severity (and point FDR mode now keys confidence off the BH cutoff). config_version bumped anomalyx-cfg/7 → /8 so agents see the change; envelope shape and PROTOCOL unchanged. Structural schema-diff findings keep their fixed priors (binary facts, not statistics); its null-rate finding joins the unified scale. Gates: proptest + cargo-mutants 0-missed on calibrate.rs (32/32) and all touched detector files (point/ctx/coll/cadence/structural/config/robustz 178 caught; dist/mv 193 caught + 10 loop-bound timeouts = detected). calibrate has exact two-x-past-threshold comparability + boundary/NaN tests. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 26 ++++++- Cargo.lock | 10 +-- Cargo.toml | 8 +-- crates/ax-detect/src/cadence.rs | 13 ++-- crates/ax-detect/src/calibrate.rs | 112 +++++++++++++++++++++++++++++ crates/ax-detect/src/coll.rs | 17 +---- crates/ax-detect/src/config.rs | 2 +- crates/ax-detect/src/ctx.rs | 4 +- crates/ax-detect/src/dist.rs | 44 ++++-------- crates/ax-detect/src/lib.rs | 1 + crates/ax-detect/src/mv.rs | 10 +-- crates/ax-detect/src/point.rs | 18 ++--- crates/ax-detect/src/robustz.rs | 15 ---- crates/ax-detect/src/structural.rs | 4 +- docs/src/envelope.md | 7 ++ 15 files changed, 189 insertions(+), 102 deletions(-) create mode 100644 crates/ax-detect/src/calibrate.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 563a36d..241024b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,26 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [0.8.0] - 2026-06-01 + +### Changed + +- **Unified confidence calibration across all detectors.** Confidence was + computed three incompatible ways (`1 − p` for the distributional/multivariate + detectors, a logistic-over-threshold for point/contextual/collective/PSI, and a + linear map for cadence), so a `0.9` meant different things depending on which + detector produced it — and severity (and `--top` / `--min-severity`) couldn't + rank across detectors. Now every detector routes through one shared function: + confidence is a logistic of how far its statistic sits past its firing + threshold, measured **relatively** so units cancel. At the threshold → `0.5`, + rising toward `1.0`; a finding "2× past threshold" earns the same confidence on + any detector. New `ax_detect::calibrate` module (`from_exceedance` / + `from_undercut`); the duplicated `shift_confidence` / `psi_confidence` / + `robustz::confidence` helpers are gone. +- This recalibrates every published confidence and severity. The `config_version` + fingerprint is bumped (`anomalyx-cfg/8`) so the change is visible to agents. + The envelope shape and `PROTOCOL` are unchanged. + ### Testing - **Parser robustness harness** (`ax-normalize/tests/robustness.rs`). Property @@ -13,8 +33,7 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). magic-prefixed-garbage, or truncated byte streams — fed both through auto-detection and straight to every registered parser — and that normalization is deterministic over fuzz inputs. Untrusted-input hardening: - a malformed file must fail cleanly, never crash. Test-only; no library change - (so no release). + a malformed file must fail cleanly, never crash. ## [0.7.0] - 2026-06-01 @@ -263,7 +282,8 @@ Initial release — a contract-first anomaly-detection CLI over arbitrary corpor gates on every push. - Dual-licensed under MIT OR Apache-2.0. -[Unreleased]: https://github.com/copyleftdev/anomalyx/compare/v0.7.0...HEAD +[Unreleased]: https://github.com/copyleftdev/anomalyx/compare/v0.8.0...HEAD +[0.8.0]: https://github.com/copyleftdev/anomalyx/compare/v0.7.0...v0.8.0 [0.7.0]: https://github.com/copyleftdev/anomalyx/compare/v0.6.0...v0.7.0 [0.6.0]: https://github.com/copyleftdev/anomalyx/compare/v0.5.0...v0.6.0 [0.5.0]: https://github.com/copyleftdev/anomalyx/compare/v0.4.1...v0.5.0 diff --git a/Cargo.lock b/Cargo.lock index ed598f2..e8ebd73 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -69,7 +69,7 @@ dependencies = [ [[package]] name = "anomalyx" -version = "0.7.0" +version = "0.8.0" dependencies = [ "anomalyx-core", "anomalyx-detect", @@ -80,7 +80,7 @@ dependencies = [ [[package]] name = "anomalyx-core" -version = "0.7.0" +version = "0.8.0" dependencies = [ "proptest", "serde", @@ -90,7 +90,7 @@ dependencies = [ [[package]] name = "anomalyx-detect" -version = "0.7.0" +version = "0.8.0" dependencies = [ "anomalyx-core", "proptest", @@ -101,7 +101,7 @@ dependencies = [ [[package]] name = "anomalyx-normalize" -version = "0.7.0" +version = "0.8.0" dependencies = [ "anomalyx-core", "apache-avro", @@ -129,7 +129,7 @@ dependencies = [ [[package]] name = "anomalyx-validate" -version = "0.7.0" +version = "0.8.0" dependencies = [ "anomalyx-core", "anomalyx-detect", diff --git a/Cargo.toml b/Cargo.toml index abbe6a5..28beebc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ members = [ ] [workspace.package] -version = "0.7.0" +version = "0.8.0" edition = "2021" rust-version = "1.90" license = "MIT OR Apache-2.0" @@ -22,9 +22,9 @@ authors = ["copyleftdev"] # `ax-*` names were taken), but the import/extern name stays `ax_core` etc. via # the dependency key + `package` rename — so no source code changes are needed. # Keep versions in sync with workspace.package.version above. -ax-core = { path = "crates/ax-core", version = "0.7.0", package = "anomalyx-core" } -ax-normalize = { path = "crates/ax-normalize", version = "0.7.0", package = "anomalyx-normalize" } -ax-detect = { path = "crates/ax-detect", version = "0.7.0", package = "anomalyx-detect" } +ax-core = { path = "crates/ax-core", version = "0.8.0", package = "anomalyx-core" } +ax-normalize = { path = "crates/ax-normalize", version = "0.8.0", package = "anomalyx-normalize" } +ax-detect = { path = "crates/ax-detect", version = "0.8.0", package = "anomalyx-detect" } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/crates/ax-detect/src/cadence.rs b/crates/ax-detect/src/cadence.rs index 2fba988..4be413a 100644 --- a/crates/ax-detect/src/cadence.rs +++ b/crates/ax-detect/src/cadence.rs @@ -11,7 +11,7 @@ //! column named by `cadence_column`, and otherwise reports honest absence. use crate::config::DetectConfig; -use crate::{Detector, Report, ScanContext}; +use crate::{calibrate, Detector, Report, ScanContext}; use ax_core::det; use ax_core::finding::Handle; use ax_core::{AnomalyClass, Finding}; @@ -78,7 +78,7 @@ impl Detector for CadenceDetector { // genuinely-killable `min_n` `<` above stay under the gate. if cfg.cad_max_cv > cv { // Lower CV ⇒ more metronomic ⇒ higher confidence. - let confidence = (1.0 - cv / cfg.cad_max_cv).clamp(0.0, 1.0); + let confidence = calibrate::from_undercut(cv, cfg.cad_max_cv); out.push(Finding::new( self.id(), AnomalyClass::Cadence, @@ -202,10 +202,10 @@ mod tests { } #[test] - fn confidence_is_one_minus_cv_over_threshold() { + fn confidence_is_calibrated_from_cv_under_threshold() { // Alternating 100 / 100.1 gaps → a small but non-zero CV, well below the - // 0.05 threshold. Pins confidence = 1 − cv/threshold exactly (the score - // is the cv), catching the `/` → `*` / `%` mutations. + // 0.05 threshold. Confidence is the unified undercut calibration of the + // CV against the max-CV threshold (the score is the cv). let mut ts = vec![0.0]; for i in 0..40 { ts.push(ts[i] + if i % 2 == 0 { 100.0 } else { 100.1 }); @@ -218,8 +218,9 @@ mod tests { "cv is small but positive: {}", f.score ); - let expected = 1.0 - f.score / DetectConfig::default().cad_max_cv; + let expected = crate::calibrate::from_undercut(f.score, DetectConfig::default().cad_max_cv); assert!((f.confidence - expected).abs() < 1e-12); + assert!(f.confidence > 0.5, "CV under threshold ⇒ above 0.5"); } #[test] diff --git a/crates/ax-detect/src/calibrate.rs b/crates/ax-detect/src/calibrate.rs new file mode 100644 index 0000000..4680aa8 --- /dev/null +++ b/crates/ax-detect/src/calibrate.rs @@ -0,0 +1,112 @@ +//! Unified confidence calibration. +//! +//! Every detector ultimately reduces to "a statistic crossed a firing +//! threshold." To make a confidence of `0.9` mean the same *strength of evidence* +//! regardless of which detector produced it, confidence is a single shared +//! logistic of how far the statistic sits past its threshold, measured +//! **relatively** so the detector's units cancel: right at the threshold → `0.5`, +//! rising toward `1.0` as the statistic grows more extreme. A finding at "twice +//! its threshold" earns the same confidence whether it came from a modified +//! z-score, a KS p-value, a PSI, or an inter-arrival coefficient of variation — +//! which is what lets severity (and `--top` / `--min-severity`) rank findings +//! from different detectors on one scale. + +/// Logistic steepness in the relative excess. Larger ⇒ confidence climbs faster +/// past the threshold. Tuned so ~2× past threshold reads as "high" and ~3×+ as +/// "critical" on the severity ladder. A fixed calibration constant (like the +/// modified-z `MODZ_K`), not a tunable — changing it is a tool-version change. +const STEEPNESS: f64 = 2.0; + +fn logistic(x: f64) -> f64 { + 1.0 / (1.0 + (-x).exp()) +} + +/// Confidence for a detector that fires when `statistic` rises **above** a +/// positive `threshold` (modified z-score, CUSUM shift, PSI, null fraction): +/// logistic in the relative excess `statistic/threshold − 1`. At the threshold +/// → `0.5`; far above → → `1.0`. +pub fn from_exceedance(statistic: f64, threshold: f64) -> f64 { + if threshold <= 0.0 { + // A zero (or invalid) bar: any positive statistic is maximally past it. + return if statistic > 0.0 { 1.0 } else { 0.5 }; + } + logistic(STEEPNESS * (statistic / threshold - 1.0)) +} + +/// Confidence for a detector that fires when `statistic` falls **below** a +/// positive `threshold` (a p-value under alpha, a CV under the max): logistic in +/// the relative excess `threshold/statistic − 1`. At the threshold → `0.5`; as +/// the statistic approaches `0` → → `1.0` (maximally significant). +pub fn from_undercut(statistic: f64, threshold: f64) -> f64 { + if statistic <= 0.0 { + // p == 0 / CV == 0: as far past the bar as it is possible to be. + return 1.0; + } + if threshold <= 0.0 { + return 0.0; + } + logistic(STEEPNESS * (threshold / statistic - 1.0)) +} + +#[cfg(test)] +mod tests { + use super::*; + + // logistic(2·1) = 1/(1+e^-2) ≈ 0.880797; the canonical "2× past threshold". + const TWO_X: f64 = 0.8807970779778823; + + #[test] + fn at_threshold_is_one_half() { + assert_eq!(from_exceedance(3.5, 3.5), 0.5); + assert_eq!(from_exceedance(0.2, 0.2), 0.5); + assert_eq!(from_undercut(0.05, 0.05), 0.5); + assert_eq!(from_undercut(0.001, 0.001), 0.5); + } + + #[test] + fn exceedance_rises_above_threshold_falls_below() { + assert!(from_exceedance(7.0, 3.5) > 0.5); // 2× ⇒ above 0.5 + assert!(from_exceedance(2.0, 3.5) < 0.5); // under the bar ⇒ below 0.5 + assert!(from_exceedance(10.0, 3.5) > from_exceedance(5.0, 3.5)); // monotone + } + + #[test] + fn undercut_rises_as_statistic_shrinks() { + assert!(from_undercut(0.025, 0.05) > 0.5); // half the bar ⇒ above 0.5 + assert!(from_undercut(0.10, 0.05) < 0.5); // over the bar ⇒ below 0.5 + assert!(from_undercut(0.001, 0.05) > from_undercut(0.01, 0.05)); // monotone + } + + #[test] + fn boundaries_are_saturated_not_nan() { + assert_eq!(from_undercut(0.0, 0.05), 1.0); // p == 0 + assert_eq!(from_undercut(0.0, 0.0), 1.0); // degenerate p == cutoff == 0 + assert_eq!(from_exceedance(1.0, 0.0), 1.0); // positive past a zero bar + assert_eq!(from_exceedance(0.0, 0.0), 0.5); + assert_eq!(from_undercut(0.5, 0.0), 0.0); + } + + #[test] + fn two_x_past_threshold_is_identical_across_directions_and_units() { + // The comparability guarantee: "2× past the threshold" yields the same + // confidence regardless of detector direction or units. + let a = from_exceedance(7.0, 3.5); // 2× above (z-score-like) + let b = from_exceedance(0.4, 0.2); // 2× above (PSI-like) + let c = from_undercut(0.025, 0.05); // half the bar = 2× past (p-value-like) + assert!((a - TWO_X).abs() < 1e-12); + assert!((b - TWO_X).abs() < 1e-12); + assert!((c - TWO_X).abs() < 1e-12); + } + + #[test] + fn stays_in_unit_interval() { + for &(s, t) in &[(1e9, 3.5), (3.5, 3.5), (0.0, 3.5)] { + let c = from_exceedance(s, t); + assert!((0.0..=1.0).contains(&c)); + } + for &(s, t) in &[(1e-12, 0.05), (0.05, 0.05), (1.0, 0.05)] { + let c = from_undercut(s, t); + assert!((0.0..=1.0).contains(&c)); + } + } +} diff --git a/crates/ax-detect/src/coll.rs b/crates/ax-detect/src/coll.rs index bc407b5..d3e6bd1 100644 --- a/crates/ax-detect/src/coll.rs +++ b/crates/ax-detect/src/coll.rs @@ -11,7 +11,7 @@ //! point chosen by a stable arg-max (first maximal split wins). use crate::config::DetectConfig; -use crate::{Detector, Report, ScanContext}; +use crate::{calibrate, Detector, Report, ScanContext}; use ax_core::det; use ax_core::finding::Handle; use ax_core::{AnomalyClass, Finding}; @@ -45,12 +45,6 @@ pub fn standardized_shift(mean_l: f64, mean_r: f64, sigma: f64, nl: usize, nr: u (mean_l - mean_r).abs() / se } -/// Logistic confidence in the excess of the standardized shift over `threshold` -/// (shift == threshold ⇒ 0.5), strictly increasing in the shift. -fn shift_confidence(shift: f64, threshold: f64) -> f64 { - 1.0 / (1.0 + (-(shift - threshold)).exp()) -} - impl Detector for CusumDetector { fn id(&self) -> &'static str { "coll.cusum" @@ -101,7 +95,7 @@ impl Detector for CusumDetector { start, end, }, - shift_confidence(shift, cfg.coll_threshold), + calibrate::from_exceedance(shift, cfg.coll_threshold), shift, format!( "{}: level shift at row {start} — mean {mean_l:.4} → {mean_r:.4}, standardized shift {shift:.3} > {:.3}", @@ -173,13 +167,6 @@ mod tests { assert!((standardized_shift(0.0, 2.0, 1.0, 1, 3) - 3.0_f64.sqrt()).abs() < 1e-12); } - #[test] - fn shift_confidence_is_half_at_threshold_and_monotonic() { - assert_eq!(shift_confidence(5.0, 5.0), 0.5); - assert!(shift_confidence(4.0, 5.0) < 0.5); - assert!(shift_confidence(8.0, 5.0) > shift_confidence(6.0, 5.0)); - } - #[test] fn sustained_level_shift_is_flagged_as_a_range() { // Clean 10→40 step at row 20 (no noise) so segment means are exactly diff --git a/crates/ax-detect/src/config.rs b/crates/ax-detect/src/config.rs index 6366e4b..4e085e6 100644 --- a/crates/ax-detect/src/config.rs +++ b/crates/ax-detect/src/config.rs @@ -110,7 +110,7 @@ impl DetectConfig { /// Deterministic: no wall-clock, no environment. pub fn version(&self) -> String { format!( - "anomalyx-cfg/7;pt={:.4};ptn={};cr={};pfdr={};da={:.4};psi={:.4};psib={};dmn={};snr={:.4};mva={:.5};mvn={};mvr={:e};cxp={};cxt={:.4};cxm={};cln={};clt={:.4};cdc={};cdcv={:.4};cdn={}", + "anomalyx-cfg/8;pt={:.4};ptn={};cr={};pfdr={};da={:.4};psi={:.4};psib={};dmn={};snr={:.4};mva={:.5};mvn={};mvr={:e};cxp={};cxt={:.4};cxm={};cln={};clt={:.4};cdc={};cdcv={:.4};cdn={}", self.point_threshold, self.point_min_n, self.column_roles, diff --git a/crates/ax-detect/src/ctx.rs b/crates/ax-detect/src/ctx.rs index c25d4a3..134fa91 100644 --- a/crates/ax-detect/src/ctx.rs +++ b/crates/ax-detect/src/ctx.rs @@ -10,7 +10,7 @@ //! honest absence rather than inventing one. use crate::config::DetectConfig; -use crate::{robustz, Detector, Report, ScanContext}; +use crate::{calibrate, robustz, Detector, Report, ScanContext}; use ax_core::finding::Handle; use ax_core::{AnomalyClass, Column, Finding, RecordSet}; @@ -53,7 +53,7 @@ impl SeasonalDetector { column: col.name.clone(), row, }, - robustz::confidence(modz, cfg.ctx_threshold), + calibrate::from_exceedance(modz, cfg.ctx_threshold), modz, format!( "{} = {v:.6} at row {row} (phase {phase}/{p}): modified z-score {modz:.3} within its seasonal subseries exceeds {:.3}", diff --git a/crates/ax-detect/src/dist.rs b/crates/ax-detect/src/dist.rs index 9aaf8ea..90c9924 100644 --- a/crates/ax-detect/src/dist.rs +++ b/crates/ax-detect/src/dist.rs @@ -14,6 +14,7 @@ //! The statistics are deterministic: sorted samples, fixed bin edges derived //! from baseline quantiles, and order-independent reductions via [`ax_core::det`]. +use crate::calibrate; use crate::config::DetectConfig; use crate::{Detector, Report, ScanContext}; use ax_core::det; @@ -128,12 +129,6 @@ pub fn psi(baseline: &[f64], current: &[f64], bins: usize) -> Option { Some(det::det_sum(&terms)) } -/// Maps a PSI value to a confidence in `[0, 1]`, logistic around `threshold` -/// (PSI == threshold ⇒ 0.5). Strictly increasing in PSI. -fn psi_confidence(psi: f64, threshold: f64) -> f64 { - 1.0 / (1.0 + (-(psi - threshold) / 0.1).exp()) -} - // --------------------------------------------------------------------------- // Chi-square over categories // --------------------------------------------------------------------------- @@ -254,7 +249,7 @@ impl Detector for KsDetector { Handle::Dist { column: name.to_string(), }, - 1.0 - p, + calibrate::from_undercut(p, cfg.dist_alpha), d, format!( "{name}: KS D={d:.4}, p={p:.4} < α={:.4} — distribution shifted", @@ -302,7 +297,7 @@ impl Detector for PsiDetector { Handle::Dist { column: name.to_string(), }, - psi_confidence(value, cfg.psi_threshold), + calibrate::from_exceedance(value, cfg.psi_threshold), value, format!( "{name}: PSI={value:.4} > {:.4} — population shifted", @@ -361,7 +356,7 @@ impl Detector for Chi2Detector { Handle::Dist { column: col.name.clone(), }, - 1.0 - p, + calibrate::from_undercut(p, cfg.dist_alpha), stat, format!( "{}: χ²={stat:.3} (dof={dof}), p={p:.4} < α={:.4} — category mix changed", @@ -451,13 +446,6 @@ mod tests { assert_eq!(psi(&[1.0, 2.0, 3.0], &[1.0, 2.0], 10), None); } - #[test] - fn psi_confidence_is_half_at_threshold_and_monotonic() { - assert_eq!(psi_confidence(0.2, 0.2), 0.5); - assert!(psi_confidence(0.1, 0.2) < 0.5); - assert!(psi_confidence(0.5, 0.2) > psi_confidence(0.3, 0.2)); - } - #[test] fn chi2_zero_for_identical_category_mix() { let counts: BTreeMap = [("a".to_string(), 10), ("b".to_string(), 30)] @@ -610,12 +598,6 @@ mod tests { assert_eq!(psi(&[1.0, 2.0, 3.0], &[1.0], 1), None); // bins < 2 } - #[test] - fn psi_confidence_exact() { - // (0.3 − 0.2)/0.1 = 1.0 ⇒ sigmoid(1) = 0.7310585786300049. - assert!((psi_confidence(0.3, 0.2) - 0.7310585786300049).abs() < 1e-15); - } - #[test] fn chi2_categorical_exact_stat() { let base: BTreeMap = [("a".to_string(), 10), ("b".to_string(), 30)] @@ -633,22 +615,22 @@ mod tests { // ----- detector boundary & confidence behavior ----- #[test] - fn ks_detector_confidence_is_one_minus_pvalue() { + fn ks_detector_confidence_is_calibrated_from_pvalue() { // Mild drift (shift 12) → flagged with a *meaningful* p-value, so the - // exact `1 − p` confidence is distinguishable from `1 + p` / `1 / p`. + // calibrated confidence is the unified undercut mapping of (p vs alpha), + // distinguishable from a degenerate constant. let base: Vec = (0..40).map(|i| i as f64).collect(); let cur: Vec = (0..40).map(|i| i as f64 + 12.0).collect(); let (b, c) = baseline_vs_current(vec![ncol("x", &base)], vec![ncol("x", &cur)]); + let cfg = DetectConfig::default(); let mut out = Report::new(); - KsDetector.detect( - &ScanContext::compared(&b, &c), - &DetectConfig::default(), - &mut out, - ); + KsDetector.detect(&ScanContext::compared(&b, &c), &cfg, &mut out); assert_eq!(out.findings.len(), 1); let p = ks_pvalue(ks_statistic(&base, &cur).unwrap(), 40, 40); - assert!((out.findings[0].confidence - (1.0 - p)).abs() < 1e-12); - assert!(out.findings[0].confidence < 0.99); + let expected = calibrate::from_undercut(p, cfg.dist_alpha); + assert!((out.findings[0].confidence - expected).abs() < 1e-12); + // p well below alpha ⇒ confidence above the at-threshold 0.5. + assert!(out.findings[0].confidence > 0.5); } #[test] diff --git a/crates/ax-detect/src/lib.rs b/crates/ax-detect/src/lib.rs index 66eef64..a76a361 100644 --- a/crates/ax-detect/src/lib.rs +++ b/crates/ax-detect/src/lib.rs @@ -14,6 +14,7 @@ use ax_core::envelope::Absence; use ax_core::{AnomalyClass, Finding, RecordSet}; pub mod cadence; +pub mod calibrate; pub mod coll; pub mod config; pub mod ctx; diff --git a/crates/ax-detect/src/mv.rs b/crates/ax-detect/src/mv.rs index fb0c6bd..eaeb17a 100644 --- a/crates/ax-detect/src/mv.rs +++ b/crates/ax-detect/src/mv.rs @@ -15,7 +15,7 @@ use crate::config::DetectConfig; use crate::dist::chi2_pvalue; -use crate::{linalg, Detector, Report, ScanContext}; +use crate::{calibrate, linalg, Detector, Report, ScanContext}; use ax_core::finding::Handle; use ax_core::{det, AnomalyClass, Finding, RecordSet}; @@ -153,7 +153,7 @@ impl Detector for MahalanobisDetector { self.id(), AnomalyClass::Multivariate, Handle::Row { row }, - 1.0 - p, + calibrate::from_undercut(p, cfg.mv_alpha), dsq, format!( "row {row}: Mahalanobis D²={dsq:.3} over {} numeric columns, p={p:.3e} < α={:.3e} — joint outlier", @@ -254,9 +254,9 @@ mod tests { assert_eq!(report.findings.len(), 1); assert!(matches!(report.findings[0].handle, Handle::Row { row: 40 })); assert_eq!(report.findings[0].class, AnomalyClass::Multivariate); - // confidence is 1 − p with p > 0, so strictly below 1.0 (pins `1 - p`). - assert!(report.findings[0].confidence < 1.0); - assert!(report.findings[0].confidence > 0.99); + // A clear joint outlier sits far past the alpha bar, so the unified + // undercut calibration drives confidence to the critical end. + assert!(report.findings[0].confidence > 0.95); } #[test] diff --git a/crates/ax-detect/src/point.rs b/crates/ax-detect/src/point.rs index 0b66a92..c695525 100644 --- a/crates/ax-detect/src/point.rs +++ b/crates/ax-detect/src/point.rs @@ -12,7 +12,7 @@ //! exactly what the property tests pin down. use crate::config::DetectConfig; -use crate::{fdr, robustz, Detector, Report, ScanContext}; +use crate::{calibrate, fdr, robustz, Detector, Report, ScanContext}; use ax_core::finding::Handle; use ax_core::{AnomalyClass, Column, Finding, Role, Value}; @@ -81,7 +81,7 @@ impl PointDetector { }; match cfg.point_fdr_q { - Some(q) => self.scan_column_fdr(col, center, scale, q, cfg, out), + Some(q) => self.scan_column_fdr(col, center, scale, q, out), None => self.scan_column_threshold(col, center, scale, k, cfg, out), } } @@ -114,7 +114,7 @@ impl PointDetector { col, row, modz, - robustz::confidence(modz, cfg.point_threshold), + calibrate::from_exceedance(modz, cfg.point_threshold), reason, out, ); @@ -125,15 +125,7 @@ impl PointDetector { /// flag only the cells that survive Benjamini–Hochberg control at level `q` /// *within this column*. A column that is really just noise rejects nothing, /// so it stops contributing chance flags; the fixed threshold is bypassed. - fn scan_column_fdr( - &self, - col: &Column, - center: f64, - scale: f64, - q: f64, - cfg: &DetectConfig, - out: &mut Report, - ) { + fn scan_column_fdr(&self, col: &Column, center: f64, scale: f64, q: f64, out: &mut Report) { // First pass: (row, value, |z|, p) for every finite cell, in cell order. // `z = (x − center)/scale` is the consistent-σ standardized deviation // (≈ N(0, 1) under the null in both the MAD and σ branches) — unlike @@ -169,7 +161,7 @@ impl PointDetector { col, row, z, - robustz::confidence(z, cfg.point_threshold), + calibrate::from_undercut(p, cutoff), reason, out, ); diff --git a/crates/ax-detect/src/robustz.rs b/crates/ax-detect/src/robustz.rs index 766ba55..7d0d8de 100644 --- a/crates/ax-detect/src/robustz.rs +++ b/crates/ax-detect/src/robustz.rs @@ -34,14 +34,6 @@ pub fn score(x: f64, center: f64, scale: f64, k: f64) -> f64 { (k * (x - center) / scale).abs() } -/// Maps a modified z-score to a calibrated confidence in `[0, 1]`: logistic in -/// the excess `modz − threshold`, so `modz == threshold` gives 0.5 and it rises -/// monotonically toward 1.0. -pub fn confidence(modz: f64, threshold: f64) -> f64 { - let excess = modz - threshold; - 1.0 / (1.0 + (-excess).exp()) -} - #[cfg(test)] mod tests { use super::*; @@ -53,13 +45,6 @@ mod tests { assert_eq!(score(4.0, 10.0, 3.0, 1.0), 2.0); } - #[test] - fn confidence_is_half_at_threshold_and_monotonic() { - assert_eq!(confidence(3.5, 3.5), 0.5); - assert!(confidence(2.0, 3.5) < 0.5); - assert!(confidence(6.0, 3.5) > confidence(4.0, 3.5)); - } - #[test] fn center_scale_prefers_mad_then_sigma_then_none() { // spread data → MAD path, k = MODZ_K diff --git a/crates/ax-detect/src/structural.rs b/crates/ax-detect/src/structural.rs index 076c027..77b91b6 100644 --- a/crates/ax-detect/src/structural.rs +++ b/crates/ax-detect/src/structural.rs @@ -11,7 +11,7 @@ //! reports absence; it simply emits fewer findings when there is no baseline. use crate::config::DetectConfig; -use crate::{Detector, Report, ScanContext}; +use crate::{calibrate, Detector, Report, ScanContext}; use ax_core::finding::Handle; use ax_core::{AnomalyClass, ColType, Finding, RecordSet}; @@ -51,7 +51,7 @@ impl SchemaDetector { self.id(), AnomalyClass::Structural, Self::col_handle(&col.name), - frac, + calibrate::from_exceedance(frac, cfg.struct_null_rate), frac, format!("column '{}' is {:.0}% null", col.name, frac * 100.0), ) diff --git a/docs/src/envelope.md b/docs/src/envelope.md index d062b93..29ed7ba 100644 --- a/docs/src/envelope.md +++ b/docs/src/envelope.md @@ -35,6 +35,13 @@ JSON Schema. - **`columns`** — the fixed column order of each dense finding row. - **`rows`** — one array per finding, aligned to `columns`: `[detector_idx, class_idx, handle_idx, confidence, severity_idx, score, reason_idx]`. + `confidence` is **calibrated to one scale across every detector**: a logistic of + how far the detector's statistic sits past its firing threshold, measured + relatively (so units cancel) — `0.5` at the threshold, rising toward `1.0`. A + finding "2× past threshold" earns the same confidence whether it came from a + modified z-score, a KS p-value, a PSI, or a cadence CV, so `severity` (derived + from confidence) ranks findings from different detectors on one scale. `score` + is the detector's raw statistic (uncalibrated), for drill-down. - **`absent`** — detectors that declined to run, each with a machine-readable reason. See [honest absence](./determinism.md). - **`summary`** — total count, max severity, and per-class counts for at-a-glance