feat: unified cross-detector confidence calibration (0.8.0)

CHANGELOG.md

@@ -6,15 +6,34 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
 
+## [0.8.0] - 2026-06-01
+
+### Changed
+
+- **Unified confidence calibration across all detectors.** Confidence was
+  computed three incompatible ways (`1 − p` for the distributional/multivariate
+  detectors, a logistic-over-threshold for point/contextual/collective/PSI, and a
+  linear map for cadence), so a `0.9` meant different things depending on which
+  detector produced it — and severity (and `--top` / `--min-severity`) couldn't
+  rank across detectors. Now every detector routes through one shared function:
+  confidence is a logistic of how far its statistic sits past its firing
+  threshold, measured **relatively** so units cancel. At the threshold → `0.5`,
+  rising toward `1.0`; a finding "2× past threshold" earns the same confidence on
+  any detector. New `ax_detect::calibrate` module (`from_exceedance` /
+  `from_undercut`); the duplicated `shift_confidence` / `psi_confidence` /
+  `robustz::confidence` helpers are gone.
+- This recalibrates every published confidence and severity. The `config_version`
+  fingerprint is bumped (`anomalyx-cfg/8`) so the change is visible to agents.
+  The envelope shape and `PROTOCOL` are unchanged.
+
 ### Testing
 
 - **Parser robustness harness** (`ax-normalize/tests/robustness.rs`). Property
   tests assert that no parser panics, hangs, or over-allocates on arbitrary,
   magic-prefixed-garbage, or truncated byte streams — fed both through
   auto-detection and straight to every registered parser — and that
   normalization is deterministic over fuzz inputs. Untrusted-input hardening:
-  a malformed file must fail cleanly, never crash. Test-only; no library change
-  (so no release).
+  a malformed file must fail cleanly, never crash.
 
 ## [0.7.0] - 2026-06-01
 
@@ -263,7 +282,8 @@ Initial release — a contract-first anomaly-detection CLI over arbitrary corpor
   gates on every push.
 - Dual-licensed under MIT OR Apache-2.0.
 
-[Unreleased]: https://github.com/copyleftdev/anomalyx/compare/v0.7.0...HEAD
+[Unreleased]: https://github.com/copyleftdev/anomalyx/compare/v0.8.0...HEAD
+[0.8.0]: https://github.com/copyleftdev/anomalyx/compare/v0.7.0...v0.8.0
 [0.7.0]: https://github.com/copyleftdev/anomalyx/compare/v0.6.0...v0.7.0
 [0.6.0]: https://github.com/copyleftdev/anomalyx/compare/v0.5.0...v0.6.0
 [0.5.0]: https://github.com/copyleftdev/anomalyx/compare/v0.4.1...v0.5.0

Cargo.toml

@@ -9,7 +9,7 @@ members = [
 ]
 
 [workspace.package]
-version = "0.7.0"
+version = "0.8.0"
 edition = "2021"
 rust-version = "1.90"
 license = "MIT OR Apache-2.0"
@@ -22,9 +22,9 @@ authors = ["copyleftdev"]
 # `ax-*` names were taken), but the import/extern name stays `ax_core` etc. via
 # the dependency key + `package` rename — so no source code changes are needed.
 # Keep versions in sync with workspace.package.version above.
-ax-core = { path = "crates/ax-core", version = "0.7.0", package = "anomalyx-core" }
-ax-normalize = { path = "crates/ax-normalize", version = "0.7.0", package = "anomalyx-normalize" }
-ax-detect = { path = "crates/ax-detect", version = "0.7.0", package = "anomalyx-detect" }
+ax-core = { path = "crates/ax-core", version = "0.8.0", package = "anomalyx-core" }
+ax-normalize = { path = "crates/ax-normalize", version = "0.8.0", package = "anomalyx-normalize" }
+ax-detect = { path = "crates/ax-detect", version = "0.8.0", package = "anomalyx-detect" }
 
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"

crates/ax-detect/src/cadence.rs

@@ -11,7 +11,7 @@
 //! column named by `cadence_column`, and otherwise reports honest absence.
 
 use crate::config::DetectConfig;
-use crate::{Detector, Report, ScanContext};
+use crate::{calibrate, Detector, Report, ScanContext};
 use ax_core::det;
 use ax_core::finding::Handle;
 use ax_core::{AnomalyClass, Finding};
@@ -78,7 +78,7 @@ impl Detector for CadenceDetector {
         // genuinely-killable `min_n` `<` above stay under the gate.
         if cfg.cad_max_cv > cv {
             // Lower CV ⇒ more metronomic ⇒ higher confidence.
-            let confidence = (1.0 - cv / cfg.cad_max_cv).clamp(0.0, 1.0);
+            let confidence = calibrate::from_undercut(cv, cfg.cad_max_cv);
             out.push(Finding::new(
                 self.id(),
                 AnomalyClass::Cadence,
@@ -202,10 +202,10 @@ mod tests {
     }
 
     #[test]
-    fn confidence_is_one_minus_cv_over_threshold() {
+    fn confidence_is_calibrated_from_cv_under_threshold() {
         // Alternating 100 / 100.1 gaps → a small but non-zero CV, well below the
-        // 0.05 threshold. Pins confidence = 1 − cv/threshold exactly (the score
-        // is the cv), catching the `/` → `*` / `%` mutations.
+        // 0.05 threshold. Confidence is the unified undercut calibration of the
+        // CV against the max-CV threshold (the score is the cv).
         let mut ts = vec![0.0];
         for i in 0..40 {
             ts.push(ts[i] + if i % 2 == 0 { 100.0 } else { 100.1 });
@@ -218,8 +218,9 @@ mod tests {
             "cv is small but positive: {}",
             f.score
         );
-        let expected = 1.0 - f.score / DetectConfig::default().cad_max_cv;
+        let expected = crate::calibrate::from_undercut(f.score, DetectConfig::default().cad_max_cv);
         assert!((f.confidence - expected).abs() < 1e-12);
+        assert!(f.confidence > 0.5, "CV under threshold ⇒ above 0.5");
     }
 
     #[test]

crates/ax-detect/src/calibrate.rs

@@ -0,0 +1,112 @@
+//! Unified confidence calibration.
+//!
+//! Every detector ultimately reduces to "a statistic crossed a firing
+//! threshold." To make a confidence of `0.9` mean the same *strength of evidence*
+//! regardless of which detector produced it, confidence is a single shared
+//! logistic of how far the statistic sits past its threshold, measured
+//! **relatively** so the detector's units cancel: right at the threshold → `0.5`,
+//! rising toward `1.0` as the statistic grows more extreme. A finding at "twice
+//! its threshold" earns the same confidence whether it came from a modified
+//! z-score, a KS p-value, a PSI, or an inter-arrival coefficient of variation —
+//! which is what lets severity (and `--top` / `--min-severity`) rank findings
+//! from different detectors on one scale.
+
+/// Logistic steepness in the relative excess. Larger ⇒ confidence climbs faster
+/// past the threshold. Tuned so ~2× past threshold reads as "high" and ~3×+ as
+/// "critical" on the severity ladder. A fixed calibration constant (like the
+/// modified-z `MODZ_K`), not a tunable — changing it is a tool-version change.
+const STEEPNESS: f64 = 2.0;
+
+fn logistic(x: f64) -> f64 {
+    1.0 / (1.0 + (-x).exp())
+}
+
+/// Confidence for a detector that fires when `statistic` rises **above** a
+/// positive `threshold` (modified z-score, CUSUM shift, PSI, null fraction):
+/// logistic in the relative excess `statistic/threshold − 1`. At the threshold
+/// → `0.5`; far above → → `1.0`.
+pub fn from_exceedance(statistic: f64, threshold: f64) -> f64 {
+    if threshold <= 0.0 {
+        // A zero (or invalid) bar: any positive statistic is maximally past it.
+        return if statistic > 0.0 { 1.0 } else { 0.5 };
+    }
+    logistic(STEEPNESS * (statistic / threshold - 1.0))
+}
+
+/// Confidence for a detector that fires when `statistic` falls **below** a
+/// positive `threshold` (a p-value under alpha, a CV under the max): logistic in
+/// the relative excess `threshold/statistic − 1`. At the threshold → `0.5`; as
+/// the statistic approaches `0` → → `1.0` (maximally significant).
+pub fn from_undercut(statistic: f64, threshold: f64) -> f64 {
+    if statistic <= 0.0 {
+        // p == 0 / CV == 0: as far past the bar as it is possible to be.
+        return 1.0;
+    }
+    if threshold <= 0.0 {
+        return 0.0;
+    }
+    logistic(STEEPNESS * (threshold / statistic - 1.0))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    // logistic(2·1) = 1/(1+e^-2) ≈ 0.880797; the canonical "2× past threshold".
+    const TWO_X: f64 = 0.8807970779778823;
+
+    #[test]
+    fn at_threshold_is_one_half() {
+        assert_eq!(from_exceedance(3.5, 3.5), 0.5);
+        assert_eq!(from_exceedance(0.2, 0.2), 0.5);
+        assert_eq!(from_undercut(0.05, 0.05), 0.5);
+        assert_eq!(from_undercut(0.001, 0.001), 0.5);
+    }
+
+    #[test]
+    fn exceedance_rises_above_threshold_falls_below() {
+        assert!(from_exceedance(7.0, 3.5) > 0.5); // 2× ⇒ above 0.5
+        assert!(from_exceedance(2.0, 3.5) < 0.5); // under the bar ⇒ below 0.5
+        assert!(from_exceedance(10.0, 3.5) > from_exceedance(5.0, 3.5)); // monotone
+    }
+
+    #[test]
+    fn undercut_rises_as_statistic_shrinks() {
+        assert!(from_undercut(0.025, 0.05) > 0.5); // half the bar ⇒ above 0.5
+        assert!(from_undercut(0.10, 0.05) < 0.5); // over the bar ⇒ below 0.5
+        assert!(from_undercut(0.001, 0.05) > from_undercut(0.01, 0.05)); // monotone
+    }
+
+    #[test]
+    fn boundaries_are_saturated_not_nan() {
+        assert_eq!(from_undercut(0.0, 0.05), 1.0); // p == 0
+        assert_eq!(from_undercut(0.0, 0.0), 1.0); // degenerate p == cutoff == 0
+        assert_eq!(from_exceedance(1.0, 0.0), 1.0); // positive past a zero bar
+        assert_eq!(from_exceedance(0.0, 0.0), 0.5);
+        assert_eq!(from_undercut(0.5, 0.0), 0.0);
+    }
+
+    #[test]
+    fn two_x_past_threshold_is_identical_across_directions_and_units() {
+        // The comparability guarantee: "2× past the threshold" yields the same
+        // confidence regardless of detector direction or units.
+        let a = from_exceedance(7.0, 3.5); // 2× above (z-score-like)
+        let b = from_exceedance(0.4, 0.2); // 2× above (PSI-like)
+        let c = from_undercut(0.025, 0.05); // half the bar = 2× past (p-value-like)
+        assert!((a - TWO_X).abs() < 1e-12);
+        assert!((b - TWO_X).abs() < 1e-12);
+        assert!((c - TWO_X).abs() < 1e-12);
+    }
+
+    #[test]
+    fn stays_in_unit_interval() {
+        for &(s, t) in &[(1e9, 3.5), (3.5, 3.5), (0.0, 3.5)] {
+            let c = from_exceedance(s, t);
+            assert!((0.0..=1.0).contains(&c));
+        }
+        for &(s, t) in &[(1e-12, 0.05), (0.05, 0.05), (1.0, 0.05)] {
+            let c = from_undercut(s, t);
+            assert!((0.0..=1.0).contains(&c));
+        }
+    }
+}

crates/ax-detect/src/coll.rs

@@ -11,7 +11,7 @@
 //! point chosen by a stable arg-max (first maximal split wins).
 
 use crate::config::DetectConfig;
-use crate::{Detector, Report, ScanContext};
+use crate::{calibrate, Detector, Report, ScanContext};
 use ax_core::det;
 use ax_core::finding::Handle;
 use ax_core::{AnomalyClass, Finding};
@@ -45,12 +45,6 @@ pub fn standardized_shift(mean_l: f64, mean_r: f64, sigma: f64, nl: usize, nr: u
     (mean_l - mean_r).abs() / se
 }
 
-/// Logistic confidence in the excess of the standardized shift over `threshold`
-/// (shift == threshold ⇒ 0.5), strictly increasing in the shift.
-fn shift_confidence(shift: f64, threshold: f64) -> f64 {
-    1.0 / (1.0 + (-(shift - threshold)).exp())
-}
-
 impl Detector for CusumDetector {
     fn id(&self) -> &'static str {
         "coll.cusum"
@@ -101,7 +95,7 @@ impl Detector for CusumDetector {
                         start,
                         end,
                     },
-                    shift_confidence(shift, cfg.coll_threshold),
+                    calibrate::from_exceedance(shift, cfg.coll_threshold),
                     shift,
                     format!(
                         "{}: level shift at row {start} — mean {mean_l:.4} → {mean_r:.4}, standardized shift {shift:.3} > {:.3}",
@@ -173,13 +167,6 @@ mod tests {
         assert!((standardized_shift(0.0, 2.0, 1.0, 1, 3) - 3.0_f64.sqrt()).abs() < 1e-12);
     }
 
-    #[test]
-    fn shift_confidence_is_half_at_threshold_and_monotonic() {
-        assert_eq!(shift_confidence(5.0, 5.0), 0.5);
-        assert!(shift_confidence(4.0, 5.0) < 0.5);
-        assert!(shift_confidence(8.0, 5.0) > shift_confidence(6.0, 5.0));
-    }
-
     #[test]
     fn sustained_level_shift_is_flagged_as_a_range() {
         // Clean 10→40 step at row 20 (no noise) so segment means are exactly

crates/ax-detect/src/config.rs

@@ -110,7 +110,7 @@ impl DetectConfig {
     /// Deterministic: no wall-clock, no environment.
     pub fn version(&self) -> String {
         format!(
-            "anomalyx-cfg/7;pt={:.4};ptn={};cr={};pfdr={};da={:.4};psi={:.4};psib={};dmn={};snr={:.4};mva={:.5};mvn={};mvr={:e};cxp={};cxt={:.4};cxm={};cln={};clt={:.4};cdc={};cdcv={:.4};cdn={}",
+            "anomalyx-cfg/8;pt={:.4};ptn={};cr={};pfdr={};da={:.4};psi={:.4};psib={};dmn={};snr={:.4};mva={:.5};mvn={};mvr={:e};cxp={};cxt={:.4};cxm={};cln={};clt={:.4};cdc={};cdcv={:.4};cdn={}",
             self.point_threshold,
             self.point_min_n,
             self.column_roles,

crates/ax-detect/src/ctx.rs

@@ -10,7 +10,7 @@
 //! honest absence rather than inventing one.
 
 use crate::config::DetectConfig;
-use crate::{robustz, Detector, Report, ScanContext};
+use crate::{calibrate, robustz, Detector, Report, ScanContext};
 use ax_core::finding::Handle;
 use ax_core::{AnomalyClass, Column, Finding, RecordSet};
 
@@ -53,7 +53,7 @@ impl SeasonalDetector {
                         column: col.name.clone(),
                         row,
                     },
-                    robustz::confidence(modz, cfg.ctx_threshold),
+                    calibrate::from_exceedance(modz, cfg.ctx_threshold),
                     modz,
                     format!(
                         "{} = {v:.6} at row {row} (phase {phase}/{p}): modified z-score {modz:.3} within its seasonal subseries exceeds {:.3}",