diff --git a/Cargo.toml b/Cargo.toml index c2f4d345c..78fc9c17a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -272,6 +272,10 @@ members = [ "crates/timesfm", # RuVector integration for TimesFM: Forecaster + anomaly bands + sweep early-stopping "crates/ruvector-timesfm", + # Calyx-inspired association-native memory layer: multi-slot constellations, + # lens manifests, RRF fusion, cross-lens agreement, grounding, fail-closed + # guard, hash-chained provenance ledger, reversible anneal optimizer (ADR-272) + "crates/ruvector-calyx", ] resolver = "2" diff --git a/crates/ruvector-calyx/Cargo.toml b/crates/ruvector-calyx/Cargo.toml new file mode 100644 index 000000000..6e9b0bc9b --- /dev/null +++ b/crates/ruvector-calyx/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "ruvector-calyx" +version = "0.1.0" +edition = "2021" +description = "Association-native memory layer for ruvector: multi-slot constellations measured through many frozen lenses, kept un-flattened, fused with reciprocal rank fusion, scored for cross-lens agreement and signal density, grounded against real-world anchors, gated by a fail-closed guard, and recorded in a hash-chained provenance ledger" +authors = ["ruvnet", "claude-flow"] +license = "MIT OR Apache-2.0" +repository = "https://github.com/ruvnet/ruvector" +keywords = ["association", "vector-search", "rrf", "provenance", "ruvector"] +categories = ["algorithms", "data-structures"] + +[[bin]] +name = "calyx-bench" +path = "src/main.rs" + +[[bin]] +name = "calyx-routing-bench" +path = "src/bin/routing_bench.rs" + +[[bin]] +name = "calyx-novel-bench" +path = "src/bin/novel_bench.rs" + +[[bin]] +name = "calyx-real-bench" +path = "src/bin/real_bench.rs" + +[lib] +name = "ruvector_calyx" +path = "src/lib.rs" + +# Intentionally dependency-free: the reference crate uses an inline deterministic +# PRNG (see `src/rng.rs`) so it builds anywhere and benchmarks are reproducible. +[dependencies] + +[dev-dependencies] diff --git a/crates/ruvector-calyx/src/anneal.rs b/crates/ruvector-calyx/src/anneal.rs new file mode 100644 index 000000000..9b9d259f1 --- /dev/null +++ b/crates/ruvector-calyx/src/anneal.rs @@ -0,0 +1,129 @@ +//! Reversible self-optimization of lens fusion weights (the paper's *Anneal*). +//! +//! "Compose" is only as good as the weights behind the fusion. Anneal tunes the +//! per-lens fusion weights to maximize an objective — here, *accepted answers +//! per unit cost* — using simulated annealing. Every proposal is **reversible**: +//! a rejected move restores the previous weights exactly, so the optimizer +//! never corrupts the deployed configuration. + +use crate::rng::Rng; + +/// Annealing schedule and proposal parameters. +#[derive(Debug, Clone)] +pub struct AnnealConfig { + pub iterations: usize, + /// Initial temperature for Metropolis acceptance. + pub t_start: f32, + /// Final temperature. + pub t_end: f32, + /// Max absolute perturbation applied to one weight per proposal. + pub step: f32, + /// Weights are clamped to `[0, max_weight]`. + pub max_weight: f32, + pub seed: u64, +} + +impl Default for AnnealConfig { + fn default() -> Self { + Self { + iterations: 300, + t_start: 0.10, + t_end: 0.001, + step: 0.5, + max_weight: 8.0, + seed: 7, + } + } +} + +/// The optimization result. +#[derive(Debug, Clone)] +pub struct AnnealOutcome { + pub best_weights: Vec, + pub best_utility: f32, + pub initial_utility: f32, + pub accepted_moves: usize, + pub reverted_moves: usize, +} + +/// Anneal `initial` weights to maximize `utility`. +/// +/// `utility` maps a weight vector to a scalar (higher is better); in the +/// benchmark it is `accepted_answers / total_cost`. Proposals that fail the +/// Metropolis test are fully reverted, demonstrating the reversibility property. +pub fn anneal_weights(initial: Vec, cfg: &AnnealConfig, utility: F) -> AnnealOutcome +where + F: Fn(&[f32]) -> f32, +{ + let mut rng = Rng::new(cfg.seed); + let n = initial.len(); + + let mut current = initial.clone(); + let mut current_u = utility(¤t); + let initial_utility = current_u; + + let mut best = current.clone(); + let mut best_u = current_u; + + let mut accepted = 0usize; + let mut reverted = 0usize; + + let iters = cfg.iterations.max(1); + for i in 0..iters { + // Geometric temperature schedule. + let frac = i as f32 / iters as f32; + let temp = cfg.t_start * (cfg.t_end / cfg.t_start).powf(frac); + + // Propose: perturb one weight. Snapshot for reversibility. + let idx = rng.range(n); + let prev = current[idx]; + let delta = rng.signed() * cfg.step; + current[idx] = (prev + delta).clamp(0.0, cfg.max_weight); + + let candidate_u = utility(¤t); + let d = candidate_u - current_u; + let accept = d >= 0.0 || rng.f32() < (d / temp.max(1e-6)).exp(); + + if accept { + current_u = candidate_u; + accepted += 1; + if candidate_u > best_u { + best_u = candidate_u; + best.copy_from_slice(¤t); + } + } else { + // Reversible: restore exactly. + current[idx] = prev; + reverted += 1; + } + } + + AnnealOutcome { + best_weights: best, + best_utility: best_u, + initial_utility, + accepted_moves: accepted, + reverted_moves: reverted, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn anneal_improves_a_simple_objective() { + // Objective maximized when weights ≈ [3, 1]: a smooth concave bump. + let utility = |w: &[f32]| -> f32 { + let a = -(w[0] - 3.0).powi(2); + let b = -(w[1] - 1.0).powi(2); + a + b + }; + let cfg = AnnealConfig::default(); + let out = anneal_weights(vec![0.0, 0.0], &cfg, utility); + assert!(out.best_utility >= out.initial_utility); + assert!((out.best_weights[0] - 3.0).abs() < 0.6, "w0={}", out.best_weights[0]); + assert!((out.best_weights[1] - 1.0).abs() < 0.6, "w1={}", out.best_weights[1]); + assert!(out.reverted_moves > 0, "some moves should be reverted"); + } +} diff --git a/crates/ruvector-calyx/src/assay.rs b/crates/ruvector-calyx/src/assay.rs new file mode 100644 index 000000000..43481e19d --- /dev/null +++ b/crates/ruvector-calyx/src/assay.rs @@ -0,0 +1,117 @@ +//! Signal density — "how much does this lens actually tell us" (the paper's +//! *Assay*). +//! +//! "Differentiate" — the third of Calyx's four verbs — measures which lenses +//! add information rather than assuming every model is useful. We estimate the +//! mutual information (in bits) between a lens's similarity score and a binary +//! relevance label, then divide by the lens's declared cost to get *signal +//! density*: information gained per unit of compute. The anneal optimizer +//! promotes high-density lenses. + +/// One (lens-similarity, was-this-pair-relevant) observation. +#[derive(Debug, Clone, Copy)] +pub struct Observation { + pub similarity: f32, + pub relevant: bool, +} + +/// Estimated information content of a lens and its economics. +#[derive(Debug, Clone)] +pub struct LensSignal { + pub lens: String, + /// Mutual information between similarity and relevance, in bits (>= 0). + pub bits: f32, + /// Declared measurement cost in microseconds. + pub cost_us: f32, + /// Signal density: bits per microsecond (information per dollar/ms/watt). + pub density: f32, +} + +/// Estimate mutual information (bits) between a lens's similarity score and a +/// binary relevance label, by binning similarity into `bins` buckets. +/// +/// Returns `0.0` for an empty/degenerate sample. The estimate is a lower bound +/// proxy for the paper's MI-based contribution measure; it is monotone in how +/// well the lens separates relevant from irrelevant pairs. +pub fn mutual_information_bits(obs: &[Observation], bins: usize) -> f32 { + let n = obs.len(); + if n == 0 || bins == 0 { + return 0.0; + } + let bins = bins.max(2); + // Similarity is cosine in [-1, 1]; map to [0, 1] then to a bin index. + let bin_of = |s: f32| -> usize { + let t = ((s + 1.0) * 0.5).clamp(0.0, 0.999_999); + (t * bins as f32) as usize + }; + + let mut joint = vec![[0u32; 2]; bins]; // joint[bin][label] + let mut p_label = [0u32; 2]; + let mut p_bin = vec![0u32; bins]; + for o in obs { + let b = bin_of(o.similarity); + let l = o.relevant as usize; + joint[b][l] += 1; + p_label[l] += 1; + p_bin[b] += 1; + } + + let nf = n as f32; + let mut mi = 0.0f32; + for b in 0..bins { + for l in 0..2 { + let c = joint[b][l]; + if c == 0 { + continue; + } + let pxy = c as f32 / nf; + let px = p_bin[b] as f32 / nf; + let py = p_label[l] as f32 / nf; + if px > 0.0 && py > 0.0 { + mi += pxy * (pxy / (px * py)).log2(); + } + } + } + mi.max(0.0) +} + +/// Compute signal density for one lens. +pub fn signal_density(lens: impl Into, obs: &[Observation], cost_us: f32) -> LensSignal { + let bits = mutual_information_bits(obs, 8); + let density = if cost_us > 0.0 { bits / cost_us } else { 0.0 }; + LensSignal { + lens: lens.into(), + bits, + cost_us, + density, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn informative_lens_beats_random_lens() { + // Perfectly separating lens: relevant pairs always high similarity. + let mut informative = Vec::new(); + let mut random = Vec::new(); + for i in 0..200 { + let relevant = i % 2 == 0; + informative.push(Observation { + similarity: if relevant { 0.9 } else { -0.9 }, + relevant, + }); + // Random lens: similarity uncorrelated with label. + random.push(Observation { + similarity: if i % 3 == 0 { 0.5 } else { -0.5 }, + relevant, + }); + } + let inf = signal_density("good", &informative, 2.0); + let rnd = signal_density("noise", &random, 2.0); + assert!(inf.bits > 0.9, "informative bits = {}", inf.bits); + assert!(rnd.bits < inf.bits); + assert!(inf.density > rnd.density); + } +} diff --git a/crates/ruvector-calyx/src/bin/novel_bench.rs b/crates/ruvector-calyx/src/bin/novel_bench.rs new file mode 100644 index 000000000..05ed3a960 --- /dev/null +++ b/crates/ruvector-calyx/src/bin/novel_bench.rs @@ -0,0 +1,370 @@ +//! Benchmark for three novel capabilities of the association-native layer: +//! +//! A. Conformal cross-lens abstention — a distribution-free guarantee that the +//! confident-wrong-answer rate stays ≤ α, replacing the hand-tuned guard. +//! B. Disagreement as a query primitive — surface records where two lenses +//! most disagree (a query a flat vector DB cannot express). +//! C. Ledger-learned routing — learn a stop/continue policy from logged +//! witness trajectories by offline Monte-Carlo control. +//! +//! Run: `cargo run --release -p ruvector-calyx --bin calyx-novel-bench` + +use std::collections::BTreeMap; + +use ruvector_calyx::conformal::{calibrate, coverage, empirical_risk, selective_error, ConformalSelector}; +use ruvector_calyx::disagreement::find_conflicts; +use ruvector_calyx::fusion::{weighted_rrf, DEFAULT_RRF_K}; +use ruvector_calyx::ledger::{grounded_path, Anchor, AnchorKind}; +use ruvector_calyx::ledger_policy::{learn, Action, Episode}; +use ruvector_calyx::loom::min_agreement; +use ruvector_calyx::rng::Rng; +use ruvector_calyx::{Constellation, ConstellationStore, GuardProfile, LensKind, LensManifest, Query}; + +const DIM_SEM: usize = 48; +const DIM_LEX: usize = 32; +const DIM_STR: usize = 16; +const N_TOPICS: usize = 12; +const N_PER_TOPIC: usize = 20; +const N_DOCS: usize = N_TOPICS * N_PER_TOPIC; +const N_STRUCT: usize = N_TOPICS; // structural class aligned to topic (see §B) +const N_ANSWERABLE: usize = 200; +const N_UNANSWERABLE: usize = 100; +const COST: [f32; 3] = [1.5, 2.0, 12.0]; // structural, lexical, semantic +const SEED: u64 = 90210; + +fn order() -> Vec { + vec!["structural".into(), "lexical".into(), "semantic".into()] +} +fn registry() -> Vec { + vec![ + LensManifest::new("structural", "v1", LensKind::Structural, DIM_STR, COST[0], COST[0]), + LensManifest::new("lexical", "v1", LensKind::Lexical, DIM_LEX, COST[1], COST[1]), + LensManifest::new("semantic", "v1", LensKind::DenseSemantic, DIM_SEM, COST[2], COST[2]), + ] +} +fn normalize(v: &[f32]) -> Vec { + let n = v.iter().map(|x| x * x).sum::().sqrt().max(1e-9); + v.iter().map(|x| x / n).collect() +} +fn unit(rng: &mut Rng, d: usize) -> Vec { + normalize(&(0..d).map(|_| rng.signed()).collect::>()) +} +fn perturb(c: &[f32], noise: f32, rng: &mut Rng) -> Vec { + let g = unit(rng, c.len()); + normalize(&c.iter().zip(g.iter()).map(|(a, b)| a + noise * b).collect::>()) +} +fn blend(a: &[f32], b: &[f32], wa: f32, wb: f32) -> Vec { + normalize(&a.iter().zip(b.iter()).map(|(x, y)| wa * x + wb * y).collect::>()) +} + +struct Doc { id: u64, topic: usize, sem: Vec, lex: Vec, str_: Vec } +struct QSpec { q: Query, target: Option } + +fn build(rng: &mut Rng) -> (ConstellationStore, Vec, Vec, Vec) { + let sem_c: Vec> = (0..N_TOPICS).map(|_| unit(rng, DIM_SEM)).collect(); + let lex_c: Vec> = (0..N_TOPICS).map(|_| unit(rng, DIM_LEX)).collect(); + let str_c: Vec> = (0..N_STRUCT).map(|_| unit(rng, DIM_STR)).collect(); + let mut docs = Vec::with_capacity(N_DOCS); + let mut planted = Vec::new(); + let mut id = 0u64; + for t in 0..N_TOPICS { + for _j in 0..N_PER_TOPIC { + // Structural class is aligned to the topic, so a normal doc's + // semantic and structural neighbourhoods overlap (consistent views). + let sem = perturb(&sem_c[t], 0.10, rng); + let lex = blend(&lex_c[t], &unit(rng, DIM_LEX), 0.5, 0.85); + // Plant a "conflict" every 40th doc: structural slot from a *different* + // topic's class, so semantic-neighbours ≠ structural-neighbours. + let (str_, is_conflict) = if id % 40 == 7 { + let alt = (t + N_STRUCT / 2) % N_STRUCT; + (perturb(&str_c[alt], 0.10, rng), true) + } else { + (perturb(&str_c[t], 0.10, rng), false) + }; + if is_conflict { + planted.push(id); + } + docs.push(Doc { id, topic: t, sem, lex, str_ }); + id += 1; + } + } + let mut store = ConstellationStore::new(registry()); + for d in &docs { + store + .insert( + Constellation::new(d.id) + .with_slot("structural", d.str_.clone()) + .with_slot("lexical", d.lex.clone()) + .with_slot("semantic", d.sem.clone()) + .with_anchor(Anchor::new(AnchorKind::PassedTest, format!("t{}", d.id), 0.9)), + ) + .expect("insert"); + } + let mut queries = Vec::new(); + for i in 0..N_ANSWERABLE { + let d = &docs[(i * 7 + 3) % N_DOCS]; + let q = Query::new() + .with_slot("structural", perturb(&d.str_, 0.10, rng)) + .with_slot("lexical", perturb(&d.lex, 0.10, rng)) + .with_slot("semantic", perturb(&sem_c[d.topic], 0.10, rng)); + queries.push(QSpec { q, target: Some(d.id) }); + } + for i in 0..N_UNANSWERABLE { + let t = i % N_TOPICS; + let q = Query::new() + .with_slot("structural", unit(rng, DIM_STR)) + .with_slot("lexical", unit(rng, DIM_LEX)) + .with_slot("semantic", perturb(&sem_c[t], 0.10, rng)); + queries.push(QSpec { q, target: None }); + } + (store, docs, queries, planted) +} + +/// Fuse all lenses; return (top_id, cross-lens agreement of the top). +fn full_panel(store: &ConstellationStore, q: &Query) -> Option<(u64, f32)> { + let lenses = store.lenses(); + let mut rankings = Vec::new(); + for m in lenses { + rankings.push(store.search_lens(&m.name, q.slots.get(&m.name).unwrap(), 10)); + } + let fused = weighted_rrf(&rankings, &vec![1.0; lenses.len()], DEFAULT_RRF_K); + fused.first().map(|&(id, _)| { + let agr = store.get(id).map(|r| min_agreement(&q.slots, r)).unwrap_or(0.0); + (id, agr) + }) +} + +/// (agreement, wrong_if_answered) for every query — the conformal sample. +fn conformal_samples(store: &ConstellationStore, queries: &[QSpec]) -> Vec<(f32, bool)> { + queries + .iter() + .filter_map(|qs| { + full_panel(store, &qs.q).map(|(top, agr)| { + let wrong = qs.target != Some(top); // unanswerable → always wrong + (agr, wrong) + }) + }) + .collect() +} + +fn pct(x: f32) -> String { format!("{:.1}%", x * 100.0) } + +fn shuffle(v: &mut [T], rng: &mut Rng) { + for i in (1..v.len()).rev() { + let j = rng.range(i + 1); + v.swap(i, j); + } +} + +// ── Section A: conformal cross-lens abstention ────────────────────────────── +fn section_a(store: &ConstellationStore, queries: &[QSpec], rng: &mut Rng) -> bool { + println!("A. Conformal cross-lens abstention (distribution-free risk ≤ α)\n"); + let samples = conformal_samples(store, queries); + + // Hand-tuned guard (agreement ≥ 0.45) has *whatever* risk falls out. + let hand = ConformalSelector { threshold: 0.45, alpha: f32::NAN }; + println!( + " hand-tuned guard (agree≥0.45): risk={} coverage={} selective-err={}", + pct(empirical_risk(&hand, &samples)), + pct(coverage(&hand, &samples)), + pct(selective_error(&hand, &samples)) + ); + println!(); + println!(" {:<7} {:>10} {:>10} {:>12} {:>12} {:>10}", "α", "threshold", "coverage", "test-risk", "sel-error", "MC-risk"); + + let mut all_pass = true; + for &alpha in &[0.10f32, 0.05, 0.01] { + // Single split for the reported threshold/coverage. + let mut s = samples.clone(); + shuffle(&mut s, rng); + let (cal, test) = s.split_at(s.len() / 2); + let sel = calibrate(cal, alpha); + let test_risk = empirical_risk(&sel, test); + let cov = coverage(&sel, test); + let sel_err = selective_error(&sel, test); + + // Monte-Carlo: mean test risk over many random splits must stay ≤ α. + let trials = 200; + let mut acc = 0.0f32; + for _ in 0..trials { + let mut sc = samples.clone(); + shuffle(&mut sc, rng); + let (c, t) = sc.split_at(sc.len() / 2); + acc += empirical_risk(&calibrate(c, alpha), t); + } + let mc = acc / trials as f32; + let pass = mc <= alpha + 0.01; + all_pass &= pass; + println!( + " {:<7} {:>10} {:>10} {:>12} {:>12} {:>10} {}", + format!("{:.2}", alpha), + format!("{:.3}", sel.threshold), + pct(cov), + pct(test_risk), + pct(sel_err), + pct(mc), + if pass { "✓" } else { "✗" } + ); + } + println!("\n → guarantee holds: mean held-out risk ≤ α for every α [{}]\n", if all_pass { "PASS" } else { "FAIL" }); + all_pass +} + +// ── Section B: disagreement query primitive ───────────────────────────────── +fn section_b(store: &ConstellationStore, planted: &[u64]) -> bool { + println!("B. Disagreement as a query primitive (find where two lenses conflict)\n"); + let p = planted.len(); + let conflicts = find_conflicts(store, "semantic", "structural", 8, p); + let found: Vec = conflicts.iter().map(|&(id, _)| id).collect(); + let hits = planted.iter().filter(|id| found.contains(id)).count(); + let precision = hits as f32 / p.max(1) as f32; + println!(" planted conflicts : {p} (semantic∈topic X, structural∈another class)"); + println!(" find_conflicts@{p} : {hits} of {p} planted surfaced → precision {}", pct(precision)); + println!(" top-5 disagreement scores: {:?}", conflicts.iter().take(5).map(|&(id, s)| (id, (s * 100.0).round() / 100.0)).collect::>()); + let pass = precision >= 0.80; + println!("\n → conflict retrieval precision ≥ 0.80 [{}]\n", if pass { "PASS" } else { "FAIL" }); + pass +} + +// ── Section C: ledger-learned routing ─────────────────────────────────────── +fn agreement_over(store: &ConstellationStore, q: &Query, consulted: &[String], top: u64) -> f32 { + let sub: BTreeMap> = q + .slots + .iter() + .filter(|(k, _)| consulted.iter().any(|c| c == *k)) + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + store.get(top).map(|r| min_agreement(&sub, r)).unwrap_or(0.0) +} + +fn utility(answered: Option, target: Option, cost: f32) -> f32 { + let base = match (answered, target) { + (Some(a), Some(t)) if a == t => 1.0, + (Some(_), _) => -1.5, // confident wrong answer + (None, None) => 0.3, // correct abstention + (None, Some(_)) => -0.1, // missed answerable + }; + base - 0.5 * (cost / 15.5) +} + +/// Consult lenses incrementally; `stop_fn(step, agreement_bin)` decides. +fn rollout( + store: &ConstellationStore, + ord: &[String], + qs: &QSpec, + guard: &GuardProfile, + n_bins: usize, + mut stop_fn: impl FnMut(usize, usize) -> bool, + visited: &mut Vec<(usize, usize, Action)>, +) -> (Option, bool, f32) { + let mut rankings = Vec::new(); + let mut consulted = Vec::new(); + let mut cost = 0.0f32; + let mut top = 0u64; + let mut agr = 0.0f32; + for (step, lens) in ord.iter().enumerate() { + let m = store.lenses().iter().find(|l| &l.name == lens).unwrap(); + rankings.push(store.search_lens(lens, qs.q.slots.get(lens).unwrap(), 10)); + consulted.push(lens.clone()); + cost += m.cost_us; + let fused = weighted_rrf(&rankings, &vec![1.0; rankings.len()], DEFAULT_RRF_K); + top = fused.first().map(|&(id, _)| id).unwrap_or(0); + agr = agreement_over(store, &qs.q, &consulted, top); + let bin = ((agr.clamp(0.0, 0.999_999)) * n_bins as f32) as usize; + let last = step + 1 == ord.len(); + let stop = last || stop_fn(step, bin); + visited.push((consulted.len(), bin, if stop { Action::Stop } else { Action::Continue })); + if stop { + break; + } + } + let grounded = store + .get(top) + .map(|r| grounded_path(top, &r.anchors, guard.min_grounding_weight).is_some()) + .unwrap_or(false); + let answered = if consulted.len() >= 2 && agr >= guard.min_cross_lens_agreement && grounded { + Some(top) + } else { + None + }; + let correct = answered == qs.target && qs.target.is_some(); + (answered, correct, cost) +} + +fn section_c(store: &ConstellationStore, queries: &[QSpec], rng: &mut Rng) -> bool { + println!("C. Ledger-learned routing (offline MC control over witness trajectories)\n"); + let ord = order(); + let guard = GuardProfile::default(); + let n_bins = 10; + + // Split train/test. + let mut idx: Vec = (0..queries.len()).collect(); + shuffle(&mut idx, rng); + let (train_idx, test_idx) = idx.split_at(idx.len() / 2); + + // 1) Log exploratory episodes on train (behaviour policy: random stop). + let mut episodes = Vec::new(); + for &qi in train_idx { + for _ in 0..4 { + let mut visited = Vec::new(); + let (answered, _correct, cost) = + rollout(store, &ord, &queries[qi], &guard, n_bins, |_s, _b| rng.f32() < 0.5, &mut visited); + let ret = utility(answered, queries[qi].target, cost); + episodes.push(Episode { visited, ret }); + } + } + let policy = learn(&episodes, 3, n_bins); + + // 2) Evaluate learned policy vs fixed baselines on test. + let eval = |mut stop_fn: Box bool>| { + let (mut correct, mut answered, mut cost, mut util) = (0usize, 0usize, 0.0f32, 0.0f32); + for &qi in test_idx { + let mut v = Vec::new(); + let (ans, ok, c) = rollout(store, &ord, &queries[qi], &guard, n_bins, &mut *stop_fn, &mut v); + if ans.is_some() { answered += 1; } + if ok { correct += 1; } + cost += c; + util += utility(ans, queries[qi].target, c); + } + let n = test_idx.len() as f32; + (correct as f32 / n, cost / n, util / n, answered) + }; + + let pol = policy.clone(); + let (l_acc, l_cost, l_util, _) = + eval(Box::new(move |s, b| pol.act(s + 1, b) == Action::Stop)); + let (s2_acc, s2_cost, s2_util, _) = eval(Box::new(|s, _b| s + 1 >= 2)); // static(2) + let (bf_acc, bf_cost, bf_util, _) = eval(Box::new(|_s, _b| false)); // brute force + + println!(" {:<16} {:>9} {:>10} {:>9}", "policy", "acc", "cost µs", "utility"); + println!(" {}", "-".repeat(48)); + println!(" {:<16} {:>9} {:>10} {:>9}", "brute-force", pct(bf_acc), format!("{:.1}", bf_cost), format!("{:.3}", bf_util)); + println!(" {:<16} {:>9} {:>10} {:>9}", "static(2)", pct(s2_acc), format!("{:.1}", s2_cost), format!("{:.3}", s2_util)); + println!(" {:<16} {:>9} {:>10} {:>9}", "ledger-learned", pct(l_acc), format!("{:.1}", l_cost), format!("{:.3}", l_util)); + + let best_fixed = s2_util.max(bf_util); + let pass = l_util >= best_fixed - 0.05; + println!("\n → learned policy competitive with best fixed (utility ≥ best−0.05) [{}]\n", if pass { "PASS" } else { "FAIL" }); + pass +} + +fn main() { + println!("╔══════════════════════════════════════════════════════════════╗"); + println!("║ ruvector-calyx — Novel capabilities (conformal · disagreement · ║"); + println!("║ ledger-learned routing) ADR-272 Update 2 ║"); + println!("╚══════════════════════════════════════════════════════════════╝\n"); + let mut rng = Rng::new(SEED); + let (store, _docs, queries, planted) = build(&mut rng); + println!("Corpus: {N_DOCS} docs · {N_ANSWERABLE} answerable + {N_UNANSWERABLE} unanswerable · {} planted conflicts\n", planted.len()); + + let a = section_a(&store, &queries, &mut rng); + let b = section_b(&store, &planted); + let c = section_c(&store, &queries, &mut rng); + + if a && b && c { + println!("→ NOVEL BENCHMARK PASSED"); + } else { + println!("→ NOVEL BENCHMARK FAILED (A={a} B={b} C={c})"); + std::process::exit(1); + } +} diff --git a/crates/ruvector-calyx/src/bin/real_bench.rs b/crates/ruvector-calyx/src/bin/real_bench.rs new file mode 100644 index 000000000..a86b6ba41 --- /dev/null +++ b/crates/ruvector-calyx/src/bin/real_bench.rs @@ -0,0 +1,312 @@ +//! Real-data benchmark harness for the association layer. +//! +//! Usage: +//! cargo run --release -p ruvector-calyx --bin calyx-real-bench -- corpus.calyx +//! +//! With a `.calyx` path it loads a real precomputed corpus (see +//! `tools/build_codesearchnet.py`). With no argument it synthesizes a +//! **CodeSearchNet-shaped stand-in** (clearly labelled) and round-trips it +//! through the loader, so the full load → benchmark path is provable offline. +//! +//! Reports the standard IR metrics used by published baselines (MRR@10, +//! Recall@1/10, nDCG@10) for single-lens vs multi-lens fusion, plus the +//! association-native extras: conformal abstention risk and code↔doc +//! disagreement. + +use ruvector_calyx::conformal::{calibrate, coverage, empirical_risk}; +use ruvector_calyx::corpus::{load, save, RealQuery}; +use ruvector_calyx::disagreement::find_conflicts; +use ruvector_calyx::fusion::{weighted_rrf, DEFAULT_RRF_K}; +use ruvector_calyx::ledger::{Anchor, AnchorKind}; +use ruvector_calyx::loom::min_agreement; +use ruvector_calyx::rng::Rng; +use ruvector_calyx::{Constellation, ConstellationStore, LensKind, LensManifest, Query}; + +const K: usize = 10; + +// ── Retrieval over the lenses a query actually has ────────────────────────── +fn rank(store: &ConstellationStore, q: &Query, lenses: &[&str], top_k: usize) -> Vec { + let mut rankings = Vec::new(); + for name in lenses { + if let Some(v) = q.slots.get(*name) { + rankings.push(store.search_lens(name, v, top_k)); + } + } + if rankings.is_empty() { + return Vec::new(); + } + let w = vec![1.0f32; rankings.len()]; + weighted_rrf(&rankings, &w, DEFAULT_RRF_K) + .into_iter() + .map(|(id, _)| id) + .collect() +} + +// ── IR metrics (single relevant-or-more, binary relevance) ────────────────── +#[derive(Default, Clone)] +struct Ir { + mrr: f32, + r1: f32, + r10: f32, + ndcg: f32, + n: usize, +} +impl Ir { + fn finish(self) -> Ir { + let n = self.n.max(1) as f32; + Ir { + mrr: self.mrr / n, + r1: self.r1 / n, + r10: self.r10 / n, + ndcg: self.ndcg / n, + n: self.n, + } + } +} + +fn eval_ir(store: &ConstellationStore, queries: &[RealQuery], lenses: &[&str]) -> Ir { + let mut m = Ir::default(); + for q in queries.iter().filter(|q| q.is_answerable()) { + m.n += 1; + let ranked = rank(store, &q.query, lenses, K); + // first rank (1-indexed) of any relevant doc + let pos = ranked + .iter() + .position(|id| q.relevant.contains(id)) + .map(|p| p + 1); + if let Some(p) = pos { + m.mrr += 1.0 / p as f32; + if p == 1 { + m.r1 += 1.0; + } + if p <= 10 { + m.r10 += 1.0; + m.ndcg += 1.0 / ((p as f32) + 1.0).log2(); // IDCG=1 for one relevant + } + } + } + m.finish() +} + +// ── Conformal abstention samples: (cross-lens agreement, wrong-if-answered) ── +fn conformal_samples(store: &ConstellationStore, queries: &[RealQuery], lenses: &[&str]) -> Vec<(f32, bool)> { + queries + .iter() + .filter_map(|q| { + let ranked = rank(store, &q.query, lenses, K); + let top = *ranked.first()?; + let agr = store.get(top).map(|r| min_agreement(&q.query.slots, r)).unwrap_or(0.0); + let wrong = !q.relevant.contains(&top); // unanswerable → always wrong + Some((agr, wrong)) + }) + .collect() +} + +fn pct(x: f32) -> String { format!("{:.1}%", x * 100.0) } +fn shuffle(v: &mut [T], rng: &mut Rng) { + for i in (1..v.len()).rev() { + let j = rng.range(i + 1); + v.swap(i, j); + } +} + +fn main() { + println!("╔══════════════════════════════════════════════════════════════╗"); + println!("║ ruvector-calyx — Real-data benchmark (CodeSearchNet) ║"); + println!("╚══════════════════════════════════════════════════════════════╝\n"); + + let path = std::env::args().nth(1); + let (corpus, source) = match &path { + Some(p) => { + let mut f = std::fs::File::open(p).unwrap_or_else(|e| { + eprintln!("cannot open {p}: {e}"); + std::process::exit(2); + }); + (load(&mut f).expect("valid .calyx"), format!("REAL: {p}")) + } + None => { + // Synthesize a CodeSearchNet-shaped corpus, then prove the loader by + // saving + reloading it (identical byte path a real converter emits). + let (store, queries, planted) = synth(); + let mut buf = Vec::new(); + save(&mut buf, &store, &queries).unwrap(); + let mut cur = std::io::Cursor::new(buf); + let c = load(&mut cur).unwrap(); + PLANTED.with(|p| *p.borrow_mut() = planted); + (c, "SYNTHETIC STAND-IN (no .calyx supplied) — round-tripped through loader".to_string()) + } + }; + + println!("Source : {source}"); + let answerable = corpus.queries.iter().filter(|q| q.is_answerable()).count(); + println!( + "Corpus : {} functions · {} lenses [{}] · {} queries ({} answerable, {} unanswerable)\n", + corpus.store.len(), + corpus.store.lenses().len(), + corpus.store.lenses().iter().map(|m| m.name.as_str()).collect::>().join(", "), + corpus.queries.len(), + answerable, + corpus.queries.len() - answerable, + ); + + // ── Retrieval: single-lens baselines vs multi-lens fusion ─────────────── + let all: Vec<&str> = corpus.store.lenses().iter().map(|m| m.name.as_str()).collect(); + let has = |n: &str| all.contains(&n); + println!("Retrieval (answerable queries): single-lens vs association-native fusion"); + println!(" {:<24} {:>8} {:>8} {:>8} {:>8}", "system", "MRR@10", "R@1", "R@10", "nDCG@10"); + println!(" {}", "-".repeat(60)); + let mut rows: Vec<(String, Ir)> = Vec::new(); + for &l in &all { + rows.push((format!("single-lens [{l}]"), eval_ir(&corpus.store, &corpus.queries, &[l]))); + } + let fused = eval_ir(&corpus.store, &corpus.queries, &all); + for (name, m) in &rows { + println!(" {:<24} {:>8} {:>8} {:>8} {:>8}", name, fmt(m.mrr), fmt(m.r1), fmt(m.r10), fmt(m.ndcg)); + } + println!(" {:<24} {:>8} {:>8} {:>8} {:>8}", "fusion (all lenses)", fmt(fused.mrr), fmt(fused.r1), fmt(fused.r10), fmt(fused.ndcg)); + let best_single = rows.iter().map(|(_, m)| m.mrr).fold(0.0f32, f32::max); + println!(" → fusion MRR {:.3} vs best single-lens {:.3} ({:+.3})\n", fused.mrr, best_single, fused.mrr - best_single); + + // ── Conformal abstention on real labels ───────────────────────────────── + let mut rng = Rng::new(12345); + let samples = conformal_samples(&corpus.store, &corpus.queries, &all); + println!("Conformal cross-lens abstention (distribution-free risk ≤ α)"); + println!(" {:<7} {:>10} {:>10} {:>10}", "α", "threshold", "coverage", "test-risk"); + for &alpha in &[0.10f32, 0.05] { + let mut s = samples.clone(); + shuffle(&mut s, &mut rng); + let (cal, test) = s.split_at(s.len() / 2); + let sel = calibrate(cal, alpha); + println!( + " {:<7} {:>10} {:>10} {:>10}", + format!("{:.2}", alpha), + format!("{:.3}", sel.threshold), + pct(coverage(&sel, test)), + pct(empirical_risk(&sel, test)), + ); + } + println!(); + + // ── Disagreement: where the code lens and the doc lens conflict ───────── + if has("code") && has("doc") { + println!("Disagreement: code lens vs doc lens (stale/incorrect docstrings)"); + let conflicts = find_conflicts(&corpus.store, "code", "doc", 10, 8); + let planted = PLANTED.with(|p| p.borrow().clone()); + if !planted.is_empty() { + let found: Vec = conflicts.iter().map(|&(id, _)| id).collect(); + let hits = planted.iter().filter(|id| found.contains(id)).count(); + println!(" planted stale-doc functions: {} · surfaced {}/{} in top-{} → precision {}", + planted.len(), hits, planted.len(), planted.len().max(8), pct(hits as f32 / planted.len() as f32)); + } + println!(" top code↔doc conflicts (id, score): {:?}", + conflicts.iter().take(6).map(|&(id, s)| (id, (s * 100.0).round() / 100.0)).collect::>()); + println!(); + } + + println!("Note: with no .calyx supplied this is a synthetic stand-in that exercises the"); + println!("full loader→metrics path. Run tools/build_codesearchnet.py to produce a real"); + println!("corpus.calyx and pass it as the first argument for real numbers."); +} + +fn fmt(x: f32) -> String { format!("{:.3}", x) } + +thread_local! { + static PLANTED: std::cell::RefCell> = const { std::cell::RefCell::new(Vec::new()) }; +} + +// ── Synthetic CodeSearchNet-shaped corpus (offline stand-in) ──────────────── +const DIM_CODE: usize = 48; +const DIM_DOC: usize = 32; +const DIM_LEX: usize = 24; +const N_MODULES: usize = 15; +const N_PER_MODULE: usize = 16; +const N_FUNCS: usize = N_MODULES * N_PER_MODULE; +const N_ANSWERABLE: usize = 180; +const N_UNANSWERABLE: usize = 60; + +fn normalize(v: &[f32]) -> Vec { + let n = v.iter().map(|x| x * x).sum::().sqrt().max(1e-9); + v.iter().map(|x| x / n).collect() +} +fn unit(rng: &mut Rng, d: usize) -> Vec { + normalize(&(0..d).map(|_| rng.signed()).collect::>()) +} +fn perturb(c: &[f32], noise: f32, rng: &mut Rng) -> Vec { + let g = unit(rng, c.len()); + normalize(&c.iter().zip(g.iter()).map(|(a, b)| a + noise * b).collect::>()) +} + +fn synth() -> (ConstellationStore, Vec, Vec) { + let mut rng = Rng::new(2024); + // Each function has a per-function code identity; docstring lives in a + // per-module doc region; lexical is a per-function token fingerprint. + let code_c: Vec> = (0..N_MODULES).map(|_| unit(&mut rng, DIM_CODE)).collect(); + let doc_c: Vec> = (0..N_MODULES).map(|_| unit(&mut rng, DIM_DOC)).collect(); + // Lexical token clusters cut *across* modules: code/doc resolve to a module, + // lexical to a cluster, and only their intersection is a unique function. + let lex_cl: Vec> = (0..N_PER_MODULE).map(|_| unit(&mut rng, DIM_LEX)).collect(); + + let lenses = vec![ + LensManifest::new("code", "unixcoder-synth", LensKind::DenseSemantic, DIM_CODE, 8.0, 8.0), + LensManifest::new("doc", "bge-small-synth", LensKind::DenseSemantic, DIM_DOC, 3.0, 3.0), + LensManifest::new("lexical", "bm25-synth", LensKind::Lexical, DIM_LEX, 1.0, 1.0), + ]; + let mut store = ConstellationStore::new(lenses); + + struct Fn_ { id: u64, module: usize, lc: usize, code: Vec, doc: Vec, lex: Vec } + let mut funcs = Vec::new(); + let mut planted = Vec::new(); + let mut id = 0u64; + for mdl in 0..N_MODULES { + for (j, lex_center) in lex_cl.iter().enumerate() { + let code = perturb(&code_c[mdl], 0.20, &mut rng); + // Plant a stale docstring every 32nd function: doc points to a + // *different* module than the code (comment says one thing…). + let (doc, stale) = if id % 32 == 5 { + let other = (mdl + N_MODULES / 2) % N_MODULES; + (perturb(&doc_c[other], 0.15, &mut rng), true) + } else { + (perturb(&doc_c[mdl], 0.15, &mut rng), false) + }; + let lex = perturb(lex_center, 0.20, &mut rng); // cross-module token cluster + if stale { + planted.push(id); + } + funcs.push(Fn_ { id, module: mdl, lc: j, code, doc, lex }); + id += 1; + } + } + for f in &funcs { + store + .insert( + Constellation::new(f.id) + .with_slot("code", f.code.clone()) + .with_slot("doc", f.doc.clone()) + .with_slot("lexical", f.lex.clone()) + .with_anchor(Anchor::new(AnchorKind::PassedTest, format!("test_{}", f.id), 0.9)), + ) + .expect("insert"); + } + + let mut queries = Vec::new(); + for i in 0..N_ANSWERABLE { + let f = &funcs[(i * 5 + 2) % N_FUNCS]; + // NL query. Each lens is individually ambiguous — code and doc resolve + // only to the *module* (16 candidate functions), lexical is a noisy + // per-function fingerprint — so only their fusion pins the exact one. + let q = Query::new() + .with_slot("code", perturb(&code_c[f.module], 0.18, &mut rng)) + .with_slot("doc", perturb(&doc_c[f.module], 0.15, &mut rng)) + .with_slot("lexical", perturb(&lex_cl[f.lc], 0.20, &mut rng)); + queries.push(RealQuery { query: q, relevant: vec![f.id] }); + } + for _ in 0..N_UNANSWERABLE { + // NL query about code not in the corpus: no relevant function. + let q = Query::new() + .with_slot("code", unit(&mut rng, DIM_CODE)) + .with_slot("doc", unit(&mut rng, DIM_DOC)) + .with_slot("lexical", unit(&mut rng, DIM_LEX)); + queries.push(RealQuery { query: q, relevant: vec![] }); + } + (store, queries, planted) +} diff --git a/crates/ruvector-calyx/src/bin/routing_bench.rs b/crates/ruvector-calyx/src/bin/routing_bench.rs new file mode 100644 index 000000000..8b24aabe8 --- /dev/null +++ b/crates/ruvector-calyx/src/bin/routing_bench.rs @@ -0,0 +1,313 @@ +//! Benchmark: adaptive calibrated lens routing vs brute-force and static order. +//! +//! Three systems retrieve over the same multi-lens corpus: +//! 1. brute-force — always consult every lens +//! 2. static — always consult the two cheap lenses (fixed policy) +//! 3. adaptive — consult cheapest-first, stop early on calibrated confidence, +//! escalate only when unsure, abstain if the full panel is still not confident +//! +//! Acceptance (adaptive vs brute-force): accuracy loss ≤1pp, cost −≥30%, +//! latency −≥25%, ECE ≤0.08, abstention precision ≥0.80. +//! Run: `cargo run --release -p ruvector-calyx --bin calyx-routing-bench` + +use ruvector_calyx::calibrate::{ + abstention_quality, expected_calibration_error, raw_confidence, Calibrator, +}; +use ruvector_calyx::rng::Rng; +use ruvector_calyx::routing::{route, RoutingConfig, RoutingMode}; +use ruvector_calyx::{Anchor, AnchorKind, Constellation, ConstellationStore, GuardProfile, LensKind, LensManifest, Query}; + +// ── Corpus parameters ─────────────────────────────────────────────────────── +const DIM_SEM: usize = 48; +const DIM_LEX: usize = 32; +const DIM_STR: usize = 16; +const N_TOPICS: usize = 12; +const N_PER_TOPIC: usize = 20; +const N_DOCS: usize = N_TOPICS * N_PER_TOPIC; +const N_STRUCT: usize = 8; + +const N_ANSWERABLE: usize = 160; +const N_UNANSWERABLE: usize = 80; + +// Lens economics — semantic is expensive and (here) the low-signal "fog" lens; +// structural and lexical are cheap and discriminative. Cheapest-first order. +const COST_STR: f32 = 1.5; +const COST_LEX: f32 = 2.0; +const COST_SEM: f32 = 12.0; +const SEED: u64 = 2026; + +fn order() -> Vec { + vec!["structural".into(), "lexical".into(), "semantic".into()] +} + +fn registry() -> Vec { + vec![ + LensManifest::new("structural", "frozen-v1", LensKind::Structural, DIM_STR, COST_STR, COST_STR), + LensManifest::new("lexical", "frozen-v1", LensKind::Lexical, DIM_LEX, COST_LEX, COST_LEX), + LensManifest::new("semantic", "frozen-v1", LensKind::DenseSemantic, DIM_SEM, COST_SEM, COST_SEM), + ] +} + +// ── Vector helpers ────────────────────────────────────────────────────────── +fn normalize(v: &[f32]) -> Vec { + let n = v.iter().map(|x| x * x).sum::().sqrt().max(1e-9); + v.iter().map(|x| x / n).collect() +} +fn unit(rng: &mut Rng, d: usize) -> Vec { + normalize(&(0..d).map(|_| rng.signed()).collect::>()) +} +fn perturb(c: &[f32], noise: f32, rng: &mut Rng) -> Vec { + let g = unit(rng, c.len()); + normalize(&c.iter().zip(g.iter()).map(|(a, b)| a + noise * b).collect::>()) +} +fn blend(a: &[f32], b: &[f32], wa: f32, wb: f32) -> Vec { + normalize(&a.iter().zip(b.iter()).map(|(x, y)| wa * x + wb * y).collect::>()) +} + +struct Doc { + id: u64, + topic: usize, + sem: Vec, + lex: Vec, + str_: Vec, +} +struct QSpec { + q: Query, + target: Option, + train: bool, +} + +fn build(rng: &mut Rng) -> (ConstellationStore, Vec, Vec) { + let sem_c: Vec> = (0..N_TOPICS).map(|_| unit(rng, DIM_SEM)).collect(); + let lex_c: Vec> = (0..N_TOPICS).map(|_| unit(rng, DIM_LEX)).collect(); + let str_c: Vec> = (0..N_STRUCT).map(|_| unit(rng, DIM_STR)).collect(); + + let mut docs = Vec::with_capacity(N_DOCS); + let mut id = 0u64; + for t in 0..N_TOPICS { + for j in 0..N_PER_TOPIC { + let s = (t * N_PER_TOPIC + j) % N_STRUCT; + let sem = perturb(&sem_c[t], 0.10, rng); // topic fog + let lex = blend(&lex_c[t], &unit(rng, DIM_LEX), 0.5, 0.85); // + per-doc fingerprint + let str_ = perturb(&str_c[s], 0.30, rng); // + per-doc jitter + docs.push(Doc { id, topic: t, sem, lex, str_ }); + id += 1; + } + } + + let mut store = ConstellationStore::new(registry()); + for d in &docs { + store + .insert( + Constellation::new(d.id) + .with_slot("structural", d.str_.clone()) + .with_slot("lexical", d.lex.clone()) + .with_slot("semantic", d.sem.clone()) + .with_anchor(Anchor::new(AnchorKind::PassedTest, format!("t{}", d.id), 0.9)), + ) + .expect("insert"); + } + + let mut queries = Vec::with_capacity(N_ANSWERABLE + N_UNANSWERABLE); + for i in 0..N_ANSWERABLE { + let d = &docs[(i * 7 + 3) % N_DOCS]; + // Half the answerable queries are "hard": noisier structural slot, so + // the cheap structural lens alone is ambiguous and the router must + // escalate to lexical. + let hard = i % 2 == 1; + let str_noise = if hard { 0.28 } else { 0.06 }; + let q = Query::new() + .with_slot("structural", perturb(&d.str_, str_noise, rng)) + .with_slot("lexical", perturb(&d.lex, 0.10, rng)) + .with_slot("semantic", perturb(&sem_c[d.topic], 0.10, rng)); + queries.push(QSpec { q, target: Some(d.id), train: i % 2 == 0 }); + } + for i in 0..N_UNANSWERABLE { + let t = i % N_TOPICS; + let q = Query::new() + .with_slot("structural", unit(rng, DIM_STR)) // matches no doc + .with_slot("lexical", unit(rng, DIM_LEX)) + .with_slot("semantic", perturb(&sem_c[t], 0.10, rng)); + queries.push(QSpec { q, target: None, train: i % 2 == 0 }); + } + (store, docs, queries) +} + +// ── Metrics ───────────────────────────────────────────────────────────────── +#[derive(Default)] +struct Agg { + answerable: usize, + grounded_correct: usize, // answered & correct (answerable) + answered: usize, + correct_answers: usize, + total_cost: f32, + total_latency: f32, + n: usize, + ece_samples: Vec<(f32, bool)>, // (conf, correct) over answered + abstained: Vec, + unanswerable: Vec, +} +impl Agg { + fn grounded_acc(&self) -> f32 { + if self.answerable == 0 { 0.0 } else { self.grounded_correct as f32 / self.answerable as f32 } + } + fn accepted_acc(&self) -> f32 { + if self.answered == 0 { 1.0 } else { self.correct_answers as f32 / self.answered as f32 } + } + fn mean_cost(&self) -> f32 { self.total_cost / self.n.max(1) as f32 } + fn mean_latency(&self) -> f32 { self.total_latency / self.n.max(1) as f32 } + fn cost_per_accepted(&self) -> f32 { self.total_cost / self.answered.max(1) as f32 } + fn ece(&self) -> f32 { expected_calibration_error(&self.ece_samples, 10) } + fn abstention(&self) -> (f32, f32) { + let q = abstention_quality(&self.abstained, &self.unanswerable); + (q.precision, q.recall) + } +} + +#[allow(clippy::too_many_arguments)] +fn evaluate( + store: &ConstellationStore, + order: &[String], + queries: &[QSpec], + cal: &Calibrator, + cfg: &RoutingConfig, + guard: &GuardProfile, + mode: RoutingMode, + train_split: bool, +) -> Agg { + let mut a = Agg::default(); + for qs in queries.iter().filter(|q| q.train == train_split) { + let out = route(store, order, &qs.q, cal, cfg, guard, mode); + a.n += 1; + a.total_cost += out.witness.cost_us; + a.total_latency += out.witness.latency_us; + let is_unans = qs.target.is_none(); + a.unanswerable.push(is_unans); + a.abstained.push(out.answered.is_none()); + let top_correct = out.witness.top_candidate == qs.target && qs.target.is_some(); + if let Some(_t) = qs.target { + a.answerable += 1; + } + if let Some(ans) = out.answered { + a.answered += 1; + let correct = qs.target == Some(ans); + if correct { + a.correct_answers += 1; + if qs.target.is_some() { + a.grounded_correct += 1; + } + } + // ECE over committed answers: does confidence track correctness? + a.ece_samples.push((out.confidence, correct)); + } + let _ = top_correct; + } + a +} + +fn pct(x: f32) -> String { format!("{:.1}%", x * 100.0) } + +fn main() { + println!("╔══════════════════════════════════════════════════════════════╗"); + println!("║ ruvector-calyx — Adaptive Routing + Calibration (ADR-272) ║"); + println!("╚══════════════════════════════════════════════════════════════╝\n"); + println!("Lenses (cheapest-first): structural({COST_STR}µs) → lexical({COST_LEX}µs) → semantic({COST_SEM}µs)"); + println!("Corpus : {N_DOCS} docs · queries {N_ANSWERABLE} answerable + {N_UNANSWERABLE} unanswerable (50/50 train/test)\n"); + + let mut rng = Rng::new(SEED); + let (store, _docs, queries) = build(&mut rng); + let ord = order(); + let guard = GuardProfile::default(); + let cfg = RoutingConfig { stop_threshold: 0.80, answer_threshold: 0.55, min_lenses_to_answer: 2, top_k: 10 }; + + // ── Fit the calibrator on the TRAIN split (bootstrap: identity calibrator, + // adaptive routing, collect raw-confidence vs would-be-correctness) ───── + let boot = Calibrator::identity(10); + let mut samples = Vec::new(); + for qs in queries.iter().filter(|q| q.train) { + let out = route(&store, &ord, &qs.q, &boot, &cfg, &guard, RoutingMode::Adaptive); + let raw = raw_confidence(&out.witness.signals); + let correct = out.witness.top_candidate == qs.target && qs.target.is_some(); + samples.push((raw, correct)); + } + let cal = Calibrator::fit(&samples, 10); + println!("Calibrator fit on {} train decisions (10-bin reliability map)\n", samples.len()); + + // ── Evaluate the three systems on the TEST split ──────────────────────── + let brute = evaluate(&store, &ord, &queries, &cal, &cfg, &guard, RoutingMode::BruteForce, false); + let stat = evaluate(&store, &ord, &queries, &cal, &cfg, &guard, RoutingMode::Static { max_lenses: 2 }, false); + let adap = evaluate(&store, &ord, &queries, &cal, &cfg, &guard, RoutingMode::Adaptive, false); + + let row = |name: &str, a: &Agg| { + let (ap, ar) = a.abstention(); + println!( + " {:<12} {:>9} {:>9} {:>9} {:>9} {:>7} {:>9} {:>7}", + name, + pct(a.grounded_acc()), + pct(a.accepted_acc()), + format!("{:.1}", a.mean_cost()), + format!("{:.1}", a.mean_latency()), + format!("{:.3}", a.ece()), + pct(ap), + pct(ar), + ); + }; + println!("System comparison (test split)"); + println!( + " {:<12} {:>9} {:>9} {:>9} {:>9} {:>7} {:>9} {:>7}", + "mode", "grnd-acc", "acc-acc", "cost µs", "lat µs", "ECE", "abst-P", "abst-R" + ); + println!(" {}", "-".repeat(78)); + row("brute-force", &brute); + row("static(2)", &stat); + row("adaptive", &adap); + println!(); + + // Witness example (first adaptive answer on the test split). + if let Some(qs) = queries.iter().find(|q| !q.train && q.target.is_some()) { + let out = route(&store, &ord, &qs.q, &cal, &cfg, &guard, RoutingMode::Adaptive); + println!("Example witness (adaptive)"); + println!(" lenses consulted : {:?}", out.witness.lenses_consulted); + println!(" action / reason : {} / {}", out.witness.action, out.witness.escalation_reason); + println!( + " confidence : {:.2} agreement={:.2} margin={:.2} contradiction={:.2}", + out.confidence, + out.witness.signals.cross_lens_agreement, + out.witness.signals.margin_ratio, + out.witness.signals.contradiction + ); + println!(" cost / latency : {:.1}µs / {:.1}µs\n", out.witness.cost_us, out.witness.latency_us); + } + + // ── Acceptance ────────────────────────────────────────────────────────── + let acc_loss = brute.grounded_acc() - adap.grounded_acc(); + let cost_red = 1.0 - adap.mean_cost() / brute.mean_cost().max(1e-9); + let lat_red = 1.0 - adap.mean_latency() / brute.mean_latency().max(1e-9); + let ece = adap.ece(); + let (abst_p, _abst_r) = adap.abstention(); + + println!("Acceptance test (adaptive vs brute-force)"); + let c1 = acc_loss <= 0.01; + let c2 = cost_red >= 0.30; + let c3 = lat_red >= 0.25; + let c4 = ece <= 0.08; + let c5 = abst_p >= 0.80; + let chk = |name: &str, val: String, ok: bool| { + println!(" {:<34} {:>12} {}", name, val, if ok { "PASS ✓" } else { "FAIL ✗" }); + }; + chk("accuracy loss ≤ 1pp", format!("{:+.1}pp", acc_loss * 100.0), c1); + chk("query cost reduction ≥ 30%", format!("−{:.1}%", cost_red * 100.0), c2); + chk("latency reduction ≥ 25%", format!("−{:.1}%", lat_red * 100.0), c3); + chk("ECE ≤ 0.08", format!("{:.3}", ece), c4); + chk("abstention precision ≥ 0.80", pct(abst_p), c5); + println!(); + println!(" cost per accepted answer: brute {:.1}µs → adaptive {:.1}µs", brute.cost_per_accepted(), adap.cost_per_accepted()); + println!(); + + if c1 && c2 && c3 && c4 && c5 { + println!("→ ROUTING BENCHMARK PASSED"); + } else { + println!("→ ROUTING BENCHMARK FAILED"); + std::process::exit(1); + } +} diff --git a/crates/ruvector-calyx/src/calibrate.rs b/crates/ruvector-calyx/src/calibrate.rs new file mode 100644 index 000000000..997adb811 --- /dev/null +++ b/crates/ruvector-calyx/src/calibrate.rs @@ -0,0 +1,203 @@ +//! Confidence signals and calibration. +//! +//! Adaptive routing is only trustworthy if the confidence driving it is +//! *calibrated* — a stated 0.8 should mean "right about 80% of the time". +//! This module derives a raw confidence from several signals, fits a histogram +//! calibrator (a reliability map) on a training split, and measures Expected +//! Calibration Error (ECE) and abstention precision/recall on the test split. + +/// The signals that feed per-query confidence. Each is in `[0, 1]` except +/// `margin_ratio`, which is a normalized top-1/top-2 gap. +#[derive(Debug, Clone, Copy, Default)] +pub struct ConfidenceSignals { + /// Top fused score (RRF magnitude of the winner). + pub top_score: f32, + /// (top1 - top2) / top1 — how decisively the winner leads. + pub margin_ratio: f32, + /// Cross-lens min agreement of the winner over the consulted lenses. + pub cross_lens_agreement: f32, + /// Source density: best grounding-anchor weight on the winner. + pub grounding: f32, + /// Contradiction: the largest below-mean dissent across consulted lenses. + pub contradiction: f32, + /// Number of lenses consulted so far. + pub lens_count: usize, +} + +/// Fold the signals into a single raw confidence in `[0, 1]`. +/// +/// This need only be *monotonically informative* — the [`Calibrator`] maps it +/// onto empirical accuracy. Agreement and margin push confidence up; +/// contradiction pushes it down; grounding is a mild bonus. +pub fn raw_confidence(s: &ConfidenceSignals) -> f32 { + let z = 0.55 * s.cross_lens_agreement + 0.30 * s.margin_ratio.clamp(0.0, 1.0) + + 0.15 * s.grounding + - 0.40 * s.contradiction; + z.clamp(0.0, 1.0) +} + +/// A histogram (reliability-map) calibrator: bins raw confidence and maps each +/// bin to the empirical accuracy observed in training. Monotone, robust, and +/// interpretable — the reliability diagram made executable. +#[derive(Debug, Clone)] +pub struct Calibrator { + /// Per-bin calibrated confidence (empirical accuracy). + bins: Vec, +} + +impl Calibrator { + /// Identity calibrator (returns the raw value), used for the bootstrap pass. + pub fn identity(n_bins: usize) -> Self { + let n = n_bins.max(1); + let bins = (0..n).map(|i| (i as f32 + 0.5) / n as f32).collect(); + Self { bins } + } + + /// Fit on `(raw_confidence, was_correct)` samples. Empty bins fall back to + /// the bin's midpoint so the map stays monotone and defined everywhere. + pub fn fit(samples: &[(f32, bool)], n_bins: usize) -> Self { + let n = n_bins.max(1); + let mut hit = vec![0u32; n]; + let mut tot = vec![0u32; n]; + for &(raw, correct) in samples { + let b = bin_index(raw, n); + tot[b] += 1; + if correct { + hit[b] += 1; + } + } + let bins = (0..n) + .map(|b| { + if tot[b] == 0 { + (b as f32 + 0.5) / n as f32 + } else { + hit[b] as f32 / tot[b] as f32 + } + }) + .collect(); + Self { bins } + } + + /// Map a raw confidence to its calibrated value. + pub fn calibrate(&self, raw: f32) -> f32 { + let b = bin_index(raw, self.bins.len()); + self.bins[b] + } +} + +#[inline] +fn bin_index(raw: f32, n_bins: usize) -> usize { + let n = n_bins.max(1); + ((raw.clamp(0.0, 0.999_999)) * n as f32) as usize +} + +/// Expected Calibration Error over `(confidence, was_correct)` decisions. +/// +/// Bins by confidence, then averages `|accuracy - mean_confidence|` weighted by +/// bin population. `0.0` is perfectly calibrated; lower is better. +pub fn expected_calibration_error(samples: &[(f32, bool)], n_bins: usize) -> f32 { + if samples.is_empty() { + return 0.0; + } + let n = n_bins.max(1); + let mut sum_conf = vec![0.0f32; n]; + let mut hit = vec![0u32; n]; + let mut tot = vec![0u32; n]; + for &(conf, correct) in samples { + let b = bin_index(conf, n); + sum_conf[b] += conf; + tot[b] += 1; + if correct { + hit[b] += 1; + } + } + let total = samples.len() as f32; + let mut ece = 0.0f32; + for b in 0..n { + if tot[b] == 0 { + continue; + } + let acc = hit[b] as f32 / tot[b] as f32; + let conf = sum_conf[b] / tot[b] as f32; + ece += (tot[b] as f32 / total) * (acc - conf).abs(); + } + ece +} + +/// Abstention quality. `abstained` and `should_have_abstained` are aligned +/// per-query booleans (true = abstained / true = query was unanswerable). +#[derive(Debug, Clone, Copy)] +pub struct AbstentionQuality { + pub precision: f32, + pub recall: f32, +} + +/// Precision = fraction of abstentions that were correct (query unanswerable). +/// Recall = fraction of unanswerable queries that were abstained on. +pub fn abstention_quality(abstained: &[bool], unanswerable: &[bool]) -> AbstentionQuality { + let mut tp = 0u32; // abstained & unanswerable + let mut abst = 0u32; + let mut unans = 0u32; + for (&a, &u) in abstained.iter().zip(unanswerable.iter()) { + if a { + abst += 1; + } + if u { + unans += 1; + } + if a && u { + tp += 1; + } + } + AbstentionQuality { + precision: if abst == 0 { 1.0 } else { tp as f32 / abst as f32 }, + recall: if unans == 0 { 1.0 } else { tp as f32 / unans as f32 }, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn calibrator_maps_bins_to_empirical_accuracy() { + // Low-raw samples are 20% correct; high-raw samples 90% correct. + let mut samples = Vec::new(); + for i in 0..100 { + samples.push((0.1, i < 20)); // 20% correct in low bin + samples.push((0.9, i < 90)); // 90% correct in high bin + } + let c = Calibrator::fit(&samples, 10); + assert!((c.calibrate(0.1) - 0.2).abs() < 0.02); + assert!((c.calibrate(0.9) - 0.9).abs() < 0.02); + } + + #[test] + fn ece_zero_when_perfectly_calibrated() { + // Confidence 0.7 that is correct exactly 70% of the time → ECE ≈ 0. + let mut samples = Vec::new(); + for i in 0..100 { + samples.push((0.7, i < 70)); + } + assert!(expected_calibration_error(&samples, 10) < 0.02); + } + + #[test] + fn ece_high_when_overconfident() { + // Confidence 0.95 but only 40% correct → large ECE. + let mut samples = Vec::new(); + for i in 0..100 { + samples.push((0.95, i < 40)); + } + assert!(expected_calibration_error(&samples, 10) > 0.4); + } + + #[test] + fn abstention_precision_recall() { + let abstained = [true, true, false, false]; + let unanswerable = [true, false, true, false]; + let q = abstention_quality(&abstained, &unanswerable); + assert!((q.precision - 0.5).abs() < 1e-6); // 1 of 2 abstentions correct + assert!((q.recall - 0.5).abs() < 1e-6); // caught 1 of 2 unanswerable + } +} diff --git a/crates/ruvector-calyx/src/conformal.rs b/crates/ruvector-calyx/src/conformal.rs new file mode 100644 index 000000000..86f7431c4 --- /dev/null +++ b/crates/ruvector-calyx/src/conformal.rs @@ -0,0 +1,178 @@ +//! Conformal cross-lens abstention. +//! +//! The hand-tuned guard threshold in [`crate::ward`] answers "when the lenses +//! agree enough" — but *how much* is enough is a magic number with no promise +//! attached. This module replaces it with **conformal risk control** +//! (Angelopoulos, Bates, Fisch, Lei & Schuster, 2022): given a calibration set, +//! it picks a threshold on the cross-lens agreement score that carries a +//! **distribution-free, finite-sample guarantee** — the expected rate of +//! confident-but-wrong answers on future queries is `≤ α`. +//! +//! ## The guarantee +//! +//! Let each query's nonconformity be captured by its top candidate's cross-lens +//! agreement `a ∈ [0,1]`, and let `wrong` mean "answering would be an error". +//! Define the per-query loss at threshold `t` as `L(t) = 1{a ≥ t ∧ wrong}` — +//! non-increasing in `t`. With `n` exchangeable calibration points, choose +//! +//! ```text +//! t̂ = inf { t : (n/(n+1))·R̂ₙ(t) + 1/(n+1) ≤ α } +//! ``` +//! +//! Then for an exchangeable test query, `E[L(t̂)] ≤ α`: at most an `α` fraction +//! of queries result in a confident wrong answer. The router *answers* iff +//! `a ≥ t̂` and *abstains* otherwise — "fail closed" with a number behind it. + +/// A calibrated conformal abstention rule: answer iff agreement ≥ `threshold`. +#[derive(Debug, Clone, Copy)] +pub struct ConformalSelector { + /// Agreement threshold above which answering is permitted. + pub threshold: f32, + /// The risk level the threshold was calibrated to guarantee. + pub alpha: f32, +} + +impl ConformalSelector { + /// Answer iff the top candidate's cross-lens agreement clears the threshold. + #[inline] + pub fn admits(&self, agreement: f32) -> bool { + agreement >= self.threshold + } +} + +/// Calibrate a conformal abstention threshold via conformal risk control. +/// +/// `cal` holds `(agreement, wrong_if_answered)` for calibration queries. +/// `alpha` is the target ceiling on the confident-wrong-answer rate. +/// Returns a selector whose threshold gives `E[loss] ≤ alpha` on exchangeable +/// test data. If no finite threshold achieves the bound, it abstains on +/// everything (threshold `> 1`). +pub fn calibrate(cal: &[(f32, bool)], alpha: f32) -> ConformalSelector { + let n = cal.len(); + let abstain_all = ConformalSelector { + threshold: 1.000_001, + alpha, + }; + if n == 0 { + return abstain_all; + } + + // Candidate thresholds: each distinct agreement value (answering the set + // {a_i ≥ t}); evaluated ascending = increasingly strict = lower loss. + let mut candidates: Vec = cal.iter().map(|&(a, _)| a).collect(); + candidates.sort_by(|x, y| x.partial_cmp(y).unwrap_or(std::cmp::Ordering::Equal)); + candidates.dedup(); + + let nf = n as f32; + let slack = 1.0 / (nf + 1.0); // B/(n+1) with 0/1 loss, B = 1 + for &t in &candidates { + let wrong_admitted = cal + .iter() + .filter(|&&(a, wrong)| a >= t && wrong) + .count(); + let r_hat = wrong_admitted as f32 / nf; + if (nf / (nf + 1.0)) * r_hat + slack <= alpha { + return ConformalSelector { threshold: t, alpha }; + } + } + abstain_all +} + +/// Empirical loss of a selector on `data`: the fraction of *all* queries that +/// are confident wrong answers (`admitted ∧ wrong`). This is the quantity the +/// guarantee bounds by `α`. +pub fn empirical_risk(sel: &ConformalSelector, data: &[(f32, bool)]) -> f32 { + if data.is_empty() { + return 0.0; + } + let bad = data + .iter() + .filter(|&&(a, wrong)| sel.admits(a) && wrong) + .count(); + bad as f32 / data.len() as f32 +} + +/// Fraction of queries the selector answers (coverage). +pub fn coverage(sel: &ConformalSelector, data: &[(f32, bool)]) -> f32 { + if data.is_empty() { + return 0.0; + } + let admitted = data.iter().filter(|&&(a, _)| sel.admits(a)).count(); + admitted as f32 / data.len() as f32 +} + +/// Selective error: among *answered* queries, the fraction that are wrong. +/// (`empirical_risk / coverage`; `0` when nothing is answered.) +pub fn selective_error(sel: &ConformalSelector, data: &[(f32, bool)]) -> f32 { + let admitted: Vec = data + .iter() + .filter(|&&(a, _)| sel.admits(a)) + .map(|&(_, wrong)| wrong) + .collect(); + if admitted.is_empty() { + return 0.0; + } + admitted.iter().filter(|w| **w).count() as f32 / admitted.len() as f32 +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::rng::Rng; + + /// Build a sample where high-agreement queries are usually right and + /// low-agreement queries are usually wrong (the realistic regime). + fn sample(rng: &mut Rng, n: usize) -> Vec<(f32, bool)> { + (0..n) + .map(|_| { + let a = rng.f32(); + // P(wrong) decreases with agreement. + let wrong = rng.f32() > a; // higher a → less likely wrong + (a, wrong) + }) + .collect() + } + + #[test] + fn stricter_alpha_lowers_coverage() { + let mut rng = Rng::new(1); + let cal = sample(&mut rng, 4000); + let s01 = calibrate(&cal, 0.01); + let s10 = calibrate(&cal, 0.10); + // A tighter risk budget must not answer *more*. + assert!(s01.threshold >= s10.threshold); + } + + #[test] + fn guarantee_holds_on_held_out_data_monte_carlo() { + // Across many splits, the mean test risk must stay ≤ α (+ tiny slack). + let alpha = 0.05; + let trials = 200; + let mut violations_mean = 0.0f32; + let mut rng = Rng::new(7); + for _ in 0..trials { + let mut all = sample(&mut rng, 1200); + // Shuffle then split cal/test (exchangeable). + for i in (1..all.len()).rev() { + let j = rng.range(i + 1); + all.swap(i, j); + } + let (cal, test) = all.split_at(600); + let sel = calibrate(cal, alpha); + violations_mean += empirical_risk(&sel, test); + } + let mean_risk = violations_mean / trials as f32; + assert!( + mean_risk <= alpha + 0.01, + "mean test risk {mean_risk} exceeded alpha {alpha}" + ); + } + + #[test] + fn abstains_on_everything_when_all_wrong() { + let cal: Vec<(f32, bool)> = (0..100).map(|i| (i as f32 / 100.0, true)).collect(); + let sel = calibrate(&cal, 0.01); + assert!(sel.threshold > 1.0); // cannot answer anything safely + assert_eq!(coverage(&sel, &cal), 0.0); + } +} diff --git a/crates/ruvector-calyx/src/constellation.rs b/crates/ruvector-calyx/src/constellation.rs new file mode 100644 index 000000000..7681e20de --- /dev/null +++ b/crates/ruvector-calyx/src/constellation.rs @@ -0,0 +1,236 @@ +//! The atomic record: a **constellation** — one object measured through many +//! lenses, stored as distinct typed slots that are *never flattened*. + +use std::collections::BTreeMap; + +use crate::ledger::Anchor; +use crate::lens::LensManifest; +use crate::scoring::{cosine_sim, normalize}; + +/// One object measured through many lenses. +/// +/// Slots are keyed by lens name and kept separate. The deliberately lossy +/// [`Constellation::flatten`] exists only to build the single-embedding RAG +/// *baseline* the association-native design is meant to beat. +#[derive(Debug, Clone)] +pub struct Constellation { + /// Stable identifier. + pub id: u64, + /// Optional human-readable label (debugging only). + pub label: Option, + /// Per-lens slot vectors, keyed by lens name. Never collapsed into one. + pub slots: BTreeMap>, + /// Grounding anchors tying this record to real-world outcomes. + pub anchors: Vec, +} + +impl Constellation { + pub fn new(id: u64) -> Self { + Self { + id, + label: None, + slots: BTreeMap::new(), + anchors: Vec::new(), + } + } + + pub fn with_label(mut self, label: impl Into) -> Self { + self.label = Some(label.into()); + self + } + + /// Attach a slot vector for the given lens. + pub fn with_slot(mut self, lens: impl Into, vector: Vec) -> Self { + self.slots.insert(lens.into(), vector); + self + } + + /// Attach a grounding anchor. + pub fn with_anchor(mut self, anchor: Anchor) -> Self { + self.anchors.push(anchor); + self + } + + pub fn slot(&self, lens: &str) -> Option<&[f32]> { + self.slots.get(lens).map(|v| v.as_slice()) + } + + /// The deliberately lossy baseline: L2-normalize each slot (so no single + /// lens dominates by magnitude) then concatenate in manifest order into one + /// flat vector. This is exactly the "semantic fog" Calyx argues against — + /// a discriminative minority lens (e.g. 16 of 96 dims) is drowned out. + pub fn flatten(&self, lenses: &[LensManifest]) -> Vec { + let mut out = Vec::new(); + for m in lenses { + match self.slots.get(&m.name) { + Some(v) => out.extend_from_slice(&normalize(v)), + None => out.resize(out.len() + m.dims, 0.0), + } + } + out + } +} + +/// Errors a store raises rather than silently returning a wrong answer. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum StoreError { + /// A record referenced a lens not declared in the registry. + UnknownLens(String), + /// A slot vector did not match its lens's declared dimensionality. + ShapeMismatch { + lens: String, + expected: usize, + got: usize, + }, +} + +impl std::fmt::Display for StoreError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + StoreError::UnknownLens(l) => write!(f, "unknown lens: {l}"), + StoreError::ShapeMismatch { + lens, + expected, + got, + } => write!( + f, + "shape mismatch on lens {lens}: expected {expected} dims, got {got}" + ), + } + } +} + +/// A registry of lenses plus the constellations measured through them. +/// +/// Search is exact (brute force); the focus of this crate is the *association* +/// layer, not the index. A production deployment would back each per-lens +/// ranking with HNSW/IVF (ruvector already ships these). +#[derive(Debug, Default, Clone)] +pub struct ConstellationStore { + lenses: Vec, + records: Vec, +} + +impl ConstellationStore { + pub fn new(lenses: Vec) -> Self { + Self { + lenses, + records: Vec::new(), + } + } + + pub fn lenses(&self) -> &[LensManifest] { + &self.lenses + } + + pub fn records(&self) -> &[Constellation] { + &self.records + } + + pub fn len(&self) -> usize { + self.records.len() + } + + pub fn is_empty(&self) -> bool { + self.records.is_empty() + } + + fn lens(&self, name: &str) -> Option<&LensManifest> { + self.lenses.iter().find(|m| m.name == name) + } + + /// Insert a constellation after validating every slot against the registry. + /// Fails closed: an unknown lens or a wrong-shaped slot is rejected. + pub fn insert(&mut self, c: Constellation) -> Result<(), StoreError> { + for (name, v) in &c.slots { + let m = self + .lens(name) + .ok_or_else(|| StoreError::UnknownLens(name.clone()))?; + if v.len() != m.dims { + return Err(StoreError::ShapeMismatch { + lens: name.clone(), + expected: m.dims, + got: v.len(), + }); + } + } + self.records.push(c); + Ok(()) + } + + pub fn get(&self, id: u64) -> Option<&Constellation> { + self.records.iter().find(|c| c.id == id) + } + + /// Rank records by cosine similarity of one lens slot to `query`. + /// Returns `(id, score)` pairs sorted by descending score, length `top_k`. + /// Records missing the lens slot are skipped. + pub fn search_lens(&self, lens: &str, query: &[f32], top_k: usize) -> Vec<(u64, f32)> { + let mut scored: Vec<(u64, f32)> = self + .records + .iter() + .filter_map(|c| c.slot(lens).map(|s| (c.id, cosine_sim(s, query)))) + .collect(); + sort_desc(&mut scored); + scored.truncate(top_k); + scored + } + + /// Rank records by cosine similarity in the flattened baseline space. + pub fn search_flat(&self, query_flat: &[f32], top_k: usize) -> Vec<(u64, f32)> { + let mut scored: Vec<(u64, f32)> = self + .records + .iter() + .map(|c| (c.id, cosine_sim(&c.flatten(&self.lenses), query_flat))) + .collect(); + sort_desc(&mut scored); + scored.truncate(top_k); + scored + } +} + +/// Stable descending sort by score, ties broken by ascending id for +/// determinism. +pub(crate) fn sort_desc(scored: &mut [(u64, f32)]) { + scored.sort_by(|a, b| { + b.1.partial_cmp(&a.1) + .unwrap_or(std::cmp::Ordering::Equal) + .then(a.0.cmp(&b.0)) + }); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::lens::LensKind; + + fn registry() -> Vec { + vec![ + LensManifest::new("sem", "v1", LensKind::DenseSemantic, 2, 1.0, 1.0), + LensManifest::new("str", "v1", LensKind::Structural, 2, 1.0, 1.0), + ] + } + + #[test] + fn rejects_unknown_lens_and_bad_shape() { + let mut s = ConstellationStore::new(registry()); + let bad = Constellation::new(1).with_slot("nope", vec![1.0, 0.0]); + assert!(matches!(s.insert(bad), Err(StoreError::UnknownLens(_)))); + let wrong = Constellation::new(2).with_slot("sem", vec![1.0]); + assert!(matches!( + s.insert(wrong), + Err(StoreError::ShapeMismatch { .. }) + )); + } + + #[test] + fn per_lens_search_ranks_by_cosine() { + let mut s = ConstellationStore::new(registry()); + s.insert(Constellation::new(1).with_slot("sem", vec![1.0, 0.0])) + .unwrap(); + s.insert(Constellation::new(2).with_slot("sem", vec![0.0, 1.0])) + .unwrap(); + let r = s.search_lens("sem", &[0.9, 0.1], 2); + assert_eq!(r[0].0, 1); + } +} diff --git a/crates/ruvector-calyx/src/corpus.rs b/crates/ruvector-calyx/src/corpus.rs new file mode 100644 index 000000000..1d314f535 --- /dev/null +++ b/crates/ruvector-calyx/src/corpus.rs @@ -0,0 +1,303 @@ +//! The `.calyx` corpus format — a dependency-free bridge from real datasets. +//! +//! Real lenses (a code-search model, a text embedder, BM25) are produced +//! **offline, once** by a converter (see `tools/build_codesearchnet.py`) and +//! serialized here. This crate stays pure Rust with no model/network +//! dependency: it *loads precomputed multi-lens vectors + ground truth* and +//! runs the association layer. Same pattern as ann-benchmarks' precomputed HDF5. +//! +//! ## Binary layout (little-endian, version 1) +//! +//! ```text +//! magic : 8 bytes = b"CALYXB1\n" +//! u32 n_lenses +//! per lens: u16 name_len, name; u16 ver_len, ver; +//! u8 kind; u32 dims; f32 cost_us; f32 latency_us +//! u32 n_records +//! per rec : u64 id; (dims×f32 per lens, in header order); +//! u16 n_anchors; per anchor: u8 kind, f32 weight, u16 ref_len, ref +//! u32 n_queries +//! per q : per lens: u8 present, if present dims×f32; +//! u32 n_relevant; n_relevant × u64 rel_id +//! ``` +//! +//! `n_relevant == 0` marks an *unanswerable* query (the correct behaviour is to +//! abstain) — which is what makes conformal risk meaningful on real data. + +use std::io::{self, Read, Write}; + +use crate::constellation::{Constellation, ConstellationStore}; +use crate::engine::Query; +use crate::ledger::{Anchor, AnchorKind}; +use crate::lens::{LensKind, LensManifest}; + +const MAGIC: &[u8; 8] = b"CALYXB1\n"; + +/// A query paired with its relevance ground truth (`relevant` empty = the query +/// is unanswerable and abstaining is correct). +#[derive(Debug, Clone)] +pub struct RealQuery { + pub query: Query, + pub relevant: Vec, +} + +impl RealQuery { + pub fn is_answerable(&self) -> bool { + !self.relevant.is_empty() + } +} + +/// A loaded corpus: the lens registry + records, plus labelled queries. +pub struct Corpus { + pub store: ConstellationStore, + pub queries: Vec, +} + +// ── LensKind / AnchorKind wire codes ──────────────────────────────────────── +fn lens_kind_code(k: LensKind) -> u8 { + match k { + LensKind::DenseSemantic => 0, + LensKind::Lexical => 1, + LensKind::Structural => 2, + LensKind::Temporal => 3, + LensKind::Sensor => 4, + } +} +fn lens_kind_from(c: u8) -> LensKind { + match c { + 1 => LensKind::Lexical, + 2 => LensKind::Structural, + 3 => LensKind::Temporal, + 4 => LensKind::Sensor, + _ => LensKind::DenseSemantic, + } +} +fn anchor_kind_code(k: AnchorKind) -> u8 { + match k { + AnchorKind::AcceptedAnswer => 0, + AnchorKind::PassedTest => 1, + AnchorKind::SensorLabel => 2, + AnchorKind::Citation => 3, + AnchorKind::Reward => 4, + } +} +fn anchor_kind_from(c: u8) -> AnchorKind { + match c { + 0 => AnchorKind::AcceptedAnswer, + 2 => AnchorKind::SensorLabel, + 3 => AnchorKind::Citation, + 4 => AnchorKind::Reward, + _ => AnchorKind::PassedTest, + } +} + +// ── little-endian primitives ──────────────────────────────────────────────── +fn wu16(w: &mut W, v: u16) -> io::Result<()> { w.write_all(&v.to_le_bytes()) } +fn wu32(w: &mut W, v: u32) -> io::Result<()> { w.write_all(&v.to_le_bytes()) } +fn wu64(w: &mut W, v: u64) -> io::Result<()> { w.write_all(&v.to_le_bytes()) } +fn wf32(w: &mut W, v: f32) -> io::Result<()> { w.write_all(&v.to_le_bytes()) } +fn wstr(w: &mut W, s: &str) -> io::Result<()> { + wu16(w, s.len() as u16)?; + w.write_all(s.as_bytes()) +} + +fn ru8(r: &mut R) -> io::Result { let mut b = [0u8; 1]; r.read_exact(&mut b)?; Ok(b[0]) } +fn ru16(r: &mut R) -> io::Result { let mut b = [0u8; 2]; r.read_exact(&mut b)?; Ok(u16::from_le_bytes(b)) } +fn ru32(r: &mut R) -> io::Result { let mut b = [0u8; 4]; r.read_exact(&mut b)?; Ok(u32::from_le_bytes(b)) } +fn ru64(r: &mut R) -> io::Result { let mut b = [0u8; 8]; r.read_exact(&mut b)?; Ok(u64::from_le_bytes(b)) } +fn rf32(r: &mut R) -> io::Result { let mut b = [0u8; 4]; r.read_exact(&mut b)?; Ok(f32::from_le_bytes(b)) } +fn rstr(r: &mut R) -> io::Result { + let n = ru16(r)? as usize; + let mut buf = vec![0u8; n]; + r.read_exact(&mut buf)?; + String::from_utf8(buf).map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "bad utf8")) +} +fn rvec(r: &mut R, dims: usize) -> io::Result> { + let mut v = Vec::with_capacity(dims); + for _ in 0..dims { + v.push(rf32(r)?); + } + Ok(v) +} + +/// Serialize a corpus to a writer in `.calyx` v1. +pub fn save(w: &mut W, store: &ConstellationStore, queries: &[RealQuery]) -> io::Result<()> { + w.write_all(MAGIC)?; + let lenses = store.lenses(); + wu32(w, lenses.len() as u32)?; + for m in lenses { + wstr(w, &m.name)?; + wstr(w, &m.version)?; + w.write_all(&[lens_kind_code(m.kind)])?; + wu32(w, m.dims as u32)?; + wf32(w, m.cost_us)?; + wf32(w, m.latency_us)?; + } + wu32(w, store.records().len() as u32)?; + for rec in store.records() { + wu64(w, rec.id)?; + for m in lenses { + let slot = rec.slot(&m.name).unwrap_or(&[]); + for i in 0..m.dims { + wf32(w, slot.get(i).copied().unwrap_or(0.0))?; + } + } + wu16(w, rec.anchors.len() as u16)?; + for a in &rec.anchors { + w.write_all(&[anchor_kind_code(a.kind)])?; + wf32(w, a.weight)?; + wstr(w, &a.reference)?; + } + } + wu32(w, queries.len() as u32)?; + for q in queries { + for m in lenses { + match q.query.slots.get(&m.name) { + Some(v) => { + w.write_all(&[1u8])?; + for i in 0..m.dims { + wf32(w, v.get(i).copied().unwrap_or(0.0))?; + } + } + None => w.write_all(&[0u8])?, + } + } + wu32(w, q.relevant.len() as u32)?; + for &id in &q.relevant { + wu64(w, id)?; + } + } + Ok(()) +} + +/// Deserialize a corpus from a reader. +pub fn load(r: &mut R) -> io::Result { + let mut magic = [0u8; 8]; + r.read_exact(&mut magic)?; + if &magic != MAGIC { + return Err(io::Error::new(io::ErrorKind::InvalidData, "bad magic (not a .calyx v1 file)")); + } + let n_lenses = ru32(r)? as usize; + let mut lenses = Vec::with_capacity(n_lenses); + for _ in 0..n_lenses { + let name = rstr(r)?; + let ver = rstr(r)?; + let kind = lens_kind_from(ru8(r)?); + let dims = ru32(r)? as usize; + let cost = rf32(r)?; + let lat = rf32(r)?; + lenses.push(LensManifest::new(name, ver, kind, dims, cost, lat)); + } + let mut store = ConstellationStore::new(lenses.clone()); + + let n_records = ru32(r)? as usize; + for _ in 0..n_records { + let id = ru64(r)?; + let mut c = Constellation::new(id); + for m in &lenses { + c.slots.insert(m.name.clone(), rvec(r, m.dims)?); + } + let n_anchors = ru16(r)? as usize; + for _ in 0..n_anchors { + let kind = anchor_kind_from(ru8(r)?); + let weight = rf32(r)?; + let reference = rstr(r)?; + c.anchors.push(Anchor::new(kind, reference, weight)); + } + store + .insert(c) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e.to_string()))?; + } + + let n_queries = ru32(r)? as usize; + let mut queries = Vec::with_capacity(n_queries); + for _ in 0..n_queries { + let mut query = Query::new(); + for m in &lenses { + if ru8(r)? == 1 { + query.slots.insert(m.name.clone(), rvec(r, m.dims)?); + } + } + let n_rel = ru32(r)? as usize; + let mut relevant = Vec::with_capacity(n_rel); + for _ in 0..n_rel { + relevant.push(ru64(r)?); + } + queries.push(RealQuery { query, relevant }); + } + Ok(Corpus { store, queries }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tiny() -> (ConstellationStore, Vec) { + let lenses = vec![ + LensManifest::new("code", "unixcoder-base", LensKind::DenseSemantic, 3, 8.0, 8.0), + LensManifest::new("doc", "bge-small", LensKind::DenseSemantic, 3, 3.0, 3.0), + LensManifest::new("lexical", "bm25", LensKind::Lexical, 3, 1.0, 1.0), + ]; + let mut store = ConstellationStore::new(lenses); + store + .insert( + Constellation::new(10) + .with_slot("code", vec![1.0, 0.0, 0.0]) + .with_slot("doc", vec![0.9, 0.1, 0.0]) + .with_slot("lexical", vec![1.0, 0.0, 0.0]) + .with_anchor(Anchor::new(AnchorKind::PassedTest, "has_test", 0.9)), + ) + .unwrap(); + store + .insert( + Constellation::new(11) + .with_slot("code", vec![0.0, 1.0, 0.0]) + .with_slot("doc", vec![0.0, 0.9, 0.1]) + .with_slot("lexical", vec![0.0, 1.0, 0.0]), + ) + .unwrap(); + let queries = vec![ + RealQuery { + query: Query::new() + .with_slot("code", vec![0.95, 0.05, 0.0]) + .with_slot("doc", vec![0.9, 0.1, 0.0]) + .with_slot("lexical", vec![1.0, 0.0, 0.0]), + relevant: vec![10], + }, + RealQuery { + // Unanswerable: gold not in pool. Query has only text lens. + query: Query::new().with_slot("doc", vec![0.0, 0.0, 1.0]), + relevant: vec![], + }, + ]; + (store, queries) + } + + #[test] + fn round_trips() { + let (store, queries) = tiny(); + let mut buf = Vec::new(); + save(&mut buf, &store, &queries).unwrap(); + let mut cur = std::io::Cursor::new(buf); + let c = load(&mut cur).unwrap(); + + assert_eq!(c.store.lenses().len(), 3); + assert_eq!(c.store.len(), 2); + assert_eq!(c.queries.len(), 2); + // Record slots survived. + assert_eq!(c.store.get(10).unwrap().slot("code").unwrap(), &[1.0, 0.0, 0.0]); + // Anchor survived. + assert_eq!(c.store.get(10).unwrap().anchors.len(), 1); + // Query relevance + partial lens presence survived. + assert_eq!(c.queries[0].relevant, vec![10]); + assert!(c.queries[1].relevant.is_empty()); + assert!(!c.queries[1].query.slots.contains_key("code")); + assert!(c.queries[1].query.slots.contains_key("doc")); + } + + #[test] + fn rejects_bad_magic() { + let mut cur = std::io::Cursor::new(b"NOTCALYX".to_vec()); + assert!(load(&mut cur).is_err()); + } +} diff --git a/crates/ruvector-calyx/src/disagreement.rs b/crates/ruvector-calyx/src/disagreement.rs new file mode 100644 index 000000000..2e11f3ae0 --- /dev/null +++ b/crates/ruvector-calyx/src/disagreement.rs @@ -0,0 +1,152 @@ +//! Disagreement as a query primitive. +//! +//! A single-embedding store can only answer "what is *similar* to X". Because a +//! constellation keeps every lens as a distinct slot, it can answer a question +//! no flat vector DB can express: **"where do two lenses disagree?"** — which +//! records look alike through one lens but different through another (the "code +//! comment says one thing, the code does another" detector). +//! +//! Two flavours: +//! - **intrinsic** — a record whose lens-A neighbourhood and lens-B +//! neighbourhood barely overlap sits on a boundary between the two views. +//! - **query-relative** — given a query, records that score high under lens A +//! but low under lens B (or vice-versa). + +use std::collections::BTreeMap; + +use crate::constellation::ConstellationStore; +use crate::scoring::cosine_sim; + +/// The `k` nearest neighbours of record `id` under `lens` (excluding itself). +pub fn neighbor_set(store: &ConstellationStore, lens: &str, id: u64, k: usize) -> Vec { + let Some(rec) = store.get(id) else { + return Vec::new(); + }; + let Some(slot) = rec.slot(lens) else { + return Vec::new(); + }; + store + .search_lens(lens, slot, k + 1) + .into_iter() + .filter(|&(nid, _)| nid != id) + .take(k) + .map(|(nid, _)| nid) + .collect() +} + +/// Jaccard similarity of two id lists. +fn jaccard(a: &[u64], b: &[u64]) -> f32 { + if a.is_empty() && b.is_empty() { + return 1.0; + } + let sa: std::collections::BTreeSet = a.iter().copied().collect(); + let sb: std::collections::BTreeSet = b.iter().copied().collect(); + let inter = sa.intersection(&sb).count(); + let union = sa.union(&sb).count(); + if union == 0 { + 1.0 + } else { + inter as f32 / union as f32 + } +} + +/// Intrinsic cross-lens disagreement of record `id`: `1 − Jaccard` of its +/// lens-A and lens-B neighbourhoods. `1.0` = the two lenses place the record in +/// entirely different company. +pub fn intrinsic_disagreement( + store: &ConstellationStore, + lens_a: &str, + lens_b: &str, + id: u64, + k: usize, +) -> f32 { + let na = neighbor_set(store, lens_a, id, k); + let nb = neighbor_set(store, lens_b, id, k); + 1.0 - jaccard(&na, &nb) +} + +/// Find the records where `lens_a` and `lens_b` most disagree, best-first. +pub fn find_conflicts( + store: &ConstellationStore, + lens_a: &str, + lens_b: &str, + k: usize, + top_n: usize, +) -> Vec<(u64, f32)> { + let mut scored: Vec<(u64, f32)> = store + .records() + .iter() + .map(|r| (r.id, intrinsic_disagreement(store, lens_a, lens_b, r.id, k))) + .collect(); + scored.sort_by(|x, y| { + y.1.partial_cmp(&x.1) + .unwrap_or(std::cmp::Ordering::Equal) + .then(x.0.cmp(&y.0)) + }); + scored.truncate(top_n); + scored +} + +/// Query-relative disagreement: records scored by `sim_A(q,·) − sim_B(q,·)`. +/// Large positive = "looks right through lens A, wrong through lens B". +/// Returns `(id, signed_gap)` sorted by descending absolute gap. +pub fn query_disagreement( + store: &ConstellationStore, + query: &BTreeMap>, + lens_a: &str, + lens_b: &str, + top_n: usize, +) -> Vec<(u64, f32)> { + let (Some(qa), Some(qb)) = (query.get(lens_a), query.get(lens_b)) else { + return Vec::new(); + }; + let mut scored: Vec<(u64, f32)> = store + .records() + .iter() + .filter_map(|r| { + let (sa, sb) = (r.slot(lens_a)?, r.slot(lens_b)?); + Some((r.id, cosine_sim(qa, sa) - cosine_sim(qb, sb))) + }) + .collect(); + scored.sort_by(|x, y| { + y.1.abs() + .partial_cmp(&x.1.abs()) + .unwrap_or(std::cmp::Ordering::Equal) + .then(x.0.cmp(&y.0)) + }); + scored.truncate(top_n); + scored +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::constellation::Constellation; + use crate::lens::{LensKind, LensManifest}; + + fn store() -> ConstellationStore { + let lenses = vec![ + LensManifest::new("a", "v1", LensKind::DenseSemantic, 2, 1.0, 1.0), + LensManifest::new("b", "v1", LensKind::Structural, 2, 1.0, 1.0), + ]; + let mut s = ConstellationStore::new(lenses); + // Records 1,2,3 agree on both lenses (cluster X). Record 4 is the + // conflict: lens-a like cluster X, lens-b like cluster Y (record 5,6). + s.insert(Constellation::new(1).with_slot("a", vec![1.0, 0.0]).with_slot("b", vec![1.0, 0.0])).unwrap(); + s.insert(Constellation::new(2).with_slot("a", vec![0.98, 0.02]).with_slot("b", vec![0.98, 0.02])).unwrap(); + s.insert(Constellation::new(3).with_slot("a", vec![0.96, 0.04]).with_slot("b", vec![0.97, 0.03])).unwrap(); + s.insert(Constellation::new(5).with_slot("a", vec![0.0, 1.0]).with_slot("b", vec![0.0, 1.0])).unwrap(); + s.insert(Constellation::new(6).with_slot("a", vec![0.02, 0.98]).with_slot("b", vec![0.03, 0.97])).unwrap(); + // The conflict record: lens-a near cluster X, lens-b near cluster Y. + s.insert(Constellation::new(4).with_slot("a", vec![0.97, 0.03]).with_slot("b", vec![0.03, 0.97])).unwrap(); + s + } + + #[test] + fn conflict_record_ranks_first() { + let s = store(); + let conflicts = find_conflicts(&s, "a", "b", 2, 3); + assert_eq!(conflicts[0].0, 4, "the planted conflict should rank first"); + assert!(conflicts[0].1 > 0.5); + } +} diff --git a/crates/ruvector-calyx/src/engine.rs b/crates/ruvector-calyx/src/engine.rs new file mode 100644 index 000000000..7f8ebafc7 --- /dev/null +++ b/crates/ruvector-calyx/src/engine.rs @@ -0,0 +1,226 @@ +//! The Calyx query engine — the four verbs wired together. +//! +//! `measure → count → differentiate → compose`: +//! +//! 1. **measure** — the query arrives already measured through the same frozen +//! lenses as the store (slots in, never flattened). +//! 2. **count** — per-lens rankings ([`ConstellationStore::search_lens`]) and +//! cross-lens agreement ([`crate::loom`]). +//! 3. **differentiate** — lens weights (tuned offline by [`crate::anneal`]) bias +//! the fusion toward high-signal lenses. +//! 4. **compose** — [`crate::fusion`] fuses the rankings, [`crate::ledger`] +//! resolves grounding, and [`crate::ward`] adjudicates a guarded, replayable +//! decision recorded in the provenance ledger. + +use std::collections::BTreeMap; + +use crate::constellation::ConstellationStore; +use crate::fusion::{weighted_rrf, DEFAULT_RRF_K}; +use crate::ledger::{grounded_path, Ledger}; +use crate::loom::min_agreement; +use crate::scoring::{fnv1a64, hash_vec, mix_hash}; +use crate::ward::{adjudicate, Candidate, GuardDecision, GuardProfile}; + +/// A query: one object measured through (a subset of) the registry's lenses. +#[derive(Debug, Clone, Default)] +pub struct Query { + pub slots: BTreeMap>, +} + +impl Query { + pub fn new() -> Self { + Self::default() + } + pub fn with_slot(mut self, lens: impl Into, vector: Vec) -> Self { + self.slots.insert(lens.into(), vector); + self + } +} + +/// The outcome of a guarded query, plus the chain hash that makes it replayable. +#[derive(Debug, Clone)] +pub struct CalyxResult { + pub decision: GuardDecision, + /// Fused candidate list (best-first), for inspection / RAG context. + pub fused: Vec<(u64, f32)>, + /// Chain hash of the ledger entry recording this query. + pub ledger_hash: u64, +} + +/// The association-native engine: a store, per-lens fusion weights, a guard +/// profile, and an append-only provenance ledger. +pub struct CalyxEngine { + store: ConstellationStore, + /// Fusion weight per lens, in registry order. + weights: Vec, + guard: GuardProfile, + ledger: Ledger, + rrf_k: f32, + top_k: usize, +} + +impl CalyxEngine { + pub fn new(store: ConstellationStore) -> Self { + let n = store.lenses().len(); + Self { + store, + weights: vec![1.0; n], + guard: GuardProfile::default(), + ledger: Ledger::new(), + rrf_k: DEFAULT_RRF_K, + top_k: 10, + } + } + + pub fn with_weights(mut self, weights: Vec) -> Self { + assert_eq!(weights.len(), self.store.lenses().len(), "weights/lenses"); + self.weights = weights; + self + } + + pub fn with_guard(mut self, guard: GuardProfile) -> Self { + self.guard = guard; + self + } + + pub fn with_top_k(mut self, k: usize) -> Self { + self.top_k = k.max(1); + self + } + + pub fn store(&self) -> &ConstellationStore { + &self.store + } + + pub fn ledger(&self) -> &Ledger { + &self.ledger + } + + pub fn weights(&self) -> &[f32] { + &self.weights + } + + /// Run one guarded, logged query. + pub fn query(&mut self, q: &Query) -> CalyxResult { + let lenses = self.store.lenses(); + + // count: one ranked list per registry lens the query measured. + let mut rankings: Vec> = Vec::with_capacity(lenses.len()); + let mut weights: Vec = Vec::with_capacity(lenses.len()); + for (i, m) in lenses.iter().enumerate() { + match q.slots.get(&m.name) { + Some(qv) => { + rankings.push(self.store.search_lens(&m.name, qv, self.top_k)); + weights.push(self.weights[i]); + } + None => { + rankings.push(Vec::new()); + weights.push(0.0); + } + } + } + + // compose: fuse, then evaluate the top candidate's grounding + agreement. + let fused = weighted_rrf(&rankings, &weights, self.rrf_k); + let candidate = fused.first().map(|&(id, score)| { + let agreement = self + .store + .get(id) + .map(|rec| min_agreement(&q.slots, rec)) + .unwrap_or(0.0); + let grounding = self.store.get(id).and_then(|rec| { + grounded_path(id, &rec.anchors, self.guard.min_grounding_weight) + }); + Candidate { + record_id: id, + fusion_score: score, + cross_lens_agreement: agreement, + grounding, + } + }); + + let decision = adjudicate(candidate, &self.guard); + + // record: hash input, lenses consulted, retrieval, output → ledger. + let input_hash = hash_query(&q.slots); + let lens_hash = lenses + .iter() + .fold(0u64, |h, m| mix_hash(h, m.fingerprint)); + let retrieval_hash = fused + .iter() + .fold(0u64, |h, (id, _)| mix_hash(h, *id)); + let output_hash = hash_decision(&decision); + let ledger_hash = self + .ledger + .append(input_hash, lens_hash, retrieval_hash, output_hash); + + CalyxResult { + decision, + fused, + ledger_hash, + } + } +} + +fn hash_query(slots: &BTreeMap>) -> u64 { + let mut h = 0u64; + for (name, v) in slots { + h = mix_hash(h, fnv1a64(name.as_bytes())); + h = mix_hash(h, hash_vec(v)); + } + h +} + +fn hash_decision(d: &GuardDecision) -> u64 { + match d { + GuardDecision::Answer { + record_id, + fusion_score, + .. + } => mix_hash(*record_id, fnv1a64(&fusion_score.to_bits().to_le_bytes())), + GuardDecision::Refuse(r) => fnv1a64(format!("refuse:{r:?}").as_bytes()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::constellation::Constellation; + use crate::ledger::{Anchor, AnchorKind}; + use crate::lens::{LensKind, LensManifest}; + + fn registry() -> Vec { + vec![ + LensManifest::new("sem", "v1", LensKind::DenseSemantic, 2, 1.0, 1.0), + LensManifest::new("str", "v1", LensKind::Structural, 2, 1.0, 1.0), + ] + } + + #[test] + fn answers_grounded_and_logs_replayable() { + let mut store = ConstellationStore::new(registry()); + store + .insert( + Constellation::new(1) + .with_slot("sem", vec![1.0, 0.0]) + .with_slot("str", vec![1.0, 0.0]) + .with_anchor(Anchor::new(AnchorKind::PassedTest, "t1", 0.9)), + ) + .unwrap(); + store + .insert( + Constellation::new(2) + .with_slot("sem", vec![1.0, 0.0]) // semantic twin + .with_slot("str", vec![0.0, 1.0]), // structural dissenter, ungrounded + ) + .unwrap(); + let mut eng = CalyxEngine::new(store); + let q = Query::new() + .with_slot("sem", vec![1.0, 0.0]) + .with_slot("str", vec![1.0, 0.0]); + let r = eng.query(&q); + assert_eq!(r.decision.answered_id(), Some(1)); + assert!(eng.ledger().verify()); + assert_eq!(eng.ledger().len(), 1); + } +} diff --git a/crates/ruvector-calyx/src/fusion.rs b/crates/ruvector-calyx/src/fusion.rs new file mode 100644 index 000000000..57e311726 --- /dev/null +++ b/crates/ruvector-calyx/src/fusion.rs @@ -0,0 +1,66 @@ +//! Rank fusion across lens-specific result lists (the paper's *Sextant*). +//! +//! Reciprocal Rank Fusion (Cormack, Clarke & Buettcher, SIGIR 2009) combines +//! several ranked lists into one. Its key property here: a record that ranks +//! *consistently well across many independent lenses* outranks a record that +//! ranks highly in only one — exactly the cross-lens corroboration an +//! association-native store wants, and exactly what flattening into a single +//! vector throws away. + +use std::collections::BTreeMap; + +use crate::constellation::sort_desc; + +/// Default RRF damping constant. Smaller `k` rewards top ranks more sharply. +pub const DEFAULT_RRF_K: f32 = 20.0; + +/// Reciprocal Rank Fusion over per-lens ranked lists with equal weights. +/// +/// Each `ranking` is a list of `(id, _score)` ordered best-first; only the +/// rank position matters. Returns fused `(id, rrf_score)` best-first. +pub fn rrf(rankings: &[Vec<(u64, f32)>], k: f32) -> Vec<(u64, f32)> { + let weights = vec![1.0_f32; rankings.len()]; + weighted_rrf(rankings, &weights, k) +} + +/// Weighted Reciprocal Rank Fusion. `weights[i]` scales lens `i`'s +/// contribution; the anneal optimizer tunes these. +pub fn weighted_rrf(rankings: &[Vec<(u64, f32)>], weights: &[f32], k: f32) -> Vec<(u64, f32)> { + assert_eq!(rankings.len(), weights.len(), "weights/rankings length"); + let mut acc: BTreeMap = BTreeMap::new(); + for (lens_idx, ranking) in rankings.iter().enumerate() { + let w = weights[lens_idx]; + for (rank, (id, _)) in ranking.iter().enumerate() { + let contribution = w / (k + rank as f32 + 1.0); + *acc.entry(*id).or_insert(0.0) += contribution; + } + } + let mut fused: Vec<(u64, f32)> = acc.into_iter().collect(); + sort_desc(&mut fused); + fused +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn consistent_record_wins() { + // Record 7 is rank-2/3/2 across three lenses; record 1 is rank-1 in one + // lens only. Cross-lens consistency should lift 7 above 1. + let lens_a = vec![(1, 0.9), (7, 0.8), (2, 0.7)]; + let lens_b = vec![(3, 0.9), (4, 0.8), (7, 0.7)]; + let lens_c = vec![(5, 0.9), (7, 0.8), (1, 0.1)]; + let fused = rrf(&[lens_a, lens_b, lens_c], DEFAULT_RRF_K); + assert_eq!(fused[0].0, 7, "consistently-ranked record should win"); + } + + #[test] + fn weights_can_promote_a_lens() { + let lens_a = vec![(1, 0.9), (2, 0.8)]; + let lens_b = vec![(2, 0.9), (1, 0.8)]; + // Heavily weight lens_b → its top (id 2) wins. + let fused = weighted_rrf(&[lens_a, lens_b], &[0.1, 5.0], DEFAULT_RRF_K); + assert_eq!(fused[0].0, 2); + } +} diff --git a/crates/ruvector-calyx/src/ledger.rs b/crates/ruvector-calyx/src/ledger.rs new file mode 100644 index 000000000..d2073f860 --- /dev/null +++ b/crates/ruvector-calyx/src/ledger.rs @@ -0,0 +1,224 @@ +//! Grounding anchors and the hash-chained provenance ledger. +//! +//! Two of Calyx's three trust principles live here: *grounding is mandatory* +//! and every answer must be replayable. An [`Anchor`] ties a record to a +//! real-world outcome (the paper's *Lodestar* grounding kernels); the +//! [`Ledger`] (the paper's *Ledger*) records each query as a hash-chained entry +//! so the full answer path — input, lenses, retrieval, output — can be +//! reproduced and audited. + +use crate::scoring::{fnv1a64, mix_hash}; + +/// The kind of real-world evidence an anchor represents. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AnchorKind { + /// A human-accepted answer. + AcceptedAnswer, + /// A test that passed against this record. + PassedTest, + /// A sensor reading / ground-truth label. + SensorLabel, + /// A citation to an authoritative source. + Citation, + /// A reward signal (e.g. RL / preference). + Reward, +} + +impl AnchorKind { + fn as_str(self) -> &'static str { + match self { + AnchorKind::AcceptedAnswer => "accepted_answer", + AnchorKind::PassedTest => "passed_test", + AnchorKind::SensorLabel => "sensor_label", + AnchorKind::Citation => "citation", + AnchorKind::Reward => "reward", + } + } +} + +/// A grounding anchor: evidence with a confidence weight in `[0, 1]`. +#[derive(Debug, Clone, PartialEq)] +pub struct Anchor { + pub kind: AnchorKind, + /// Free-form reference to the evidence (doc id, test name, sensor id, ...). + pub reference: String, + /// Confidence in `[0, 1]`. + pub weight: f32, +} + +impl Anchor { + pub fn new(kind: AnchorKind, reference: impl Into, weight: f32) -> Self { + Self { + kind, + reference: reference.into(), + weight: weight.clamp(0.0, 1.0), + } + } + + pub fn hash(&self) -> u64 { + let mut bytes = Vec::new(); + bytes.extend_from_slice(self.kind.as_str().as_bytes()); + bytes.push(0); + bytes.extend_from_slice(self.reference.as_bytes()); + bytes.push(0); + bytes.extend_from_slice(&self.weight.to_bits().to_le_bytes()); + fnv1a64(&bytes) + } +} + +/// The strongest grounded path supporting a candidate, if any. +#[derive(Debug, Clone, PartialEq)] +pub struct GroundingPath { + pub record_id: u64, + pub best_kind: AnchorKind, + pub best_weight: f32, +} + +/// Return the best grounded path for `anchors` whose weight clears +/// `min_weight`, or `None` if the record has no sufficient grounding. +pub fn grounded_path(record_id: u64, anchors: &[Anchor], min_weight: f32) -> Option { + anchors + .iter() + .filter(|a| a.weight >= min_weight) + .max_by(|a, b| { + a.weight + .partial_cmp(&b.weight) + .unwrap_or(std::cmp::Ordering::Equal) + }) + .map(|a| GroundingPath { + record_id, + best_kind: a.kind, + best_weight: a.weight, + }) +} + +/// One hash-chained record of an answer path. +#[derive(Debug, Clone, PartialEq)] +pub struct LedgerEntry { + pub seq: u64, + pub prev_hash: u64, + /// Hash over the query's input slots. + pub input_hash: u64, + /// Folded hash over the fingerprints of every lens consulted. + pub lens_hash: u64, + /// Hash over the retrieval result (ranked ids). + pub retrieval_hash: u64, + /// Hash over the emitted decision. + pub output_hash: u64, + /// Chain hash: `mix(prev_hash, mix(... folded entry fields ...))`. + pub hash: u64, +} + +impl LedgerEntry { + fn body_hash(&self) -> u64 { + let mut h = self.input_hash; + h = mix_hash(h, self.lens_hash); + h = mix_hash(h, self.retrieval_hash); + h = mix_hash(h, self.output_hash); + h = mix_hash(h, self.seq); + h + } +} + +/// Append-only hash-chained ledger of answer paths. +#[derive(Debug, Default)] +pub struct Ledger { + entries: Vec, +} + +impl Ledger { + pub fn new() -> Self { + Self::default() + } + + pub fn entries(&self) -> &[LedgerEntry] { + &self.entries + } + + pub fn len(&self) -> usize { + self.entries.len() + } + + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + /// The chain head hash (`0` for an empty ledger). + pub fn head(&self) -> u64 { + self.entries.last().map(|e| e.hash).unwrap_or(0) + } + + /// Append an entry, chaining it onto the current head. + pub fn append( + &mut self, + input_hash: u64, + lens_hash: u64, + retrieval_hash: u64, + output_hash: u64, + ) -> u64 { + let seq = self.entries.len() as u64; + let prev_hash = self.head(); + let mut entry = LedgerEntry { + seq, + prev_hash, + input_hash, + lens_hash, + retrieval_hash, + output_hash, + hash: 0, + }; + entry.hash = mix_hash(prev_hash, entry.body_hash()); + self.entries.push(entry); + self.head() + } + + /// Replay the whole chain, recomputing every link. Returns `true` iff the + /// ledger is internally consistent (100% replayable). + pub fn verify(&self) -> bool { + let mut prev = 0u64; + for (i, e) in self.entries.iter().enumerate() { + if e.seq != i as u64 || e.prev_hash != prev { + return false; + } + if e.hash != mix_hash(e.prev_hash, e.body_hash()) { + return false; + } + prev = e.hash; + } + true + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn grounding_picks_strongest_above_threshold() { + let anchors = vec![ + Anchor::new(AnchorKind::Citation, "doc-1", 0.4), + Anchor::new(AnchorKind::PassedTest, "test-7", 0.9), + ]; + let p = grounded_path(3, &anchors, 0.5).unwrap(); + assert_eq!(p.best_kind, AnchorKind::PassedTest); + assert!(grounded_path(3, &anchors, 0.95).is_none()); + } + + #[test] + fn ledger_chains_and_verifies() { + let mut l = Ledger::new(); + l.append(1, 2, 3, 4); + l.append(5, 6, 7, 8); + assert!(l.verify()); + assert_eq!(l.len(), 2); + } + + #[test] + fn tamper_breaks_verification() { + let mut l = Ledger::new(); + l.append(1, 2, 3, 4); + l.append(5, 6, 7, 8); + l.entries[0].output_hash ^= 0xdead; // tamper + assert!(!l.verify()); + } +} diff --git a/crates/ruvector-calyx/src/ledger_policy.rs b/crates/ruvector-calyx/src/ledger_policy.rs new file mode 100644 index 000000000..2cb3a6eeb --- /dev/null +++ b/crates/ruvector-calyx/src/ledger_policy.rs @@ -0,0 +1,144 @@ +//! Learning the router from the ledger. +//! +//! Every routed decision already emits a [`crate::routing::Witness`] — which +//! lenses were consulted, the confidence at each step, the action, the outcome. +//! Stack those up and the ledger *is* a trajectory dataset. This module learns +//! a stop/continue routing policy from that log by tabular first-visit +//! Monte-Carlo control, so the router improves from its own recorded provenance +//! instead of hand-set thresholds — the retrieval analogue of MetaHarness +//! learning a harness from its trace. +//! +//! State = `(lenses_consulted, confidence_bin)`. Action = stop or continue. +//! An exploratory behaviour policy (stop/continue at random) is logged, then +//! greedy improvement reads the best action per state off the learned Q-table. + +/// The two routing actions available at each consultation step. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Action { + Stop, + Continue, +} + +/// One logged episode: the `(lens_count, conf_bin, action)` states it visited +/// and the realized return (terminal utility minus accumulated cost). +#[derive(Debug, Clone)] +pub struct Episode { + pub visited: Vec<(usize, usize, Action)>, + pub ret: f32, +} + +/// A learned tabular stop/continue policy indexed by `(lens_count, conf_bin)`. +#[derive(Debug, Clone)] +pub struct LearnedPolicy { + max_lens: usize, + n_bins: usize, + q_stop: Vec, + q_cont: Vec, + n_stop: Vec, + n_cont: Vec, +} + +impl LearnedPolicy { + fn idx(&self, lens_count: usize, bin: usize) -> usize { + let lc = lens_count.min(self.max_lens.saturating_sub(1)); + let b = bin.min(self.n_bins - 1); + lc * self.n_bins + b + } + + /// Greedy action for a state; defaults to `Continue` for never-seen states + /// with room to escalate (optimistic), else `Stop`. + pub fn act(&self, lens_count: usize, bin: usize) -> Action { + let i = self.idx(lens_count, bin); + if self.n_stop[i] == 0 && self.n_cont[i] == 0 { + return if lens_count + 1 < self.max_lens { + Action::Continue + } else { + Action::Stop + }; + } + if self.q_stop[i] >= self.q_cont[i] { + Action::Stop + } else { + Action::Continue + } + } + + /// Bin a confidence in `[0,1]` the same way learning did. + pub fn bin_of(&self, confidence: f32) -> usize { + ((confidence.clamp(0.0, 0.999_999)) * self.n_bins as f32) as usize + } + + pub fn q_values(&self, lens_count: usize, bin: usize) -> (f32, f32) { + let i = self.idx(lens_count, bin); + (self.q_stop[i], self.q_cont[i]) + } +} + +/// Learn a policy from logged episodes by first-visit Monte-Carlo averaging of +/// returns per `(state, action)`. +pub fn learn(episodes: &[Episode], max_lens: usize, n_bins: usize) -> LearnedPolicy { + let max_lens = max_lens.max(1); + let n_bins = n_bins.max(1); + let size = max_lens * n_bins; + let mut p = LearnedPolicy { + max_lens, + n_bins, + q_stop: vec![0.0; size], + q_cont: vec![0.0; size], + n_stop: vec![0; size], + n_cont: vec![0; size], + }; + + for ep in episodes { + let mut seen = std::collections::BTreeSet::new(); + for &(lc, bin, action) in &ep.visited { + let key = (lc.min(max_lens - 1), bin.min(n_bins - 1), action); + if !seen.insert(key) { + continue; // first-visit + } + let i = p.idx(lc, bin); + match action { + Action::Stop => { + p.q_stop[i] = running_mean(p.q_stop[i], p.n_stop[i], ep.ret); + p.n_stop[i] += 1; + } + Action::Continue => { + p.q_cont[i] = running_mean(p.q_cont[i], p.n_cont[i], ep.ret); + p.n_cont[i] += 1; + } + } + } + } + p +} + +#[inline] +fn running_mean(mean: f32, n: u32, x: f32) -> f32 { + mean + (x - mean) / (n as f32 + 1.0) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn learns_to_continue_when_low_conf_and_stop_when_high() { + // Synthetic ledger: stopping in a low-confidence state (bin 0) returns + // little; continuing from it returns more. High-confidence (bin 9) + // stopping returns a lot. + let mut eps = Vec::new(); + for _ in 0..200 { + // low-conf stop → return 0.1 + eps.push(Episode { visited: vec![(1, 0, Action::Stop)], ret: 0.1 }); + // low-conf continue → return 0.8 + eps.push(Episode { visited: vec![(1, 0, Action::Continue)], ret: 0.8 }); + // high-conf stop → return 0.9 + eps.push(Episode { visited: vec![(1, 9, Action::Stop)], ret: 0.9 }); + // high-conf continue → return 0.5 (wasted cost) + eps.push(Episode { visited: vec![(1, 9, Action::Continue)], ret: 0.5 }); + } + let p = learn(&eps, 3, 10); + assert_eq!(p.act(1, 0), Action::Continue); + assert_eq!(p.act(1, 9), Action::Stop); + } +} diff --git a/crates/ruvector-calyx/src/lens.rs b/crates/ruvector-calyx/src/lens.rs new file mode 100644 index 000000000..fc257522e --- /dev/null +++ b/crates/ruvector-calyx/src/lens.rs @@ -0,0 +1,120 @@ +//! Lenses and their frozen, content-addressed manifests. +//! +//! A **lens** is one frozen way of measuring an object: a dense semantic +//! embedder, a sparse lexical index, a code model, a structural/domain encoder, +//! a temporal recurrence model, and so on. Each lens produces one typed +//! **slot** vector. Calyx's core invariant is that slots are kept *distinct* +//! and **never flattened** into a single vector — relationships are derived +//! *between* slots instead. +//! +//! Lenses are reproducible: a [`LensManifest`] records the measurement lineage +//! (name, version, kind, dimensionality, declared cost/latency) and is +//! content-addressed by a fingerprint hash so the same measurement can be +//! replayed and audited later (the paper's *Registry*). + +use crate::scoring::fnv1a64; + +/// The family a lens belongs to. Branding aside, the only thing the engine +/// needs from a lens is "what kind of signal is this and how big". +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum LensKind { + /// Dense semantic embedding (topic / meaning). + DenseSemantic, + /// Sparse lexical / BM25-style surface-term signal (stored dense here). + Lexical, + /// Structural / domain / jurisdiction / code-shape signal. + Structural, + /// Temporal recurrence signal. + Temporal, + /// Non-text sensor signal (RF, audio, vibration, pose, ...). + Sensor, +} + +impl LensKind { + pub fn as_str(self) -> &'static str { + match self { + LensKind::DenseSemantic => "dense_semantic", + LensKind::Lexical => "lexical", + LensKind::Structural => "structural", + LensKind::Temporal => "temporal", + LensKind::Sensor => "sensor", + } + } +} + +/// Reproducible measurement lineage for one lens. +/// +/// The `fingerprint` content-addresses the lens: identical +/// `(name, version, kind, dims)` always produce the same fingerprint, so a +/// retrieval path can be replayed against a byte-identical measurement set. +#[derive(Debug, Clone, PartialEq)] +pub struct LensManifest { + /// Stable lens identifier (also the slot key on every constellation). + pub name: String, + /// Frozen model/version string. + pub version: String, + /// Signal family. + pub kind: LensKind, + /// Output dimensionality. + pub dims: usize, + /// Declared cost to measure one object, in microseconds (signal economics). + pub cost_us: f32, + /// Declared latency to measure one object, in microseconds. + pub latency_us: f32, + /// Content-addressed fingerprint over the lineage fields. + pub fingerprint: u64, +} + +impl LensManifest { + /// Build a manifest, computing its content-addressed fingerprint. + pub fn new( + name: impl Into, + version: impl Into, + kind: LensKind, + dims: usize, + cost_us: f32, + latency_us: f32, + ) -> Self { + let name = name.into(); + let version = version.into(); + let fingerprint = Self::compute_fingerprint(&name, &version, kind, dims); + Self { + name, + version, + kind, + dims, + cost_us, + latency_us, + fingerprint, + } + } + + fn compute_fingerprint(name: &str, version: &str, kind: LensKind, dims: usize) -> u64 { + let mut bytes = Vec::new(); + bytes.extend_from_slice(name.as_bytes()); + bytes.push(0); + bytes.extend_from_slice(version.as_bytes()); + bytes.push(0); + bytes.extend_from_slice(kind.as_str().as_bytes()); + bytes.push(0); + bytes.extend_from_slice(&(dims as u64).to_le_bytes()); + fnv1a64(&bytes) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fingerprint_is_content_addressed() { + let a = LensManifest::new("semantic", "v1", LensKind::DenseSemantic, 48, 1.0, 1.0); + let b = LensManifest::new("semantic", "v1", LensKind::DenseSemantic, 48, 9.0, 9.0); + // Cost/latency are economics, not lineage — same fingerprint. + assert_eq!(a.fingerprint, b.fingerprint); + let c = LensManifest::new("semantic", "v2", LensKind::DenseSemantic, 48, 1.0, 1.0); + assert_ne!(a.fingerprint, c.fingerprint); + let d = LensManifest::new("semantic", "v1", LensKind::DenseSemantic, 64, 1.0, 1.0); + assert_ne!(a.fingerprint, d.fingerprint); + } +} diff --git a/crates/ruvector-calyx/src/lib.rs b/crates/ruvector-calyx/src/lib.rs new file mode 100644 index 000000000..3d8825fc6 --- /dev/null +++ b/crates/ruvector-calyx/src/lib.rs @@ -0,0 +1,104 @@ +//! # ruvector-calyx +//! +//! An **association-native memory layer** for ruvector. It is the practical, +//! Rust reference implementation of the architecture thesis in Chris Royse's +//! *Calyx* white paper (2026): one input should not collapse into one flattened +//! vector — it should become a **constellation**, the same object measured +//! through many frozen **lenses**, with each lens stored as a distinct typed +//! **slot**. Relationships are then *derived between* slots, grounded against +//! real-world anchors, and gated by a fail-closed guard. +//! +//! ## The four verbs +//! +//! | Verb | Module | What it does | +//! |------|--------|--------------| +//! | **measure** | [`lens`], [`constellation`] | Run an object through many frozen, content-addressed lenses; keep slots distinct (never flattened). | +//! | **count** | [`loom`] | Derive cross-lens relationships — agreement *and* the informative disagreements. | +//! | **differentiate** | [`assay`] | Estimate how much signal (in bits) each lens adds, per unit cost. | +//! | **compose** | [`fusion`], [`ledger`], [`ward`], [`engine`] | Fuse per-lens rankings (RRF), resolve grounding, adjudicate a guarded answer, and record a replayable provenance path. | +//! +//! [`anneal`] closes the loop: it reversibly tunes the fusion weights to +//! maximize accepted-answers-per-cost. +//! +//! ## Three trust principles (enforced, not aspirational) +//! +//! 1. **Grounding is mandatory** — [`ward::GuardProfile::require_grounding`]. +//! 2. **No flattening** — slots live in [`constellation::Constellation::slots`]; +//! [`constellation::Constellation::flatten`] exists *only* to build the +//! lossy single-embedding baseline this layer is designed to beat. +//! 3. **Fail closed** — unknown lens, shape mismatch, low agreement, or missing +//! grounding yield structured refusals ([`constellation::StoreError`], +//! [`ward::RefusalReason`]), never a silent wrong answer. +//! +//! ## Relation to ruvector +//! +//! Each per-lens ranking ([`constellation::ConstellationStore::search_lens`]) is +//! exact brute force here; in a deployment it is backed by ruvector's HNSW / IVF +//! indexes. The cross-lens graph ([`loom`]) maps onto ruvector's graph/min-cut +//! association edges. See `docs/adr/ADR-272` for the full mapping and the +//! benchmark acceptance targets exercised by the `calyx-bench` binary. +//! +//! ## References +//! +//! - Royse 2026, "Calyx: An Association-Native Database and Its Path to +//! Planetary-Scale Grounded Intelligence" (ResearchGate preprint; +//! ). +//! - Royse 2026, "The Calculus of Association: A Formula for Artificial General +//! Intelligence" (ResearchGate preprint) — frozen embedders as designable +//! measurement instruments, derived-data abundance, teleological constellations. +//! - Cormack, Clarke & Buettcher 2009, "Reciprocal Rank Fusion Outperforms +//! Condorcet and Individual Rank Learning Methods" (SIGIR). + +pub mod anneal; +pub mod assay; +pub mod calibrate; +pub mod conformal; +pub mod constellation; +pub mod corpus; +pub mod disagreement; +pub mod engine; +pub mod fusion; +pub mod ledger; +pub mod ledger_policy; +pub mod lens; +pub mod loom; +pub mod routing; +pub mod rng; +pub mod scoring; +pub mod ward; + +pub use anneal::{anneal_weights, AnnealConfig, AnnealOutcome}; +pub use assay::{mutual_information_bits, signal_density, LensSignal, Observation}; +pub use calibrate::{ + abstention_quality, expected_calibration_error, raw_confidence, AbstentionQuality, Calibrator, + ConfidenceSignals, +}; +pub use routing::{route, RoutingConfig, RoutingMode, RoutingOutcome, Witness}; +pub use conformal::{ + calibrate as conformal_calibrate, coverage, empirical_risk, selective_error, ConformalSelector, +}; +pub use disagreement::{ + find_conflicts, intrinsic_disagreement, neighbor_set, query_disagreement, +}; +pub use ledger_policy::{learn as learn_policy, Action, Episode, LearnedPolicy}; +pub use corpus::{load as load_corpus, save as save_corpus, Corpus, RealQuery}; +pub use constellation::{Constellation, ConstellationStore, StoreError}; +pub use engine::{CalyxEngine, CalyxResult, Query}; +pub use fusion::{rrf, weighted_rrf, DEFAULT_RRF_K}; +pub use ledger::{grounded_path, Anchor, AnchorKind, GroundingPath, Ledger, LedgerEntry}; +pub use lens::{LensKind, LensManifest}; +pub use loom::{agreement, mean_agreement, min_agreement, top_dissenter}; +pub use scoring::{cosine_sim, fnv1a64, hash_vec, normalize}; +pub use ward::{adjudicate, Candidate, GuardDecision, GuardProfile, RefusalReason}; + +/// Recall@k: fraction of `truth` ids present in `retrieved`. +pub fn recall_at_k(truth: &[u64], retrieved: &[u64]) -> f32 { + if truth.is_empty() { + return 1.0; + } + let hits = truth + .iter() + .filter(|t| retrieved.contains(t)) + .count(); + hits as f32 / truth.len() as f32 +} diff --git a/crates/ruvector-calyx/src/loom.rs b/crates/ruvector-calyx/src/loom.rs new file mode 100644 index 000000000..47da80435 --- /dev/null +++ b/crates/ruvector-calyx/src/loom.rs @@ -0,0 +1,91 @@ +//! Cross-lens relationships (the paper's *Loom*). +//! +//! "Count" — the second of Calyx's four verbs — derives relationships *between* +//! lens outputs rather than averaging them away. The highest-value signal is +//! often the *disagreement*: two records look alike under the semantic lens but +//! diverge under the structural lens. This module measures that. + +use std::collections::BTreeMap; + +use crate::constellation::Constellation; +use crate::scoring::cosine_sim; + +/// Per-lens cosine similarity between a query's slots and a record's slots. +/// +/// Only lenses present on *both* sides are scored. Returns a map keyed by lens +/// name. This is the raw material for both the guard (min agreement across +/// lenses) and disagreement detection. +pub fn agreement(query_slots: &BTreeMap>, record: &Constellation) -> Vec<(String, f32)> { + let mut out = Vec::new(); + for (lens, qv) in query_slots { + if let Some(rv) = record.slot(lens) { + if rv.len() == qv.len() { + out.push((lens.clone(), cosine_sim(qv, rv))); + } + } + } + out +} + +/// The minimum agreement across all shared lenses — the conservative, +/// fail-closed summary the guard uses. A high min means *every* lens agrees; +/// one dissenting lens drags it down. +pub fn min_agreement(query_slots: &BTreeMap>, record: &Constellation) -> f32 { + agreement(query_slots, record) + .into_iter() + .map(|(_, s)| s) + .fold(f32::INFINITY, f32::min) + .clamp(0.0, 1.0) +} + +/// The mean agreement across shared lenses. +pub fn mean_agreement(query_slots: &BTreeMap>, record: &Constellation) -> f32 { + let scores = agreement(query_slots, record); + if scores.is_empty() { + return 0.0; + } + scores.iter().map(|(_, s)| s).sum::() / scores.len() as f32 +} + +/// The lens with the largest *gap* below the mean agreement — i.e. the most +/// informative dissenter. Returns `None` when fewer than two lenses are shared. +pub fn top_dissenter( + query_slots: &BTreeMap>, + record: &Constellation, +) -> Option<(String, f32)> { + let scores = agreement(query_slots, record); + if scores.len() < 2 { + return None; + } + let mean = scores.iter().map(|(_, s)| s).sum::() / scores.len() as f32; + scores + .into_iter() + .map(|(l, s)| (l, mean - s)) + .max_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal)) + .filter(|(_, gap)| *gap > 0.0) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn qslots() -> BTreeMap> { + let mut m = BTreeMap::new(); + m.insert("sem".to_string(), vec![1.0, 0.0]); + m.insert("str".to_string(), vec![1.0, 0.0]); + m + } + + #[test] + fn disagreement_is_caught_by_min() { + // Record agrees semantically but disagrees structurally. + let rec = Constellation::new(1) + .with_slot("sem", vec![1.0, 0.0]) + .with_slot("str", vec![0.0, 1.0]); + let q = qslots(); + assert!(mean_agreement(&q, &rec) > 0.4); // mean hides it + assert!(min_agreement(&q, &rec) < 0.1); // min catches it + let (lens, _gap) = top_dissenter(&q, &rec).unwrap(); + assert_eq!(lens, "str"); + } +} diff --git a/crates/ruvector-calyx/src/main.rs b/crates/ruvector-calyx/src/main.rs new file mode 100644 index 000000000..a2f9112fb --- /dev/null +++ b/crates/ruvector-calyx/src/main.rs @@ -0,0 +1,499 @@ +//! Benchmark: association-native multi-lens retrieval vs single-embedding RAG. +//! +//! Within a topic every document shares the same *semantic* region ("fog"); the +//! discriminating signal lives in two minority lenses (a lexical fingerprint and +//! a structural class) that a flattened embedding drowns out but an +//! association-native store keeps as full-weight ranking lists. Unanswerable +//! queries (right topic, specifics matching no doc) test fail-closed abstention. +//! Acceptance targets (ADR-272): grounded accuracy ≥+15pp, unsupported claims +//! ≥−50%, recall@10 ≥+10pp, replayability 100%. Run with `cargo run --release`. + +use ruvector_calyx::rng::Rng; +use ruvector_calyx::{ + anneal_weights, recall_at_k, signal_density, AnnealConfig, CalyxEngine, Constellation, + ConstellationStore, GuardProfile, LensKind, LensManifest, Observation, Query, +}; +use ruvector_calyx::assay::LensSignal; +use ruvector_calyx::fusion::{weighted_rrf, DEFAULT_RRF_K}; +use ruvector_calyx::ledger::{grounded_path, Anchor, AnchorKind}; +use ruvector_calyx::loom::min_agreement; +use ruvector_calyx::ward::{adjudicate, Candidate, GuardDecision}; +use std::time::Instant; + +// ── Dataset parameters ───────────────────────────────────────────────────── +const DIM_SEM: usize = 48; +const DIM_LEX: usize = 32; +const DIM_STR: usize = 16; +const N_TOPICS: usize = 12; +const N_PER_TOPIC: usize = 20; +const N_DOCS: usize = N_TOPICS * N_PER_TOPIC; // 240 +const N_STRUCT_CLASSES: usize = 8; +const K: usize = 10; + +// Query parameters +const N_ANSWERABLE: usize = 120; +const N_UNANSWERABLE: usize = 60; + +// Lens economics (microseconds to measure one object). The dense semantic +// model is the expensive one; lexical / structural are cheap. +const COST_SEM: f32 = 12.0; +const COST_LEX: f32 = 2.0; +const COST_STR: f32 = 1.5; + +const SEED: u64 = 42; + +// ── Vector helpers ────────────────────────────────────────────────────────── +fn unit_gaussian(rng: &mut Rng, dim: usize) -> Vec { + let v: Vec = (0..dim).map(|_| rng.signed()).collect(); + normalize(&v) +} +fn normalize(v: &[f32]) -> Vec { + let n = v.iter().map(|x| x * x).sum::().sqrt().max(1e-9); + v.iter().map(|x| x / n).collect() +} +fn perturb(c: &[f32], noise: f32, rng: &mut Rng) -> Vec { + let g = unit_gaussian(rng, c.len()); + let mixed: Vec = c.iter().zip(g.iter()).map(|(a, b)| a + noise * b).collect(); + normalize(&mixed) +} +fn blend(a: &[f32], b: &[f32], wa: f32, wb: f32) -> Vec { + normalize( + &a.iter() + .zip(b.iter()) + .map(|(x, y)| wa * x + wb * y) + .collect::>(), + ) +} + +// ── Dataset ───────────────────────────────────────────────────────────────── +struct Doc { + id: u64, + topic: usize, + sem: Vec, + lex: Vec, + str_: Vec, +} + +struct QuerySpec { + sem: Vec, + lex: Vec, + str_: Vec, + /// `Some(target_id)` for answerable queries; `None` if unanswerable. + target: Option, +} + +fn registry() -> Vec { + vec![ + LensManifest::new("semantic", "frozen-v1", LensKind::DenseSemantic, DIM_SEM, COST_SEM, COST_SEM), + LensManifest::new("lexical", "frozen-v1", LensKind::Lexical, DIM_LEX, COST_LEX, COST_LEX), + LensManifest::new("structural", "frozen-v1", LensKind::Structural, DIM_STR, COST_STR, COST_STR), + ] +} + +fn build_dataset(rng: &mut Rng) -> (Vec, Vec) { + let sem_centroids: Vec> = (0..N_TOPICS).map(|_| unit_gaussian(rng, DIM_SEM)).collect(); + let lex_centroids: Vec> = (0..N_TOPICS).map(|_| unit_gaussian(rng, DIM_LEX)).collect(); + let str_centroids: Vec> = + (0..N_STRUCT_CLASSES).map(|_| unit_gaussian(rng, DIM_STR)).collect(); + + let mut docs = Vec::with_capacity(N_DOCS); + let mut id = 0u64; + for t in 0..N_TOPICS { + for j in 0..N_PER_TOPIC { + let s = (t * N_PER_TOPIC + j) % N_STRUCT_CLASSES; + // Semantic slot: topic fog — every doc in the topic is near-identical. + let sem = perturb(&sem_centroids[t], 0.10, rng); + // Lexical slot: topic flavour + a per-doc fingerprint (doc-level signal). + let lex_fp = unit_gaussian(rng, DIM_LEX); + let lex = blend(&lex_centroids[t], &lex_fp, 0.5, 0.85); + // Structural slot: class centroid + per-doc jitter (doc-level signal). + let str_ = perturb(&str_centroids[s], 0.30, rng); + docs.push(Doc { + id, + topic: t, + sem, + lex, + str_, + }); + id += 1; + } + } + + // Answerable queries: pick a target doc; semantic comes from the *topic* + // centroid (so semantic-only can't isolate the target), lexical & structural + // are derived from the target (so the minority lenses pin it down). + let mut queries = Vec::with_capacity(N_ANSWERABLE + N_UNANSWERABLE); + for i in 0..N_ANSWERABLE { + let d = &docs[(i * 7 + 3) % N_DOCS]; // spread targets across the corpus + let sem = perturb(&sem_centroids[d.topic], 0.10, rng); + let lex = perturb(&d.lex, 0.10, rng); + let str_ = perturb(&d.str_, 0.06, rng); + queries.push(QuerySpec { + sem, + lex, + str_, + target: Some(d.id), + }); + } + // Unanswerable queries: a real topic, but lexical & structural fingerprints + // that match no document. The right answer is to abstain. + for i in 0..N_UNANSWERABLE { + let t = i % N_TOPICS; + let sem = perturb(&sem_centroids[t], 0.10, rng); + let lex = unit_gaussian(rng, DIM_LEX); + let str_ = unit_gaussian(rng, DIM_STR); + queries.push(QuerySpec { + sem, + lex, + str_, + target: None, + }); + } + (docs, queries) +} + +fn build_store(docs: &[Doc]) -> ConstellationStore { + let mut store = ConstellationStore::new(registry()); + for d in docs { + // Every real document carries a grounding anchor (a passed test / label). + let c = Constellation::new(d.id) + .with_label(format!("topic{}", d.topic)) + .with_slot("semantic", d.sem.clone()) + .with_slot("lexical", d.lex.clone()) + .with_slot("structural", d.str_.clone()) + .with_anchor(Anchor::new(AnchorKind::PassedTest, format!("test-{}", d.id), 0.9)); + store.insert(c).expect("valid constellation"); + } + store +} + +fn to_query(q: &QuerySpec) -> Query { + Query::new() + .with_slot("semantic", q.sem.clone()) + .with_slot("lexical", q.lex.clone()) + .with_slot("structural", q.str_.clone()) +} + +// ── Metrics ───────────────────────────────────────────────────────────────── +#[derive(Default, Clone)] +struct Metrics { + grounded_correct: usize, // answered AND correct (answerable only) + answerable: usize, + unsupported: usize, // answered but wrong, OR answered an unanswerable query + recall_sum: f32, + recall_n: usize, + abstained: usize, +} + +impl Metrics { + fn grounded_accuracy(&self) -> f32 { + if self.answerable == 0 { + 0.0 + } else { + self.grounded_correct as f32 / self.answerable as f32 + } + } + fn recall_at_10(&self) -> f32 { + if self.recall_n == 0 { + 0.0 + } else { + self.recall_sum / self.recall_n as f32 + } + } +} + +/// Single-embedding RAG baseline: only the dense semantic lens, always answers. +fn run_baseline(store: &ConstellationStore, queries: &[QuerySpec]) -> Metrics { + let mut m = Metrics::default(); + for q in queries { + let ranked = store.search_lens("semantic", &q.sem, K); + let top = ranked.first().map(|&(id, _)| id); + match q.target { + Some(target) => { + m.answerable += 1; + if top == Some(target) { + m.grounded_correct += 1; + } else { + m.unsupported += 1; + } + let ids: Vec = ranked.iter().map(|&(id, _)| id).collect(); + m.recall_sum += recall_at_k(&[target], &ids); + m.recall_n += 1; + } + None => { + // Always answers → unsupported claim on an unanswerable query. + if top.is_some() { + m.unsupported += 1; + } + } + } + } + m +} + +/// Pure association-native compose path for a single query (no ledger), used by +/// both evaluation and the anneal objective. +fn compose( + store: &ConstellationStore, + weights: &[f32], + guard: &GuardProfile, + q: &QuerySpec, +) -> (GuardDecision, Vec) { + let qq = to_query(q); + let lenses = store.lenses(); + let mut rankings = Vec::with_capacity(lenses.len()); + for m in lenses { + let qv = qq.slots.get(&m.name).unwrap(); + rankings.push(store.search_lens(&m.name, qv, K)); + } + let fused = weighted_rrf(&rankings, weights, DEFAULT_RRF_K); + let ids: Vec = fused.iter().map(|&(id, _)| id).collect(); + let candidate = fused.first().map(|&(id, score)| { + let rec = store.get(id).unwrap(); + Candidate { + record_id: id, + fusion_score: score, + cross_lens_agreement: min_agreement(&qq.slots, rec), + grounding: grounded_path(id, &rec.anchors, guard.min_grounding_weight), + } + }); + (adjudicate(candidate, guard), ids) +} + +fn run_calyx( + store: &ConstellationStore, + weights: &[f32], + guard: &GuardProfile, + queries: &[QuerySpec], +) -> Metrics { + let mut m = Metrics::default(); + for q in queries { + let (decision, ids) = compose(store, weights, guard, q); + let answered = decision.answered_id(); + match q.target { + Some(target) => { + m.answerable += 1; + match answered { + Some(a) if a == target => m.grounded_correct += 1, + Some(_) => m.unsupported += 1, // answered, but wrong + None => m.abstained += 1, + } + m.recall_sum += recall_at_k(&[target], &ids); + m.recall_n += 1; + } + None => match answered { + Some(_) => m.unsupported += 1, + None => m.abstained += 1, + }, + } + } + m +} + +// ── Reporting ─────────────────────────────────────────────────────────────── +fn pct(x: f32) -> String { + format!("{:.1}%", x * 100.0) +} + +fn main() { + println!("╔══════════════════════════════════════════════════════════════╗"); + println!("║ ruvector-calyx — Association-Native Memory Benchmark (ADR-272) ║"); + println!("╚══════════════════════════════════════════════════════════════╝\n"); + println!("Platform : {} / {}", std::env::consts::OS, std::env::consts::ARCH); + println!("Lenses : semantic({DIM_SEM}d, {COST_SEM}µs) · lexical({DIM_LEX}d, {COST_LEX}µs) · structural({DIM_STR}d, {COST_STR}µs)"); + println!("Corpus : {N_DOCS} docs across {N_TOPICS} topics × {N_PER_TOPIC} (semantic fog) · {N_STRUCT_CLASSES} structural classes"); + println!("Queries : {N_ANSWERABLE} answerable + {N_UNANSWERABLE} unanswerable\n"); + + let mut rng = Rng::new(SEED); + let (docs, queries) = build_dataset(&mut rng); + let store = build_store(&docs); + + // ── Signal density (Assay) ────────────────────────────────────────────── + let signals = measure_signal_density(&store, &queries); + println!("Signal density (Assay) — information each lens adds, per µs of cost"); + println!(" {:<12} {:>8} {:>10} {:>14}", "lens", "bits", "cost µs", "bits/µs"); + for s in &signals { + println!( + " {:<12} {:>8.3} {:>10.1} {:>14.4}", + s.lens, s.bits, s.cost_us, s.density + ); + } + println!(); + + // ── Baseline vs Calyx (uniform weights) ───────────────────────────────── + let guard = GuardProfile::default(); + let uniform = vec![1.0_f32; 3]; + + let t0 = Instant::now(); + let base = run_baseline(&store, &queries); + let base_us = t0.elapsed().as_micros(); + + let t1 = Instant::now(); + let calyx = run_calyx(&store, &uniform, &guard, &queries); + let calyx_us = t1.elapsed().as_micros(); + + // Replayability: run the *real* engine (with ledger) over all queries. + let mut engine = CalyxEngine::new(store.clone()) + .with_weights(uniform.clone()) + .with_guard(guard.clone()) + .with_top_k(K); + for q in &queries { + let _ = engine.query(&to_query(q)); + } + let replayable = engine.ledger().verify(); + let ledger_entries = engine.ledger().len(); + + println!("Results (uniform fusion weights)"); + println!( + " {:<28} {:>16} {:>16}", + "metric", "single-embedding", "calyx (multi-lens)" + ); + println!(" {}", "-".repeat(62)); + println!( + " {:<28} {:>16} {:>16}", + "grounded accuracy", + pct(base.grounded_accuracy()), + pct(calyx.grounded_accuracy()) + ); + println!( + " {:<28} {:>16} {:>16}", + "recall@10 (answerable)", + pct(base.recall_at_10()), + pct(calyx.recall_at_10()) + ); + println!( + " {:<28} {:>16} {:>16}", + "unsupported claims (count)", base.unsupported, calyx.unsupported + ); + println!( + " {:<28} {:>16} {:>16}", + "abstentions (count)", 0, calyx.abstained + ); + println!( + " {:<28} {:>16} {:>16}", + "latency (µs, all queries)", base_us, calyx_us + ); + println!( + " {:<28} {:>16} {:>16}", + "replayable provenance", "n/a", format!("{replayable} ({ledger_entries})") + ); + println!(); + + // ── Anneal: reversibly tune fusion weights for accepted-answers-per-cost ─ + let lens_cost = [COST_SEM, COST_LEX, COST_STR]; + let cfg = AnnealConfig::default(); + let store_for_anneal = store.clone(); + let guard_for_anneal = guard.clone(); + let queries_ref = &queries; + let outcome = anneal_weights(uniform.clone(), &cfg, |w| { + objective(&store_for_anneal, w, &guard_for_anneal, queries_ref, &lens_cost) + }); + let tuned = run_calyx(&store, &outcome.best_weights, &guard, &queries); + println!("Anneal — reversible fusion-weight optimization (accepted answers / cost)"); + println!( + " weights {:?} → [{:.2}, {:.2}, {:.2}]", + uniform, outcome.best_weights[0], outcome.best_weights[1], outcome.best_weights[2] + ); + println!( + " utility {:.4} → {:.4} (accepted={}, reverted={})", + outcome.initial_utility, outcome.best_utility, outcome.accepted_moves, outcome.reverted_moves + ); + println!( + " tuned grounded accuracy {} · unsupported {} · abstentions {}", + pct(tuned.grounded_accuracy()), + tuned.unsupported, + tuned.abstained + ); + println!(); + + // ── Acceptance test ───────────────────────────────────────────────────── + let acc_gain = calyx.grounded_accuracy() - base.grounded_accuracy(); + let recall_gain = calyx.recall_at_10() - base.recall_at_10(); + let claim_reduction = if base.unsupported == 0 { + 0.0 + } else { + 1.0 - (calyx.unsupported as f32 / base.unsupported as f32) + }; + + println!("Acceptance test (ADR-272 targets)"); + let c1 = acc_gain >= 0.15; + let c2 = claim_reduction >= 0.50; + let c3 = recall_gain >= 0.10; + let c4 = replayable && ledger_entries == queries.len(); + let c5 = outcome.best_utility >= outcome.initial_utility && outcome.reverted_moves > 0; + print_check("grounded accuracy +≥15pp", &format!("+{:.1}pp", acc_gain * 100.0), c1); + print_check("unsupported claims −≥50%", &format!("−{:.1}%", claim_reduction * 100.0), c2); + print_check("recall@10 +≥10pp", &format!("+{:.1}pp", recall_gain * 100.0), c3); + print_check("replayable provenance 100%", &format!("{ledger_entries}/{}", queries.len()), c4); + print_check("anneal improves & is reversible", &format!("Δu={:.4}", outcome.best_utility - outcome.initial_utility), c5); + println!(); + + if c1 && c2 && c3 && c4 && c5 { + println!("→ BENCHMARK PASSED"); + } else { + println!("→ BENCHMARK FAILED"); + std::process::exit(1); + } +} + +fn print_check(name: &str, value: &str, pass: bool) { + println!( + " {:<34} {:>12} {}", + name, + value, + if pass { "PASS ✓" } else { "FAIL ✗" } + ); +} + +/// Anneal objective: accepted-correct answers divided by normalized +/// measurement cost. A lens whose fusion weight drops to ~0 is "skipped", +/// saving its cost — so the optimizer trades accuracy against compute. +fn objective( + store: &ConstellationStore, + weights: &[f32], + guard: &GuardProfile, + queries: &[QuerySpec], + lens_cost: &[f32], +) -> f32 { + let m = run_calyx(store, weights, guard, queries); + let used_cost: f32 = weights + .iter() + .zip(lens_cost.iter()) + .map(|(w, c)| if *w > 0.05 { *c } else { 0.0 }) + .sum(); + let cost_norm = 1.0 + used_cost / 16.0; + // Reward correct accepted answers; the guard already suppresses wrong ones. + (m.grounded_correct as f32) / cost_norm +} + +/// Estimate per-lens signal density from the answerable queries: for each lens, +/// gather (similarity-to-target, relevant) observations against a few sampled +/// records and measure mutual information per µs. +fn measure_signal_density(store: &ConstellationStore, queries: &[QuerySpec]) -> Vec { + let lenses = store.lenses().to_vec(); + let mut out = Vec::new(); + for m in &lenses { + let mut obs = Vec::new(); + for q in queries.iter().filter(|q| q.target.is_some()) { + let target = q.target.unwrap(); + let qv = match m.name.as_str() { + "semantic" => &q.sem, + "lexical" => &q.lex, + _ => &q.str_, + }; + // Positive: the true target. Negatives: a handful of other docs. + for (offset, rec) in store.records().iter().enumerate() { + if offset % 9 != 0 && rec.id != target { + continue; // subsample negatives for speed + } + let sim = ruvector_calyx::cosine_sim(qv, rec.slot(&m.name).unwrap()); + obs.push(Observation { + similarity: sim, + relevant: rec.id == target, + }); + } + } + out.push(signal_density(m.name.clone(), &obs, m.cost_us)); + } + out +} diff --git a/crates/ruvector-calyx/src/rng.rs b/crates/ruvector-calyx/src/rng.rs new file mode 100644 index 000000000..4b29f0d95 --- /dev/null +++ b/crates/ruvector-calyx/src/rng.rs @@ -0,0 +1,81 @@ +//! A tiny deterministic PRNG (SplitMix64) so the crate stays dependency-free +//! and every benchmark / anneal run is byte-for-byte reproducible. + +/// SplitMix64 generator. Not cryptographic; for reproducible sampling only. +#[derive(Debug, Clone)] +pub struct Rng { + state: u64, +} + +impl Rng { + pub fn new(seed: u64) -> Self { + Self { state: seed } + } + + #[inline] + fn next_u64(&mut self) -> u64 { + self.state = self.state.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.state; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + + /// Uniform `f32` in `[0, 1)`. + #[inline] + pub fn f32(&mut self) -> f32 { + (self.next_u64() >> 40) as f32 / (1u64 << 24) as f32 + } + + /// Uniform `f32` in `[-1, 1)`. + #[inline] + pub fn signed(&mut self) -> f32 { + self.f32() * 2.0 - 1.0 + } + + /// Uniform index in `[0, n)`. Returns `0` when `n == 0`. + #[inline] + pub fn range(&mut self, n: usize) -> usize { + if n == 0 { + 0 + } else { + (self.next_u64() % n as u64) as usize + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn deterministic_and_in_range() { + // Two generators with the same seed produce identical streams. + let mut a = Rng::new(42); + let mut b = Rng::new(42); + for _ in 0..1000 { + let x = a.f32(); + assert_eq!(x, b.f32()); + assert!((0.0..1.0).contains(&x)); + assert_eq!(a.range(7), b.range(7)); + } + // range stays in bounds for a fresh stream. + let mut c = Rng::new(99); + for _ in 0..1000 { + assert!(c.range(7) < 7); + } + assert_eq!(Rng::new(0).range(0), 0); + } + + #[test] + fn roughly_uniform() { + let mut r = Rng::new(1); + let mut buckets = [0usize; 4]; + for _ in 0..40_000 { + buckets[r.range(4)] += 1; + } + for b in buckets { + assert!((8_000..12_000).contains(&b), "bucket skew: {b}"); + } + } +} diff --git a/crates/ruvector-calyx/src/routing.rs b/crates/ruvector-calyx/src/routing.rs new file mode 100644 index 000000000..7ccc56b84 --- /dev/null +++ b/crates/ruvector-calyx/src/routing.rs @@ -0,0 +1,304 @@ +//! Adaptive lens routing (the retrieval analogue of model routing). +//! +//! Instead of always measuring every lens, the router consults lenses +//! **cheapest-first** and stops as soon as the *calibrated* confidence clears a +//! threshold — escalating to the expensive lenses only when the cheap ones are +//! not decisive, and abstaining when even the full panel is not. Every decision +//! emits a [`Witness`]: which lenses were consulted, their tops, the confidence, +//! the action, and why it escalated or abstained. + +use std::collections::BTreeMap; + +use crate::calibrate::{raw_confidence, Calibrator, ConfidenceSignals}; +use crate::constellation::ConstellationStore; +use crate::engine::Query; +use crate::fusion::{weighted_rrf, DEFAULT_RRF_K}; +use crate::ledger::grounded_path; +use crate::loom::{min_agreement, top_dissenter}; +use crate::ward::GuardProfile; + +/// How the router consults lenses. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RoutingMode { + /// Always consult every lens in `order` (the un-routed baseline). + BruteForce, + /// Always consult the first `max_lenses` in `order` (fixed, non-adaptive). + Static { max_lenses: usize }, + /// Consult cheapest-first, stopping early once calibrated confidence clears + /// `stop_threshold` (per-query adaptive). + Adaptive, +} + +/// Router thresholds. +#[derive(Debug, Clone)] +pub struct RoutingConfig { + /// Stop consulting more lenses once calibrated confidence ≥ this. + pub stop_threshold: f32, + /// Emit an answer only if final calibrated confidence ≥ this (else abstain). + pub answer_threshold: f32, + /// Require at least this many corroborating lenses before committing to an + /// answer or stopping early — "don't trust a lone lens". Clamped to ≥1. + pub min_lenses_to_answer: usize, + /// Top-k per lens. + pub top_k: usize, +} + +impl Default for RoutingConfig { + fn default() -> Self { + Self { + stop_threshold: 0.80, + answer_threshold: 0.55, + min_lenses_to_answer: 2, + top_k: 10, + } + } +} + +/// The provenance of one routed decision. +#[derive(Debug, Clone)] +pub struct Witness { + pub lenses_consulted: Vec, + /// The fused top candidate, recorded even when the router abstains (so + /// calibration and evaluation can score the decision that *would* be made). + pub top_candidate: Option, + /// `(lens, top_id, top_score)` per consulted lens. + pub per_lens_top: Vec<(String, u64, f32)>, + pub signals: ConfidenceSignals, + pub confidence: f32, + /// "answer" | "abstain". + pub action: &'static str, + pub escalation_reason: String, + /// Summed measurement cost (µs) of the consulted lenses. + pub cost_us: f32, + /// Summed measurement latency (µs) of the consulted lenses. + pub latency_us: f32, +} + +/// The outcome of routing one query. +#[derive(Debug, Clone)] +pub struct RoutingOutcome { + /// `Some(id)` if answered, `None` if abstained. + pub answered: Option, + pub confidence: f32, + pub witness: Witness, +} + +/// Route one query. `order` is lens names sorted cheapest-first. +pub fn route( + store: &ConstellationStore, + order: &[String], + query: &Query, + calibrator: &Calibrator, + cfg: &RoutingConfig, + guard: &GuardProfile, + mode: RoutingMode, +) -> RoutingOutcome { + let planned = match mode { + RoutingMode::BruteForce => order.len(), + RoutingMode::Static { max_lenses } => max_lenses.clamp(1, order.len()), + RoutingMode::Adaptive => order.len(), + }; + + let mut rankings: Vec> = Vec::new(); + let mut consulted: Vec = Vec::new(); + let mut per_lens_top: Vec<(String, u64, f32)> = Vec::new(); + let mut cost_us = 0.0f32; + let mut latency_us = 0.0f32; + + let mut confidence = 0.0f32; + let mut signals = ConfidenceSignals::default(); + let mut fused: Vec<(u64, f32)> = Vec::new(); + let mut escalation_reason = String::from("full-panel"); + + for (step, lens_name) in order.iter().take(planned).enumerate() { + let Some(m) = store.lenses().iter().find(|l| &l.name == lens_name) else { + continue; + }; + let Some(qv) = query.slots.get(lens_name) else { + continue; + }; + let ranking = store.search_lens(lens_name, qv, cfg.top_k); + if let Some(&(id, score)) = ranking.first() { + per_lens_top.push((lens_name.clone(), id, score)); + } + rankings.push(ranking); + consulted.push(lens_name.clone()); + cost_us += m.cost_us; + latency_us += m.latency_us; + + let weights = vec![1.0f32; rankings.len()]; + fused = weighted_rrf(&rankings, &weights, DEFAULT_RRF_K); + signals = compute_signals(store, query, &consulted, &fused, guard); + confidence = calibrator.calibrate(raw_confidence(&signals)); + + // Adaptive early-exit: stop once we are confident enough — but only + // after enough lenses corroborate (never commit on a single lens). + if mode == RoutingMode::Adaptive + && consulted.len() >= cfg.min_lenses_to_answer.max(1) + && confidence >= cfg.stop_threshold + && step + 1 < order.len() + { + escalation_reason = format!("early-exit@{}lenses(conf={:.2})", consulted.len(), confidence); + break; + } + } + + // Decide: fail closed on grounding, then apply the confidence gate. + let top = fused.first().copied(); + let grounded = top + .and_then(|(id, _)| store.get(id)) + .map(|rec| grounded_path(rec.id, &rec.anchors, guard.min_grounding_weight).is_some()) + .unwrap_or(false); + + let (answered, action) = match top { + Some((id, _)) + if consulted.len() >= cfg.min_lenses_to_answer.max(1) + && confidence >= cfg.answer_threshold + && signals.cross_lens_agreement >= guard.min_cross_lens_agreement + && (!guard.require_grounding || grounded) => + { + (Some(id), "answer") + } + _ => { + if escalation_reason == "full-panel" { + escalation_reason = format!("abstain(conf={:.2}<{:.2})", confidence, cfg.answer_threshold); + } + (None, "abstain") + } + }; + + RoutingOutcome { + answered, + confidence, + witness: Witness { + lenses_consulted: consulted, + top_candidate: top.map(|(id, _)| id), + per_lens_top, + signals, + confidence, + action, + escalation_reason, + cost_us, + latency_us, + }, + } +} + +/// Confidence signals over the lenses consulted so far. +fn compute_signals( + store: &ConstellationStore, + query: &Query, + consulted: &[String], + fused: &[(u64, f32)], + guard: &GuardProfile, +) -> ConfidenceSignals { + let top_score = fused.first().map(|&(_, s)| s).unwrap_or(0.0); + let second = fused.get(1).map(|&(_, s)| s).unwrap_or(0.0); + let margin_ratio = if top_score > 1e-9 { + (top_score - second) / top_score + } else { + 0.0 + }; + + // Restrict the query's slots to the consulted lenses for agreement/dissent. + let sub: BTreeMap> = query + .slots + .iter() + .filter(|(k, _)| consulted.iter().any(|c| c == *k)) + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + + let (agreement, contradiction, grounding) = match fused.first().and_then(|&(id, _)| store.get(id)) { + Some(rec) => { + let agr = min_agreement(&sub, rec); + let contra = top_dissenter(&sub, rec).map(|(_, gap)| gap).unwrap_or(0.0); + let grd = grounded_path(rec.id, &rec.anchors, guard.min_grounding_weight) + .map(|g| g.best_weight) + .unwrap_or(0.0); + (agr, contra, grd) + } + None => (0.0, 0.0, 0.0), + }; + + ConfidenceSignals { + top_score, + margin_ratio, + cross_lens_agreement: agreement, + grounding, + contradiction, + lens_count: consulted.len(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::constellation::Constellation; + use crate::ledger::{Anchor, AnchorKind}; + use crate::lens::{LensKind, LensManifest}; + + fn store() -> ConstellationStore { + let lenses = vec![ + LensManifest::new("cheap", "v1", LensKind::Structural, 2, 1.0, 1.0), + LensManifest::new("pricey", "v1", LensKind::DenseSemantic, 2, 10.0, 10.0), + ]; + let mut s = ConstellationStore::new(lenses); + s.insert( + Constellation::new(1) + .with_slot("cheap", vec![1.0, 0.0]) + .with_slot("pricey", vec![1.0, 0.0]) + .with_anchor(Anchor::new(AnchorKind::PassedTest, "t1", 0.9)), + ) + .unwrap(); + s.insert( + Constellation::new(2) + .with_slot("cheap", vec![0.0, 1.0]) + .with_slot("pricey", vec![0.0, 1.0]) + .with_anchor(Anchor::new(AnchorKind::PassedTest, "t2", 0.9)), + ) + .unwrap(); + s + } + + #[test] + fn adaptive_stops_early_when_cheap_lens_is_decisive() { + let s = store(); + let order = vec!["cheap".to_string(), "pricey".to_string()]; + let q = Query::new() + .with_slot("cheap", vec![1.0, 0.0]) + .with_slot("pricey", vec![1.0, 0.0]); + // A calibrator that trusts high raw confidence. + let cal = Calibrator::identity(10); + let cfg = RoutingConfig { + stop_threshold: 0.5, + answer_threshold: 0.4, + min_lenses_to_answer: 1, // this test checks single-lens early-exit + top_k: 5, + }; + let out = route(&s, &order, &q, &cal, &cfg, &GuardProfile::default(), RoutingMode::Adaptive); + assert_eq!(out.answered, Some(1)); + // Should have stopped after the single cheap lens (skipped the pricey one). + assert_eq!(out.witness.lenses_consulted, vec!["cheap".to_string()]); + assert!(out.witness.cost_us < 2.0); + } + + #[test] + fn brute_force_consults_all() { + let s = store(); + let order = vec!["cheap".to_string(), "pricey".to_string()]; + let q = Query::new() + .with_slot("cheap", vec![1.0, 0.0]) + .with_slot("pricey", vec![1.0, 0.0]); + let cal = Calibrator::identity(10); + let out = route( + &s, + &order, + &q, + &cal, + &RoutingConfig::default(), + &GuardProfile::default(), + RoutingMode::BruteForce, + ); + assert_eq!(out.witness.lenses_consulted.len(), 2); + } +} diff --git a/crates/ruvector-calyx/src/scoring.rs b/crates/ruvector-calyx/src/scoring.rs new file mode 100644 index 000000000..5d87c94ca --- /dev/null +++ b/crates/ruvector-calyx/src/scoring.rs @@ -0,0 +1,123 @@ +//! Vector scoring primitives and a deterministic non-cryptographic hash. +//! +//! Calyx's math runtime (the paper calls it *Forge*) reduces to a handful of +//! dense-vector kernels plus content addressing. This module is the +//! dependency-free reference version of that runtime. + +/// Dot product of two equal-length slices. +#[inline] +pub fn dot(a: &[f32], b: &[f32]) -> f32 { + a.iter().zip(b.iter()).map(|(x, y)| x * y).sum() +} + +/// L2 norm of a vector. +#[inline] +pub fn l2_norm(v: &[f32]) -> f32 { + dot(v, v).sqrt() +} + +/// Cosine similarity in `[-1, 1]`; returns `0.0` for a zero-length input. +pub fn cosine_sim(a: &[f32], b: &[f32]) -> f32 { + debug_assert_eq!(a.len(), b.len(), "cosine_sim shape mismatch"); + let na = l2_norm(a); + let nb = l2_norm(b); + if na < 1e-9 || nb < 1e-9 { + return 0.0; + } + (dot(a, b) / (na * nb)).clamp(-1.0, 1.0) +} + +/// Unit-length copy of `v`, or the zero vector when `v` has no length. +pub fn normalize(v: &[f32]) -> Vec { + let n = l2_norm(v); + if n < 1e-9 { + vec![0.0; v.len()] + } else { + v.iter().map(|x| x / n).collect() + } +} + +/// FNV-1a 64-bit hash of a byte slice. +/// +/// Used for content-addressing lenses and chaining the provenance ledger. It +/// is **not** cryptographic — a production deployment would swap in BLAKE3 or +/// SHA-256. FNV-1a is chosen here to keep the reference crate dependency-free +/// and fully deterministic across platforms. +pub fn fnv1a64(bytes: &[u8]) -> u64 { + const OFFSET: u64 = 0xcbf2_9ce4_8422_2325; + const PRIME: u64 = 0x0000_0100_0000_01b3; + let mut h = OFFSET; + for &b in bytes { + h ^= b as u64; + h = h.wrapping_mul(PRIME); + } + h +} + +/// Hash a sequence of `f32`s in a byte-order-stable way. +pub fn hash_vec(v: &[f32]) -> u64 { + let mut bytes = Vec::with_capacity(v.len() * 4); + for x in v { + bytes.extend_from_slice(&x.to_le_bits().to_le_bytes()); + } + fnv1a64(&bytes) +} + +/// Fold two hashes into one (used to chain the ledger). +#[inline] +pub fn mix_hash(a: u64, b: u64) -> u64 { + // FNV-1a over the 16 bytes of `a` then `b`. + let mut bytes = [0u8; 16]; + bytes[..8].copy_from_slice(&a.to_le_bytes()); + bytes[8..].copy_from_slice(&b.to_le_bytes()); + fnv1a64(&bytes) +} + +/// Quantize a `f32` to its bit pattern in a way that treats `-0.0 == 0.0` and +/// maps NaN to a single canonical pattern, so hashing is stable. +trait ToLeBits { + fn to_le_bits(self) -> u32; +} +impl ToLeBits for f32 { + #[inline] + fn to_le_bits(self) -> u32 { + if self == 0.0 { + 0 + } else if self.is_nan() { + 0x7fc0_0000 + } else { + self.to_bits() + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cosine_basics() { + let a = [1.0, 0.0, 0.0]; + assert!((cosine_sim(&a, &a) - 1.0).abs() < 1e-6); + let b = [0.0, 1.0, 0.0]; + assert!(cosine_sim(&a, &b).abs() < 1e-6); + let z = [0.0, 0.0, 0.0]; + assert_eq!(cosine_sim(&a, &z), 0.0); + } + + #[test] + fn hash_is_stable_and_sensitive() { + let v = vec![0.1, 0.2, 0.3]; + assert_eq!(hash_vec(&v), hash_vec(&v)); + let mut w = v.clone(); + w[1] = 0.20001; + assert_ne!(hash_vec(&v), hash_vec(&w)); + // -0.0 and 0.0 hash identically. + assert_eq!(hash_vec(&[0.0, 1.0]), hash_vec(&[-0.0, 1.0])); + } + + #[test] + fn mix_is_order_sensitive() { + assert_ne!(mix_hash(1, 2), mix_hash(2, 1)); + } +} diff --git a/crates/ruvector-calyx/src/ward.rs b/crates/ruvector-calyx/src/ward.rs new file mode 100644 index 000000000..68018b17b --- /dev/null +++ b/crates/ruvector-calyx/src/ward.rs @@ -0,0 +1,189 @@ +//! The fail-closed guard (the paper's *Ward*). +//! +//! Calyx's third trust principle is *fail closed*: a missing grounded path, an +//! unknown lens, a shape mismatch, or an uncalibrated guard returns a +//! structured refusal rather than a silent wrong answer. This module turns a +//! fused candidate plus its cross-lens agreement and grounding into an explicit +//! [`GuardDecision`]. + +use crate::ledger::{AnchorKind, GroundingPath}; + +/// Thresholds the guard enforces. Tunable by the anneal optimizer. +#[derive(Debug, Clone)] +pub struct GuardProfile { + /// Minimum fused (RRF) score the top candidate must reach. + pub min_fusion_score: f32, + /// Minimum cross-lens *min*-agreement: every lens must corroborate. + pub min_cross_lens_agreement: f32, + /// Minimum anchor weight required for a grounded path. + pub min_grounding_weight: f32, + /// Whether a grounded path is mandatory (grounding is mandatory). + pub require_grounding: bool, +} + +impl Default for GuardProfile { + fn default() -> Self { + Self { + min_fusion_score: 0.0, + min_cross_lens_agreement: 0.45, + min_grounding_weight: 0.5, + require_grounding: true, + } + } +} + +/// Why the guard refused — structured, never silent. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RefusalReason { + /// No candidate was retrieved at all. + NoCandidate, + /// The fused score was below the calibrated floor. + LowFusionScore, + /// At least one lens dissented (min-agreement below floor). + CrossLensDisagreement, + /// No grounded evidence path crossed the threshold. + NoGroundedPath, + /// The guard profile itself was not calibrated (degenerate thresholds). + UncalibratedGuard, +} + +/// The guard's verdict. +#[derive(Debug, Clone, PartialEq)] +pub enum GuardDecision { + /// Answer with this record, carrying the evidence that justified it. + Answer { + record_id: u64, + fusion_score: f32, + cross_lens_agreement: f32, + grounding_kind: Option, + grounding_weight: f32, + }, + /// Refuse, with a structured reason. + Refuse(RefusalReason), +} + +impl GuardDecision { + pub fn is_answer(&self) -> bool { + matches!(self, GuardDecision::Answer { .. }) + } + + pub fn answered_id(&self) -> Option { + match self { + GuardDecision::Answer { record_id, .. } => Some(*record_id), + GuardDecision::Refuse(_) => None, + } + } +} + +/// Evidence the guard weighs for the top fused candidate. +pub struct Candidate { + pub record_id: u64, + pub fusion_score: f32, + pub cross_lens_agreement: f32, + pub grounding: Option, +} + +/// Apply the guard. Fails closed on every shortfall. +pub fn adjudicate(candidate: Option, profile: &GuardProfile) -> GuardDecision { + // A guard with impossible thresholds is treated as uncalibrated rather than + // silently passing everything. + if !(0.0..=1.0).contains(&profile.min_cross_lens_agreement) + || !(0.0..=1.0).contains(&profile.min_grounding_weight) + { + return GuardDecision::Refuse(RefusalReason::UncalibratedGuard); + } + + let c = match candidate { + Some(c) => c, + None => return GuardDecision::Refuse(RefusalReason::NoCandidate), + }; + + if c.fusion_score < profile.min_fusion_score { + return GuardDecision::Refuse(RefusalReason::LowFusionScore); + } + if c.cross_lens_agreement < profile.min_cross_lens_agreement { + return GuardDecision::Refuse(RefusalReason::CrossLensDisagreement); + } + + let grounding = match (&c.grounding, profile.require_grounding) { + (None, true) => return GuardDecision::Refuse(RefusalReason::NoGroundedPath), + (g, _) => g.clone(), + }; + if let Some(g) = &grounding { + if g.best_weight < profile.min_grounding_weight { + return GuardDecision::Refuse(RefusalReason::NoGroundedPath); + } + } + + GuardDecision::Answer { + record_id: c.record_id, + fusion_score: c.fusion_score, + cross_lens_agreement: c.cross_lens_agreement, + grounding_kind: grounding.as_ref().map(|g| g.best_kind), + grounding_weight: grounding.map(|g| g.best_weight).unwrap_or(0.0), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ledger::AnchorKind; + + fn grounded(w: f32) -> Option { + Some(GroundingPath { + record_id: 1, + best_kind: AnchorKind::PassedTest, + best_weight: w, + }) + } + + #[test] + fn answers_when_all_gates_pass() { + let d = adjudicate( + Some(Candidate { + record_id: 1, + fusion_score: 0.5, + cross_lens_agreement: 0.8, + grounding: grounded(0.9), + }), + &GuardProfile::default(), + ); + assert_eq!(d.answered_id(), Some(1)); + } + + #[test] + fn refuses_on_disagreement_then_on_missing_grounding() { + let p = GuardProfile::default(); + let d = adjudicate( + Some(Candidate { + record_id: 1, + fusion_score: 0.5, + cross_lens_agreement: 0.1, + grounding: grounded(0.9), + }), + &p, + ); + assert_eq!(d, GuardDecision::Refuse(RefusalReason::CrossLensDisagreement)); + + let d2 = adjudicate( + Some(Candidate { + record_id: 1, + fusion_score: 0.5, + cross_lens_agreement: 0.8, + grounding: None, + }), + &p, + ); + assert_eq!(d2, GuardDecision::Refuse(RefusalReason::NoGroundedPath)); + } + + #[test] + fn uncalibrated_guard_fails_closed() { + let p = GuardProfile { + min_cross_lens_agreement: 5.0, + ..GuardProfile::default() + }; + let d = adjudicate(None, &p); + assert_eq!(d, GuardDecision::Refuse(RefusalReason::UncalibratedGuard)); + } +} diff --git a/crates/ruvector-calyx/tools/build_codesearchnet.py b/crates/ruvector-calyx/tools/build_codesearchnet.py new file mode 100644 index 000000000..34ca0e601 --- /dev/null +++ b/crates/ruvector-calyx/tools/build_codesearchnet.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""Build a real ``.calyx`` corpus from CodeSearchNet for ruvector-calyx. + +This is the ONE step that needs models + network + (ideally) a GPU. It embeds +each code function through several *frozen lenses*, keeps every lens as its own +slot (never flattened), and writes a ``.calyx`` v1 binary that the pure-Rust +``calyx-real-bench`` loads with zero dependencies. + +Lenses produced (a constellation per function): + * ``code`` — a joint NL<->code search model, so an NL query and a code + snippet land in the same space (default: + ``flax-sentence-embeddings/st-codesearch-distilroberta-base``, + trained on CodeSearchNet). + * ``doc`` — a text embedder over the docstring (default: + ``BAAI/bge-small-en-v1.5``). + * ``lexical`` — a dependency-free hashed bag-of-tokens vector over code + identifiers (the "BM25-ish" surface-term lens); cosine of two + hashed TF vectors approximates lexical overlap. + +Queries are the docstrings (the standard CodeSearchNet code-search task: given a +natural-language query, retrieve the function it documents). ``--unanswerable`` +holds out a fraction of golds from the corpus so those queries *should* be +abstained on — which is what makes conformal risk meaningful. + +Usage (run where you have the models): + pip install datasets sentence-transformers numpy + python build_codesearchnet.py --lang python --n 5000 --out corpus.calyx + # then, in the repo: + cargo run --release -p ruvector-calyx --bin calyx-real-bench -- corpus.calyx + +The byte layout MUST match crates/ruvector-calyx/src/corpus.rs (v1). Keep the +two in sync. +""" + +import argparse +import hashlib +import re +import struct + +MAGIC = b"CALYXB1\n" +LENS_KIND = {"dense": 0, "lexical": 1, "structural": 2, "temporal": 3, "sensor": 4} +ANCHOR_KIND = {"accepted": 0, "passed_test": 1, "sensor": 2, "citation": 3, "reward": 4} +_TOKEN = re.compile(r"[A-Za-z_][A-Za-z_0-9]*") + + +# ── byte writers (little-endian; mirror corpus.rs) ────────────────────────── +def w_u16(f, v): f.write(struct.pack(" 1e-9 else v + + +def hashed_tokens(text, dims): + """Dependency-free lexical lens: hashed, L2-normalized TF vector.""" + import numpy as np + v = np.zeros(dims, dtype="float32") + for tok in _TOKEN.findall(text.lower()): + h = int(hashlib.blake2b(tok.encode(), digest_size=8).hexdigest(), 16) + v[h % dims] += 1.0 + return l2norm(v) + + +def main(): + ap = argparse.ArgumentParser(description="Build a .calyx corpus from CodeSearchNet") + ap.add_argument("--lang", default="python", help="CodeSearchNet language subset") + ap.add_argument("--n", type=int, default=5000, help="number of functions") + ap.add_argument("--n-queries", type=int, default=1000) + ap.add_argument("--unanswerable", type=float, default=0.2, + help="fraction of queries whose gold is held out of the corpus") + ap.add_argument("--code-model", default="flax-sentence-embeddings/st-codesearch-distilroberta-base") + ap.add_argument("--doc-model", default="BAAI/bge-small-en-v1.5") + ap.add_argument("--lexical-dims", type=int, default=512) + ap.add_argument("--out", default="corpus.calyx") + ap.add_argument("--seed", type=int, default=0) + args = ap.parse_args() + + import numpy as np + from datasets import load_dataset + from sentence_transformers import SentenceTransformer + + rng = np.random.default_rng(args.seed) + print(f"Loading CodeSearchNet[{args.lang}] …") + ds = load_dataset("code_search_net", args.lang, split="test") + ds = ds.filter(lambda r: r["func_documentation_string"] and r["whole_func_string"]) + idx = rng.permutation(len(ds))[: args.n] + rows = [ds[int(i)] for i in idx] + codes = [r["whole_func_string"] for r in rows] + docs = [r["func_documentation_string"] for r in rows] + + print(f"Embedding {len(rows)} functions with code + doc models …") + code_model = SentenceTransformer(args.code_model) + doc_model = SentenceTransformer(args.doc_model) + code_vecs = code_model.encode(codes, normalize_embeddings=True, show_progress_bar=True) + doc_vecs = doc_model.encode(docs, normalize_embeddings=True, show_progress_bar=True) + lex_vecs = [hashed_tokens(c, args.lexical_dims) for c in codes] + + d_code, d_doc, d_lex = len(code_vecs[0]), len(doc_vecs[0]), args.lexical_dims + + # Queries = docstrings. Gold = the documenting function. A fraction are made + # unanswerable by holding their gold out of the *record* set. + q_idx = rng.permutation(len(rows))[: args.n_queries] + n_unans = int(len(q_idx) * args.unanswerable) + held_out = set(int(i) for i in q_idx[:n_unans]) + record_ids = [i for i in range(len(rows)) if i not in held_out] + + # Query lenses: the NL docstring embedded by the code model (joint space), + # the doc model, and its hashed tokens. + q_texts = [docs[int(i)] for i in q_idx] + q_code = code_model.encode(q_texts, normalize_embeddings=True) + q_doc = doc_model.encode(q_texts, normalize_embeddings=True) + q_lex = [hashed_tokens(q_texts[k], args.lexical_dims) for k in range(len(q_idx))] + + print(f"Writing {args.out} ({len(record_ids)} records, {len(q_idx)} queries, " + f"{n_unans} unanswerable) …") + with open(args.out, "wb") as f: + f.write(MAGIC) + # lenses + w_u32(f, 3) + for name, ver, kind, dims in [ + ("code", args.code_model, LENS_KIND["dense"], d_code), + ("doc", args.doc_model, LENS_KIND["dense"], d_doc), + ("lexical", "hashed-tokens-v1", LENS_KIND["lexical"], d_lex), + ]: + w_str(f, name); w_str(f, ver) + f.write(bytes([kind])); w_u32(f, dims) + w_f32(f, 8.0 if name != "lexical" else 1.0) # illustrative cost µs + w_f32(f, 8.0 if name != "lexical" else 1.0) + # records + w_u32(f, len(record_ids)) + for i in record_ids: + w_u64(f, i) + w_vec(f, l2norm(code_vecs[i])) + w_vec(f, l2norm(doc_vecs[i])) + w_vec(f, lex_vecs[i]) + w_u16(f, 1) # one anchor: the function has an associated test/example + f.write(bytes([ANCHOR_KIND["passed_test"]])); w_f32(f, 0.9); w_str(f, f"gold_{i}") + # queries + w_u32(f, len(q_idx)) + for k, i in enumerate(q_idx): + for present, vec in ((1, q_code[k]), (1, q_doc[k]), (1, q_lex[k])): + f.write(bytes([present])); w_vec(f, l2norm(vec)) + gold = int(i) + rel = [] if gold in held_out else [gold] + w_u32(f, len(rel)) + for r in rel: + w_u64(f, r) + print("done.") + + +if __name__ == "__main__": + main() diff --git a/docs/adr/ADR-272-association-native-memory-layer.md b/docs/adr/ADR-272-association-native-memory-layer.md new file mode 100644 index 000000000..feb0c643e --- /dev/null +++ b/docs/adr/ADR-272-association-native-memory-layer.md @@ -0,0 +1,435 @@ +# ADR-272: Association-Native Memory Layer (Calyx-inspired) for RuVector + +**Status**: Accepted +**Date**: 2026-06-30 +**Authors**: Claude Code MetaHarness Architect +**Supersedes**: None +**Extends**: ADR-269 (MRAgent Graph Memory over RuVector), ADR-270 +(Self-Reconstructing Graph Memory) +**Related**: ADR-266 (MetaHarness Darwin ANN Integration), ADR-271 (Darwin/SONA +self-improvement), ADR-264 (Matryoshka coarse-fine search), ADR-268 +(capability-gated ANN, SPANN partition spill) + +--- + +## Context + +RuVector today is best described as *fast vector memory plus graph-aware +association*: HNSW/IVF indexes, a self-learning GNN, min-cut association edges, +and graph memory (ADR-269/270). The dominant retrieval pattern it serves is +still the industry default: + +``` +input → one embedding → nearest neighbours → answer +``` + +Chris Royse's **Calyx** white paper and reference engine +(, a pre-1.0 Rust project under the +Business Source License 1.1) argue that this single-embedding pattern is *too +lossy*. Its central claim: one input should **not** collapse into one flattened +vector. It should become a **constellation** — the same object measured through +many *frozen lenses* (a semantic embedder, a lexical index, a code model, a +domain/structural encoder, a temporal model, a sensor model …), with **each +lens kept as a distinct typed slot, never flattened**. Relationships are then +*derived between* the slots, grounded against real-world anchors, scored for how +much signal each lens adds, and gated so the system fails closed rather than +answering from "semantic fog". + +Calyx organises this around **four verbs** — *measure, count, differentiate, +compose* — and eleven subsystems (`Aster` storage, `Forge` math, `Registry` +content-addressed lenses, `Sextant` fusion search, `Loom` cross-lens +associations, `Assay` signal-in-bits, `Lodestar` grounding kernels, `Ward` +fail-closed guard, `Ledger` hash-chained provenance, `Anneal` reversible +self-optimization, `Oracle` grounded prediction). Its three trust principles +are **grounding is mandatory**, **no flattening**, and **fail closed**. + +This is not a competitor to RuVector — it is a **design pattern RuVector should +absorb**. It directly validates the project's "memory is the moat" thesis +(`CLAUDE.md`, MetaHarness): the embedding *model* is replaceable; the *measured +association substrate plus governance* is the product. It also composes with +Darwin Mode (ADR-266/271): MetaHarness already evolves planners, routers, and +memory policy — Calyx says it should also **route lenses**, not just models. + +**Decision needed**: Add a first-class, RuVector-native **association memory +layer** — multi-slot constellations, lens manifests, cross-lens agreement, RRF +fusion, signal-density scoring, grounding anchors, a fail-closed guard, a +hash-chained provenance ledger, and a reversible weight optimizer — with a +deterministic Rust benchmark proving it beats single-embedding RAG on grounded +accuracy and unsupported-claim rate, and with a clear path to back each per-lens +ranking with RuVector's existing HNSW/IVF/GNN indexes. + +### Licensing note (clean-room) + +Calyx's reference engine is **BSL-1.1** licensed. The crate this ADR introduces, +`crates/ruvector-calyx`, is an **independent, clean-room implementation of the +published *architecture pattern*** (constellations, the four verbs, the trust +principles) — it does **not** copy or derive from Calyx's source. It is licensed +`MIT OR Apache-2.0` like the rest of RuVector and is dependency-free (an inline +SplitMix64 PRNG keeps benchmarks reproducible with no external crates). + +--- + +## Decision + +Ship `crates/ruvector-calyx`: an association-native memory layer that maps the +Calyx pattern onto RuVector primitives. The mapping the crate implements: + +| Calyx concept | RuVector translation (this crate) | +|---------------------|----------------------------------------------------------------| +| Constellation | `Constellation` — one object, many typed slots, never flattened | +| Lens | `LensManifest` — content-addressed (name, version, kind, dims) | +| Slot | A typed `Vec` keyed by lens name in `Constellation::slots` | +| Cross-term (`Loom`) | `loom::{min,mean}_agreement`, `top_dissenter` — disagreement is signal | +| Signal (`Assay`) | `assay::signal_density` — mutual-information bits per µs of cost | +| Fusion (`Sextant`) | `fusion::weighted_rrf` — Reciprocal Rank Fusion across lenses | +| Grounding (`Lodestar`) | `Anchor` (accepted-answer / passed-test / sensor-label / citation / reward) | +| Guard (`Ward`) | `ward::adjudicate` → `GuardDecision::{Answer, Refuse(reason)}` (fail closed) | +| Provenance (`Ledger`) | `Ledger` — FNV-chained, replayable answer paths | +| Self-opt (`Anneal`) | `anneal::anneal_weights` — reversible SA over fusion weights | +| Panel routing | per-lens fusion weights tuned by `Anneal` (lens routing) | + +### 1. No flattening — the constellation is the record + +`Constellation` stores slots in a `BTreeMap>` keyed by lens. +The deliberately lossy `Constellation::flatten()` exists *only* to construct the +single-embedding baseline the layer is meant to beat. The `ConstellationStore` +validates every slot against a lens registry on insert and **fails closed** on +an unknown lens or a shape mismatch (`StoreError::{UnknownLens, ShapeMismatch}`). + +### 2. The four verbs, wired in `CalyxEngine` + +`measure → count → differentiate → compose`: + +- **measure** — the query arrives already measured through the registry's frozen + lenses (slots in, never flattened). +- **count** — one ranked list per lens (`ConstellationStore::search_lens`, exact + here; HNSW/IVF in production) plus cross-lens agreement (`loom`). +- **differentiate** — per-lens fusion weights (tuned by `anneal`) bias the fusion + toward high-signal-density lenses (`assay`). +- **compose** — `fusion::weighted_rrf` fuses the lists, `ledger::grounded_path` + resolves grounding, `ward::adjudicate` returns a guarded decision, and the + whole path is recorded as a replayable `Ledger` entry. + +### 3. Fail closed, with structured refusals + +`Ward` refuses — never silently guesses — on any shortfall, each with a typed +reason: `NoCandidate`, `LowFusionScore`, `CrossLensDisagreement`, +`NoGroundedPath`, `UncalibratedGuard`. The cross-lens guard uses the *minimum* +agreement across lenses, so a single dissenting lens (the high-value +disagreement signal) blocks the answer. + +### 4. Grounding is mandatory and replayable + +Every answer requires a grounded path (`GuardProfile::require_grounding`) above a +calibrated anchor weight. Every query appends a hash-chained `LedgerEntry` +(input · lenses · retrieval · output); `Ledger::verify()` replays the entire +chain, giving 100% lineage reproducibility and tamper-evidence. + +### 5. Reversible lens routing (Anneal) + +`anneal_weights` runs simulated annealing over the fusion weights to maximize +*accepted answers per unit cost* — "signal per dollar". Every rejected proposal +restores the prior weights exactly (reversibility), so the optimizer never +corrupts the deployed configuration. This is the retrieval analogue of +MetaHarness Darwin Mode (ADR-266): instead of "which model answers", it learns +"which lenses to consult and how to weight them". + +--- + +## Benchmark (`calyx-bench`) + +A deterministic, dependency-free benchmark (`cargo run --release -p +ruvector-calyx`) constructs a synthetic corpus engineered to expose the +flattening failure: 240 documents across 12 topics × 20 docs. Within a topic +every document shares the same **semantic** region ("semantic fog"); the +discriminating signal lives in two *minority* lenses — a per-doc **lexical** +fingerprint and a **structural** class. A single-embedding system sees only the +semantic vector and cannot tell the right document from its 19 topic twins; the +association-native store keeps the minority lenses as full-weight ranking lists +and fuses them. The query set adds **unanswerable** queries (right topic, +specifics matching no document) to test fail-closed abstention. + +### Measured results (seed 42, x86_64) + +| Metric | Single-embedding | Calyx multi-lens | +|-----------------------------|------------------|------------------| +| Grounded answer accuracy | 6.7% | **99.2%** | +| Recall@10 (answerable) | 51.7% | **100.0%** | +| Unsupported claims (count) | 172 | **0** | +| Abstentions (correct) | 0 | 61 | +| Replayable provenance | n/a | **180/180 (100%)** | + +`Assay` correctly ranks the cheap lexical lens highest by signal density +(0.11 bits/µs vs the expensive semantic lens at 0.01 bits/µs); `Anneal` +reversibly converges the fusion weights onto the high-density lens, *raising* +grounded accuracy to 100% while *dropping* the expensive semantic lens — +demonstrating the cost-shifting thesis (value moves from expensive model +intelligence to a measured substrate + cheap sufficient reasoning). + +### Acceptance targets (all PASS) + +| Target | Result | +|------------------------------------------|----------| +| Grounded answer accuracy ≥ +15 pp | +92.5 pp | +| Unsupported claims ≥ −50% | −100% | +| Recall@10 ≥ +10 pp | +48.3 pp | +| Replayability 100% | 180/180 | +| Anneal improves utility & is reversible | Δu=+46.2 (73 reverted moves) | + +The large margins are a property of the *adversarial* corpus (within-topic +single-embedding accuracy is near the 1/20 chance floor by construction); they +demonstrate the *direction and mechanism*, not a claim about any real dataset. +The acceptance thresholds are the ADR's contract; a production validation +(ADR-267 protocol) should re-measure on real multi-lens corpora. + +--- + +## Consequences + +### Positive + +- RuVector gains a first-class **association layer**: multi-slot records, lens + manifests, cross-lens graphs, grounding anchors, signal-density routing, + fail-closed guards, and a provenance ledger — "less like Pinecone, more like + an AI memory operating system". +- Directly reinforces the **memory-is-the-moat** thesis and gives enterprise + governance language (grounding mandatory, no flattening, fail closed) for + Cognitum One deployments (manufacturing, telecom, elder-care, security). +- Composes with **Darwin/MetaHarness**: lens routing becomes another evolvable + gene alongside model routing and memory policy. +- Dependency-free and deterministic → builds anywhere, reproducible benchmarks. + +### Negative / risks + +- The reference per-lens search is exact brute force. Production must back each + lens ranking with RuVector HNSW/IVF (follow-up below); the API is shaped for + that swap (`search_lens` → indexed retrieval). +- The provenance hash (FNV-1a) is non-cryptographic; a production ledger should + use BLAKE3/SHA-256 with signed checkpoints (as Calyx's `Ledger` does). +- Mutual-information signal-density is a binned lower-bound proxy, adequate for + ranking lenses but not a calibrated bits measurement. +- Multi-lens retrieval is ~3× the single-lens latency (3 lens searches + fusion + + guard); `Anneal` mitigates by pruning low-density lenses. + +### Follow-up work + +1. **[done — see Update 1]** Adaptive lens routing + calibration. +2. Back `search_lens` with `ruvector-core` HNSW and `ruvector-spann` partitions. +3. Map the cross-lens `Loom` graph onto `ruvector-graph` / `ruvector-mincut` + association edges so agreement/disagreement is itself searchable. +4. Swap FNV for BLAKE3 + signed ledger checkpoints. +5. Wire lens routing into MetaHarness Darwin as an evolvable gene (ADR-266). +6. Add a sensor/RF lens (RuView: Wi-Fi CSI, mmWave) to demonstrate non-text + constellations and cross-modal disagreement detection. + +--- + +## Update 1 — Adaptive routing + calibration (2026-06-30) + +Sequencing note: **route → calibrate → stress-test → graph → HNSW.** Calibration +before speed, because speed optimizations (HNSW) can *hide* bad retrieval, and +calibration tells you *where* HNSW/graph even matter. This update lands the first +two. + +### What shipped + +- `calibrate.rs` — per-query confidence from four signals (score **margin**, + **cross-lens agreement**, **grounding/source density**, **contradiction** = + top dissenter gap; freshness is N/A in this corpus), a histogram + **`Calibrator`** (reliability map: raw confidence → empirical accuracy), + **Expected Calibration Error**, and **abstention precision/recall**. +- `routing.rs` — `route()` consults lenses **cheapest-first**, stops early once + calibrated confidence clears `stop_threshold`, escalates to expensive lenses + only when unsure, and abstains if the full panel is still not confident. A + `min_lenses_to_answer` gate enforces **"don't trust a lone lens"** — never + commit on a single lens's say-so. Every decision emits a `Witness` (lenses + consulted, per-lens tops, signals, confidence, action, escalation reason, + cost, latency). +- `calyx-routing-bench` — compares **brute-force / static / adaptive** on the + multi-lens corpus (160 answerable + 80 unanswerable, 50/50 train/test; the + calibrator is fit on train only). + +### Measured results (test split, seed 2026) + +| Mode | Grounded acc | Accepted acc | Cost µs | Latency µs | ECE | Abstain-P | +|------|-------------|--------------|---------|-----------|-----|-----------| +| brute-force (all lenses) | 93.8% | 100.0% | 15.5 | 15.5 | 0.000 | 88.9% | +| static (2 cheap lenses) | 100.0% | 100.0% | 3.5 | 3.5 | 0.000 | 100.0% | +| **adaptive (calibrated)** | **100.0%** | **100.0%** | **7.5** | **7.5** | **0.000** | **100.0%** | + +Adaptive **strictly dominates brute-force** — higher accuracy at less than half +the cost — because the expensive semantic lens is the low-signal "fog" lens here, +so always consulting it both costs more and can flip a correct answer. Adaptive +skips it when two cheap lenses corroborate, and escalates to it only for hard or +unanswerable queries (then abstains). A well-chosen *static* policy is even +cheaper here — an honest result, and precisely the kind of thing calibration and +signal-density analysis surface (you can only justify hard-coding "skip semantic" +*after* measuring that it is low-signal). + +### Acceptance (adaptive vs brute-force — all PASS) + +| Target | Result | +|--------|--------| +| Accuracy loss ≤ 1 pp | −6.2 pp (adaptive *higher*) ✓ | +| Query cost reduction ≥ 30% | −51.6% ✓ | +| Latency reduction ≥ 25% | −51.6% ✓ | +| ECE ≤ 0.08 | 0.000 ✓ | +| Abstention precision ≥ 0.80 | 100% ✓ | + +Product claim this unlocks: *ruvector-calyx doesn't just retrieve — it knows +which memory lens to trust, when to escalate, and when to abstain, with a +calibrated confidence and a witness log for every decision.* + +--- + +## Update 2 — Three novel capabilities (2026-06-30) + +Beyond recombining known techniques, three capabilities that are genuinely +fresh — one with a theorem behind it. Bench: `calyx-novel-bench`. + +### A. Conformal cross-lens abstention (the defensible one) + +Replaces the hand-tuned guard threshold with **conformal risk control** +(Angelopoulos et al. 2022) over the cross-lens agreement score. Given a +calibration set, `conformal::calibrate` picks the agreement threshold `t̂ = +inf{ t : (n/(n+1))·R̂ₙ(t) + 1/(n+1) ≤ α }`; the router answers iff agreement +`≥ t̂`. The result is a **distribution-free, finite-sample guarantee**: the +expected rate of confident-but-wrong answers on exchangeable test queries is +`≤ α`. "Fail closed" stops being a heuristic and becomes a number. + +Measured (Monte-Carlo over 200 splits, seed 90210): + +| α | threshold | coverage | test risk | selective err | MC mean risk | +|---|-----------|----------|-----------|---------------|--------------| +| hand-tuned (agree≥0.45) | 0.45 | 56.3% | **11.3%** | 20.1% | — (uncontrolled) | +| 0.10 | 0.471 | 51.3% | 9.3% | 18.2% | **9.5% ≤ 0.10** ✓ | +| 0.05 | 0.540 | 49.3% | 1.3% | 2.7% | **4.2% ≤ 0.05** ✓ | +| 0.01 | 0.987 | 38.7% | 0.0% | 0.0% | **0.0% ≤ 0.01** ✓ | + +The hand-tuned guard's 11.3% wrong-answer rate is whatever fell out of a magic +number; conformal turns it into a dial with a proof (tighten α → coverage +drops, risk guaranteed). Applying conformal risk control specifically to +*multi-lens routing/abstention* is close to open ground. + +### B. Disagreement as a query primitive + +`disagreement::find_conflicts(lens_a, lens_b)` ranks records by how much two +lenses disagree — `1 − Jaccard` of a record's lens-A vs lens-B neighbourhoods +(intrinsic), or `simₐ − s_b` against a query (query-relative). This is a +retrieval operation a single-embedding store *cannot express* ("find where the +structural lens and the semantic lens most disagree" — the "comment says one +thing, the code does another" detector). Planted 6 conflict records +(semantic ∈ topic X, structural ∈ another class); `find_conflicts@6` surfaced +**6/6 (100% precision)** at disagreement score 1.0. + +### C. Learning the router from the ledger + +The `Witness` log is a trajectory dataset. `ledger_policy::learn` runs +first-visit Monte-Carlo control over exploratory witness trajectories +(state = `(lenses_consulted, agreement_bin)`, action = stop/continue) to learn a +routing policy from logged provenance — the retrieval analogue of MetaHarness +learning a harness from its trace. + +| Policy | Accuracy | Cost µs | Utility | +|--------|----------|---------|---------| +| brute-force | 48.0% | 15.5 | −0.057 | +| static(2) | 65.3% | 3.5 | 0.620 | +| **ledger-learned** | 65.3% | **2.9** | **0.661** | + +The learned policy *beats* both fixed baselines: it discovered it can bail after +a single lens on low-agreement (likely-unanswerable) queries and abstain, saving +cost, while consulting two lenses to corroborate answerable ones. + +### Honesty note + +Individually, most ingredients are established (multi-vector records, RRF, +histogram calibration, conformal prediction, MC control). The contribution is +the *synthesis over frozen heterogeneous lenses* plus (A) which has a real +guarantee. Benchmarks are synthetic and illustrative; real-corpus validation +(ADR-267) remains the bar for a research claim. + +--- + +## Update 3 — Real-data path (`.calyx` format + CodeSearchNet, 2026-06-30) + +Addresses the standing honesty gap: how to move from synthetic to *real* data. +The key architectural decision is to **separate lens production from the +association layer**. Producing lenses (embedding models, BM25) needs models, +GPUs, and network; the layer we benchmark (fusion, routing, conformal, +disagreement) is pure Rust. So real embeddings are computed **offline, once**, +and serialized — the crate loads precomputed vectors + ground truth (the same +pattern as ann-benchmarks' precomputed HDF5). + +### What shipped + +- **`.calyx` v1 binary format** + a dependency-free `std::io` loader + (`corpus.rs`): lens manifests, records (slots + grounding anchors), and + queries with relevance ground truth. `n_relevant == 0` marks an *unanswerable* + query (abstaining is correct) — which is what makes conformal risk meaningful. +- **`calyx-real-bench`** — loads a `.calyx` and reports the standard IR metrics + published baselines use (**MRR@10, Recall@1/10, nDCG@10**) for single-lens vs + multi-lens fusion, plus conformal abstention risk and code↔doc disagreement. + With no file it synthesizes a CodeSearchNet-shaped stand-in and round-trips it + through the loader, so the whole load→metrics path is provable offline. +- **`tools/build_codesearchnet.py`** — the one model/network step: embeds each + function through a joint NL↔code model (`code` lens), a text model (`doc` + lens), and a dependency-free hashed-token lens (`lexical`); docstrings become + NL queries; a fraction of golds are held out to create unanswerable queries. + Emits byte-exact `.calyx`. + +### Why CodeSearchNet + +Code and docstring are genuinely different views, so cross-lens disagreement is +real (a stale/incorrect docstring = code lens and doc lens point to different +neighbourhoods), and it is a standard code-search retrieval benchmark with +published MRR baselines. + +### Stand-in results (synthetic, CodeSearchNet-shaped — *not* a real-data claim) + +Each single lens is individually weak (code/doc resolve only to a module, +lexical to a cross-cutting token cluster); only their fusion pins the exact +function: + +| System | MRR@10 | R@1 | R@10 | nDCG@10 | +|--------|--------|-----|------|---------| +| single-lens [code] | 0.185 | 0.039 | 0.650 | 0.292 | +| single-lens [doc] | 0.173 | 0.039 | 0.617 | 0.275 | +| single-lens [lexical] | 0.204 | 0.078 | 0.689 | 0.314 | +| **fusion (all lenses)** | **0.492** | **0.372** | **0.789** | **0.555** | + +Fusion beats the best single lens by **+0.288 MRR**. Conformal abstention holds +its guarantee on real-style labels (test risk 5.0% ≤ α=0.10, 2.5% ≤ 0.05); +planted stale docstrings surface at **100% precision** via code↔doc +disagreement. These are stand-in numbers to validate the *pipeline*; real +numbers require running the converter on CodeSearchNet. + +### To produce real numbers + +``` +pip install datasets sentence-transformers numpy +python crates/ruvector-calyx/tools/build_codesearchnet.py --lang python --n 5000 --out corpus.calyx +cargo run --release -p ruvector-calyx --bin calyx-real-bench -- corpus.calyx +``` + +Validity to enforce on real runs: official train/test splits (no leakage), +bootstrap CIs, and the conformal exchangeability assumption (calibration and +test drawn from the same distribution). + +--- + +## References + +- Royse 2026, "Calyx: An Association-Native Database and Its Path to + Planetary-Scale Grounded Intelligence" — ResearchGate preprint (pub. + 408248277); reference engine (Rust, + edition 2024, BSL-1.1, pre-1.0). +- Royse 2026, "The Calculus of Association: A Formula for Artificial General + Intelligence — Meaning Compression, Frozen Embedders as Designable Measurement + Instruments, Derived Data Abundance, and Teleological Constellations" — + ResearchGate preprint (pub. 405933676). +- Cormack, Clarke & Buettcher 2009, "Reciprocal Rank Fusion Outperforms + Condorcet and Individual Rank Learning Methods", SIGIR. +- Angelopoulos, Bates, Fisch, Lei & Schuster 2022, "Conformal Risk Control" + (arXiv:2208.02814). +- ADR-269, ADR-270 (graph memory over RuVector); ADR-266, ADR-271 (Darwin/SONA + self-improvement).