feat(ruvector-calyx): association-native memory layer (ADR-272)

Cargo.toml

@@ -272,6 +272,10 @@ members = [
     "crates/timesfm",
     # RuVector integration for TimesFM: Forecaster + anomaly bands + sweep early-stopping
     "crates/ruvector-timesfm",
+    # Calyx-inspired association-native memory layer: multi-slot constellations,
+    # lens manifests, RRF fusion, cross-lens agreement, grounding, fail-closed
+    # guard, hash-chained provenance ledger, reversible anneal optimizer (ADR-272)
+    "crates/ruvector-calyx",
 ]
 resolver = "2"
 

crates/ruvector-calyx/Cargo.toml

@@ -0,0 +1,36 @@
+[package]
+name        = "ruvector-calyx"
+version     = "0.1.0"
+edition     = "2021"
+description = "Association-native memory layer for ruvector: multi-slot constellations measured through many frozen lenses, kept un-flattened, fused with reciprocal rank fusion, scored for cross-lens agreement and signal density, grounded against real-world anchors, gated by a fail-closed guard, and recorded in a hash-chained provenance ledger"
+authors     = ["ruvnet", "claude-flow"]
+license     = "MIT OR Apache-2.0"
+repository  = "https://github.com/ruvnet/ruvector"
+keywords    = ["association", "vector-search", "rrf", "provenance", "ruvector"]
+categories  = ["algorithms", "data-structures"]
+
+[[bin]]
+name = "calyx-bench"
+path = "src/main.rs"
+
+[[bin]]
+name = "calyx-routing-bench"
+path = "src/bin/routing_bench.rs"
+
+[[bin]]
+name = "calyx-novel-bench"
+path = "src/bin/novel_bench.rs"
+
+[[bin]]
+name = "calyx-real-bench"
+path = "src/bin/real_bench.rs"
+
+[lib]
+name = "ruvector_calyx"
+path = "src/lib.rs"
+
+# Intentionally dependency-free: the reference crate uses an inline deterministic
+# PRNG (see `src/rng.rs`) so it builds anywhere and benchmarks are reproducible.
+[dependencies]
+
+[dev-dependencies]

crates/ruvector-calyx/src/anneal.rs

@@ -0,0 +1,129 @@
+//! Reversible self-optimization of lens fusion weights (the paper's *Anneal*).
+//!
+//! "Compose" is only as good as the weights behind the fusion. Anneal tunes the
+//! per-lens fusion weights to maximize an objective — here, *accepted answers
+//! per unit cost* — using simulated annealing. Every proposal is **reversible**:
+//! a rejected move restores the previous weights exactly, so the optimizer
+//! never corrupts the deployed configuration.
+
+use crate::rng::Rng;
+
+/// Annealing schedule and proposal parameters.
+#[derive(Debug, Clone)]
+pub struct AnnealConfig {
+    pub iterations: usize,
+    /// Initial temperature for Metropolis acceptance.
+    pub t_start: f32,
+    /// Final temperature.
+    pub t_end: f32,
+    /// Max absolute perturbation applied to one weight per proposal.
+    pub step: f32,
+    /// Weights are clamped to `[0, max_weight]`.
+    pub max_weight: f32,
+    pub seed: u64,
+}
+
+impl Default for AnnealConfig {
+    fn default() -> Self {
+        Self {
+            iterations: 300,
+            t_start: 0.10,
+            t_end: 0.001,
+            step: 0.5,
+            max_weight: 8.0,
+            seed: 7,
+        }
+    }
+}
+
+/// The optimization result.
+#[derive(Debug, Clone)]
+pub struct AnnealOutcome {
+    pub best_weights: Vec<f32>,
+    pub best_utility: f32,
+    pub initial_utility: f32,
+    pub accepted_moves: usize,
+    pub reverted_moves: usize,
+}
+
+/// Anneal `initial` weights to maximize `utility`.
+///
+/// `utility` maps a weight vector to a scalar (higher is better); in the
+/// benchmark it is `accepted_answers / total_cost`. Proposals that fail the
+/// Metropolis test are fully reverted, demonstrating the reversibility property.
+pub fn anneal_weights<F>(initial: Vec<f32>, cfg: &AnnealConfig, utility: F) -> AnnealOutcome
+where
+    F: Fn(&[f32]) -> f32,
+{
+    let mut rng = Rng::new(cfg.seed);
+    let n = initial.len();
+
+    let mut current = initial.clone();
+    let mut current_u = utility(&current);
+    let initial_utility = current_u;
+
+    let mut best = current.clone();
+    let mut best_u = current_u;
+
+    let mut accepted = 0usize;
+    let mut reverted = 0usize;
+
+    let iters = cfg.iterations.max(1);
+    for i in 0..iters {
+        // Geometric temperature schedule.
+        let frac = i as f32 / iters as f32;
+        let temp = cfg.t_start * (cfg.t_end / cfg.t_start).powf(frac);
+
+        // Propose: perturb one weight. Snapshot for reversibility.
+        let idx = rng.range(n);
+        let prev = current[idx];
+        let delta = rng.signed() * cfg.step;
+        current[idx] = (prev + delta).clamp(0.0, cfg.max_weight);
+
+        let candidate_u = utility(&current);
+        let d = candidate_u - current_u;
+        let accept = d >= 0.0 || rng.f32() < (d / temp.max(1e-6)).exp();
+
+        if accept {
+            current_u = candidate_u;
+            accepted += 1;
+            if candidate_u > best_u {
+                best_u = candidate_u;
+                best.copy_from_slice(&current);
+            }
+        } else {
+            // Reversible: restore exactly.
+            current[idx] = prev;
+            reverted += 1;
+        }
+    }
+
+    AnnealOutcome {
+        best_weights: best,
+        best_utility: best_u,
+        initial_utility,
+        accepted_moves: accepted,
+        reverted_moves: reverted,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn anneal_improves_a_simple_objective() {
+        // Objective maximized when weights ≈ [3, 1]: a smooth concave bump.
+        let utility = |w: &[f32]| -> f32 {
+            let a = -(w[0] - 3.0).powi(2);
+            let b = -(w[1] - 1.0).powi(2);
+            a + b
+        };
+        let cfg = AnnealConfig::default();
+        let out = anneal_weights(vec![0.0, 0.0], &cfg, utility);
+        assert!(out.best_utility >= out.initial_utility);
+        assert!((out.best_weights[0] - 3.0).abs() < 0.6, "w0={}", out.best_weights[0]);
+        assert!((out.best_weights[1] - 1.0).abs() < 0.6, "w1={}", out.best_weights[1]);
+        assert!(out.reverted_moves > 0, "some moves should be reverted");
+    }
+}

crates/ruvector-calyx/src/assay.rs

@@ -0,0 +1,117 @@
+//! Signal density — "how much does this lens actually tell us" (the paper's
+//! *Assay*).
+//!
+//! "Differentiate" — the third of Calyx's four verbs — measures which lenses
+//! add information rather than assuming every model is useful. We estimate the
+//! mutual information (in bits) between a lens's similarity score and a binary
+//! relevance label, then divide by the lens's declared cost to get *signal
+//! density*: information gained per unit of compute. The anneal optimizer
+//! promotes high-density lenses.
+
+/// One (lens-similarity, was-this-pair-relevant) observation.
+#[derive(Debug, Clone, Copy)]
+pub struct Observation {
+    pub similarity: f32,
+    pub relevant: bool,
+}
+
+/// Estimated information content of a lens and its economics.
+#[derive(Debug, Clone)]
+pub struct LensSignal {
+    pub lens: String,
+    /// Mutual information between similarity and relevance, in bits (>= 0).
+    pub bits: f32,
+    /// Declared measurement cost in microseconds.
+    pub cost_us: f32,
+    /// Signal density: bits per microsecond (information per dollar/ms/watt).
+    pub density: f32,
+}
+
+/// Estimate mutual information (bits) between a lens's similarity score and a
+/// binary relevance label, by binning similarity into `bins` buckets.
+///
+/// Returns `0.0` for an empty/degenerate sample. The estimate is a lower bound
+/// proxy for the paper's MI-based contribution measure; it is monotone in how
+/// well the lens separates relevant from irrelevant pairs.
+pub fn mutual_information_bits(obs: &[Observation], bins: usize) -> f32 {
+    let n = obs.len();
+    if n == 0 || bins == 0 {
+        return 0.0;
+    }
+    let bins = bins.max(2);
+    // Similarity is cosine in [-1, 1]; map to [0, 1] then to a bin index.
+    let bin_of = |s: f32| -> usize {
+        let t = ((s + 1.0) * 0.5).clamp(0.0, 0.999_999);
+        (t * bins as f32) as usize
+    };
+
+    let mut joint = vec![[0u32; 2]; bins]; // joint[bin][label]
+    let mut p_label = [0u32; 2];
+    let mut p_bin = vec![0u32; bins];
+    for o in obs {
+        let b = bin_of(o.similarity);
+        let l = o.relevant as usize;
+        joint[b][l] += 1;
+        p_label[l] += 1;
+        p_bin[b] += 1;
+    }
+
+    let nf = n as f32;
+    let mut mi = 0.0f32;
+    for b in 0..bins {
+        for l in 0..2 {
+            let c = joint[b][l];
+            if c == 0 {
+                continue;
+            }
+            let pxy = c as f32 / nf;
+            let px = p_bin[b] as f32 / nf;
+            let py = p_label[l] as f32 / nf;
+            if px > 0.0 && py > 0.0 {
+                mi += pxy * (pxy / (px * py)).log2();
+            }
+        }
+    }
+    mi.max(0.0)
+}
+
+/// Compute signal density for one lens.
+pub fn signal_density(lens: impl Into<String>, obs: &[Observation], cost_us: f32) -> LensSignal {
+    let bits = mutual_information_bits(obs, 8);
+    let density = if cost_us > 0.0 { bits / cost_us } else { 0.0 };
+    LensSignal {
+        lens: lens.into(),
+        bits,
+        cost_us,
+        density,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn informative_lens_beats_random_lens() {
+        // Perfectly separating lens: relevant pairs always high similarity.
+        let mut informative = Vec::new();
+        let mut random = Vec::new();
+        for i in 0..200 {
+            let relevant = i % 2 == 0;
+            informative.push(Observation {
+                similarity: if relevant { 0.9 } else { -0.9 },
+                relevant,
+            });
+            // Random lens: similarity uncorrelated with label.
+            random.push(Observation {
+                similarity: if i % 3 == 0 { 0.5 } else { -0.5 },
+                relevant,
+            });
+        }
+        let inf = signal_density("good", &informative, 2.0);
+        let rnd = signal_density("noise", &random, 2.0);
+        assert!(inf.bits > 0.9, "informative bits = {}", inf.bits);
+        assert!(rnd.bits < inf.bits);
+        assert!(inf.density > rnd.density);
+    }
+}