research(nightly): MinCut-Partitioned Community Graph-RAG for Agent Memory Coherence

Cargo.toml

@@ -4,6 +4,8 @@ exclude = ["external/ruqu", "external/rvdna", "examples/OSpipe", "examples/rvf",
     "crates/emergent-time-wasm",
     # sonic-ct crates: self-contained detached workspaces
     "crates/sonic-ct", "crates/sonic-ct-wasm",
+    # ruvector-community-rag: standalone nightly PoC (ADR-272), no external deps
+    "crates/ruvector-community-rag",
     # ruvector-postgres is a pgrx-based PostgreSQL extension. Its build script
     # requires `$PGRX_HOME` set up via `cargo install cargo-pgrx --version 0.12.9`
     # and `cargo pgrx init`, which downloads and builds multiple Postgres

crates/ruvector-community-rag/Cargo.toml

@@ -0,0 +1,24 @@
+[workspace]
+resolver = "2"
+members = ["."]
+
+[package]
+name = "ruvector-community-rag"
+version = "0.1.0"
+edition = "2021"
+license = "MIT OR Apache-2.0"
+repository = "https://github.com/ruvnet/ruvector"
+description = "MinCut-partitioned community graph-RAG for coherent agent memory retrieval — FlatScan, GraphHop, CommunityRAG variants in safe Rust"
+readme = "README.md"
+keywords = ["vector-search", "graph-rag", "community-detection", "agent-memory", "ann"]
+categories = ["algorithms", "data-structures", "science"]
+
+[[bin]]
+name = "community-rag-benchmark"
+path = "src/main.rs"
+
+[dependencies]
+
+[lints.rust]
+dead_code = "allow"
+unused_variables = "allow"

crates/ruvector-community-rag/src/community.rs

@@ -0,0 +1,141 @@
+//! Lightweight community detection via threshold-based graph connectivity.
+//!
+//! Two-step process:
+//! 1. Build a sparse similarity graph: add an edge between two nodes when
+//!    their cosine similarity exceeds `edge_threshold`.
+//! 2. Run Union-Find to collect connected components → communities.
+//!
+//! This approximates the partition produced by a global mincut with the
+//! same threshold: communities are the strongly-connected subgraphs separated
+//! by cuts weaker than (1 - edge_threshold) in cosine distance.
+//!
+//! The approach is O(N²) to build (same as k-NN graph), O(N·α(N)) to label,
+//! where α is the inverse Ackermann function. Both are fast in practice for
+//! the PoC scale of N ≤ 50k.
+
+/// Union-Find with path compression and union-by-rank.
+pub struct UnionFind {
+    parent: Vec<usize>,
+    rank: Vec<usize>,
+}
+
+impl UnionFind {
+    pub fn new(n: usize) -> Self {
+        Self {
+            parent: (0..n).collect(),
+            rank: vec![0; n],
+        }
+    }
+
+    pub fn find(&mut self, mut x: usize) -> usize {
+        while self.parent[x] != x {
+            self.parent[x] = self.parent[self.parent[x]]; // path halving
+            x = self.parent[x];
+        }
+        x
+    }
+
+    pub fn union(&mut self, a: usize, b: usize) {
+        let ra = self.find(a);
+        let rb = self.find(b);
+        if ra == rb { return; }
+        match self.rank[ra].cmp(&self.rank[rb]) {
+            std::cmp::Ordering::Less => self.parent[ra] = rb,
+            std::cmp::Ordering::Greater => self.parent[rb] = ra,
+            std::cmp::Ordering::Equal => {
+                self.parent[rb] = ra;
+                self.rank[ra] += 1;
+            }
+        }
+    }
+}
+
+/// Detected community structure for a vector corpus.
+pub struct Communities {
+    /// For each vector id, its community label (0-indexed, contiguous).
+    pub labels: Vec<usize>,
+    /// Number of distinct communities found.
+    pub num_communities: usize,
+    /// Centroid of each community (mean of member vectors).
+    pub centroids: Vec<Vec<f32>>,
+}
+
+impl Communities {
+    /// Detect communities from raw vectors using cosine-similarity threshold.
+    ///
+    /// Two vectors are in the same community when `cosine_sim > edge_threshold`.
+    /// A higher threshold → smaller, more coherent communities.
+    /// A lower threshold → larger, loosely connected communities.
+    pub fn detect(vectors: &[Vec<f32>], edge_threshold: f32) -> Self {
+        let n = vectors.len();
+        let dims = vectors.first().map(|v| v.len()).unwrap_or(0);
+        let mut uf = UnionFind::new(n);
+
+        // Build edges: O(N²) — acceptable for PoC sizes.
+        for i in 0..n {
+            for j in (i + 1)..n {
+                let sim = cosine_sim(&vectors[i], &vectors[j]);
+                if sim > edge_threshold {
+                    uf.union(i, j);
+                }
+            }
+        }
+
+        // Collect labels and renumber contiguously.
+        let roots: Vec<usize> = (0..n).map(|i| uf.find(i)).collect();
+        let mut root_to_label = std::collections::HashMap::new();
+        let mut next_label = 0usize;
+        let labels: Vec<usize> = roots
+            .iter()
+            .map(|&r| {
+                *root_to_label.entry(r).or_insert_with(|| {
+                    let l = next_label;
+                    next_label += 1;
+                    l
+                })
+            })
+            .collect();
+        let num_communities = next_label;
+
+        // Compute centroids.
+        let mut sums = vec![vec![0.0f32; dims]; num_communities];
+        let mut counts = vec![0usize; num_communities];
+        for (i, &lab) in labels.iter().enumerate() {
+            for d in 0..dims {
+                sums[lab][d] += vectors[i][d];
+            }
+            counts[lab] += 1;
+        }
+        let centroids: Vec<Vec<f32>> = sums
+            .into_iter()
+            .zip(counts.iter())
+            .map(|(s, &c)| {
+                let c = c.max(1) as f32;
+                s.into_iter().map(|x| x / c).collect()
+            })
+            .collect();
+
+        Communities { labels, num_communities, centroids }
+    }
+
+    /// Find the nearest community centroid for a query vector.
+    /// Returns (community_label, distance_to_centroid).
+    pub fn nearest_community(&self, query: &[f32]) -> (usize, f32) {
+        self.centroids
+            .iter()
+            .enumerate()
+            .map(|(c, cent)| {
+                let d: f32 = query.iter().zip(cent.iter()).map(|(a, b)| (a - b) * (a - b)).sum();
+                (c, d)
+            })
+            .min_by(|a, b| a.1.partial_cmp(&b.1).unwrap())
+            .unwrap_or((0, f32::INFINITY))
+    }
+}
+
+fn cosine_sim(a: &[f32], b: &[f32]) -> f32 {
+    let dot: f32 = a.iter().zip(b.iter()).map(|(x, y)| x * y).sum();
+    let na: f32 = a.iter().map(|x| x * x).sum::<f32>().sqrt();
+    let nb: f32 = b.iter().map(|x| x * x).sum::<f32>().sqrt();
+    if na == 0.0 || nb == 0.0 { 0.0 } else { dot / (na * nb) }
+}

crates/ruvector-community-rag/src/community_rag.rs

@@ -0,0 +1,108 @@
+//! Variant 3 — CommunityRAG: community centroid routing + member rerank.
+//!
+//! At query time:
+//! 1. Find the nearest community centroid (O(C) where C = community count).
+//! 2. Retrieve all vectors assigned to that community.
+//! 3. Re-score by exact L2 distance to query.
+//! 4. Return top-k.
+//!
+//! This variant excels when queries are coherent with a task cluster —
+//! e.g., an agent asking about "Python debugging" retrieves all Python-related
+//! memories rather than a mixed bag of generically similar embeddings.
+//!
+//! **Tradeoff**: cross-community recall may be lower than FlatScan or GraphHop
+//! when the query genuinely spans multiple communities. A future extension is
+//! to query the top-C communities and merge results — but that is left as a
+//! production refinement (see ADR-272).
+
+use crate::{community::Communities, CommunitySearch, Hit, VectorMeta, l2_sq};
+
+pub struct CommunityRAG {
+    vectors: Vec<Vec<f32>>,
+    metas: Vec<VectorMeta>,
+    /// Detected community partition (None until `build()` is called).
+    communities: Option<Communities>,
+    /// Cosine similarity threshold for the community graph.
+    edge_threshold: f32,
+    /// Members of each community, indexed by community label.
+    members: Vec<Vec<usize>>,
+}
+
+impl CommunityRAG {
+    pub fn new(edge_threshold: f32) -> Self {
+        Self {
+            vectors: Vec::new(),
+            metas: Vec::new(),
+            communities: None,
+            edge_threshold,
+            members: Vec::new(),
+        }
+    }
+
+    /// Read-only access to the detected community labels (after build).
+    pub fn detected_labels(&self) -> Option<&[usize]> {
+        self.communities.as_ref().map(|c| c.labels.as_slice())
+    }
+
+    pub fn num_communities(&self) -> usize {
+        self.communities.as_ref().map(|c| c.num_communities).unwrap_or(0)
+    }
+}
+
+impl CommunitySearch for CommunityRAG {
+    fn insert(&mut self, vector: &[f32], community: usize) {
+        let id = self.vectors.len();
+        self.metas.push(VectorMeta { id, community });
+        self.vectors.push(vector.to_vec());
+    }
+
+    fn build(&mut self) {
+        let comm = Communities::detect(&self.vectors, self.edge_threshold);
+        let nc = comm.num_communities;
+
+        // Index members per community.
+        let mut members = vec![Vec::new(); nc];
+        for (i, &lab) in comm.labels.iter().enumerate() {
+            members[lab].push(i);
+        }
+
+        self.members = members;
+        self.communities = Some(comm);
+    }
+
+    fn search(&self, query: &[f32], k: usize) -> Vec<Hit> {
+        let comm = match &self.communities {
+            Some(c) => c,
+            None => return vec![],
+        };
+
+        // Route to the nearest community.
+        let (nearest_comm, _dist) = comm.nearest_community(query);
+        let candidates = &self.members[nearest_comm];
+
+        // Re-score all community members by exact L2.
+        let mut hits: Vec<Hit> = candidates
+            .iter()
+            .map(|&id| Hit { id, distance: l2_sq(query, &self.vectors[id]) })
+            .collect();
+        hits.sort_by(|a, b| a.distance.partial_cmp(&b.distance).unwrap());
+        hits.truncate(k);
+        hits
+    }
+
+    fn memory_bytes(&self) -> usize {
+        let vec_bytes: usize = self.vectors.iter().map(|v| v.len() * 4).sum();
+        let meta_bytes = self.metas.len() * std::mem::size_of::<VectorMeta>();
+        let member_bytes: usize = self.members.iter().map(|m| m.len() * 8).sum();
+        let centroid_bytes = self
+            .communities
+            .as_ref()
+            .map(|c| c.centroids.iter().map(|cv| cv.len() * 4).sum::<usize>())
+            .unwrap_or(0);
+        vec_bytes + meta_bytes + member_bytes + centroid_bytes
+    }
+
+    fn name(&self) -> &'static str {
+        "CommunityRAG"
+    }
+}

crates/ruvector-community-rag/src/flat_scan.rs

@@ -0,0 +1,58 @@
+//! Variant 1 — FlatScan: exact brute-force L2 search (baseline / ground truth oracle).
+
+use crate::{CommunitySearch, Hit, VectorMeta, l2_sq};
+
+/// Stores raw f32 vectors and performs an exhaustive linear scan.
+/// Used as the oracle for measuring recall of the other two variants.
+pub struct FlatScan {
+    vectors: Vec<Vec<f32>>,
+    metas: Vec<VectorMeta>,
+}
+
+impl FlatScan {
+    pub fn new() -> Self {
+        Self {
+            vectors: Vec::new(),
+            metas: Vec::new(),
+        }
+    }
+
+    /// Read-only access to metadata (needed for cross-variant community scoring).
+    pub fn metas(&self) -> &[VectorMeta] {
+        &self.metas
+    }
+}
+
+impl CommunitySearch for FlatScan {
+    fn insert(&mut self, vector: &[f32], community: usize) {
+        let id = self.vectors.len();
+        self.metas.push(VectorMeta { id, community });
+        self.vectors.push(vector.to_vec());
+    }
+
+    fn build(&mut self) {
+        // No index to build for brute-force scan.
+    }
+
+    fn search(&self, query: &[f32], k: usize) -> Vec<Hit> {
+        let mut heap: Vec<Hit> = self
+            .vectors
+            .iter()
+            .enumerate()
+            .map(|(id, v)| Hit { id, distance: l2_sq(query, v) })
+            .collect();
+        heap.sort_by(|a, b| a.distance.partial_cmp(&b.distance).unwrap());
+        heap.truncate(k);
+        heap
+    }
+
+    fn memory_bytes(&self) -> usize {
+        let vec_bytes: usize = self.vectors.iter().map(|v| v.len() * 4).sum();
+        let meta_bytes = self.metas.len() * std::mem::size_of::<VectorMeta>();
+        vec_bytes + meta_bytes
+    }
+
+    fn name(&self) -> &'static str {
+        "FlatScan"
+    }
+}