From 5e4d1881db679d997f7d8641cea71750a6ef8d9a Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 08:16:47 +0000 Subject: [PATCH 1/2] research: add ruvector-partition-memory nightly crate (mincut-partitioned memory consolidation) Implements and benchmarks a mincut-partitioned agent-memory retention policy against the existing global top-score CoherencePolicy baseline (ruvector-agent-memory, nightly 2026-06-14), reused directly as a dependency rather than re-implemented. Includes a from-scratch, tested weighted Stoer-Wagner min cut (mincut_exact.rs) used as the authoritative partition source, after ruvector_mincut::DynamicMinCut::partition() was found during development to return vertex splits inconsistent with its own min_cut_value() and nondeterministic across runs; ruvector_mincut's value is still queried as an independent cross-check. fixed_k_partition similarly guards GraphPartitioner's output against observed vertex loss/fabrication and is scale-gated after GraphPartitioner measured 8.4s at n=500 and did not finish in 5m42s at n=4000. Also wires the previously-orphaned ruvector-agent-memory crate into the workspace members list so it can be depended on. Co-Authored-By: claude-flow Claude-Session: https://claude.ai/code/session_01KrAfJLv2U99vvEvQqJ1Zoo --- Cargo.lock | 19 + Cargo.toml | 5 + crates/ruvector-partition-memory/Cargo.toml | 24 ++ .../examples/calibrate.rs | 53 +++ .../examples/darwin_sweep.rs | 131 +++++++ .../ruvector-partition-memory/src/corpus.rs | 277 +++++++++++++ crates/ruvector-partition-memory/src/graph.rs | 90 +++++ crates/ruvector-partition-memory/src/lib.rs | 54 +++ crates/ruvector-partition-memory/src/main.rs | 251 ++++++++++++ .../ruvector-partition-memory/src/metrics.rs | 126 ++++++ .../src/mincut_exact.rs | 366 ++++++++++++++++++ .../src/partition.rs | 241 ++++++++++++ .../src/retention.rs | 187 +++++++++ .../ruvector-partition-memory/src/search.rs | 95 +++++ .../ruvector-partition-memory/src/witness.rs | 191 +++++++++ 15 files changed, 2110 insertions(+) create mode 100644 crates/ruvector-partition-memory/Cargo.toml create mode 100644 crates/ruvector-partition-memory/examples/calibrate.rs create mode 100644 crates/ruvector-partition-memory/examples/darwin_sweep.rs create mode 100644 crates/ruvector-partition-memory/src/corpus.rs create mode 100644 crates/ruvector-partition-memory/src/graph.rs create mode 100644 crates/ruvector-partition-memory/src/lib.rs create mode 100644 crates/ruvector-partition-memory/src/main.rs create mode 100644 crates/ruvector-partition-memory/src/metrics.rs create mode 100644 crates/ruvector-partition-memory/src/mincut_exact.rs create mode 100644 crates/ruvector-partition-memory/src/partition.rs create mode 100644 crates/ruvector-partition-memory/src/retention.rs create mode 100644 crates/ruvector-partition-memory/src/search.rs create mode 100644 crates/ruvector-partition-memory/src/witness.rs diff --git a/Cargo.lock b/Cargo.lock index 895e642572..298e06c380 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8787,6 +8787,13 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "ruvector-agent-memory" +version = "0.1.0" +dependencies = [ + "rand 0.8.6", +] + [[package]] name = "ruvector-attention" version = "2.3.0" @@ -10174,6 +10181,18 @@ dependencies = [ "tracing", ] +[[package]] +name = "ruvector-partition-memory" +version = "0.1.0" +dependencies = [ + "rand 0.8.6", + "ruvector-agent-memory", + "ruvector-mincut 2.3.0", + "serde", + "serde_json", + "sha2 0.10.9", +] + [[package]] name = "ruvector-perception" version = "2.3.0" diff --git a/Cargo.toml b/Cargo.toml index b4381e6431..67824f4bd5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,10 @@ exclude = ["external/ruqu", "external/rvdna", "examples/OSpipe", "examples/rvf", members = [ "crates/ruvector-bounded-rag", "crates/ruvector-temporal-coherence", + # Coherence-weighted agent memory compaction (nightly 2026-06-14). Predates + # workspace wiring; added as a member so ruvector-partition-memory can + # depend on its CoherencePolicy baseline directly instead of duplicating it. + "crates/ruvector-agent-memory", "crates/ruvector-acorn", "crates/ruvector-acorn-wasm", "crates/ruvector-coherence-hnsw", @@ -75,6 +79,7 @@ members = [ "crates/ruvector-mincut-node", "crates/ruvector-mincut-gated-transformer", "crates/ruvector-mincut-gated-transformer-wasm", + "crates/ruvector-partition-memory", # NOTE: ruvector-postgres is in workspace `exclude` (pgrx env requirement). "crates/ruvector-nervous-system", # Iter 219 — hailo backend rejoined the workspace (closes diff --git a/crates/ruvector-partition-memory/Cargo.toml b/crates/ruvector-partition-memory/Cargo.toml new file mode 100644 index 0000000000..a1b067c9e4 --- /dev/null +++ b/crates/ruvector-partition-memory/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "ruvector-partition-memory" +version = "0.1.0" +edition = "2021" +description = "Mincut-partitioned agent-memory consolidation: coherence-graph clustering with floor-guaranteed retention to protect minority-topic memories, with a SHA-256 witness chain over partition decisions" +authors = ["ruvnet", "claude-flow"] +license = "MIT OR Apache-2.0" +repository = "https://github.com/ruvnet/ruvector" +keywords = ["agent-memory", "graph-partitioning", "min-cut", "rag", "ruvector"] +categories = ["algorithms", "data-structures"] + +[[bin]] +name = "benchmark" +path = "src/main.rs" + +[dependencies] +ruvector-mincut = { path = "../ruvector-mincut", default-features = false, features = ["exact"] } +ruvector-agent-memory = { path = "../ruvector-agent-memory" } +rand = "0.8" +sha2 = "0.10" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" + +[dev-dependencies] diff --git a/crates/ruvector-partition-memory/examples/calibrate.rs b/crates/ruvector-partition-memory/examples/calibrate.rs new file mode 100644 index 0000000000..48503fd817 --- /dev/null +++ b/crates/ruvector-partition-memory/examples/calibrate.rs @@ -0,0 +1,53 @@ +use ruvector_partition_memory::corpus::{generate, CorpusConfig}; +use ruvector_partition_memory::graph::build_knn_edges; +use ruvector_partition_memory::partition::{adaptive_partition, AdaptiveConfig}; +use std::time::Instant; + +fn main() { + for n in [500usize, 4000] { + let cfg = CorpusConfig { + n, + ..CorpusConfig::default() + }; + let corpus = generate(&cfg); + let edges = build_knn_edges(&corpus.records, 10); + let verts: Vec = corpus.records.iter().map(|r| r.id).collect(); + let mut cluster_of = std::collections::HashMap::new(); + for r in &corpus.records { + cluster_of.insert(r.id, r.cluster); + } + + for ratio in [0.5f64, 0.35, 0.2] { + let cfg2 = AdaptiveConfig { + coherence_ratio: ratio, + ..AdaptiveConfig::default() + }; + let t0 = Instant::now(); + let result = adaptive_partition(&edges, &verts, &cfg2); + let us = t0.elapsed().as_micros(); + let sizes: Vec = result.clusters.iter().map(|c| c.len()).collect(); + // purity: for each partition, fraction belonging to its majority cluster + let purities: Vec = result + .clusters + .iter() + .map(|part| { + let mut hist = std::collections::HashMap::new(); + for v in part { + *hist.entry(cluster_of[v]).or_insert(0usize) += 1; + } + let max = *hist.values().max().unwrap_or(&0); + max as f64 / part.len() as f64 + }) + .collect(); + println!( + "n={n} ratio={ratio} partitions={} sizes={:?} purities={:?} us={us}", + result.clusters.len(), + sizes, + purities + .iter() + .map(|p| format!("{p:.2}")) + .collect::>() + ); + } + } +} diff --git a/crates/ruvector-partition-memory/examples/darwin_sweep.rs b/crates/ruvector-partition-memory/examples/darwin_sweep.rs new file mode 100644 index 0000000000..15cb312c43 --- /dev/null +++ b/crates/ruvector-partition-memory/examples/darwin_sweep.rs @@ -0,0 +1,131 @@ +//! Bounded Darwin-style parameter sweep over `floor_min`, run once, after +//! the pre-declared main.rs acceptance benchmark had already produced its +//! REJECT verdict. This does NOT retroactively change that verdict — it is +//! a separate, explicitly bounded exploration (generations=1, +//! candidates_per_generation=4, matching the nightly harness's default +//! budget) asking whether retention-budget tuning alone can rescue the +//! rejected hypothesis, or whether the failure is structural (the +//! partition step, not the retention step). The partition (coherence_ratio +//! fixed at 0.35, the value used in the accepted run) is computed once and +//! reused, since `floor_min` only affects retention, not partitioning. +//! +//! Fitness (declared before running, per ADR guidance — not fit to the +//! result): 0.5*worst_cluster_recall + 0.3*overall_recall + 0.2*correctness, +//! where correctness = 1.0 if the witness chain verifies, else 0.0. + +use ruvector_partition_memory::corpus::{generate, CorpusConfig}; +use ruvector_partition_memory::graph::build_knn_edges; +use ruvector_partition_memory::metrics::evaluate; +use ruvector_partition_memory::partition::{adaptive_partition, AdaptiveConfig}; +use ruvector_partition_memory::retention::{retain_partitioned, RetentionPolicy}; + +fn recency_context( + records: &[ruvector_partition_memory::corpus::MemoryRecord], + n_ctx: usize, +) -> Vec> { + let mut sorted: Vec<_> = records.iter().collect(); + sorted.sort_by(|a, b| b.last_accessed_tick.cmp(&a.last_accessed_tick)); + sorted + .into_iter() + .take(n_ctx) + .map(|r| r.embedding.clone()) + .collect() +} + +fn fitness(worst: f64, overall: f64, correctness: f64) -> f64 { + 0.5 * worst + 0.3 * overall + 0.2 * correctness +} + +fn main() { + let cfg = CorpusConfig { + n: 4000, + ..CorpusConfig::default() + }; + let corpus = generate(&cfg); + let target_size = (corpus.records.len() as f64 * 0.5).round() as usize; + let context = recency_context(&corpus.records, 20); + let vertices: Vec = corpus.records.iter().map(|r| r.id).collect(); + let edges = build_knn_edges(&corpus.records, 10); + + let adaptive_cfg = AdaptiveConfig { + coherence_ratio: 0.35, + ..AdaptiveConfig::default() + }; + let result = adaptive_partition(&edges, &vertices, &adaptive_cfg); + let correctness = if result.witness.verify() { 1.0 } else { 0.0 }; + println!( + "parent partition (fixed for the whole sweep): {} partitions, sizes={:?}, correctness={correctness}", + result.clusters.len(), + result.clusters.iter().map(|c| c.len()).collect::>() + ); + + let candidates = [1usize, 3, 8, 15]; + let mut best: Option<(usize, f64)> = None; + println!("gen=1 candidates_per_generation={}", candidates.len()); + for &floor_min in &candidates { + let policy = RetentionPolicy { floor_min }; + let ids = retain_partitioned( + &corpus.records, + &result.clusters, + &context, + target_size, + &policy, + ); + let report = evaluate(&corpus, &ids, &format!("floor_min={floor_min}"), 0, 0); + let f = fitness( + report.worst_cluster_recall, + report.overall_recall, + correctness, + ); + println!( + "floor_min={floor_min:<3} retained={:<5} overall_recall={:.4} worst_cluster_recall={:.4} fitness={:.4}", + report.retained_count, report.overall_recall, report.worst_cluster_recall, f + ); + if best.map(|(_, bf)| f > bf).unwrap_or(true) { + best = Some((floor_min, f)); + } + } + let (winner, winner_fitness) = best.unwrap(); + let parent_floor_min = 3usize; // the value used in the accepted (rejected) main.rs run + let beats_parent = winner != parent_floor_min + && candidates.contains(&winner) + && winner_fitness + > fitness( + { + let policy = RetentionPolicy { + floor_min: parent_floor_min, + }; + let ids = retain_partitioned( + &corpus.records, + &result.clusters, + &context, + target_size, + &policy, + ); + evaluate(&corpus, &ids, "parent", 0, 0).worst_cluster_recall + }, + { + let policy = RetentionPolicy { + floor_min: parent_floor_min, + }; + let ids = retain_partitioned( + &corpus.records, + &result.clusters, + &context, + target_size, + &policy, + ); + evaluate(&corpus, &ids, "parent", 0, 0).overall_recall + }, + correctness, + ); + println!("winner: floor_min={winner} fitness={winner_fitness:.4} beats_parent(floor_min=3)={beats_parent}"); + println!( + "DARWIN_RESULT: {}", + if beats_parent { + "PROMOTE" + } else { + "KEEP_PARENT" + } + ); +} diff --git a/crates/ruvector-partition-memory/src/corpus.rs b/crates/ruvector-partition-memory/src/corpus.rs new file mode 100644 index 0000000000..debcbddcf1 --- /dev/null +++ b/crates/ruvector-partition-memory/src/corpus.rs @@ -0,0 +1,277 @@ +//! Deterministic synthetic agent-memory corpus with unequal-size semantic +//! clusters (including a minority cluster), decoupled recency/frequency +//! signals, and a recency-biased "focus cluster" standing in for the +//! agent's most recent working topic. + +use crate::search::{cosine, normalize, top_k_by_cosine}; +use rand::rngs::StdRng; +use rand::{Rng, SeedableRng}; + +#[derive(Debug, Clone)] +pub struct MemoryRecord { + pub id: u64, + pub embedding: Vec, + /// Ground-truth cluster label. Not visible to any compaction policy — + /// used only for evaluation. + pub cluster: usize, + /// 0 = oldest, n-1 = most recently accessed. + pub last_accessed_tick: u32, + pub access_count: u32, +} + +#[derive(Debug, Clone)] +pub struct Query { + pub embedding: Vec, + pub cluster: usize, + /// True top-k nearest memory ids in the *full, uncompacted* corpus. + pub ground_truth: Vec, +} + +pub struct Corpus { + pub records: Vec, + pub queries: Vec, + pub cluster_sizes: Vec, + pub focus_cluster: usize, + pub dims: usize, +} + +pub struct CorpusConfig { + pub n: usize, + pub dims: usize, + /// Fractional size of each cluster; renormalized to sum to 1 and + /// rounded to integer counts that sum to exactly `n`. + pub cluster_fracs: Vec, + pub queries_per_cluster: usize, + pub k: usize, + pub noise_std: f32, + pub seed: u64, +} + +impl Default for CorpusConfig { + fn default() -> Self { + Self { + n: 4000, + dims: 64, + // 6 clusters: majority, three mid-size, and two minorities + // (5% and 2% of the corpus) that a global top-score compactor + // is free to drop entirely. + cluster_fracs: vec![0.35, 0.25, 0.18, 0.15, 0.05, 0.02], + queries_per_cluster: 25, + k: 10, + // Calibrated (see mincut_exact.rs / examples used during this + // nightly's development): at noise_std=0.35 the kNN graph's + // true global min cut degenerately isolates a single outlier + // vertex (normalized_cut ~0.76, no real topic boundary is the + // weakest link). At 0.25 the min cut cleanly isolates one whole + // semantic cluster (normalized_cut ~0.07) — a graph structure + // this crate's hypothesis can actually be tested against. + noise_std: 0.25, + seed: 42, + } + } +} + +fn random_unit_vector(rng: &mut StdRng, dims: usize) -> Vec { + let mut v: Vec = (0..dims).map(|_| rng.gen_range(-1.0f32..1.0)).collect(); + normalize(&mut v); + v +} + +/// Sample `k` centroids on the unit sphere, retrying (bounded) whenever a +/// new centroid is too close to an existing one so clusters stay separable. +fn well_separated_centroids( + rng: &mut StdRng, + dims: usize, + k: usize, + max_sim: f32, +) -> Vec> { + let mut centroids: Vec> = Vec::with_capacity(k); + for _ in 0..k { + let mut best = random_unit_vector(rng, dims); + for _attempt in 0..200 { + let worst_sim = centroids + .iter() + .map(|c| cosine(&best, c)) + .fold(f32::NEG_INFINITY, f32::max); + if centroids.is_empty() || worst_sim <= max_sim { + break; + } + best = random_unit_vector(rng, dims); + } + centroids.push(best); + } + centroids +} + +fn cluster_counts(n: usize, fracs: &[f64]) -> Vec { + let total: f64 = fracs.iter().sum(); + let mut counts: Vec = fracs + .iter() + .map(|f| ((f / total) * n as f64).round() as usize) + .collect(); + let assigned: usize = counts.iter().sum(); + if assigned != n { + // Reconcile rounding drift against the largest cluster so every + // other cluster keeps its intended (possibly minority) size. + let (largest_idx, _) = counts + .iter() + .enumerate() + .max_by_key(|(_, c)| **c) + .expect("cluster_fracs must be non-empty"); + let diff = n as i64 - assigned as i64; + counts[largest_idx] = (counts[largest_idx] as i64 + diff).max(0) as usize; + } + counts +} + +pub fn generate(cfg: &CorpusConfig) -> Corpus { + let mut rng = StdRng::seed_from_u64(cfg.seed); + let num_clusters = cfg.cluster_fracs.len(); + let centroids = well_separated_centroids(&mut rng, cfg.dims, num_clusters, 0.35); + let counts = cluster_counts(cfg.n, &cfg.cluster_fracs); + // The largest cluster stands in for "what the agent was just working + // on" — recency is biased toward it below, decoupled from frequency. + let focus_cluster = counts + .iter() + .enumerate() + .max_by_key(|(_, c)| **c) + .map(|(i, _)| i) + .unwrap_or(0); + + let mut records: Vec = Vec::with_capacity(cfg.n); + let mut id = 0u64; + for (cluster, &count) in counts.iter().enumerate() { + for _ in 0..count { + let mut emb: Vec = centroids[cluster] + .iter() + .map(|c| c + rng.gen_range(-cfg.noise_std..cfg.noise_std)) + .collect(); + normalize(&mut emb); + let access_count = 1 + (rng.gen::().powf(3.0) * 50.0) as u32; + records.push(MemoryRecord { + id, + embedding: emb, + cluster, + last_accessed_tick: 0, // assigned below + access_count, + }); + id += 1; + } + } + + // Recency: sample a tick-order key per record, biased for the focus + // cluster, then rank into a dense 0..n-1 permutation so ticks stay + // decoupled from frequency and only softly correlated with cluster. + let mut keyed: Vec<(usize, f64)> = records + .iter() + .enumerate() + .map(|(idx, r)| { + let key = if r.cluster == focus_cluster { + rng.gen_range(0.5f64..1.0) + } else { + rng.gen_range(0.0f64..0.85) + }; + (idx, key) + }) + .collect(); + keyed.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap()); + for (tick, (idx, _)) in keyed.into_iter().enumerate() { + records[idx].last_accessed_tick = tick as u32; + } + + // Out-of-sample queries per cluster, with brute-force ground truth + // against the *full* corpus computed once, up front. + let refs: Vec<(u64, &[f32])> = records + .iter() + .map(|r| (r.id, r.embedding.as_slice())) + .collect(); + let mut queries = Vec::with_capacity(num_clusters * cfg.queries_per_cluster); + for (cluster, centroid) in centroids.iter().enumerate() { + for _ in 0..cfg.queries_per_cluster { + let mut emb: Vec = centroid + .iter() + .map(|c| c + rng.gen_range(-cfg.noise_std * 0.5..cfg.noise_std * 0.5)) + .collect(); + normalize(&mut emb); + let ground_truth = top_k_by_cosine(&emb, &refs, cfg.k); + queries.push(Query { + embedding: emb, + cluster, + ground_truth, + }); + } + } + + Corpus { + records, + queries, + cluster_sizes: counts, + focus_cluster, + dims: cfg.dims, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn generates_requested_total_and_cluster_counts() { + let cfg = CorpusConfig { + n: 500, + ..CorpusConfig::default() + }; + let corpus = generate(&cfg); + assert_eq!(corpus.records.len(), 500); + assert_eq!(corpus.cluster_sizes.iter().sum::(), 500); + assert_eq!(corpus.cluster_sizes.len(), cfg.cluster_fracs.len()); + } + + #[test] + fn minority_cluster_is_nonempty() { + let cfg = CorpusConfig { + n: 1000, + ..CorpusConfig::default() + }; + let corpus = generate(&cfg); + let minority = *corpus.cluster_sizes.last().unwrap(); + assert!( + minority >= 15, + "minority cluster too small to test: {minority}" + ); + } + + #[test] + fn is_deterministic_for_fixed_seed() { + let cfg = CorpusConfig { + n: 200, + ..CorpusConfig::default() + }; + let a = generate(&cfg); + let b = generate(&cfg); + assert_eq!(a.records[10].embedding, b.records[10].embedding); + assert_eq!(a.queries[0].ground_truth, b.queries[0].ground_truth); + } + + #[test] + fn ground_truth_query_finds_own_cluster_neighbours() { + let cfg = CorpusConfig { + n: 400, + ..CorpusConfig::default() + }; + let corpus = generate(&cfg); + let q = &corpus.queries[0]; + let hit_own_cluster = q + .ground_truth + .iter() + .filter(|id| corpus.records[**id as usize].cluster == q.cluster) + .count(); + // Centroids are separated (cosine <= 0.35) so a query's true top-k + // should be dominated by its own cluster, not scattered randomly. + assert!( + hit_own_cluster >= cfg.k / 2, + "got {hit_own_cluster}/{}", + cfg.k + ); + } +} diff --git a/crates/ruvector-partition-memory/src/graph.rs b/crates/ruvector-partition-memory/src/graph.rs new file mode 100644 index 0000000000..8e328658f8 --- /dev/null +++ b/crates/ruvector-partition-memory/src/graph.rs @@ -0,0 +1,90 @@ +//! Build a memory similarity graph: an undirected k-NN edge list over +//! memory embeddings, weighted by cosine similarity. + +use crate::corpus::MemoryRecord; +use crate::search::cosine; +use ruvector_mincut::{VertexId, Weight}; +use std::collections::HashSet; + +/// Brute-force k-NN graph construction. O(n^2) in the number of records — +/// fine at this crate's benchmark scale (low thousands) and it is the +/// same ground-truth-quality neighbourhood the recall oracle uses, so the +/// partitioner sees the same notion of "nearby" the evaluator does. +/// Only positive-similarity neighbours become edges; each undirected pair +/// is emitted once. +pub fn build_knn_edges(records: &[MemoryRecord], k: usize) -> Vec<(VertexId, VertexId, Weight)> { + let refs: Vec<(u64, &[f32])> = records + .iter() + .map(|r| (r.id, r.embedding.as_slice())) + .collect(); + + let mut seen: HashSet<(u64, u64)> = HashSet::new(); + let mut edges = Vec::new(); + + for r in records { + let mut scored: Vec<(u64, f32)> = refs + .iter() + .filter(|(id, _)| *id != r.id) + .map(|(id, emb)| (*id, cosine(&r.embedding, emb))) + .collect(); + scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap().then(a.0.cmp(&b.0))); + + for (neighbor_id, sim) in scored.into_iter().take(k) { + if sim <= 0.0 { + continue; + } + let key = if r.id < neighbor_id { + (r.id, neighbor_id) + } else { + (neighbor_id, r.id) + }; + if seen.insert(key) { + edges.push((key.0, key.1, sim as Weight)); + } + } + } + + edges +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::corpus::{generate, CorpusConfig}; + + #[test] + fn every_vertex_gets_at_least_one_edge() { + let cfg = CorpusConfig { + n: 300, + ..CorpusConfig::default() + }; + let corpus = generate(&cfg); + let edges = build_knn_edges(&corpus.records, 10); + let mut touched: HashSet = HashSet::new(); + for (u, v, _) in &edges { + touched.insert(*u); + touched.insert(*v); + } + assert_eq!( + touched.len(), + corpus.records.len(), + "records with no positive-similarity neighbour: {}", + corpus.records.len() - touched.len() + ); + } + + #[test] + fn edges_are_deduplicated_and_undirected() { + let cfg = CorpusConfig { + n: 150, + ..CorpusConfig::default() + }; + let corpus = generate(&cfg); + let edges = build_knn_edges(&corpus.records, 8); + let mut seen = HashSet::new(); + for (u, v, _) in &edges { + assert!(u < v, "expected canonical (min,max) ordering"); + assert!(seen.insert((*u, *v)), "duplicate edge {u}-{v}"); + } + } +} diff --git a/crates/ruvector-partition-memory/src/lib.rs b/crates/ruvector-partition-memory/src/lib.rs new file mode 100644 index 0000000000..0b14b8e6d7 --- /dev/null +++ b/crates/ruvector-partition-memory/src/lib.rs @@ -0,0 +1,54 @@ +//! # ruvector-partition-memory +//! +//! Mincut-partitioned agent-memory consolidation. +//! +//! Nightly research (2026-08-17). Hypothesis: a global top-score compaction +//! policy (the winning `CoherencePolicy` from `ruvector-agent-memory`, +//! nightly 2026-06-14) is a session-context-biased scalar ranking, so a +//! semantic topic unrelated to the agent's most recent working context can +//! be evicted *in full* during compaction even at a favourable overall +//! compaction ratio. This crate tests whether partitioning the memory +//! similarity graph before applying a retention budget — guaranteeing each +//! partition a floor allocation — protects minority topics without +//! materially regressing overall recall. +//! +//! Three variants are benchmarked in `src/main.rs`: +//! +//! - `GlobalTopScore` (baseline): `ruvector_agent_memory::CoherencePolicy` +//! applied directly, scored against a recency-biased context window. +//! - `MincutFixedK` (candidate A): `ruvector_mincut::GraphPartitioner` +//! (existing, unweighted edge-count recursive bisection) with a +//! size-heuristic `K`, then floor+proportional retention per partition. +//! - `MincutAdaptive` (candidate B): a new adaptive-depth recursive +//! bisection that stops each branch once its normalized cut density +//! indicates the component is already coherent, then applies the same +//! floor+proportional retention. +//! +//! `mincut_exact.rs` exists because `ruvector_mincut::DynamicMinCut::partition()` +//! turned out, during this nightly's development, to return a vertex split +//! inconsistent with its own `min_cut_value()` (see that module's doc +//! comment and the nightly research doc for the repro). Candidate B's +//! splits are materialized by a from-scratch, tested Stoer–Wagner +//! implementation instead; `ruvector_mincut`'s value is still queried as an +//! independent cross-check, recorded in the witness chain. +//! +//! Every partition decision candidate B makes is committed to a SHA-256 +//! witness chain (`witness.rs`) so a split cannot be silently re-run to +//! favour a result after the fact. + +pub mod corpus; +pub mod graph; +pub mod metrics; +pub mod mincut_exact; +pub mod partition; +pub mod retention; +pub mod search; +pub mod witness; + +pub use corpus::{Corpus, CorpusConfig, MemoryRecord, Query}; +pub use graph::build_knn_edges; +pub use metrics::{evaluate, VariantReport}; +pub use mincut_exact::global_min_cut; +pub use partition::{adaptive_partition, fixed_k_partition, AdaptiveConfig, PartitionResult}; +pub use retention::{retain_global_top_score, retain_partitioned, RetentionPolicy}; +pub use witness::{PartitionWitnessChain, SplitRecord}; diff --git a/crates/ruvector-partition-memory/src/main.rs b/crates/ruvector-partition-memory/src/main.rs new file mode 100644 index 0000000000..5c16b4c130 --- /dev/null +++ b/crates/ruvector-partition-memory/src/main.rs @@ -0,0 +1,251 @@ +//! Benchmark binary: baseline vs. two mincut-partitioned retention +//! variants on a synthetic clustered agent-memory corpus. +//! +//! ```text +//! cargo run --release -p ruvector-partition-memory --bin benchmark -- \ +//! [n] [floor_min] [coherence_ratio] [fixed_k] [fixed_k_max_n] +//! ``` +//! Defaults: n=4000, floor_min=3, coherence_ratio=0.35, fixed_k=10, fixed_k_max_n=600. +//! +//! `fixed_k_max_n` gates candidate A (`MincutFixedK`, built on the existing +//! `ruvector_mincut::GraphPartitioner`): measured during this nightly's +//! development at 8.4s for n=500 and not finished after 5m42s at n=4000 — +//! see the nightly research doc for the repro. Above this size candidate A +//! is skipped with an explicit note rather than silently hung on or +//! silently omitted. + +use ruvector_partition_memory::corpus::{generate, CorpusConfig, MemoryRecord}; +use ruvector_partition_memory::graph::build_knn_edges; +use ruvector_partition_memory::metrics::{evaluate, VariantReport}; +use ruvector_partition_memory::partition::{adaptive_partition, fixed_k_partition, AdaptiveConfig}; +use ruvector_partition_memory::retention::{ + retain_global_top_score, retain_partitioned, RetentionPolicy, +}; +use std::time::Instant; + +struct Args { + n: usize, + floor_min: usize, + coherence_ratio: f64, + fixed_k: usize, + fixed_k_max_n: usize, +} + +fn parse_args() -> Args { + let args: Vec = std::env::args().collect(); + Args { + n: args.get(1).and_then(|s| s.parse().ok()).unwrap_or(4000), + floor_min: args.get(2).and_then(|s| s.parse().ok()).unwrap_or(3), + coherence_ratio: args.get(3).and_then(|s| s.parse().ok()).unwrap_or(0.35), + fixed_k: args.get(4).and_then(|s| s.parse().ok()).unwrap_or(10), + fixed_k_max_n: args.get(5).and_then(|s| s.parse().ok()).unwrap_or(600), + } +} + +/// The most recently accessed `n_ctx` memories, standing in for "what the +/// agent was just working on" — the realistic, non-strawman context a +/// consolidation event has available, and exactly the scenario in which a +/// pure top-score policy can starve unrelated topics. +fn recency_context(records: &[MemoryRecord], n_ctx: usize) -> Vec> { + let mut sorted: Vec<&MemoryRecord> = records.iter().collect(); + sorted.sort_by(|a, b| b.last_accessed_tick.cmp(&a.last_accessed_tick)); + sorted + .into_iter() + .take(n_ctx) + .map(|r| r.embedding.clone()) + .collect() +} + +fn print_report(r: &VariantReport) { + println!( + "{:<16} retained={:<6} overall_recall={:.4} worst_cluster_recall={:.4} coverage={:.3} partition_us={:<10} retention_us={:<8}", + r.name, r.retained_count, r.overall_recall, r.worst_cluster_recall, r.cluster_coverage, r.partition_us, r.retention_us + ); +} + +fn main() { + let args = parse_args(); + let cfg = CorpusConfig { + n: args.n, + ..CorpusConfig::default() + }; + + let t0 = Instant::now(); + let corpus = generate(&cfg); + let corpus_gen_us = t0.elapsed().as_micros(); + + let target_size = (corpus.records.len() as f64 * 0.5).round() as usize; + let context = recency_context(&corpus.records, 20); + let vertices: Vec = corpus.records.iter().map(|r| r.id).collect(); + let policy = RetentionPolicy { + floor_min: args.floor_min, + }; + + let t0 = Instant::now(); + let edges = build_knn_edges(&corpus.records, 10); + let knn_us = t0.elapsed().as_micros(); + + println!("=== ruvector-partition-memory nightly benchmark ==="); + println!( + "n={} dims={} cluster_sizes={:?} focus_cluster={} target_size={} (50% retention) queries={}", + corpus.records.len(), + corpus.dims, + corpus.cluster_sizes, + corpus.focus_cluster, + target_size, + corpus.queries.len() + ); + println!( + "params: floor_min={} coherence_ratio={} fixed_k={} fixed_k_max_n={}", + args.floor_min, args.coherence_ratio, args.fixed_k, args.fixed_k_max_n + ); + println!( + "corpus_gen_us={corpus_gen_us} knn_graph_us={knn_us} knn_edges={}", + edges.len() + ); + println!(); + + // ---- Baseline: GlobalTopScore (ruvector-agent-memory CoherencePolicy) ---- + let t0 = Instant::now(); + let baseline_ids = retain_global_top_score(&corpus.records, &context, target_size); + let baseline_us = t0.elapsed().as_micros(); + let baseline_report = evaluate(&corpus, &baseline_ids, "GlobalTopScore", 0, baseline_us); + + // ---- Candidate A: MincutFixedK (existing ruvector_mincut::GraphPartitioner) ---- + let report_a: Option = if corpus.records.len() <= args.fixed_k_max_n { + let t0 = Instant::now(); + let parts_a = fixed_k_partition(&vertices, &edges, args.fixed_k); + let partition_a_us = t0.elapsed().as_micros(); + let t0 = Instant::now(); + let ids_a = retain_partitioned(&corpus.records, &parts_a, &context, target_size, &policy); + let retention_a_us = t0.elapsed().as_micros(); + println!( + "MincutFixedK partitions: {} (K={} requested) sizes={:?}", + parts_a.len(), + args.fixed_k, + parts_a.iter().map(|p| p.len()).collect::>() + ); + Some(evaluate( + &corpus, + &ids_a, + "MincutFixedK", + partition_a_us, + retention_a_us, + )) + } else { + println!( + "MincutFixedK: SKIPPED (n={} exceeds fixed_k_max_n={}; ruvector_mincut::GraphPartitioner \ + measured at 8.4s for n=500 and did not finish in 5m42s for n=4000 during this nightly's \ + development — see the research doc for the repro)", + corpus.records.len(), + args.fixed_k_max_n + ); + None + }; + + // ---- Candidate B: MincutAdaptive (new adaptive-depth bisection) ---- + let adaptive_cfg = AdaptiveConfig { + coherence_ratio: args.coherence_ratio, + ..AdaptiveConfig::default() + }; + let t0 = Instant::now(); + let result_b = adaptive_partition(&edges, &vertices, &adaptive_cfg); + let partition_b_us = t0.elapsed().as_micros(); + let t0 = Instant::now(); + let ids_b = retain_partitioned( + &corpus.records, + &result_b.clusters, + &context, + target_size, + &policy, + ); + let retention_b_us = t0.elapsed().as_micros(); + let report_b = evaluate( + &corpus, + &ids_b, + "MincutAdaptive", + partition_b_us, + retention_b_us, + ); + println!( + "MincutAdaptive partitions: {} (coherence_ratio={}) sizes={:?}", + result_b.clusters.len(), + args.coherence_ratio, + result_b + .clusters + .iter() + .map(|p| p.len()) + .collect::>() + ); + println!( + "MincutAdaptive witness: {} split steps, chain_verify={}, head={}", + result_b.witness.steps.len(), + result_b.witness.verify(), + result_b.witness.head() + ); + println!(); + + println!("variant retained overall_recall worst_cluster_recall coverage partition_us retention_us"); + print_report(&baseline_report); + if let Some(ref r) = report_a { + print_report(r); + } + print_report(&report_b); + println!(); + + println!( + "per_cluster_recall GlobalTopScore = {:?}", + baseline_report.per_cluster_recall + ); + if let Some(ref r) = report_a { + println!( + "per_cluster_recall MincutFixedK = {:?}", + r.per_cluster_recall + ); + } + println!( + "per_cluster_recall MincutAdaptive = {:?}", + report_b.per_cluster_recall + ); + println!(); + + // ---- Pre-declared acceptance thresholds (fixed before the accepted + // run; not adjusted after seeing its results — see the nightly research + // doc. The latency budget was set from a real calibration pass on this + // machine, run before the accepted benchmark: MincutAdaptive measured + // ~11.1s at n=4000, ratio=0.35, so the threshold below is set with + // headroom above that measurement, not tightened around it.) + const WORST_CLUSTER_GAIN_THRESHOLD: f64 = 0.15; // best candidate must beat baseline worst-cluster recall by >= 15pp + const MAX_OVERALL_REGRESSION: f64 = 0.05; // candidates must not lose more than 5pp overall recall vs baseline + const MAX_LATENCY_US: u128 = 30_000_000; // partition+retention wall time per candidate, this n + + let best_candidate = match &report_a { + Some(a) if a.worst_cluster_recall > report_b.worst_cluster_recall => a, + _ => &report_b, + }; + let worst_gain = best_candidate.worst_cluster_recall - baseline_report.worst_cluster_recall; + let gain_ok = worst_gain >= WORST_CLUSTER_GAIN_THRESHOLD; + let overall_ok = report_a.as_ref().is_none_or(|a| { + a.overall_recall >= baseline_report.overall_recall - MAX_OVERALL_REGRESSION + }) && report_b.overall_recall + >= baseline_report.overall_recall - MAX_OVERALL_REGRESSION; + let latency_ok = report_a + .as_ref() + .is_none_or(|a| (a.partition_us + a.retention_us) < MAX_LATENCY_US) + && (report_b.partition_us + report_b.retention_us) < MAX_LATENCY_US; + let witness_ok = result_b.witness.verify(); + + println!( + "acceptance: best_candidate={} worst_cluster_gain_pp={:.2} (threshold_pp={:.2}) gain_ok={gain_ok} overall_ok={overall_ok} latency_ok={latency_ok} witness_ok={witness_ok}", + best_candidate.name, + worst_gain * 100.0, + WORST_CLUSTER_GAIN_THRESHOLD * 100.0 + ); + + let verdict = if gain_ok && overall_ok && latency_ok && witness_ok { + "ACCEPT" + } else { + "REJECT" + }; + println!("ACCEPTANCE_RESULT: {verdict}"); +} diff --git a/crates/ruvector-partition-memory/src/metrics.rs b/crates/ruvector-partition-memory/src/metrics.rs new file mode 100644 index 0000000000..854ffc8779 --- /dev/null +++ b/crates/ruvector-partition-memory/src/metrics.rs @@ -0,0 +1,126 @@ +//! Evaluation: recall@k against the corpus's brute-force ground truth, +//! computed overall and per-cluster, plus cluster coverage. + +use crate::corpus::Corpus; +use crate::search::{recall_at_k, top_k_by_cosine}; +use std::collections::{HashMap, HashSet}; + +#[derive(Debug, Clone)] +pub struct VariantReport { + pub name: String, + pub retained_count: usize, + pub overall_recall: f64, + pub worst_cluster_recall: f64, + pub per_cluster_recall: Vec, + pub cluster_coverage: f64, + pub partition_us: u128, + pub retention_us: u128, +} + +pub fn evaluate( + corpus: &Corpus, + retained_ids: &[u64], + name: &str, + partition_us: u128, + retention_us: u128, +) -> VariantReport { + let by_id: HashMap = + corpus.records.iter().map(|r| (r.id, r)).collect(); + let refs: Vec<(u64, &[f32])> = retained_ids + .iter() + .filter_map(|id| by_id.get(id).map(|r| (*id, r.embedding.as_slice()))) + .collect(); + + let num_clusters = corpus.cluster_sizes.len(); + let mut sum_per_cluster = vec![0.0f64; num_clusters]; + let mut count_per_cluster = vec![0usize; num_clusters]; + let mut overall_sum = 0.0f64; + + for q in &corpus.queries { + let retrieved = top_k_by_cosine(&q.embedding, &refs, q.ground_truth.len()); + let r = recall_at_k(&retrieved, &q.ground_truth); + overall_sum += r; + sum_per_cluster[q.cluster] += r; + count_per_cluster[q.cluster] += 1; + } + + let overall_recall = overall_sum / corpus.queries.len() as f64; + let per_cluster_recall: Vec = sum_per_cluster + .iter() + .zip(&count_per_cluster) + .map(|(s, c)| if *c > 0 { s / *c as f64 } else { 0.0 }) + .collect(); + let worst_cluster_recall = per_cluster_recall + .iter() + .cloned() + .fold(f64::INFINITY, f64::min); + + let present_clusters: HashSet = retained_ids + .iter() + .filter_map(|id| by_id.get(id).map(|r| r.cluster)) + .collect(); + let cluster_coverage = present_clusters.len() as f64 / num_clusters as f64; + + VariantReport { + name: name.to_string(), + retained_count: retained_ids.len(), + overall_recall, + worst_cluster_recall, + per_cluster_recall, + cluster_coverage, + partition_us, + retention_us, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::corpus::{generate, CorpusConfig}; + + #[test] + fn retaining_everything_gives_perfect_recall() { + let cfg = CorpusConfig { + n: 300, + ..CorpusConfig::default() + }; + let corpus = generate(&cfg); + let all_ids: Vec = corpus.records.iter().map(|r| r.id).collect(); + let report = evaluate(&corpus, &all_ids, "identity", 0, 0); + assert!((report.overall_recall - 1.0).abs() < 1e-9); + assert!((report.worst_cluster_recall - 1.0).abs() < 1e-9); + assert!((report.cluster_coverage - 1.0).abs() < 1e-9); + } + + #[test] + fn dropping_a_cluster_entirely_zeroes_its_recall() { + let cfg = CorpusConfig { + n: 300, + ..CorpusConfig::default() + }; + let corpus = generate(&cfg); + let minority_cluster = corpus.cluster_sizes.len() - 1; + let ids: Vec = corpus + .records + .iter() + .filter(|r| r.cluster != minority_cluster) + .map(|r| r.id) + .collect(); + let report = evaluate(&corpus, &ids, "drop-minority", 0, 0); + // Clusters are separated by cosine <= 0.35 with noise_std 0.35, so + // neighbouring clusters' true top-k can still contain a few + // surviving points even with the minority cluster gone entirely — + // recall degrades sharply but is not guaranteed to hit exactly + // zero. That partial-credit behaviour is itself realistic (real + // topic boundaries are not perfectly separable either) and is + // exactly why this crate measures *worst-cluster* recall as a + // continuous quantity rather than a binary "cluster present" flag. + assert!( + report.per_cluster_recall[minority_cluster] < 0.6, + "expected sharply degraded recall for the dropped cluster, got {}", + report.per_cluster_recall[minority_cluster] + ); + assert!(report.worst_cluster_recall <= report.per_cluster_recall[minority_cluster]); + assert!(report.cluster_coverage < 1.0); + } +} diff --git a/crates/ruvector-partition-memory/src/mincut_exact.rs b/crates/ruvector-partition-memory/src/mincut_exact.rs new file mode 100644 index 0000000000..4b8091cc5a --- /dev/null +++ b/crates/ruvector-partition-memory/src/mincut_exact.rs @@ -0,0 +1,366 @@ +//! Self-contained weighted Stoer–Wagner global minimum cut. +//! +//! `partition.rs` originally called `ruvector_mincut::DynamicMinCut::partition()` +//! directly to materialize each split. That turned out to be unsafe to trust: +//! diagnosed with a minimal repro (two triangles joined by one weak-weight +//! bridge edge, `examples/debug_mincut.rs` during this nightly's development), +//! `DynamicMinCut::min_cut_value()` reliably reports the correct cut weight, +//! but `DynamicMinCut::partition()` — and, derivatively, the unweighted +//! `RuVectorGraphAnalyzer` / `GraphPartitioner` path — returns a vertex split +//! that is *inconsistent with that value* and nondeterministic across runs: +//! of three runs on the 6-vertex repro, two returned the correct {0,1,2} vs +//! {3,4,5} split and one returned a degenerate {single vertex} vs {rest} +//! split; on a 100-vertex version (two 50-cliques joined by one weak edge) +//! every run returned the degenerate split. See the nightly research doc +//! (docs/research/nightly/2026-08-17_mincut-partitioned-memory-consolidation) +//! for the full repro and evidence — this is being reported upstream as a +//! defect, not silently patched around. +//! +//! This module is the workaround: a correct, deterministic, from-scratch +//! implementation used as the *sole* source of partition vertex sets in +//! this crate. `ruvector_mincut::DynamicMinCut::min_cut_value()` is still +//! called from `partition.rs` as an independent cross-check value (its +//! *value* output was never observed to be wrong) — its agreement or +//! disagreement with this module's result is recorded in the witness chain. + +use ruvector_mincut::{VertexId, Weight}; +use std::cmp::Ordering; +use std::collections::{BinaryHeap, HashMap}; + +#[derive(Copy, Clone)] +struct HeapItem { + weight: f64, + vertex: usize, +} + +impl PartialEq for HeapItem { + fn eq(&self, other: &Self) -> bool { + self.weight == other.weight + } +} +impl Eq for HeapItem {} +impl PartialOrd for HeapItem { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} +impl Ord for HeapItem { + fn cmp(&self, other: &Self) -> Ordering { + self.weight + .partial_cmp(&other.weight) + .unwrap_or(Ordering::Equal) + } +} + +/// Global minimum cut of a weighted undirected graph via Stoer–Wagner. +/// Returns `(cut_value, side_a, side_b)`, `side_a` being the smaller-index +/// "last vertex merged" side of the best phase found. Deterministic for a +/// fixed input ordering. `O(V) merge phases, each an O(E log V) maximum +/// adjacency search)` — the standard binary-heap formulation. +/// +/// Panics only on the precondition `vertices.len() >= 2`, checked by the +/// caller (`partition.rs` never calls this below `min_cluster_size`, which +/// defaults to 20). +pub fn global_min_cut( + vertices: &[VertexId], + edges: &[(VertexId, VertexId, Weight)], +) -> (f64, Vec, Vec) { + let n = vertices.len(); + assert!(n >= 2, "global_min_cut requires at least 2 vertices"); + let index_of: HashMap = + vertices.iter().enumerate().map(|(i, &v)| (v, i)).collect(); + + let mut adj: Vec> = vec![HashMap::new(); n]; + for &(u, v, w) in edges { + if let (Some(&iu), Some(&iv)) = (index_of.get(&u), index_of.get(&v)) { + if iu == iv { + continue; + } + *adj[iu].entry(iv).or_insert(0.0) += w; + *adj[iv].entry(iu).or_insert(0.0) += w; + } + } + + // Stoer-Wagner's maximum-adjacency search assumes a connected graph — + // at low corpus noise, an induced kNN subgraph can genuinely be + // disconnected (a discovered crash during this nightly's development: + // the heap-based phase ran out of reachable vertices). A disconnected + // graph's true global min cut is 0 (split along the existing component + // boundary for free), so detect that up front via BFS and short-circuit + // rather than feed a disconnected graph to a connected-graph algorithm. + if let Some((component, rest)) = find_disconnected_split(&adj, n) { + let side_a: Vec = component.iter().map(|&i| vertices[i]).collect(); + let side_b: Vec = rest.iter().map(|&i| vertices[i]).collect(); + return (0.0, side_a, side_b); + } + + let mut merged_members: Vec> = (0..n).map(|i| vec![i]).collect(); + let mut active = vec![true; n]; + let mut remaining = n; + + let mut best_cut = f64::INFINITY; + let mut best_side: Vec = Vec::new(); + + while remaining > 1 { + let (order, weight_to_a) = maximum_adjacency_ordering(&adj, &active, n, remaining); + let t = order[order.len() - 1]; + let s = order[order.len() - 2]; + let cut_of_phase = weight_to_a[t]; + + if cut_of_phase < best_cut { + best_cut = cut_of_phase; + best_side = merged_members[t].clone(); + } + + // Merge t into s (standard Stoer-Wagner vertex contraction). + let t_neighbors: Vec<(usize, f64)> = adj[t].iter().map(|(&k, &v)| (k, v)).collect(); + for (nb, w) in t_neighbors { + if nb == s { + continue; + } + *adj[s].entry(nb).or_insert(0.0) += w; + *adj[nb].entry(s).or_insert(0.0) += w; + adj[nb].remove(&t); + } + adj[s].remove(&t); + adj[t].clear(); + let t_members = std::mem::take(&mut merged_members[t]); + merged_members[s].extend(t_members); + active[t] = false; + remaining -= 1; + } + + let side_a_set: std::collections::HashSet = best_side.iter().copied().collect(); + let side_a: Vec = best_side.iter().map(|&i| vertices[i]).collect(); + let side_b: Vec = (0..n) + .filter(|i| !side_a_set.contains(i)) + .map(|i| vertices[i]) + .collect(); + (best_cut, side_a, side_b) +} + +/// BFS from vertex 0; if it does not reach every vertex, returns +/// `Some((reached, unreached))`. `None` means the graph is connected. +fn find_disconnected_split( + adj: &[HashMap], + n: usize, +) -> Option<(Vec, Vec)> { + let mut visited = vec![false; n]; + let mut stack = vec![0usize]; + visited[0] = true; + let mut reached = vec![0usize]; + while let Some(v) = stack.pop() { + for &nb in adj[v].keys() { + if !visited[nb] { + visited[nb] = true; + reached.push(nb); + stack.push(nb); + } + } + } + if reached.len() == n { + None + } else { + let unreached: Vec = (0..n).filter(|&i| !visited[i]).collect(); + Some((reached, unreached)) + } +} + +/// One Stoer-Wagner phase: a Prim-like maximum-adjacency traversal over the +/// currently active (super-)vertices. Returns the visitation order and each +/// vertex's final accumulated weight-to-the-visited-set (`weight_to_a`); the +/// last two entries in `order` are the phase's `s` and `t`. +fn maximum_adjacency_ordering( + adj: &[HashMap], + active: &[bool], + n: usize, + remaining: usize, +) -> (Vec, Vec) { + let mut in_a = vec![false; n]; + let mut weight_to_a = vec![0.0f64; n]; + let mut order = Vec::with_capacity(remaining); + let mut heap: BinaryHeap = BinaryHeap::new(); + + let start = (0..n) + .find(|&i| active[i]) + .expect("at least one active vertex"); + in_a[start] = true; + order.push(start); + for (&nb, &w) in &adj[start] { + if active[nb] && !in_a[nb] { + weight_to_a[nb] += w; + heap.push(HeapItem { + weight: weight_to_a[nb], + vertex: nb, + }); + } + } + + while order.len() < remaining { + let next = loop { + let top = heap + .pop() + .expect("heap exhausted before all active vertices visited"); + if !active[top.vertex] || in_a[top.vertex] { + continue; // stale: vertex merged away or already visited + } + if (top.weight - weight_to_a[top.vertex]).abs() > 1e-9 { + continue; // stale: a fresher, larger entry for this vertex exists + } + break top.vertex; + }; + in_a[next] = true; + order.push(next); + for (&nb, &w) in &adj[next] { + if active[nb] && !in_a[nb] { + weight_to_a[nb] += w; + heap.push(HeapItem { + weight: weight_to_a[nb], + vertex: nb, + }); + } + } + } + + (order, weight_to_a) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn side_of(v: VertexId, side_a: &[VertexId]) -> bool { + side_a.contains(&v) + } + + #[test] + fn finds_the_weak_bridge_between_two_triangles() { + let vertices: Vec = vec![0, 1, 2, 3, 4, 5]; + let edges = vec![ + (0, 1, 1.0), + (1, 2, 1.0), + (2, 0, 1.0), + (3, 4, 1.0), + (4, 5, 1.0), + (5, 3, 1.0), + (2, 3, 0.05), + ]; + let (value, side_a, side_b) = global_min_cut(&vertices, &edges); + assert!((value - 0.05).abs() < 1e-9, "got {value}"); + assert_eq!(side_a.len(), 3); + assert_eq!(side_b.len(), 3); + // {0,1,2} must be together, {3,4,5} must be together. + let a_has_0 = side_of(0, &side_a); + for v in [0u64, 1, 2] { + assert_eq!( + side_of(v, &side_a), + a_has_0, + "vertex {v} split from its triangle" + ); + } + for v in [3u64, 4, 5] { + assert_eq!( + side_of(v, &side_a), + !a_has_0, + "vertex {v} split from its triangle" + ); + } + } + + #[test] + fn finds_the_weak_bridge_at_larger_scale() { + let mut edges = Vec::new(); + for i in 0..50u64 { + for j in (i + 1)..50u64 { + edges.push((i, j, 1.0)); + } + } + for i in 50..100u64 { + for j in (i + 1)..100u64 { + edges.push((i, j, 1.0)); + } + } + edges.push((0, 50, 0.01)); + let vertices: Vec = (0..100).collect(); + let (value, side_a, side_b) = global_min_cut(&vertices, &edges); + assert!((value - 0.01).abs() < 1e-9, "got {value}"); + assert_eq!(side_a.len().min(side_b.len()), 50); + assert_eq!(side_a.len().max(side_b.len()), 50); + } + + #[test] + fn disconnected_graph_returns_zero_cut_without_panicking() { + // Two triangles with NO edge between them at all: the classic + // Stoer-Wagner phase (which assumes connectivity) would exhaust its + // heap before visiting every vertex. This was an actual crash + // discovered while calibrating this crate's corpus generator at + // low noise, where an induced kNN subgraph came out disconnected. + let vertices: Vec = vec![0, 1, 2, 3, 4, 5]; + let edges = vec![ + (0, 1, 1.0), + (1, 2, 1.0), + (2, 0, 1.0), + (3, 4, 1.0), + (4, 5, 1.0), + (5, 3, 1.0), + ]; + let (value, side_a, side_b) = global_min_cut(&vertices, &edges); + assert_eq!(value, 0.0); + assert_eq!(side_a.len().min(side_b.len()), 3); + assert_eq!(side_a.len().max(side_b.len()), 3); + } + + #[test] + fn single_edge_two_vertices() { + let vertices: Vec = vec![7, 9]; + let edges = vec![(7, 9, 2.5)]; + let (value, side_a, side_b) = global_min_cut(&vertices, &edges); + assert!((value - 2.5).abs() < 1e-9); + assert_eq!(side_a.len(), 1); + assert_eq!(side_b.len(), 1); + } + + /// This graph's minimum cut is *unique*: isolating any single triangle + /// vertex costs >= 1.0 (two clique edges), so the 0.05 bridge cut + /// strictly dominates every alternative and {0,1,2}/{3,4,5} is the only + /// partition that achieves it. Repeated calls must therefore agree on + /// cut value and side membership every time — but not necessarily on + /// the internal `Vec` ordering within a side (which vertex plays the + /// Stoer-Wagner "s" vs "t" role on a weight tie is an implementation + /// detail, not part of the correctness contract), so this test + /// canonicalizes each side as a sorted set before comparing. + #[test] + fn deterministic_across_repeated_calls() { + let vertices: Vec = vec![0, 1, 2, 3, 4, 5]; + let edges = vec![ + (0, 1, 1.0), + (1, 2, 1.0), + (2, 0, 1.0), + (3, 4, 1.0), + (4, 5, 1.0), + (5, 3, 1.0), + (2, 3, 0.05), + ]; + fn canonical( + mut side_a: Vec, + mut side_b: Vec, + ) -> (Vec, Vec) { + side_a.sort_unstable(); + side_b.sort_unstable(); + if side_a.first() > side_b.first() { + std::mem::swap(&mut side_a, &mut side_b); + } + (side_a, side_b) + } + + let (value0, a0, b0) = global_min_cut(&vertices, &edges); + let (canon_a0, canon_b0) = canonical(a0, b0); + for _ in 0..5 { + let (value, a, b) = global_min_cut(&vertices, &edges); + assert!((value - value0).abs() < 1e-9); + let (canon_a, canon_b) = canonical(a, b); + assert_eq!(canon_a, canon_a0); + assert_eq!(canon_b, canon_b0); + } + } +} diff --git a/crates/ruvector-partition-memory/src/partition.rs b/crates/ruvector-partition-memory/src/partition.rs new file mode 100644 index 0000000000..039a4dbfcc --- /dev/null +++ b/crates/ruvector-partition-memory/src/partition.rs @@ -0,0 +1,241 @@ +//! Two ways to turn a memory similarity graph into semantic partitions. +//! +//! `fixed_k_partition` wraps the existing `ruvector_mincut::GraphPartitioner` +//! (unweighted, edge-count min cut, fixed `K`). `adaptive_partition` is new: +//! it recurses directly on `ruvector_mincut::MinCutBuilder` / +//! `DynamicMinCut` — the crate's weighted, subpolynomial exact algorithm — +//! and stops splitting a component once its cut is dense relative to its +//! internal edge weight, instead of forcing a caller-chosen `K`. + +use crate::mincut_exact::global_min_cut; +use crate::witness::PartitionWitnessChain; +use ruvector_mincut::{DynamicGraph, GraphPartitioner, MinCutBuilder, VertexId, Weight}; +use std::collections::HashSet; +use std::sync::Arc; + +pub struct AdaptiveConfig { + /// Never split a component smaller than this. + pub min_cluster_size: usize, + /// Stop splitting once `cut_value / (total_internal_weight / |verts|)` + /// meets or exceeds this ratio — the component's weakest seam is + /// already about as strong as its typical internal edge, so treat it + /// as one coherent topic. Lower = more, smaller partitions. + pub coherence_ratio: f64, + /// Hard recursion depth cap, independent of the ratio, so a pathological + /// input graph cannot blow the stack. + pub max_depth: usize, +} + +impl Default for AdaptiveConfig { + fn default() -> Self { + Self { + min_cluster_size: 20, + coherence_ratio: 0.35, + max_depth: 8, + } + } +} + +pub struct PartitionResult { + pub clusters: Vec>, + pub witness: PartitionWitnessChain, +} + +/// Existing-tool baseline partitioner: `ruvector_mincut::GraphPartitioner` +/// recursively bisects down to at most `num_partitions` leaves using +/// unweighted min-cut (fewest crossing edges), independent of how strong +/// those edges are. +/// +/// During this nightly's development `GraphPartitioner` (via +/// `RuVectorGraphAnalyzer`'s `partition()`) was observed to have two +/// distinct vertex-set failure modes, independent of the weighted-cut +/// inconsistency `mincut_exact.rs` documents: (1) at moderate scale it can +/// drop vertices outright — a 100-vertex, two-clique repro returned +/// partitions covering only 50 of the 100 input vertices; (2) with a +/// non-contiguous vertex-id space it can *fabricate* ids that were never in +/// the input graph at all (repro: ids `{1,2,3,11,12,13}` produced a +/// partition also containing invented ids like `4,5,6,7,8,9,10`). See the +/// nightly research doc for both repros. Rather than silently shrinking or +/// polluting the corpus, this wrapper (a) drops any returned id absent from +/// `all_vertices`, then (b) appends any `all_vertices` id absent from every +/// returned group as one final "uncovered" group. This does not fix the +/// partitioner's own boundary quality — it is tested and reported as-is — +/// it only prevents its output from inventing or discarding memories. +pub fn fixed_k_partition( + all_vertices: &[VertexId], + edges: &[(VertexId, VertexId, Weight)], + num_partitions: usize, +) -> Vec> { + let graph = Arc::new(DynamicGraph::new()); + for &(u, v, w) in edges { + let _ = graph.insert_edge(u, v, w); + } + let valid: HashSet = all_vertices.iter().copied().collect(); + let partitioner = GraphPartitioner::new(graph, num_partitions.max(1)); + let mut parts: Vec> = partitioner + .partition() + .into_iter() + .map(|p| { + p.into_iter() + .filter(|v| valid.contains(v)) + .collect::>() + }) + .filter(|p| !p.is_empty()) + .collect(); + + let covered: HashSet = parts.iter().flatten().copied().collect(); + let uncovered: Vec = all_vertices + .iter() + .copied() + .filter(|v| !covered.contains(v)) + .collect(); + if !uncovered.is_empty() { + parts.push(uncovered); + } + parts +} + +fn induced_edges( + all_edges: &[(VertexId, VertexId, Weight)], + verts: &HashSet, +) -> Vec<(VertexId, VertexId, Weight)> { + all_edges + .iter() + .filter(|(u, v, _)| verts.contains(u) && verts.contains(v)) + .copied() + .collect() +} + +pub fn adaptive_partition( + all_edges: &[(VertexId, VertexId, Weight)], + all_vertices: &[VertexId], + cfg: &AdaptiveConfig, +) -> PartitionResult { + let mut clusters = Vec::new(); + let mut witness = PartitionWitnessChain::new(); + recurse(all_edges, all_vertices, 0, cfg, &mut clusters, &mut witness); + PartitionResult { clusters, witness } +} + +fn recurse( + all_edges: &[(VertexId, VertexId, Weight)], + verts: &[VertexId], + depth: usize, + cfg: &AdaptiveConfig, + clusters: &mut Vec>, + witness: &mut PartitionWitnessChain, +) { + if verts.len() <= cfg.min_cluster_size || depth >= cfg.max_depth { + clusters.push(verts.to_vec()); + return; + } + + let vset: HashSet = verts.iter().copied().collect(); + let sub_edges = induced_edges(all_edges, &vset); + if sub_edges.is_empty() { + clusters.push(verts.to_vec()); + return; + } + let total_weight: f64 = sub_edges.iter().map(|(_, _, w)| w).sum(); + let avg_weight_per_vertex = (total_weight / verts.len() as f64).max(1e-9); + + // Authoritative split: our own tested Stoer-Wagner (mincut_exact.rs). + // `ruvector_mincut::MinCutBuilder::min_cut_value()` is queried purely as + // an independent cross-check — see mincut_exact.rs's doc comment for + // why its own `.partition()` output is not used here. + let (cut_value, side_a, side_b) = global_min_cut(verts, &sub_edges); + + let cross_check = MinCutBuilder::new().exact().with_edges(sub_edges).build(); + if let Ok(reference) = cross_check { + let reference_value = reference.min_cut_value(); + if (reference_value - cut_value).abs() > 1e-6 * reference_value.max(1.0) { + // Disagreement between the two independent computations of the + // *value* (not just the partition materialization bug this + // module works around) would be a correctness issue in our own + // code — surface it loudly rather than silently trusting ours. + debug_assert!( + false, + "min-cut value mismatch: ours={cut_value} ruvector_mincut={reference_value}" + ); + } + } + + let normalized_cut = cut_value / avg_weight_per_vertex; + + let should_split = side_a.len() >= cfg.min_cluster_size + && side_b.len() >= cfg.min_cluster_size + && normalized_cut < cfg.coherence_ratio; + + witness.record(verts, cut_value, &side_a, &side_b, should_split); + + if should_split { + recurse(all_edges, &side_a, depth + 1, cfg, clusters, witness); + recurse(all_edges, &side_b, depth + 1, cfg, clusters, witness); + } else { + clusters.push(verts.to_vec()); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Two tight cliques joined by a single weak bridge edge should split + /// into exactly two clusters at a permissive coherence ratio. + #[test] + fn splits_two_cliques_joined_by_weak_bridge() { + let mut edges = Vec::new(); + for &(u, v) in &[(1, 2), (2, 3), (3, 1)] { + edges.push((u, v, 1.0)); + } + for &(u, v) in &[(11, 12), (12, 13), (13, 11)] { + edges.push((u, v, 1.0)); + } + edges.push((3, 11, 0.05)); + let verts: Vec = vec![1, 2, 3, 11, 12, 13]; + let cfg = AdaptiveConfig { + min_cluster_size: 2, + coherence_ratio: 0.5, + max_depth: 4, + }; + let result = adaptive_partition(&edges, &verts, &cfg); + assert_eq!(result.clusters.len(), 2, "{:?}", result.clusters); + assert!(result.witness.verify()); + } + + #[test] + fn does_not_split_a_single_dense_cluster() { + let mut edges = Vec::new(); + for &(u, v) in &[(1, 2), (2, 3), (3, 4), (4, 1), (1, 3), (2, 4)] { + edges.push((u, v, 1.0)); + } + let verts: Vec = vec![1, 2, 3, 4]; + let cfg = AdaptiveConfig { + min_cluster_size: 2, + coherence_ratio: 0.35, + max_depth: 4, + }; + let result = adaptive_partition(&edges, &verts, &cfg); + assert_eq!(result.clusters.len(), 1); + } + + #[test] + fn fixed_k_partition_respects_k_and_covers_all_vertices() { + let mut edges = Vec::new(); + for &(u, v) in &[(1, 2), (2, 3), (3, 1)] { + edges.push((u, v, 1.0)); + } + for &(u, v) in &[(11, 12), (12, 13), (13, 11)] { + edges.push((u, v, 1.0)); + } + edges.push((3, 11, 0.05)); + let verts: Vec = vec![1, 2, 3, 11, 12, 13]; + let parts = fixed_k_partition(&verts, &edges, 2); + let total: usize = parts.iter().map(|p| p.len()).sum(); + // GraphPartitioner may not achieve exactly K groups (or, per the + // documented defect, may leave vertices uncovered before the + // fallback bucket runs) — the one invariant this test enforces is + // that fixed_k_partition itself never silently drops a vertex. + assert_eq!(total, 6, "fixed_k_partition dropped vertices: {parts:?}"); + } +} diff --git a/crates/ruvector-partition-memory/src/retention.rs b/crates/ruvector-partition-memory/src/retention.rs new file mode 100644 index 0000000000..52254e8e6d --- /dev/null +++ b/crates/ruvector-partition-memory/src/retention.rs @@ -0,0 +1,187 @@ +//! Retention policies: turn a memory set (optionally already partitioned +//! into topic clusters) plus a target size into a retained-id list. +//! +//! Every policy here scores candidates with the *same* scorer — +//! `ruvector_agent_memory::CoherencePolicy`, the winning policy from the +//! 2026-06-14 nightly — so the only independent variable under test is how +//! the retention *budget* is allocated (globally vs. per-partition with a +//! floor), not the scoring function itself. + +use crate::corpus::MemoryRecord; +use ruvector_agent_memory::{CoherencePolicy, CompactionPolicy, MemoryEntry}; +use ruvector_mincut::VertexId; +use std::collections::HashMap; + +fn to_entry(r: &MemoryRecord) -> MemoryEntry { + MemoryEntry { + id: r.id, + vector: r.embedding.clone(), + label: None, + created_at: 0, + last_accessed_at: r.last_accessed_tick as u64, + access_count: r.access_count as u64, + } +} + +/// Global top-score baseline: score every record with `CoherencePolicy` +/// and keep the `target_size` highest, with no notion of topic partitions. +/// This is the exact mechanism a session-context-biased consolidation +/// event applies today (nightly 2026-06-14) — a fair, non-strawman +/// baseline, not a deliberately weak one. +pub fn retain_global_top_score( + records: &[MemoryRecord], + context_window: &[Vec], + target_size: usize, +) -> Vec { + let entries: Vec = records.iter().map(to_entry).collect(); + let survivors = CoherencePolicy::default().select_survivors( + &entries, + target_size.min(entries.len()), + context_window, + ); + survivors.into_iter().map(|i| entries[i].id).collect() +} + +pub struct RetentionPolicy { + /// Minimum records guaranteed to each non-empty partition, capacity + /// permitting. This is the mechanism that protects minority topics: + /// a proportional-only allocation (`floor_min = 0`) can round a small + /// partition's share down to zero and drop it entirely. + pub floor_min: usize, +} + +impl Default for RetentionPolicy { + fn default() -> Self { + Self { floor_min: 3 } + } +} + +/// Largest-remainder apportionment of `total` across `weights`, capped +/// per-bucket by `caps`. Guaranteed to allocate exactly +/// `total.min(caps.iter().sum())`. +fn apportion_capped(total: usize, weights: &[f64], caps: &[usize]) -> Vec { + let n = weights.len(); + if n == 0 || total == 0 { + return vec![0; n]; + } + let sum_w: f64 = weights.iter().sum(); + let raw: Vec = if sum_w > 0.0 { + weights.iter().map(|w| total as f64 * w / sum_w).collect() + } else { + vec![0.0; n] + }; + let mut alloc: Vec = raw + .iter() + .zip(caps) + .map(|(x, &cap)| (x.floor() as usize).min(cap)) + .collect(); + + let mut fracs: Vec<(usize, f64)> = raw + .iter() + .enumerate() + .map(|(i, x)| (i, x.fract())) + .collect(); + fracs.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap()); + + let mut assigned: usize = alloc.iter().sum(); + let target = total.min(caps.iter().sum()); + + // First pass: hand out remainder units by largest fractional share. + for &(i, _) in &fracs { + if assigned >= target { + break; + } + if alloc[i] < caps[i] { + alloc[i] += 1; + assigned += 1; + } + } + // Second pass: any residual (all fractional winners were already at + // cap) goes to whoever still has spare capacity, largest weight first. + if assigned < target { + let mut order: Vec = (0..n).collect(); + order.sort_by(|&a, &b| weights[b].partial_cmp(&weights[a]).unwrap()); + for _ in 0..2 { + for &i in &order { + if assigned >= target { + break; + } + if alloc[i] < caps[i] { + alloc[i] += 1; + assigned += 1; + } + } + } + } + alloc +} + +/// Partition-aware retention: allocate `target_size` across `partitions` +/// with a per-partition floor, then rank within each partition by +/// `CoherencePolicy` and keep its allocated share. +pub fn retain_partitioned( + records: &[MemoryRecord], + partitions: &[Vec], + context_window: &[Vec], + target_size: usize, + policy: &RetentionPolicy, +) -> Vec { + let by_id: HashMap = records.iter().map(|r| (r.id, r)).collect(); + let sizes: Vec = partitions.iter().map(|p| p.len()).collect(); + let floors: Vec = sizes.iter().map(|&s| policy.floor_min.min(s)).collect(); + let sum_floor: usize = floors.iter().sum(); + let remaining = target_size.saturating_sub(sum_floor); + let capacities: Vec = sizes.iter().zip(&floors).map(|(&s, &f)| s - f).collect(); + let weights: Vec = capacities.iter().map(|&c| c as f64).collect(); + let extra = apportion_capped(remaining, &weights, &capacities); + + let mut retained = Vec::new(); + for (i, part) in partitions.iter().enumerate() { + let budget = (floors[i] + extra[i]).min(part.len()); + if budget == 0 { + continue; + } + let entries: Vec = part + .iter() + .filter_map(|id| by_id.get(id).map(|r| to_entry(r))) + .collect(); + let survivors = + CoherencePolicy::default().select_survivors(&entries, budget, context_window); + for idx in survivors { + retained.push(entries[idx].id); + } + } + retained +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn apportion_respects_caps_and_hits_target() { + let weights = vec![10.0, 1.0, 1.0]; + let caps = vec![10, 1, 1]; + let alloc = apportion_capped(8, &weights, &caps); + assert_eq!(alloc.iter().sum::(), 8); + assert!(alloc.iter().zip(&caps).all(|(a, c)| a <= c)); + } + + #[test] + fn apportion_never_exceeds_total_capacity() { + let weights = vec![1.0, 1.0]; + let caps = vec![2, 2]; + let alloc = apportion_capped(10, &weights, &caps); + assert_eq!(alloc.iter().sum::(), 4); + } + + #[test] + fn floor_protects_a_tiny_partition_from_proportional_zero() { + // Partition sizes 970 / 30 at 5% overall retention (target=50): + // pure proportional would give the 30-record partition ~1.5 -> 1 + // or, with harsher rounding, 0. The floor guarantees >= floor_min. + let sizes = [970usize, 30usize]; + let floors: Vec = sizes.iter().map(|&s| 3usize.min(s)).collect(); + assert_eq!(floors[1], 3); + } +} diff --git a/crates/ruvector-partition-memory/src/search.rs b/crates/ruvector-partition-memory/src/search.rs new file mode 100644 index 0000000000..0e5687b127 --- /dev/null +++ b/crates/ruvector-partition-memory/src/search.rs @@ -0,0 +1,95 @@ +//! Shared brute-force cosine top-k search. +//! +//! N in this crate's benchmarks stays in the low thousands, so brute force +//! is the right tool: it is the ground-truth oracle everything else is +//! measured against, and introducing an approximate index here would let +//! the evaluator's own approximation error leak into the acceptance test. + +/// Cosine similarity between two equal-length vectors. Returns 0.0 for a +/// zero vector rather than panicking; inputs in this crate are always +/// normalized on construction so this only guards degenerate edges. +pub fn cosine(a: &[f32], b: &[f32]) -> f32 { + let mut dot = 0.0f32; + let mut na = 0.0f32; + let mut nb = 0.0f32; + for i in 0..a.len() { + dot += a[i] * b[i]; + na += a[i] * a[i]; + nb += b[i] * b[i]; + } + let denom = na.sqrt() * nb.sqrt(); + if denom < 1e-9 { + 0.0 + } else { + (dot / denom).clamp(-1.0, 1.0) + } +} + +pub fn normalize(v: &mut [f32]) { + let norm: f32 = v.iter().map(|x| x * x).sum::().sqrt(); + if norm > 1e-9 { + for x in v.iter_mut() { + *x /= norm; + } + } +} + +/// Brute-force top-k by cosine similarity. `candidates` is `(id, embedding)`. +/// Ties broken by ascending id for determinism. +pub fn top_k_by_cosine(query: &[f32], candidates: &[(u64, &[f32])], k: usize) -> Vec { + let mut scored: Vec<(u64, f32)> = candidates + .iter() + .map(|(id, emb)| (*id, cosine(query, emb))) + .collect(); + scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap().then(a.0.cmp(&b.0))); + scored.into_iter().take(k).map(|(id, _)| id).collect() +} + +/// Recall@k of `retrieved` against `ground_truth`, both treated as sets. +pub fn recall_at_k(retrieved: &[u64], ground_truth: &[u64]) -> f64 { + if ground_truth.is_empty() { + return 1.0; + } + let retrieved_set: std::collections::HashSet<_> = retrieved.iter().collect(); + let hits = ground_truth + .iter() + .filter(|id| retrieved_set.contains(id)) + .count(); + hits as f64 / ground_truth.len() as f64 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cosine_identical_is_one() { + let v = vec![0.6, 0.8]; + assert!((cosine(&v, &v) - 1.0).abs() < 1e-6); + } + + #[test] + fn top_k_orders_by_similarity() { + let q = vec![1.0, 0.0]; + let a = vec![1.0, 0.0]; + let b = vec![0.0, 1.0]; + let c = vec![0.9, 0.1]; + let cands = vec![ + (1u64, a.as_slice()), + (2u64, b.as_slice()), + (3u64, c.as_slice()), + ]; + let top = top_k_by_cosine(&q, &cands, 2); + assert_eq!(top, vec![1, 3]); + } + + #[test] + fn recall_full_overlap_is_one() { + assert_eq!(recall_at_k(&[1, 2, 3], &[3, 2, 1]), 1.0); + } + + #[test] + fn recall_no_overlap_is_zero() { + assert_eq!(recall_at_k(&[4, 5, 6], &[1, 2, 3]), 0.0); + } +} diff --git a/crates/ruvector-partition-memory/src/witness.rs b/crates/ruvector-partition-memory/src/witness.rs new file mode 100644 index 0000000000..c0b5095d05 --- /dev/null +++ b/crates/ruvector-partition-memory/src/witness.rs @@ -0,0 +1,191 @@ +//! SHA-256 hash-chain witness over `adaptive_partition`'s split decisions. +//! +//! Each recursion step commits `(prev_hash, hash(parent_vertices), +//! cut_value, hash(side_a), hash(side_b), accepted)` into the next link. +//! A verifier who only has the recorded chain (not the raw partition) can +//! confirm the chain is internally consistent — no step's recorded cut +//! value or vertex-set hash was edited after the fact without breaking +//! every subsequent link — and, given the actual vertex sets, can confirm +//! a specific step's hashes were genuinely derived from them via +//! `verify_step_matches`. This does not certify the *min cut itself* is +//! optimal (that guarantee comes from `ruvector_mincut`'s algorithm, not +//! from hashing); it certifies the partition decision used the same +//! vertex sets and cut value throughout, the same property +//! `ruvector-retrieval-receipt` (nightly 2026-08-13) established for +//! query result sets. + +use ruvector_mincut::VertexId; +use serde::Serialize; +use sha2::{Digest, Sha256}; + +const GENESIS: &str = "genesis"; + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +fn hash_vertex_set(verts: &[VertexId]) -> String { + let mut sorted = verts.to_vec(); + sorted.sort_unstable(); + let mut hasher = Sha256::new(); + for v in &sorted { + hasher.update(v.to_le_bytes()); + } + hex(&hasher.finalize()) +} + +#[derive(Debug, Clone, Serialize)] +pub struct SplitRecord { + pub parent_hash: String, + pub parent_size: usize, + pub cut_value: f64, + pub side_a_hash: String, + pub side_a_size: usize, + pub side_b_hash: String, + pub side_b_size: usize, + pub accepted: bool, + pub step_hash: String, +} + +fn step_hash( + prev_hash: &str, + parent_hash: &str, + cut_value: f64, + side_a_hash: &str, + side_b_hash: &str, + accepted: bool, +) -> String { + let mut hasher = Sha256::new(); + hasher.update(prev_hash.as_bytes()); + hasher.update(parent_hash.as_bytes()); + hasher.update(cut_value.to_le_bytes()); + hasher.update(side_a_hash.as_bytes()); + hasher.update(side_b_hash.as_bytes()); + hasher.update([accepted as u8]); + hex(&hasher.finalize()) +} + +#[derive(Debug, Clone, Default, Serialize)] +pub struct PartitionWitnessChain { + pub steps: Vec, +} + +impl PartitionWitnessChain { + pub fn new() -> Self { + Self { steps: Vec::new() } + } + + pub fn record( + &mut self, + parent: &[VertexId], + cut_value: f64, + side_a: &[VertexId], + side_b: &[VertexId], + accepted: bool, + ) -> &SplitRecord { + let prev = self + .steps + .last() + .map(|s| s.step_hash.clone()) + .unwrap_or_else(|| GENESIS.to_string()); + let parent_hash = hash_vertex_set(parent); + let side_a_hash = hash_vertex_set(side_a); + let side_b_hash = hash_vertex_set(side_b); + let hash = step_hash( + &prev, + &parent_hash, + cut_value, + &side_a_hash, + &side_b_hash, + accepted, + ); + self.steps.push(SplitRecord { + parent_hash, + parent_size: parent.len(), + cut_value, + side_a_hash, + side_a_size: side_a.len(), + side_b_hash, + side_b_size: side_b.len(), + accepted, + step_hash: hash, + }); + self.steps.last().unwrap() + } + + pub fn head(&self) -> &str { + self.steps + .last() + .map(|s| s.step_hash.as_str()) + .unwrap_or(GENESIS) + } + + /// Recompute every link from the stored fields and confirm it matches + /// the recorded `step_hash`. Detects any post-hoc edit to a step. + pub fn verify(&self) -> bool { + let mut prev = GENESIS.to_string(); + for step in &self.steps { + let recomputed = step_hash( + &prev, + &step.parent_hash, + step.cut_value, + &step.side_a_hash, + &step.side_b_hash, + step.accepted, + ); + if recomputed != step.step_hash { + return false; + } + prev = step.step_hash.clone(); + } + true + } + + /// Confirm step `idx`'s recorded hashes were genuinely derived from + /// these vertex sets (not just internally self-consistent). + pub fn verify_step_matches( + &self, + idx: usize, + parent: &[VertexId], + side_a: &[VertexId], + side_b: &[VertexId], + ) -> bool { + match self.steps.get(idx) { + Some(step) => { + step.parent_hash == hash_vertex_set(parent) + && step.side_a_hash == hash_vertex_set(side_a) + && step.side_b_hash == hash_vertex_set(side_b) + } + None => false, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn chain_verifies_when_untampered() { + let mut chain = PartitionWitnessChain::new(); + chain.record(&[1, 2, 3, 4], 1.5, &[1, 2], &[3, 4], true); + chain.record(&[1, 2], 2.0, &[1], &[2], false); + assert!(chain.verify()); + assert!(chain.verify_step_matches(0, &[1, 2, 3, 4], &[1, 2], &[3, 4])); + } + + #[test] + fn chain_breaks_when_a_field_is_edited_after_the_fact() { + let mut chain = PartitionWitnessChain::new(); + chain.record(&[1, 2, 3, 4], 1.5, &[1, 2], &[3, 4], true); + chain.steps[0].cut_value = 99.0; // tamper + assert!(!chain.verify()); + } + + #[test] + fn step_does_not_match_a_different_vertex_set() { + let mut chain = PartitionWitnessChain::new(); + chain.record(&[1, 2, 3, 4], 1.5, &[1, 2], &[3, 4], true); + assert!(!chain.verify_step_matches(0, &[1, 2, 3, 5], &[1, 2], &[3, 5])); + } +} From 3bec308ed10078820c61753ffe182baf18d82509 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 08:16:57 +0000 Subject: [PATCH 2/2] docs: add ADR-305 and nightly research writeup for mincut-partitioned memory consolidation Documents the rejected hypothesis (worst-cluster recall gain 0.00pp against a pre-declared 15pp threshold, at both n=4000 and n=500, and across a bounded floor_min sweep), the partial positive signal (4/6 clusters individually gained 15-23pp, overall recall +6.8pp), and the two independently reproducible ruvector-mincut defects discovered along the way (partition()/min_cut_value() inconsistency + nondeterminism; GraphPartitioner vertex loss/fabrication + severe latency). Raw benchmark output preserved under evidence/. Co-Authored-By: claude-flow Claude-Session: https://claude.ai/code/session_01KrAfJLv2U99vvEvQqJ1Zoo --- ...mincut-partitioned-memory-consolidation.md | 256 ++++++++ .../README.md | 585 ++++++++++++++++++ .../evidence/bench_n4000.txt | 18 + .../evidence/bench_n500_with_fixedk.txt | 20 + .../evidence/calibration.txt | 6 + .../evidence/darwin_sweep.txt | 8 + .../gist.md | 123 ++++ 7 files changed, 1016 insertions(+) create mode 100644 docs/adr/ADR-305-mincut-partitioned-memory-consolidation.md create mode 100644 docs/research/nightly/2026-08-17_mincut-partitioned-memory-consolidation/README.md create mode 100644 docs/research/nightly/2026-08-17_mincut-partitioned-memory-consolidation/evidence/bench_n4000.txt create mode 100644 docs/research/nightly/2026-08-17_mincut-partitioned-memory-consolidation/evidence/bench_n500_with_fixedk.txt create mode 100644 docs/research/nightly/2026-08-17_mincut-partitioned-memory-consolidation/evidence/calibration.txt create mode 100644 docs/research/nightly/2026-08-17_mincut-partitioned-memory-consolidation/evidence/darwin_sweep.txt create mode 100644 docs/research/nightly/2026-08-17_mincut-partitioned-memory-consolidation/gist.md diff --git a/docs/adr/ADR-305-mincut-partitioned-memory-consolidation.md b/docs/adr/ADR-305-mincut-partitioned-memory-consolidation.md new file mode 100644 index 0000000000..eee0859baf --- /dev/null +++ b/docs/adr/ADR-305-mincut-partitioned-memory-consolidation.md @@ -0,0 +1,256 @@ +# ADR-305: Mincut-Partitioned Agent-Memory Consolidation + +## Status + +**Proposed, hypothesis REJECTED by measurement.** Experimental crate +(`ruvector-partition-memory`), not wired into any production compaction +path. Retained for its evidence, its two documented `ruvector-mincut` +defects, and its from-scratch correct min-cut implementation +(`mincut_exact.rs`), which is a candidate for reuse in future partitioning +work regardless of this ADR's own outcome. + +## Context + +Nightly 2026-06-14 (`crates/ruvector-agent-memory`) introduced +`CoherencePolicy`: a global top-score compaction rule scoring every stored +memory by `α·recency + β·frequency + γ·coherence(context)` and keeping the +top `target_size`. It measured 100% recall after 50% compaction on its +test corpus and remains the best-performing policy in the ecosystem. + +`CoherencePolicy` is, by construction, a single global ranking scored +against one context window (in production, the agent's most recent working +context). That is also its structural risk: a memory topic unrelated to +the current context competes on the same scale as everything else, so a +minority topic can be evicted **in full** during a single consolidation +event, at a compaction ratio the aggregate recall number reports as +favorable. Nightly 2026-06-14 did not measure this — it reports mean +recall and LRU/LFU comparisons, not worst-topic behavior. + +`ruvector-mincut` provides a subpolynomial dynamic minimum-cut engine +(Jin–Sun–Thorup) and graph-partitioning utilities +(`GraphPartitioner`, `RuVectorGraphAnalyzer`) that had not previously been +applied to agent memory. This ADR's premise: partitioning the memory +similarity graph before applying a retention budget, with a +per-partition floor, should protect a topic from being evicted in full +even when it loses on the global score — because that topic's competition +for its floor allocation is only the rest of its own partition, not the +whole corpus. + +## Hypothesis + +```text +Given a 4,000-memory corpus with 6 semantic clusters of unequal size +(1400/1000/720/600/200/80 — the two smallest are 5% and 2% of the corpus), +scored against a recency-biased context drawn from the largest cluster +(the realistic "what the agent was just working on" scenario), + +when a partition-aware retention policy (floor + proportional budget per +graph partition) is used instead of CoherencePolicy's global top-score +ranking, at 50% compaction, + +then the best candidate's worst-cluster recall@10 should exceed the +baseline's by >= 15 percentage points, + +subject to: no candidate's overall recall@10 regressing more than 5pp +below baseline; partition+retention wall time staying under 30s per +candidate at this n; and the partition witness chain verifying. +``` + +Declared before the accepted run (see the research doc's Pass 2/3 and +calibration section); not modified afterward. + +## Decision + +Implement two partitioning strategies and compare both against the +`CoherencePolicy` baseline, using **the same scorer** in every retention +step so the only independent variable is budget allocation, not scoring: + +- **Candidate A — `MincutFixedK`**: wraps the existing + `ruvector_mincut::GraphPartitioner` (unweighted, edge-count recursive + bisection, fixed `K`). +- **Candidate B — `MincutAdaptive`**: a new adaptive-depth recursive + bisection that stops splitting a component once its cut is dense + relative to its internal edge weight (no caller-chosen `K`). +- **Retention**: floor + largest-remainder proportional budget per + partition (`retention.rs`), each partition ranked internally by + `ruvector_agent_memory::CoherencePolicy` — reused as a library + dependency, not re-implemented. + +## Evidence + +### A defect discovered before the hypothesis could be tested + +`DynamicMinCut::partition()` (and the `GraphPartitioner` / +`RuVectorGraphAnalyzer` path built on it) was found, during this +candidate's own development, to return vertex splits **inconsistent with +its own `min_cut_value()`**, and nondeterministically so: + +- 6-vertex repro (two triangles joined by one weak `0.05`-weight bridge; + true min cut is uniquely `{0,1,2}` vs `{3,4,5}` at value `0.05`): of + three runs, two returned the correct split, one returned a degenerate + `{single vertex}` vs `{rest}` split — while `min_cut_value()` reported + `0.05` correctly on **every** run. +- 100-vertex version (two 50-cliques, one `0.01`-weight bridge): every run + returned the degenerate split; `min_cut_value()` still correctly + reported `0.01`. +- `GraphPartitioner` was separately found to (a) drop vertices outright at + n=100 (returned partitions covering only 50 of 100 vertices) and (b) + fabricate vertex ids that were never in the input graph at all, when the + id space is non-contiguous. +- `GraphPartitioner` was also measured to be severely slow: **8.4s at + n=500**, and **did not finish in 5m42s at n=4000** (killed). + +Full repro commands are in the research doc. This crate works around the +correctness defects with a from-scratch, tested Stoer–Wagner +implementation (`mincut_exact.rs`) used as the sole source of partition +vertex sets; `ruvector_mincut`'s `min_cut_value()` is still queried as an +independent cross-check (its *value* output, as opposed to its +*partition*, was never observed wrong). It works around the performance +defect by scale-gating candidate A (`fixed_k_max_n`, default 600) rather +than hanging the benchmark or silently omitting the comparison. + +### The accepted hypothesis run (n=4000, `coherence_ratio=0.35`, `floor_min=3`) + +```text +variant overall_recall worst_cluster_recall coverage +GlobalTopScore 0.4193 0.1520 1.000 +MincutAdaptive 0.4873 0.1520 1.000 + +per_cluster_recall GlobalTopScore = [0.996, 0.152, 0.216, 0.316, 0.396, 0.440] +per_cluster_recall MincutAdaptive = [0.792, 0.380, 0.504, 0.152, 0.556, 0.540] + +worst_cluster_gain_pp = -0.00 (threshold: +15.00) +ACCEPTANCE_RESULT: REJECT +``` + +Overall recall improved (+6.8pp) and 4 of 6 clusters gained materially +(+15 to +23pp each), but the specific cluster that was *worst* under the +baseline (cluster 3, 600 members / 15% of the corpus) is **also** worst +under `MincutAdaptive`, at the identical value — because the partitioner +left cluster 3 merged with the 1400-member majority cluster (the 2000-size +partition in `sizes=[200, 2000, 1000, 720, 80]`), so its retention budget +was decided by the same global-style competition the hypothesis set out +to avoid. The `coherence_ratio=0.35` stopping rule, calibrated before this +run against the corpus's true global min cut (see the research doc), does +correctly find and isolate the genuinely weak seams — but cluster 0/3's +separation was not one of them at this threshold. + +At n=500, both candidates were run (`fixed_k_max_n=600` admits n=500): +`MincutFixedK` reached `worst_cluster_gain_pp=8.00`, `MincutAdaptive` +reached a *worse* worst-cluster recall than baseline (`0.0` vs `0.10`, +because the true 10-member minority cluster is below `min_cluster_size` +(20) and can never be isolated on its own). Both REJECT. + +A bounded, pre-declared-fitness sweep of `floor_min` over `{1,3,8,15}` at +n=4000, holding the same partition fixed, left `worst_cluster_recall` +essentially flat (`0.152`/`0.148` across all four values) — confirming +the bottleneck is the **partition step**, not the **retention-budget +step**: no floor value can protect a cluster the partitioner never +separated from the majority in the first place. + +## Consequences + +- **Do not promote** `MincutAdaptive`/`MincutFixedK` retention to + production. The pre-registered hypothesis (worst-cluster recall + protection) is rejected by direct measurement. +- `mincut_exact.rs`'s correct, tested Stoer–Wagner implementation is a + reusable asset independent of this ADR's outcome — any future graph-cut + work in this ecosystem needing a trustworthy partition should use it, or + a fixed `ruvector-mincut`, in preference to `DynamicMinCut::partition()` + as it stands today. +- The `ruvector-mincut` defects (partition/value inconsistency, + nondeterminism, vertex loss/fabrication, severe `GraphPartitioner` + latency) should be filed and fixed upstream in that crate; they affect + every existing consumer of `DynamicMinCut::partition()` / + `GraphPartitioner`, not just this experiment. +- A follow-up hypothesis worth testing (not implemented here): a + **per-branch, not global**, stopping criterion — e.g. always attempt at + least one more level of recursion on the largest remaining partition + before accepting `coherence_ratio`'s verdict, or size-weight the + threshold — might separate cluster 0/3 where the flat threshold did + not. This is a new hypothesis, not a retroactive change to the one + tested above. + +## Alternatives + +- **Ship `CoherencePolicy` unchanged.** Current state; the measured + overall-recall improvement here (+6.8pp) does not offset a rejected + primary hypothesis and a partitioner with two unresolved upstream + correctness defects and a severe latency defect. +- **Global top-score with a per-cluster-label floor** (using a cheap + clustering method like k-means on embeddings instead of graph min-cut) + was considered but not implemented; it would sidestep `ruvector-mincut` + entirely and is a reasonable next candidate. + +## Implementation plan + +Not applicable — hypothesis rejected; no production migration. + +## API shape + +`ruvector-partition-memory` (experimental, workspace member, not +re-exported by any production crate): `corpus`, `graph`, `mincut_exact`, +`partition`, `retention`, `metrics`, `witness`, `search` modules; see +`src/lib.rs` for the full surface. + +## Feature flags + +None; the crate is not on any production feature-gated path. + +## Benchmark evidence + +`docs/research/nightly/2026-08-17_mincut-partitioned-memory-consolidation/evidence/` +— raw, unedited command output: `bench_n4000.txt`, `bench_n500_with_fixedk.txt`, +`darwin_sweep.txt`, `calibration.txt`. + +## Security + +No new attack surface: the crate is a standalone research binary/library +operating on synthetic data, not wired into any request path. The witness +chain (`witness.rs`) is a correctness/audit mechanism, not an access +control mechanism, and makes no such claim. + +## Governance + +None of this crate's code should be treated as validated production +guidance for `ruvector-mincut` usage beyond the specific defects +documented above; those defects should be independently verified by +whoever owns that crate before any fix lands. + +## Failure modes + +- `DynamicMinCut::partition()` / `GraphPartitioner` defects: see Evidence. +- `AdaptiveConfig::min_cluster_size` (default 20) structurally prevents + isolating any true topic smaller than that absolute count — observed + directly at n=500 (10-member cluster, worst_cluster_recall=0.0). +- A coarse, single-threshold stopping rule can leave two clusters merged + even when one is a minority worth protecting, if their graph-structural + separation is weaker than the threshold demands elsewhere in the same + corpus (observed at n=4000, clusters 0/3). + +## Migration + +None. + +## Rollback + +None — nothing shipped to a production path. + +## Rejection criteria + +Met: worst-cluster recall gain (0.00pp, both n=4000 and n=500) fell short +of the pre-declared 15pp threshold in every configuration tested, +including a bounded post-hoc sweep of the one parameter (`floor_min`) +that could plausibly have rescued it without changing the hypothesis +itself. + +## Open questions + +- Would a per-branch/size-weighted stopping criterion (see Consequences) + cross the threshold? Untested — a genuinely new hypothesis for a future + nightly, not this one. +- Do the two `ruvector-mincut` defects reproduce on that crate's own + existing test suite, or does no existing test exercise + `DynamicMinCut::partition()` / `GraphPartitioner::partition()`'s output + against ground truth? Not investigated here; worth checking before + filing upstream. diff --git a/docs/research/nightly/2026-08-17_mincut-partitioned-memory-consolidation/README.md b/docs/research/nightly/2026-08-17_mincut-partitioned-memory-consolidation/README.md new file mode 100644 index 0000000000..b204f00cd8 --- /dev/null +++ b/docs/research/nightly/2026-08-17_mincut-partitioned-memory-consolidation/README.md @@ -0,0 +1,585 @@ +# Mincut-Partitioned Agent-Memory Consolidation + +**Date**: 2026-08-17 +**Crate**: `ruvector-partition-memory` (`crates/ruvector-partition-memory`) +**Status**: PoC complete — **hypothesis REJECTED by measurement**, plus two documented defects discovered in `ruvector-mincut` +**ADR**: [ADR-305](../../../adr/ADR-305-mincut-partitioned-memory-consolidation.md) + +--- + +## Summary of Outcome + +The hypothesis — that partitioning the agent-memory similarity graph +before applying a retention budget protects a minority topic from being +evicted in full by a global top-score compactor — is **rejected** on the +pre-declared metric (worst-cluster recall@10 gain ≥ 15pp) at every scale +tested: + +| Run | Best candidate | Worst-cluster gain | Threshold | Verdict | +|---|---|---|---|---| +| n=4000, coherence_ratio=0.35 | MincutAdaptive | **0.00pp** | 15pp | REJECT | +| n=500, coherence_ratio=0.35 | MincutFixedK | 8.00pp | 15pp | REJECT | +| n=4000, floor_min sweep {1,3,8,15} | (all) | 0.00pp (flat) | 15pp | REJECT | + +The mechanism is not worthless — 4 of 6 clusters gained 15–23pp recall +each and overall recall improved +6.8pp at n=4000 — but the specific +cluster the hypothesis exists to protect (the one a global score would +otherwise starve) was, in the accepted run, left merged with the majority +cluster by the partitioner, so it received no protection at all. A bounded +sweep of the retention floor confirmed this is a **partitioning** +shortfall, not a **retention-budget** shortfall: no floor value moved the +worst-cluster number. + +Along the way, developing this candidate against `ruvector-mincut` +surfaced two independent, reproducible defects in that crate (not +previously known to this nightly process — see below), which this run +worked around rather than silently absorbed. + +--- + +## Abstract + +`ruvector-agent-memory` (nightly 2026-06-14) scores every stored memory +against a global importance formula — `α·recency + β·frequency + +γ·coherence(context)` — and keeps the top-N at compaction time. It +measured excellent aggregate recall, but a global ranking is, by +construction, blind to topic diversity: a memory topic the agent is not +currently working on competes on the same scale as everything else, and +can be evicted **in full**. + +This nightly asks whether partitioning the memory similarity graph first +— using `ruvector-mincut`, previously unused for agent memory — and +retaining a guaranteed floor per partition, fixes that. It does not, at +least not with the threshold-based partitioner tested here; the write-up +below explains why, with per-cluster evidence. + +--- + +## Hypothesis + +```text +Given a 4,000-memory corpus with 6 semantic clusters of unequal size +(1400/1000/720/600/200/80 — two minorities at 5% and 2% of the corpus), +scored against a recency-biased context drawn from the largest cluster, + +when a partition-aware retention policy (floor + proportional budget per +graph partition) replaces CoherencePolicy's global top-score ranking, +at 50% compaction, + +then the best candidate's worst-cluster recall@10 should exceed the +baseline's by >= 15 percentage points, + +subject to: no candidate's overall recall@10 regressing more than 5pp +below baseline; partition+retention wall time under 30s per candidate at +this n; and the partition witness chain verifying. +``` + +This threshold, and the corpus/graph calibration below, were fixed +**before** the accepted run in `evidence/bench_n4000.txt`. They were not +adjusted afterward. + +--- + +## Why This Matters for RuVector + +RuVector is a Rust-native substrate for agent memory, not just a vector +store. Long-running agents accumulate memories across many unrelated +topics; a compaction policy that silently loses whole topics degrades +retrieval quality in a way aggregate recall numbers hide. This nightly +connects: + +| Component | Role | +|---|---| +| `ruvector-agent-memory` | Reused directly as a library dependency — the baseline scorer, and the within-partition scorer for both candidates. Not re-implemented. | +| `ruvector-mincut` | Source of the graph-partitioning primitives this crate builds on (`GraphPartitioner`) and cross-checks against (`DynamicMinCut::min_cut_value()`). | +| `ruvector-retrieval-receipt` (2026-08-13) | Precedent this crate follows for `witness.rs`'s SHA-256 hash-chain design — tamper-evident commitments over a decision, not a signature over correctness. | +| ruFlo | A real production path for this class of policy (if a future variant is accepted) would run as a scheduled memory-consolidation workflow, not inline on the write path. | +| MCP | A future accepted policy's natural interface is a narrow `memory_consolidate(target_pct)` tool, mirroring 2026-06-14's suggested `memory_compact`. | + +--- + +## Architecture + +```mermaid +flowchart TD + A[Memory corpus
4000 records, 6 clusters] --> B[k-NN similarity graph
graph.rs, k=10, cosine weights] + B --> C1[GlobalTopScore baseline
ruvector_agent_memory::CoherencePolicy] + B --> C2[MincutFixedK candidate A
ruvector_mincut::GraphPartitioner] + B --> C3[MincutAdaptive candidate B
mincut_exact.rs Stoer-Wagner] + C3 --> W[PartitionWitnessChain
witness.rs — SHA-256 hash chain] + C1 --> R1[retain_global_top_score] + C2 --> R2[retain_partitioned
floor + proportional budget] + C3 --> R2 + R1 --> M[metrics.rs
overall + per-cluster + worst-cluster recall@10] + R2 --> M + M --> ACC[Pre-declared acceptance gate
main.rs] +``` + +`mincut_exact.rs` exists because `ruvector_mincut::DynamicMinCut::partition()` +was found, during development, to disagree with its own `min_cut_value()` +— see **Defects Discovered** below. Candidate B's splits are materialized +by a from-scratch, tested Stoer–Wagner implementation instead; +`ruvector_mincut`'s value is still queried as an independent cross-check +and logged. + +--- + +## Implementation + +Three variants, one shared scorer: + +- **`GlobalTopScore`** (baseline): `ruvector_agent_memory::CoherencePolicy::default()` + applied to the whole corpus. +- **`MincutFixedK`** (candidate A): `ruvector_mincut::GraphPartitioner` + (existing tool, unweighted edge-count recursive bisection to a + caller-chosen `K`), then `retain_partitioned`. +- **`MincutAdaptive`** (candidate B): a new recursive bisection + (`partition.rs::recurse`) using `mincut_exact::global_min_cut` at each + level, stopping once a component's cut is dense relative to its + internal edge weight (`coherence_ratio`, calibrated below), then + `retain_partitioned`. + +`retain_partitioned` (`retention.rs`) allocates the retention budget +per-partition via a floor (`floor_min`, default 3) plus largest-remainder +proportional split of the remainder, then ranks each partition internally +with the same `CoherencePolicy` the baseline uses — isolating the +independent variable to *budget allocation*, not *scoring*. + +The corpus (`corpus.rs`) is a deterministic, seeded synthetic generator: +6 clusters on the unit sphere (rejection-sampled to cosine separation +≤ 0.35), Gaussian noise (`noise_std`), decoupled recency/frequency +signals, and a recency-biased "focus cluster" standing in for what the +agent was just working on — the realistic scenario in which +`CoherencePolicy`'s context window is biased away from other topics. +Ground truth is brute-force top-k cosine search against the full, +uncompacted corpus, computed once at generation time. + +### Calibration (before the accepted run) + +At the originally-planned `noise_std=0.35`, the corpus's true global min +cut degenerately isolated a single outlier vertex +(`normalized_cut≈0.76` — no real topic boundary was the graph's weakest +seam). At `noise_std=0.25`, the min cut cleanly isolated one whole +semantic cluster (`normalized_cut≈0.07`), confirming a graph structure +the hypothesis could actually be tested against. `noise_std=0.25` and +`coherence_ratio=0.35` were fixed from this calibration pass, before the +accepted run — see `evidence/calibration.txt`. + +--- + +## Defects Discovered in `ruvector-mincut` + +Two independent, reproducible issues, found while building candidate B, +neither previously known to this nightly process: + +### 1. `DynamicMinCut::partition()` is inconsistent with its own `min_cut_value()`, and nondeterministic + +Minimal repro: two triangles `{0,1,2}` and `{3,4,5}`, joined by one +`weight=0.05` bridge edge. The true global minimum cut is unique — value +`0.05`, split `{0,1,2}`/`{3,4,5}` (isolating any single triangle vertex +costs ≥ `1.0`). + +```rust +let mincut = MinCutBuilder::new().exact().with_edges(edges).build().unwrap(); +mincut.min_cut_value() // always 0.05, every run — correct +mincut.partition() // sometimes {0,1,2}/{3,4,5} (correct), + // sometimes {single vertex}/{rest} (wrong: that + // split's actual crossing weight is >= 1.0, not 0.05) +``` + +Of three runs: two returned the correct split, one returned the +degenerate split — same code, same input, different process invocations. +At 100 vertices (two 50-cliques, one `0.01` bridge), **every** run +returned the degenerate split, while `min_cut_value()` still correctly +reported `0.01` every time. `cut_edges()` (derived from `.partition()`) +was cross-checked to independently confirm the mismatch: for the +degenerate split, summed crossing-edge weight was `2.0`, not the reported +`0.05`. + +### 2. `GraphPartitioner` / `RuVectorGraphAnalyzer`: vertex loss, vertex fabrication, and severe latency + +- At n=100 (two 50-cliques + weak bridge), `GraphPartitioner::partition()` + returned partitions covering only 50 of the 100 input vertices. +- With a non-contiguous vertex-id space (`{1,2,3,11,12,13}`), + `RuVectorGraphAnalyzer::partition()` returned a side containing ids + (`4,5,6,7,8,9,10`) that were never in the input graph. +- **Latency**: `GraphPartitioner::partition()` (K=10) measured **8.4s at + n=500**, and had not finished after **5m42s at n=4000** (process + killed). This crate's own `mincut_exact::global_min_cut` measured + **167ms at n=500** and **~11.1s for the full adaptive recursion at + n=4000** — the same order of magnitude for *one* global min cut, + suggesting `GraphPartitioner`'s recursive re-wrapping (`RuVectorGraphAnalyzer::new` + per subgraph, itself built on the fully-dynamic `MinCutWrapper`) pays a + large, likely superlinear, overhead for what is fundamentally a + one-shot static computation at each level. + +**Workaround used in this crate**: `mincut_exact.rs` — a from-scratch, +tested, deterministic weighted Stoer–Wagner implementation — is the sole +source of partition vertex sets for candidate B. +`ruvector_mincut::DynamicMinCut::min_cut_value()` is still called as an +independent cross-check (`partition.rs`), logged via a `debug_assert!` on +disagreement; it was never observed wrong in this crate's testing, only +its *partition* output was. `fixed_k_partition` (candidate A) filters +`GraphPartitioner`'s output against the known-valid vertex set and +appends any uncovered vertex as a fallback group, so it cannot silently +drop or fabricate a memory — and is scale-gated (`fixed_k_max_n`, default +600) so a benchmark run cannot hang on it. + +**Not filed upstream as part of this nightly** (no `ruvector-mincut` +maintainer sign-off in scope here) — recorded as an open question in +ADR-305 for whoever owns that crate to verify and file. + +--- + +## Benchmark Methodology + +- Release build (`cargo build --release`), `rustc 1.94.1`, `cargo 1.94.1`. +- Hardware: x86-64, 4 logical CPUs, 15GiB RAM, Linux 6.18.5. +- Deterministic seed (`seed=42`) for corpus generation; ground truth + computed once per corpus via brute-force cosine search, not resampled + per variant. +- 150 out-of-sample queries (25 per cluster), recall@10 against the full + uncompacted corpus. +- Single run per configuration (no repeated-trial variance reporting — + see Limitations). +- Exact commands and raw, unedited output: `evidence/*.txt`. + +```bash +cargo run --release -p ruvector-partition-memory --bin benchmark -- 4000 3 0.35 10 600 +cargo run --release -p ruvector-partition-memory --bin benchmark -- 500 3 0.35 10 600 +cargo run --release -p ruvector-partition-memory --example darwin_sweep +cargo run --release -p ruvector-partition-memory --example calibrate +``` + +## Benchmark Results + +### n=4000 (accepted run) + +```text +variant retained overall_recall worst_cluster_recall coverage partition_us retention_us +GlobalTopScore 2000 0.4193 0.1520 1.000 0 12948 +MincutAdaptive 2000 0.4873 0.1520 1.000 11164029 13482 + +per_cluster_recall GlobalTopScore = [0.996, 0.152, 0.216, 0.316, 0.396, 0.440] +per_cluster_recall MincutAdaptive = [0.792, 0.380, 0.504, 0.152, 0.556, 0.540] + +MincutFixedK: SKIPPED (n=4000 exceeds fixed_k_max_n=600; see Defects Discovered) +MincutAdaptive partitions: 5, sizes=[200, 2000, 1000, 720, 80] + ^^^^ cluster0(1400)+cluster3(600) stayed merged +worst_cluster_gain_pp = -0.00 (threshold 15.00) ACCEPTANCE_RESULT: REJECT +``` + +Full raw output: `evidence/bench_n4000.txt`. + +### n=500 (both candidates) + +```text +variant overall_recall worst_cluster_recall coverage +GlobalTopScore 0.3440 0.1000 1.000 +MincutFixedK 0.4467 0.1800 1.000 +MincutAdaptive 0.3500 0.0000 0.833 <- 10-member cluster below min_cluster_size(20) + +worst_cluster_gain_pp (best=MincutFixedK) = 8.00 (threshold 15.00) ACCEPTANCE_RESULT: REJECT +``` + +Full raw output: `evidence/bench_n500_with_fixedk.txt`. + +### Bounded Darwin-style sweep (n=4000, partition fixed, `floor_min` varied) + +```text +floor_min=1 overall_recall=0.4880 worst_cluster_recall=0.1520 fitness=0.4224 +floor_min=3 overall_recall=0.4873 worst_cluster_recall=0.1520 fitness=0.4222 +floor_min=8 overall_recall=0.5000 worst_cluster_recall=0.1480 fitness=0.4260 +floor_min=15 overall_recall=0.5067 worst_cluster_recall=0.1480 fitness=0.4260 + +winner: floor_min=15 DARWIN_RESULT: PROMOTE (composite fitness only — see Darwin section) +``` + +`worst_cluster_recall` is flat (within noise) across every `floor_min` +tested — direct evidence the shortfall is structural (partitioning), not +a retention-budget tuning problem. Full raw output: +`evidence/darwin_sweep.txt`. + +--- + +## Memory Math + +At n=4000, d=64: corpus embeddings are `4000 × 64 × 4 bytes ≈ 1.0MB`. +The k-NN graph (k=10, deduplicated undirected) holds ~31,000 edges; +stored as `(u64, u64, f64)` triples, `~744KB`. `mincut_exact`'s working +set during a single `global_min_cut` call is `O(V)` `HashMap`s of degree +`~2k`; peak additional memory is a small multiple of the edge list, not +separately measured in this run (see Limitations). + +## Performance Math + +`MincutAdaptive`'s ~11.1s at n=4000 is dominated by the top-level +`global_min_cut` call over the full ~4000-vertex, ~31000-edge graph +(subsequent recursion levels operate on rapidly shrinking subgraphs). +This is consistent with the `O(V·E·log V)`-ish binary-heap Stoer–Wagner +formulation used here (not the theoretically tighter but more complex +`O(VE + V² log V)` Nagamochi–Ibaraki-style variant) — acceptable for a +one-time nightly consolidation event, not for an inline write-path +operation at this scale without further optimization. + +## Failure Modes + +- Partitioner leaves the true worst cluster merged with the majority + (this run's actual failure mode — see per-cluster evidence above). +- `min_cluster_size` floor structurally prevents isolating any topic + smaller than that absolute count (n=500 run). +- `ruvector-mincut` defects (see above) — worked around, not fixed. + +## Rejected Alternatives + +- **K-means-based partitioning** instead of graph min-cut: not + implemented; a reasonable next candidate that sidesteps + `ruvector-mincut` entirely (see ADR-305 Alternatives). +- **Forcing `GraphPartitioner` to be candidate A at full scale**: rejected + after direct measurement (5m42s, unfinished) — reported honestly as a + scale-gated skip rather than silently hidden or waited out indefinitely. + +--- + +## Security + +No new attack surface. This crate is a standalone research binary/library +over synthetic data; nothing in it is wired into a request-serving path. +`witness.rs` (SHA-256 hash chain over partition decisions) is a +tamper-evidence mechanism for *auditing a partition decision after the +fact* — it proves a step's recorded cut value and vertex-set hashes were +not edited post-hoc — it is **not** a correctness proof of the underlying +min cut and makes no access-control claim, matching the threat-model +framing `ruvector-retrieval-receipt` (2026-08-13) established for reads. + +## Governance + +Hypothesis rejected; no promotion, no production migration, no rollback +needed. The two `ruvector-mincut` defects are recorded as an open +question in ADR-305, not filed upstream from within this nightly run — +that requires the owning maintainer's verification. + +## MCP Implications + +None planned — the underlying policy is rejected. Had it been accepted, +the natural interface would mirror the 2026-06-14 nightly's suggested +`memory_compact(context, target_pct)` tool, narrowly scoped, read/write +on the agent's own memory store only. + +## WASM / Edge Implications + +Not evaluated. `mincut_exact.rs` has zero non-`ruvector_mincut` type +dependencies beyond `std` collections and would very likely compile to +WASM (no unsafe, no platform-specific code) if this policy is revisited, +but binary-size and edge-memory impact were not measured in this run — +no deployment claim is made. + +## RVF Implications + +A future accepted consolidation policy's output (retained memory ids + +partition witness chain) is a natural fit for an RVF portable snapshot: +the witness chain already produces the kind of signed-lineage evidence +RVF snapshots want. Not implemented — analysis only, per the mandatory +(implementation optional) requirement for RVF fit. + +## RVM Implications + +No RVM fit identified: this policy does not need isolated execution, +capability boundaries, or proof-gated mutation beyond what its own +witness chain already provides for its one internal decision (the +partition). Not forced. + +## ruFlo Implications + +If a future variant of this hypothesis is accepted, ruFlo's natural role +is a scheduled memory-consolidation workflow (analogous to the "memory +maintenance" workflow class in the harness's own role list) — triggered +on a cadence or storage-pressure signal, not run inline on the write +path, given the measured ~11s latency at n=4000. + +--- + +## Practical Applications + +1. **Long-running coding agents** — memory: prior debugging sessions + across unrelated modules; problem: a burst of work on module A can + starve retained memory of module B at consolidation time; RuVector + capability: (if a future variant is accepted) partition-aware + retention; ecosystem integration: ruFlo scheduled consolidation; + business value: fewer "the agent forgot X" regressions; main risk: + this run shows the naive version does not reliably deliver that + protection; time horizon: near-term, pending a revised hypothesis. +2. **Customer-support agent memory** — user: support bot; problem: a busy + week on one product line can evict memory of a rarely-escalated + product line; capability: same as above; risk: same; horizon: near-term. +3. **Multi-project assistant memory** — user: an assistant used across + several unrelated user projects; problem: intense work on project A + crowds out project B's memory; horizon: near-term. +4. **Scientific literature agents** — user: research assistant tracking + several research threads; problem: an active thread's queries bias + consolidation away from a dormant-but-still-relevant thread; horizon: + medium-term. +5. **Enterprise Graph RAG** — user: internal knowledge agent; problem: + department-specific knowledge clusters compete unevenly for retention + budget; horizon: medium-term. +6. **Robotics/edge agent memory** — user: an embedded agent with a hard + memory cap; problem: same starvation risk, higher stakes given no + "just don't compact" fallback; horizon: long-term, pending edge + feasibility work not done here. +7. **Security/anomaly-memory agents** — user: a SOC assistant; problem: + a high-volume alert category can crowd out memory of a rare-but-severe + category; horizon: medium-term. +8. **Local-first personal assistants** — user: a device-resident + assistant; problem: identical starvation risk under a tight local + memory budget; horizon: long-term. + +## Long Horizon Applications + +1. **Self-healing graph memory** — thesis: agent memory graphs that + detect and repair their own topic-starvation without a human noticing; + requires: a stopping criterion that reliably finds every weak seam, not + just some of them (this run's central gap); RuVector role: the + substrate the repair loop runs against; why this experiment matters: + it is the first measured evidence of *where* a naive version of this + idea fails; primary uncertainty: whether any single global threshold + can ever reliably separate every minority topic, or whether a + per-branch/adaptive criterion is required; falsification: repeat this + benchmark with a per-branch stopping rule and measure worst-cluster + gain again. +2. **Synthetic nervous systems for agent fleets** — thesis: fleets of + agents sharing a partitioned memory substrate, each fleet member + effectively "owning" a partition; requires: partition stability under + concurrent writes, not evaluated here; RuVector role: shared substrate; + uncertainty: whether partition boundaries stay stable as memory grows; + falsification: a delete/insert-churn variant of this benchmark. +3. **Agent operating systems** — thesis: memory partitioning as a kernel + primitive analogous to process isolation; requires: much stronger + correctness guarantees than this run's underlying library currently + provides (see Defects Discovered); uncertainty: whether the two + documented `ruvector-mincut` defects are fixable without an API + change; falsification: the fix either lands and this crate's + `mincut_exact.rs` workaround becomes redundant, or it doesn't. +4. **Swarm memory** — thesis: partition-aware consolidation as the memory + layer for multi-agent swarms; requires: partitioning at swarm scale + (this run only reached n=4000 at ~11s per full run); uncertainty: + scaling behavior beyond n=4000, not measured; falsification: repeat at + n=40,000 and check wall time stays sub-linear-ish. +5. **Dynamic world models** — thesis: topic partitions as a proxy for + distinct "world model" facets an agent maintains; requires: partition + labels that are stable and interpretable over time, not evaluated; + uncertainty: whether graph min-cut partitions correspond to anything a + human would call a coherent "facet"; falsification: qualitative review + of partition contents against human-labeled topics. +6. **Proof-gated autonomous infrastructure** — thesis: the witness chain + here generalizes to a general "prove this maintenance decision wasn't + silently gamed" primitive for autonomous infra; requires: extending + `witness.rs`'s pattern beyond partition decisions; uncertainty: + whether the pattern holds up under adversarial (not just accidental) + tampering; falsification: an explicit red-team pass against the + witness chain, not performed in this run. +7. **RVM coherence domains** — thesis: partitions as RVM coherence-domain + boundaries; requires: the RVM fit analysis above to change from "not + identified" to "identified," which would need a concrete isolation + requirement this policy does not currently have; uncertainty: high; + falsification: N/A until a concrete requirement exists. +8. **Robotics memory** — thesis: partition-aware retention for + resource-constrained robot memory; requires: the edge/WASM + measurements this run explicitly did not make; uncertainty: whether + `mincut_exact.rs`'s ~11s at n=4000 is remotely feasible on embedded + hardware; falsification: run `mincut_exact` benchmarks on target + hardware. + +--- + +## Competitor Comparison + +Not materially applicable — no public vector database documents a +graph-partition-aware memory *compaction* policy comparable to this +experiment's scope (agent-memory lifecycle management, not ANN indexing). +`documented_external_capability`: none found for this specific mechanism +in Milvus/Qdrant/Weaviate/Pinecone/LanceDB/FAISS/pgvector/Chroma/Vespa. +`directly_measured_capability`: N/A (nothing external to measure against). +`unknown`: whether any of these systems' internal (undocumented) +compaction logic does something structurally similar. + +--- + +## Evolution Results (Darwin) + +- **Executed**: yes, bounded (generations=1, candidates_per_generation=4, + matching the harness's default budget), over `floor_min ∈ {1,3,8,15}`, + partition held fixed (only retention depends on `floor_min`). +- **Fitness** (declared before running): `0.5·worst_cluster_recall + + 0.3·overall_recall + 0.2·correctness`. +- **Winner**: `floor_min=15`, `fitness=0.4260` vs parent + (`floor_min=3`) `fitness=0.4222` — `DARWIN_RESULT: PROMOTE` **on this + composite fitness metric only**. `worst_cluster_recall` itself did not + improve (0.148 vs 0.152 — marginally *worse*); the promotion is driven + by `floor_min=15`'s better overall recall. This is reported precisely + so it is not mistaken for the primary ACCEPTANCE_RESULT, which remains + REJECT. +- **Parent retained**: yes — this Darwin promotion is not wired into + `main.rs`'s defaults; ADR-305 does not recommend shipping it. + +## Witness Evidence + +`MincutAdaptive`'s partition witness chain: 9 split steps at n=4000, +`chain_verify=true`, head +`a15b77949d3d26928fc84cd89b0dcb749c4b16359b3caa08320967a8bffa8469` +(`evidence/bench_n4000.txt`). `witness.rs` unit tests additionally verify +the chain detects post-hoc tampering of a recorded step +(`chain_breaks_when_a_field_is_edited_after_the_fact`). + +## Production Path + +None — hypothesis rejected. See ADR-305 Consequences for the specific +follow-up direction (per-branch stopping criterion) that would need to be +tested as a new hypothesis before any production consideration. + +## Falsification Criteria + +Met, per the pre-declared acceptance gate: worst-cluster recall gain did +not reach +15pp in any tested configuration, including a bounded sweep of +the one parameter most likely to rescue it. + +## Limitations + +- **Single run per configuration** — no repeated-trial variance reporting + (Step 13's "prefer multiple repetitions" was not followed here, given + ~11s per n=4000 run and the time budget for one nightly cycle). The + measured numbers should be read as point estimates, not + variance-characterized results. +- **One corpus generator, one seed family** — results are specific to + this synthetic corpus's cluster-separation and noise characteristics; + not validated against a real agent-memory trace. +- **`ruvector-mincut` defects not filed upstream** from within this run — + recorded as an open question, not resolved. +- **No WASM/edge measurement**, despite the mandatory-analysis + requirement being satisfied by the qualitative section above. +- **`mincut_exact.rs` is not asymptotically optimal** Stoer–Wagner + (a Nagamochi–Ibaraki-style formulation would be faster); it was + sufficient for this run's n=4000 but was not tuned for larger scale. + +## Next Research + +1. Test a per-branch/size-weighted adaptive stopping criterion against + the same corpus and acceptance gate, as a genuinely new hypothesis. +2. Test a k-means-based (non-graph) partition baseline, sidestepping + `ruvector-mincut` entirely, as a cheaper alternative worth comparing. +3. Verify the two `ruvector-mincut` defects against that crate's own test + suite and, if confirmed absent from existing coverage, file them + upstream with the repros in this doc. +4. Repeat this benchmark with repeated trials and variance reporting if + a revised hypothesis clears the first-pass bar above. + +## References + +- Nightly 2026-06-14, `crates/ruvector-agent-memory` — `CoherencePolicy`, + reused directly here. +- Nightly 2026-08-13, `crates/ruvector-retrieval-receipt`, ADR-304 — + witness-chain design precedent for `witness.rs`. +- Jin, Sun, Thorup, "Fully Dynamic Exact Minimum Cut in Subpolynomial + Time" (SODA 2024) — the algorithm `ruvector-mincut`'s `witness` module + cites; not itself re-verified in this run. +- Stoer, Wagner, "A Simple Min-Cut Algorithm" (1997) — the algorithm + implemented from scratch in `mincut_exact.rs`. diff --git a/docs/research/nightly/2026-08-17_mincut-partitioned-memory-consolidation/evidence/bench_n4000.txt b/docs/research/nightly/2026-08-17_mincut-partitioned-memory-consolidation/evidence/bench_n4000.txt new file mode 100644 index 0000000000..c7b2cb2e9e --- /dev/null +++ b/docs/research/nightly/2026-08-17_mincut-partitioned-memory-consolidation/evidence/bench_n4000.txt @@ -0,0 +1,18 @@ +=== ruvector-partition-memory nightly benchmark === +n=4000 dims=64 cluster_sizes=[1400, 1000, 720, 600, 200, 80] focus_cluster=0 target_size=2000 (50% retention) queries=150 +params: floor_min=3 coherence_ratio=0.35 fixed_k=10 fixed_k_max_n=600 +corpus_gen_us=119649 knn_graph_us=2691799 knn_edges=31061 + +MincutFixedK: SKIPPED (n=4000 exceeds fixed_k_max_n=600; ruvector_mincut::GraphPartitioner measured at 8.4s for n=500 and did not finish in 5m42s for n=4000 during this nightly's development — see the research doc for the repro) +MincutAdaptive partitions: 5 (coherence_ratio=0.35) sizes=[200, 2000, 1000, 720, 80] +MincutAdaptive witness: 9 split steps, chain_verify=true, head=a15b77949d3d26928fc84cd89b0dcb749c4b16359b3caa08320967a8bffa8469 + +variant retained overall_recall worst_cluster_recall coverage partition_us retention_us +GlobalTopScore retained=2000 overall_recall=0.4193 worst_cluster_recall=0.1520 coverage=1.000 partition_us=0 retention_us=12948 +MincutAdaptive retained=2000 overall_recall=0.4873 worst_cluster_recall=0.1520 coverage=1.000 partition_us=11164029 retention_us=13482 + +per_cluster_recall GlobalTopScore = [0.996, 0.15200000000000005, 0.21599999999999997, 0.31600000000000006, 0.39599999999999996, 0.44000000000000006] +per_cluster_recall MincutAdaptive = [0.7919999999999999, 0.38, 0.5040000000000001, 0.15200000000000002, 0.5559999999999999, 0.54] + +acceptance: best_candidate=MincutAdaptive worst_cluster_gain_pp=-0.00 (threshold_pp=15.00) gain_ok=false overall_ok=true latency_ok=true witness_ok=true +ACCEPTANCE_RESULT: REJECT diff --git a/docs/research/nightly/2026-08-17_mincut-partitioned-memory-consolidation/evidence/bench_n500_with_fixedk.txt b/docs/research/nightly/2026-08-17_mincut-partitioned-memory-consolidation/evidence/bench_n500_with_fixedk.txt new file mode 100644 index 0000000000..c76d484f11 --- /dev/null +++ b/docs/research/nightly/2026-08-17_mincut-partitioned-memory-consolidation/evidence/bench_n500_with_fixedk.txt @@ -0,0 +1,20 @@ +=== ruvector-partition-memory nightly benchmark === +n=500 dims=64 cluster_sizes=[175, 125, 90, 75, 25, 10] focus_cluster=0 target_size=250 (50% retention) queries=150 +params: floor_min=3 coherence_ratio=0.35 fixed_k=10 fixed_k_max_n=600 +corpus_gen_us=12962 knn_graph_us=35113 knn_edges=3572 + +MincutFixedK partitions: 3 (K=10 requested) sizes=[1, 346, 153] +MincutAdaptive partitions: 4 (coherence_ratio=0.35) sizes=[90, 75, 125, 210] +MincutAdaptive witness: 7 split steps, chain_verify=true, head=11910bea6063621a4290554636dc218909bc4baa4812350f4ea4de56238a7a70 + +variant retained overall_recall worst_cluster_recall coverage partition_us retention_us +GlobalTopScore retained=250 overall_recall=0.3440 worst_cluster_recall=0.1000 coverage=1.000 partition_us=0 retention_us=1533 +MincutFixedK retained=250 overall_recall=0.4467 worst_cluster_recall=0.1800 coverage=1.000 partition_us=642943 retention_us=1681 +MincutAdaptive retained=250 overall_recall=0.3500 worst_cluster_recall=0.0000 coverage=0.833 partition_us=516739 retention_us=2514 + +per_cluster_recall GlobalTopScore = [0.988, 0.2, 0.188, 0.3, 0.2879999999999999, 0.10000000000000003] +per_cluster_recall MincutFixedK = [0.848, 0.18, 0.34, 0.4640000000000001, 0.5479999999999999, 0.2999999999999999] +per_cluster_recall MincutAdaptive = [0.6039999999999999, 0.49199999999999994, 0.4040000000000001, 0.46, 0.14000000000000004, 0.0] + +acceptance: best_candidate=MincutFixedK worst_cluster_gain_pp=8.00 (threshold_pp=15.00) gain_ok=false overall_ok=true latency_ok=true witness_ok=true +ACCEPTANCE_RESULT: REJECT diff --git a/docs/research/nightly/2026-08-17_mincut-partitioned-memory-consolidation/evidence/calibration.txt b/docs/research/nightly/2026-08-17_mincut-partitioned-memory-consolidation/evidence/calibration.txt new file mode 100644 index 0000000000..575b8e43a1 --- /dev/null +++ b/docs/research/nightly/2026-08-17_mincut-partitioned-memory-consolidation/evidence/calibration.txt @@ -0,0 +1,6 @@ +n=500 ratio=0.5 partitions=4 sizes=[90, 75, 125, 210] purities=["1.00", "1.00", "1.00", "0.83"] us=480432 +n=500 ratio=0.35 partitions=4 sizes=[90, 75, 125, 210] purities=["1.00", "1.00", "1.00", "0.83"] us=524855 +n=500 ratio=0.2 partitions=2 sizes=[90, 410] purities=["1.00", "0.43"] us=315318 +n=4000 ratio=0.5 partitions=5 sizes=[200, 2000, 1000, 720, 80] purities=["1.00", "0.70", "1.00", "1.00", "1.00"] us=10784652 +n=4000 ratio=0.35 partitions=5 sizes=[200, 2000, 1000, 720, 80] purities=["1.00", "0.70", "1.00", "1.00", "1.00"] us=11097579 +n=4000 ratio=0.2 partitions=4 sizes=[2200, 1000, 720, 80] purities=["0.64", "1.00", "1.00", "1.00"] us=6667169 diff --git a/docs/research/nightly/2026-08-17_mincut-partitioned-memory-consolidation/evidence/darwin_sweep.txt b/docs/research/nightly/2026-08-17_mincut-partitioned-memory-consolidation/evidence/darwin_sweep.txt new file mode 100644 index 0000000000..338dbf0ac6 --- /dev/null +++ b/docs/research/nightly/2026-08-17_mincut-partitioned-memory-consolidation/evidence/darwin_sweep.txt @@ -0,0 +1,8 @@ +parent partition (fixed for the whole sweep): 5 partitions, sizes=[200, 2000, 1000, 720, 80], correctness=1 +gen=1 candidates_per_generation=4 +floor_min=1 retained=2000 overall_recall=0.4880 worst_cluster_recall=0.1520 fitness=0.4224 +floor_min=3 retained=2000 overall_recall=0.4873 worst_cluster_recall=0.1520 fitness=0.4222 +floor_min=8 retained=2000 overall_recall=0.5000 worst_cluster_recall=0.1480 fitness=0.4240 +floor_min=15 retained=2000 overall_recall=0.5067 worst_cluster_recall=0.1480 fitness=0.4260 +winner: floor_min=15 fitness=0.4260 beats_parent(floor_min=3)=true +DARWIN_RESULT: PROMOTE diff --git a/docs/research/nightly/2026-08-17_mincut-partitioned-memory-consolidation/gist.md b/docs/research/nightly/2026-08-17_mincut-partitioned-memory-consolidation/gist.md new file mode 100644 index 0000000000..b6e10812d6 --- /dev/null +++ b/docs/research/nightly/2026-08-17_mincut-partitioned-memory-consolidation/gist.md @@ -0,0 +1,123 @@ +# Partitioning agent memory before compaction: a negative result, and a bug it uncovered + +## Problem + +Agent memory systems that compact by a single global importance score +(recency + frequency + relevance to current context) can evict an entire +topic in one pass, even at a compaction ratio that looks fine on average. +A topic the agent isn't currently working on has no defense against a +score built for the topic it is working on. + +## Hypothesis + +Partition the memory similarity graph into topic clusters first, then +give each partition a guaranteed minimum retention share, so a topic only +competes with itself for its floor allocation instead of the whole +corpus. Tested on a synthetic 4,000-memory corpus with 6 unequal-size +semantic clusters (down to 2% of the corpus), against +`ruvector-agent-memory`'s existing `CoherencePolicy` baseline, at 50% +compaction. + +Pre-declared bar: the best partition-aware candidate's **worst-cluster** +recall@10 must beat the baseline's by ≥15 percentage points. + +## What happened + +It didn't clear the bar. Overall recall improved (+6.8pp) and 4 of 6 +clusters individually gained 15–23pp — the mechanism clearly does +something. But the specific cluster that was worst under the baseline was +*also* worst under the partitioned candidate, at the identical recall +value, because the partitioner left it merged with the majority cluster +instead of separating it out. A follow-up sweep of the retention floor +(1, 3, 8, 15) left the worst-cluster number flat across every value — +proof the gap is in *where the graph gets cut*, not *how the budget gets +split afterward*. + +```text +per_cluster_recall GlobalTopScore = [0.996, 0.152, 0.216, 0.316, 0.396, 0.440] +per_cluster_recall MincutAdaptive = [0.792, 0.380, 0.504, 0.152, 0.556, 0.540] + ^^^^^ improved a lot ^^^^^ untouched +``` + +## The bug along the way + +Before any of the above could be measured, `ruvector_mincut::DynamicMinCut::partition()` +turned out to be untrustworthy. Minimal repro: two triangles joined by a +single weak-weight bridge edge — a graph whose true minimum cut is +unique and easy to verify by hand. + +```rust +let mincut = MinCutBuilder::new().exact().with_edges(edges).build().unwrap(); +mincut.min_cut_value() // 0.05, every single run — correct +mincut.partition() // sometimes the correct split, sometimes a + // degenerate "isolate one vertex" split whose + // actual crossing weight is 40x the reported value +``` + +At 100 vertices the degenerate split happened on every run, not just +some. `GraphPartitioner` (built on the same machinery) separately dropped +vertices outright, fabricated vertex ids for non-contiguous id spaces, +and took 8.4 seconds to partition 500 vertices — with no sign of +finishing at 4,000 after nearly six minutes. + +None of that is this crate's algorithm — it's the *value* computation +that was correct, only the *partition materialization* that wasn't. The +workaround was a from-scratch, tested, deterministic weighted +Stoer–Wagner implementation (`mincut_exact.rs`, ~250 lines, zero +non-`ruvector_mincut`-type dependencies), used as the sole source of +partition vertex sets, with `ruvector_mincut`'s value still queried +purely as an independent cross-check. + +## Why report a rejected hypothesis + +Because the measurement is real and the mechanism partially works. A +future variant with a smarter stopping rule — one that doesn't let a +single global threshold decide every split — is a legitimate next +experiment, and now has a concrete, per-cluster reason to exist instead +of a hunch. And because the correctness bug this candidate ran into would +have silently produced wrong partitions for anyone else building on +`GraphPartitioner` or `DynamicMinCut::partition()` today, whether or not +this particular hypothesis had panned out. + +## Limitations + +Single run per configuration, one synthetic corpus, one seed family — no +variance characterization. The `ruvector-mincut` defects are documented +with repros but not filed upstream from within this run; that needs the +owning maintainer's independent verification. + +## Production relevance + +None yet — this is a rejected hypothesis. If a per-branch stopping +criterion clears the bar in a follow-up run, the natural production path +is a scheduled ruFlo memory-consolidation workflow, not an inline +write-path operation (the measured ~11s partitioning time at n=4000 rules +that out regardless). + +## RuVector ecosystem implications + +`mincut_exact.rs` is a reusable, correctness-tested min-cut +implementation independent of this ADR's own rejected hypothesis — a +better foundation for any future RuVector graph-partitioning work than +`DynamicMinCut::partition()` as it stands today. + +## Future direction + +Test a per-branch/size-weighted stopping criterion (attempt at least one +more split on the largest remaining partition before accepting a global +threshold's verdict) against the same corpus and the same 15pp bar, as a +new, separately pre-declared hypothesis. + +## References + +- Nightly 2026-06-14, `ruvector-agent-memory` — `CoherencePolicy`, the + baseline this experiment measured against and reused as a dependency. +- Nightly 2026-08-13, `ruvector-retrieval-receipt` — witness-chain design + precedent. +- Stoer & Wagner, "A Simple Min-Cut Algorithm" (1997). +- Jin, Sun & Thorup, "Fully Dynamic Exact Minimum Cut in Subpolynomial + Time" (SODA 2024) — the algorithm `ruvector-mincut` implements. + +Full write-up, ADR, and raw benchmark output: +`docs/research/nightly/2026-08-17_mincut-partitioned-memory-consolidation/` +in [ruvnet/ruvector](https://github.com/ruvnet/ruvector).