research(nightly): hierarchical-cluster-rag — coherence-weighted two-level cluster tree for agent memory RAG

Cargo.toml

@@ -291,6 +291,8 @@ members = [
     "crates/ruvector-timesfm",
     # Speculative ANN search: draft-verify with adaptive candidate multiplier (ADR-272)
     "crates/ruvector-speculative-ann",
+    # Hierarchical cluster-summary RAG: RAPTOR-style two-level tree with coherence-weighted scoring (ADR-300)
+    "crates/ruvector-cluster-rag",
 ]
 resolver = "2"
 

crates/ruvector-cluster-rag/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "ruvector-cluster-rag"
+version.workspace = true
+edition.workspace = true
+authors.workspace = true
+license.workspace = true
+repository.workspace = true
+description = "Hierarchical cluster-summary retrieval for RuVector: RAPTOR-style two-level tree with coherence-weighted cluster scoring for agent memory RAG"
+readme = "README.md"
+keywords = ["vector-search", "rag", "cluster", "agent-memory", "hierarchical"]
+categories = ["algorithms", "data-structures"]
+
+[[bin]]
+name = "benchmark"
+path = "src/bin/benchmark.rs"
+
+[dependencies]
+
+[lints.rust]
+dead_code = "allow"
+unused_variables = "allow"

crates/ruvector-cluster-rag/src/bench.rs

@@ -0,0 +1,118 @@
+//! Benchmark runner: latency statistics and recall measurement.
+
+use std::time::Instant;
+
+use crate::{recall_at_k, AnnVariant, Hit};
+
+/// One benchmark result row.
+pub struct BenchResult {
+    pub variant: &'static str,
+    pub n: usize,
+    pub dim: usize,
+    pub nqueries: usize,
+    pub k: usize,
+    pub mean_us: f64,
+    pub p50_us: f64,
+    pub p95_us: f64,
+    pub qps: f64,
+    pub mem_bytes: usize,
+    pub mean_recall: f64,
+}
+
+/// Run `nqueries` searches and collect latency + recall statistics.
+pub fn run_bench(
+    variant: &dyn AnnVariant,
+    queries: &[Vec<f32>],
+    k: usize,
+    ground_truth: &[Vec<Hit>],
+    n: usize,
+    dim: usize,
+) -> BenchResult {
+    let nq = queries.len();
+    assert_eq!(nq, ground_truth.len());
+
+    let mut latencies_us: Vec<f64> = Vec::with_capacity(nq);
+    let mut total_recall = 0.0f64;
+
+    for (q, gt) in queries.iter().zip(ground_truth.iter()) {
+        let t0 = Instant::now();
+        let hits = variant.search(q, k);
+        let elapsed = t0.elapsed();
+        latencies_us.push(elapsed.as_secs_f64() * 1_000_000.0);
+        total_recall += recall_at_k(&hits, gt) as f64;
+    }
+
+    latencies_us.sort_unstable_by(|a, b| a.partial_cmp(b).unwrap());
+    let mean_us = latencies_us.iter().sum::<f64>() / nq as f64;
+    let p50_us = percentile(&latencies_us, 50.0);
+    let p95_us = percentile(&latencies_us, 95.0);
+    let qps = 1_000_000.0 / mean_us;
+
+    BenchResult {
+        variant: variant.name(),
+        n,
+        dim,
+        nqueries: nq,
+        k,
+        mean_us,
+        p50_us,
+        p95_us,
+        qps,
+        mem_bytes: variant.mem_bytes(),
+        mean_recall: total_recall / nq as f64,
+    }
+}
+
+fn percentile(sorted: &[f64], pct: f64) -> f64 {
+    if sorted.is_empty() {
+        return 0.0;
+    }
+    let idx = ((pct / 100.0) * (sorted.len() - 1) as f64).round() as usize;
+    sorted[idx.min(sorted.len() - 1)]
+}
+
+/// Print a single result row.
+pub fn print_header() {
+    println!(
+        "{:<18} {:>8} {:>6} {:>6} {:>10} {:>10} {:>10} {:>10} {:>12} {:>8}",
+        "Variant", "Mean µs", "p50 µs", "p95 µs", "QPS", "Memory", "Recall@K", "", "", ""
+    );
+    println!("{}", "-".repeat(100));
+}
+
+pub fn print_row(r: &BenchResult) {
+    println!(
+        "{:<18} {:>8.1} {:>6.1} {:>6.1} {:>10.0} {:>12} {:>8.3}",
+        r.variant,
+        r.mean_us,
+        r.p50_us,
+        r.p95_us,
+        r.qps,
+        format_bytes(r.mem_bytes),
+        r.mean_recall,
+    );
+}
+
+pub fn format_bytes(b: usize) -> String {
+    if b >= 1_048_576 {
+        format!("{:.1} MB", b as f64 / 1_048_576.0)
+    } else if b >= 1024 {
+        format!("{:.1} KB", b as f64 / 1024.0)
+    } else {
+        format!("{} B", b)
+    }
+}
+
+/// Acceptance gate: all variants must exceed `min_recall`.
+pub fn acceptance_gate(results: &[BenchResult], min_recall: f64) -> bool {
+    results.iter().all(|r| {
+        let pass = r.mean_recall >= min_recall;
+        if !pass {
+            eprintln!(
+                "FAIL: {} recall {:.3} < threshold {:.3}",
+                r.variant, r.mean_recall, min_recall
+            );
+        }
+        pass
+    })
+}

crates/ruvector-cluster-rag/src/bin/benchmark.rs

@@ -0,0 +1,191 @@
+//! Benchmark binary for ruvector-cluster-rag.
+//!
+//! Runs three search variants over a deterministic synthetic dataset and prints
+//! latency, throughput, memory, and recall statistics. All numbers are real.
+//!
+//! Usage:
+//!   cargo run --release -p ruvector-cluster-rag --bin benchmark
+//!
+//! Optional env overrides:
+//!   N=20000 DIM=128 NQ=1000 K=10 K_CLUSTERS=64 NPROBE=8 LAMBDA=0.7
+
+use ruvector_cluster_rag::{
+    bench::{format_bytes, run_bench, BenchResult},
+    cluster::kmeans,
+    dataset::{generate_queries, generate_vectors},
+    search::{ClusterSearch, CoherenceTree, FlatBrute},
+    tree::ClusterTree,
+    AnnVariant, Hit,
+};
+
+fn env_usize(key: &str, default: usize) -> usize {
+    std::env::var(key)
+        .ok()
+        .and_then(|v| v.parse().ok())
+        .unwrap_or(default)
+}
+
+fn env_f32(key: &str, default: f32) -> f32 {
+    std::env::var(key)
+        .ok()
+        .and_then(|v| v.parse().ok())
+        .unwrap_or(default)
+}
+
+fn print_row(r: &BenchResult) {
+    println!(
+        "{:<18} {:>8.1} {:>8.1} {:>8.1} {:>10.0} {:>12} {:>9.3}",
+        r.variant,
+        r.mean_us,
+        r.p50_us,
+        r.p95_us,
+        r.qps,
+        format_bytes(r.mem_bytes),
+        r.mean_recall,
+    );
+}
+
+fn main() {
+    let n = env_usize("N", 10_000);
+    let dim = env_usize("DIM", 128);
+    let nq = env_usize("NQ", 500);
+    let k = env_usize("K", 10);
+    let k_clusters = env_usize("K_CLUSTERS", 40);
+    let nprobe = env_usize("NPROBE", 20);
+    let lambda = env_f32("LAMBDA", 0.70);
+
+    // ── system info ──────────────────────────────────────────────────────────
+    println!("=== ruvector-cluster-rag benchmark ===");
+    println!("OS      : {}", std::env::consts::OS);
+    println!("Arch    : {}", std::env::consts::ARCH);
+    println!();
+    println!("Config");
+    println!("  N           = {n}  (corpus vectors)");
+    println!("  DIM         = {dim}  (dimensions)");
+    println!("  NQ          = {nq}  (query vectors)");
+    println!("  K           = {k}  (top-k)");
+    println!("  K_CLUSTERS  = {k_clusters}");
+    println!(
+        "  NPROBE      = {nprobe}  ({:.0}% of clusters searched)",
+        nprobe as f64 / k_clusters as f64 * 100.0
+    );
+    println!("  LAMBDA      = {lambda:.2}  (CoherenceTree query-sim weight)");
+    println!();
+
+    // ── dataset ──────────────────────────────────────────────────────────────
+    let seed: u64 = 20260807;
+    let corpus = generate_vectors(n, dim, seed);
+    let queries = generate_queries(nq, dim, seed);
+    println!("Dataset: {n} × {dim} f32 vectors | {nq} queries");
+    println!("Raw corpus memory: {}", format_bytes(n * dim * 4));
+    println!();
+
+    // ── k-means ──────────────────────────────────────────────────────────────
+    print!("k-means (k={k_clusters}, 20 iters) ... ");
+    let t0 = std::time::Instant::now();
+    let km_cs = kmeans(&corpus, k_clusters, 20);
+    let km_ct = kmeans(&corpus, k_clusters, 20);
+    println!("done in {:.2}s", t0.elapsed().as_secs_f64());
+
+    // ── build indexes ─────────────────────────────────────────────────────────
+    let flat = FlatBrute::new(corpus.clone());
+    let cs = ClusterSearch::new(ClusterTree::new(corpus.clone(), km_cs), nprobe);
+    let ct = CoherenceTree::new(ClusterTree::new(corpus, km_ct), nprobe, lambda);
+
+    // ── ground truth ─────────────────────────────────────────────────────────
+    let ground_truth: Vec<Vec<Hit>> = queries.iter().map(|q| flat.search(q, k)).collect();
+
+    // ── benchmark each variant ────────────────────────────────────────────────
+    struct Named {
+        name: &'static str,
+        idx: Box<dyn AnnVariant>,
+    }
+    let variants: Vec<Named> = vec![
+        Named {
+            name: "FlatBrute",
+            idx: Box::new(FlatBrute::new(generate_vectors(n, dim, seed))),
+        },
+        Named {
+            name: "ClusterSearch",
+            idx: Box::new(cs),
+        },
+        Named {
+            name: "CoherenceTree",
+            idx: Box::new(ct),
+        },
+    ];
+
+    let mut results: Vec<BenchResult> = Vec::new();
+    for v in &variants {
+        print!("  Benchmarking {} ...", v.name);
+        let r = run_bench(v.idx.as_ref(), &queries, k, &ground_truth, n, dim);
+        println!(" {:.1} µs/query", r.mean_us);
+        results.push(r);
+    }
+
+    // ── results table ─────────────────────────────────────────────────────────
+    println!();
+    println!(
+        "Results  (n={n}, dim={dim}, nq={nq}, k={k}, k_clusters={k_clusters}, nprobe={nprobe})"
+    );
+    println!();
+    println!(
+        "{:<18} {:>8} {:>8} {:>8} {:>10} {:>12} {:>9}",
+        "Variant", "Mean µs", "p50 µs", "p95 µs", "QPS", "Memory", "Recall@K"
+    );
+    println!("{}", "─".repeat(82));
+    for r in &results {
+        print_row(r);
+    }
+    println!();
+
+    // ── memory breakdown ──────────────────────────────────────────────────────
+    let leaf_bytes = n * dim * 4;
+    let centroid_bytes = k_clusters * dim * 4;
+    let inv_bytes = n * 8;
+    println!("Memory breakdown:");
+    println!("  Leaf vectors       : {}", format_bytes(leaf_bytes));
+    println!("  Centroids (level-1): {}", format_bytes(centroid_bytes));
+    println!("  Inverted lists     : {}", format_bytes(inv_bytes));
+    println!(
+        "  Overhead           : {:.1}%",
+        (centroid_bytes + inv_bytes) as f64 / leaf_bytes as f64 * 100.0
+    );
+    println!();
+
+    // ── acceptance gate ───────────────────────────────────────────────────────
+    // FlatBrute must achieve recall = 1.0 (it is ground truth).
+    // ClusterSearch and CoherenceTree must achieve ≥ 0.70 recall@10
+    // with nprobe/k_clusters = 20% of the corpus searched.
+    let min_recall_cluster = 0.70;
+
+    let flat_r = results.iter().find(|r| r.variant == "FlatBrute").unwrap();
+    assert!(
+        flat_r.mean_recall >= 0.999,
+        "FlatBrute recall {:.4} must equal 1.0",
+        flat_r.mean_recall
+    );
+
+    let mut all_pass = true;
+    for r in results.iter().filter(|r| r.variant != "FlatBrute") {
+        if r.mean_recall >= min_recall_cluster {
+            println!(
+                "ACCEPTANCE PASS: {} recall {:.3} ≥ {min_recall_cluster:.2}",
+                r.variant, r.mean_recall
+            );
+        } else {
+            println!(
+                "ACCEPTANCE FAIL: {} recall {:.3} < {min_recall_cluster:.2}",
+                r.variant, r.mean_recall
+            );
+            all_pass = false;
+        }
+    }
+    println!();
+    if all_pass {
+        println!("All acceptance criteria met. Benchmark complete.");
+    } else {
+        eprintln!("One or more variants failed the acceptance gate.");
+        std::process::exit(1);
+    }
+}