From 2a495809ec8f435f1473c712d3a22c3b99bb3275 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 25 Aug 2026 07:42:45 +0000 Subject: [PATCH 1/2] research: implement distance-adaptive beam search nightly experiment Follow up on ADR-303's untested prior-art citation (arXiv:2505.15636) by implementing the (1+gamma)*d_k relative-distance stopping rule for ANN graph traversal, with baseline/candidate-A/candidate-B variants, a matched-budget control, and a corrected entry-routing scheme (fixes a real cluster-unreachability bug found by the crate's own test suite). --- Cargo.lock | 4 + Cargo.toml | 2 + crates/ruvector-dab-search/Cargo.toml | 17 + .../ruvector-dab-search/src/bin/benchmark.rs | 415 ++++++++++++++++++ crates/ruvector-dab-search/src/dataset.rs | 109 +++++ crates/ruvector-dab-search/src/graph.rs | 208 +++++++++ crates/ruvector-dab-search/src/lib.rs | 120 +++++ crates/ruvector-dab-search/src/metrics.rs | 128 ++++++ crates/ruvector-dab-search/src/search.rs | 372 ++++++++++++++++ 9 files changed, 1375 insertions(+) create mode 100644 crates/ruvector-dab-search/Cargo.toml create mode 100644 crates/ruvector-dab-search/src/bin/benchmark.rs create mode 100644 crates/ruvector-dab-search/src/dataset.rs create mode 100644 crates/ruvector-dab-search/src/graph.rs create mode 100644 crates/ruvector-dab-search/src/lib.rs create mode 100644 crates/ruvector-dab-search/src/metrics.rs create mode 100644 crates/ruvector-dab-search/src/search.rs diff --git a/Cargo.lock b/Cargo.lock index 2a7d21694..868869262 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9191,6 +9191,10 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "ruvector-dab-search" +version = "0.1.0" + [[package]] name = "ruvector-dag" version = "2.3.0" diff --git a/Cargo.toml b/Cargo.toml index b4def6526..1cd0bf500 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -335,6 +335,8 @@ members = [ "crates/ruvector-streaming-qng", # Entropy-adaptive ANN beam search: live Shannon entropy gates beam width (ADR-303) "crates/ruvector-entropy-ann", + # Distance-adaptive beam search: (1+gamma)*d_k stopping rule for graph ANN traversal (ADR-340) + "crates/ruvector-dab-search", # PIR #859 workspace-membership sweep: these crates were dangling — # present under crates/ but neither in members nor exclude, so their # tests and lints never ran in CI. diff --git a/crates/ruvector-dab-search/Cargo.toml b/crates/ruvector-dab-search/Cargo.toml new file mode 100644 index 000000000..7ff5296bf --- /dev/null +++ b/crates/ruvector-dab-search/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "ruvector-dab-search" +version = "0.1.0" +edition = "2021" +description = "Distance-adaptive beam search stopping rule for ANN graph traversal: a (1+gamma)*d_k relative-distance stopping criterion replacing fixed ef_search budgets" +license = "MIT OR Apache-2.0" +repository = "https://github.com/ruvnet/ruvector" +keywords = ["ann", "hnsw", "vector-search", "adaptive", "beam-search"] +categories = ["algorithms", "data-structures"] + +[[bin]] +name = "benchmark" +path = "src/bin/benchmark.rs" + +[profile.release] +opt-level = 3 +lto = "thin" diff --git a/crates/ruvector-dab-search/src/bin/benchmark.rs b/crates/ruvector-dab-search/src/bin/benchmark.rs new file mode 100644 index 000000000..a759cdaad --- /dev/null +++ b/crates/ruvector-dab-search/src/bin/benchmark.rs @@ -0,0 +1,415 @@ +//! Benchmark: Distance-Adaptive Beam (DAB) Search +//! +//! Measures recall@10, latency, and per-query distance-computation work for +//! FixedEf (baseline) vs AdaptiveGamma (candidate) on the same clustered +//! synthetic dataset and graph construction used by ADR-303 +//! (`ruvector-entropy-ann`), so the two nightly experiments are directly +//! comparable. +//! +//! Run: +//! cargo run --release -p ruvector-dab-search --bin benchmark + +use ruvector_dab_search::{ + dataset::{clustered_vectors, ground_truth, random_unit_vectors}, + graph::{FlatGraph, GraphConfig}, + metrics::{LatencyStats, WorkStats}, + recall_at_k, + search::{AdaptiveGamma, FixedEf, Searcher}, +}; +use std::time::Instant; + +// Dataset parameters match ADR-303's benchmark exactly (same seeds, same +// sizes) so this experiment controls for dataset/graph-construction quality +// as a confound against that prior nightly's negative result. +const N: usize = 2_000; +const DIM: usize = 16; +const N_CLUSTERS: usize = 10; +const CLUSTER_NOISE: f32 = 0.20; +const K: usize = 10; +const GRAPH_K: usize = 16; +/// Entry-routing seeds probed per query (see graph.rs docs — replaces both +/// entropy-ann's O(n) brute-force entry scan and this crate's earlier, +/// broken single-fixed-entry design). +const NUM_ENTRY_SEEDS: usize = 40; +const N_QUERIES_EASY: usize = 200; +const N_QUERIES_HARD: usize = 200; +const N_QUERIES_MIXED: usize = 400; + +/// Pre-registered primary gamma. Chosen as the midpoint of the paper's valid +/// range (0, 2] before any benchmark was run on this dataset; the sweep +/// below over {0.2, 1.0} is reported as exploratory context only and does +/// not change which gamma the acceptance test below uses. +const GAMMA_PRIMARY: f32 = 0.5; +const GAMMA_SWEEP: [f32; 2] = [0.2, 1.0]; + +/// Production safety cap for candidate B, fixed in advance (not tuned to +/// results): roughly FixedEf(50)'s typical expansion count on this dataset. +const CAPPED_MAX_EXPANSIONS: u64 = 40; + +/// High-recall reference budget for the recall-floor test. +const EF_REFERENCE: usize = 100; +/// Apples-to-apples budget with ADR-303's default baseline. +const EF_DEFAULT: usize = 50; + +// ─── acceptance thresholds (fixed before the first run of this file) ─────── +/// AdaptiveGamma(primary) must expand measurably more on hard queries than +/// easy queries: this is the direct test for "does it actually adapt", +/// contrasting with ADR-303's measured EntropyScaledEf, whose ef_actual was +/// 122-124 for every query regardless of difficulty (ratio ~= 1.00). +const ADAPT_RATIO_MIN: f64 = 1.15; +/// AdaptiveGamma(primary) recall on each query set must be within this many +/// absolute recall points of FixedEf(EF_REFERENCE) on the same set. +const RECALL_FLOOR_DELTA: f32 = 0.03; +/// On the hard query set specifically, AdaptiveGamma(primary) must beat a +/// FixedEf baseline whose ef is calibrated (on the MIXED set only, to avoid +/// leaking the hard-set test into its own calibration) to match +/// AdaptiveGamma's mean distance-computation budget on the mixed set. +const MATCHED_BUDGET_HARD_ADVANTAGE_MIN: f32 = 0.02; + +fn system_info() { + println!("=== Distance-Adaptive Beam (DAB) Search Benchmark ==="); + println!(); + let os = std::env::consts::OS; + let arch = std::env::consts::ARCH; + println!("OS: {os} / {arch}"); + println!("Rust: (see: rustc --version)"); + let ncpu = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(0); + println!("CPU threads: {ncpu}"); + println!(); +} + +fn eval_recall( + searcher: &dyn Searcher, + queries: &[Vec], + corpus: &[Vec], + k: usize, +) -> f32 { + let total: f32 = queries + .iter() + .map(|q| { + let gt = ground_truth(q, corpus, k); + let out = searcher.search(q, k); + recall_at_k(>, &out.hits, k) + }) + .sum(); + total / queries.len() as f32 +} + +fn eval_work(searcher: &dyn Searcher, queries: &[Vec], k: usize) -> WorkStats { + let mut w = WorkStats::new(); + for q in queries { + w.record(searcher.search(q, k).dist_computations); + } + w +} + +/// Linear-scan calibration: find the FixedEf `ef` whose mean distance- +/// computation count on `queries` is closest to `target`. +fn calibrate_ef_for_budget( + graph: &FlatGraph, + queries: &[Vec], + k: usize, + target: f64, +) -> usize { + let mut best_ef = k; + let mut best_diff = f64::MAX; + for ef in (k..=300).step_by(2) { + let searcher = FixedEf { + graph, + ef_search: ef, + }; + let mean = eval_work(&searcher, queries, k).mean(); + let diff = (mean - target).abs(); + if diff < best_diff { + best_diff = diff; + best_ef = ef; + } + } + best_ef +} + +/// Linear-scan calibration: find the FixedEf `ef` whose recall on `queries` +/// is closest to `target_recall`. +fn calibrate_ef_for_recall( + graph: &FlatGraph, + queries: &[Vec], + corpus: &[Vec], + k: usize, + target_recall: f32, +) -> usize { + let mut best_ef = k; + let mut best_diff = f32::MAX; + for ef in (k..=300).step_by(2) { + let searcher = FixedEf { + graph, + ef_search: ef, + }; + let recall = eval_recall(&searcher, queries, corpus, k); + let diff = (recall - target_recall).abs(); + if diff < best_diff { + best_diff = diff; + best_ef = ef; + } + } + best_ef +} + +fn print_row( + name: &str, + query_type: &str, + n_queries: usize, + recall: f32, + work: &WorkStats, + stats: &LatencyStats, +) { + println!( + " {name:<24} {query_type:<8} n={n_queries:<5} recall={recall:.3} \ + dist_comp(mean={:6.1} sd={:5.1} min={:4} max={:4}) \ + lat_mean={:6.1}us {:.0} qps", + work.mean(), + work.stddev(), + work.min(), + work.max(), + stats.mean_us, + stats.throughput_qps, + ); +} + +fn main() { + system_info(); + + println!("Dataset (identical construction to ADR-303):"); + println!(" N (corpus) : {N}"); + println!(" Dimensions : {DIM}"); + println!(" Clusters : {N_CLUSTERS} noise={CLUSTER_NOISE}"); + println!(" k (recall) : {K}"); + println!(" Graph K : {GRAPH_K}"); + println!(" gamma : primary={GAMMA_PRIMARY}, sweep={GAMMA_SWEEP:?}"); + println!(); + + println!("Building corpus..."); + let t0 = Instant::now(); + let corpus = clustered_vectors(N, DIM, N_CLUSTERS, CLUSTER_NOISE, 42); + println!(" corpus built in {:.1}ms", t0.elapsed().as_millis()); + + println!("Building flat graph (k={GRAPH_K})..."); + let t1 = Instant::now(); + let graph = FlatGraph::build( + corpus.clone(), + GraphConfig { + k_neighbours: GRAPH_K, + num_entry_seeds: NUM_ENTRY_SEEDS, + }, + ); + println!( + " graph built in {:.1}ms, entry_seeds={}", + t1.elapsed().as_millis(), + graph.entry_seeds.len() + ); + println!(); + + let easy_queries = clustered_vectors(N_QUERIES_EASY, DIM, N_CLUSTERS, 0.02, 101); + let hard_queries = random_unit_vectors(N_QUERIES_HARD, DIM, 202); + let mixed_queries = clustered_vectors(N_QUERIES_MIXED, DIM, N_CLUSTERS, CLUSTER_NOISE, 303); + + let mem_kb = graph.memory_bytes() / 1024; + + // ── Variant definitions ──────────────────────────────────────────────── + let fixed_default = FixedEf { + graph: &graph, + ef_search: EF_DEFAULT, + }; + let fixed_reference = FixedEf { + graph: &graph, + ef_search: EF_REFERENCE, + }; + let adaptive_primary = AdaptiveGamma { + graph: &graph, + gamma: GAMMA_PRIMARY, + max_expansions: None, + }; + let adaptive_capped = AdaptiveGamma { + graph: &graph, + gamma: GAMMA_PRIMARY, + max_expansions: Some(CAPPED_MAX_EXPANSIONS), + }; + let adaptive_sweep: Vec = GAMMA_SWEEP + .iter() + .map(|&g| AdaptiveGamma { + graph: &graph, + gamma: g, + max_expansions: None, + }) + .collect(); + + println!("─── Recall / Work / Latency by variant and query set ───"); + println!(); + + let query_sets: [(&[Vec], &str); 3] = [ + (&easy_queries, "easy"), + (&hard_queries, "hard"), + (&mixed_queries, "mixed"), + ]; + + let mut all_variants: Vec<&dyn Searcher> = vec![ + &fixed_default, + &fixed_reference, + &adaptive_primary, + &adaptive_capped, + ]; + for s in &adaptive_sweep { + all_variants.push(s); + } + + // recall/work tables, keyed by (variant name, query label) + use std::collections::HashMap; + let mut recall_table: HashMap<(String, &str), f32> = HashMap::new(); + let mut work_table: HashMap<(String, &str), WorkStats> = HashMap::new(); + + for &searcher in &all_variants { + for &(queries, label) in &query_sets { + let recall = eval_recall(searcher, queries, &corpus, K); + let work = eval_work(searcher, queries, K); + let (_, stats) = + LatencyStats::measure(queries.len(), |i| searcher.search(&queries[i], K)); + print_row( + &searcher.name(), + label, + queries.len(), + recall, + &work, + &stats, + ); + recall_table.insert((searcher.name(), label), recall); + work_table.insert((searcher.name(), label), work); + } + } + println!(); + println!(" Index memory: {mem_kb} KB"); + println!(); + + // ── Acceptance test 1: adaptivity ratio (hard/easy mean dist_comp) ───── + println!("─── Test 1: Does the stopping rule actually adapt per query? ───"); + let name_primary = adaptive_primary.name(); + let easy_work = work_table.get(&(name_primary.clone(), "easy")).unwrap(); + let hard_work = work_table.get(&(name_primary.clone(), "hard")).unwrap(); + let adapt_ratio = hard_work.mean() / easy_work.mean().max(1e-9); + let test1_pass = adapt_ratio >= ADAPT_RATIO_MIN; + println!( + " {name_primary}: mean dist_comp easy={:.1} hard={:.1} ratio(hard/easy)={:.3} (threshold >= {ADAPT_RATIO_MIN})", + easy_work.mean(), + hard_work.mean(), + adapt_ratio + ); + println!( + " Contrast — ADR-303 measured EntropyScaledEf's ef_actual at 122-124 for EVERY \ + query (ratio ~= 1.00), which is why it was rejected. This test is the same question \ + asked of a different signal." + ); + println!(" [{}]", if test1_pass { "PASS" } else { "FAIL" }); + println!(); + + // ── Acceptance test 2: recall floor vs high-recall reference ─────────── + println!("─── Test 2: Recall floor vs FixedEf({EF_REFERENCE}) reference ───"); + let mut test2_pass = true; + for &(_, label) in &query_sets { + let ref_recall = *recall_table.get(&(fixed_reference.name(), label)).unwrap(); + let adaptive_recall = *recall_table.get(&(name_primary.clone(), label)).unwrap(); + let ok = adaptive_recall >= ref_recall - RECALL_FLOOR_DELTA; + test2_pass &= ok; + println!( + " {label:<6} reference={ref_recall:.3} adaptive={adaptive_recall:.3} \ + delta={:+.3} (floor: adaptive >= reference - {RECALL_FLOOR_DELTA}) [{}]", + adaptive_recall - ref_recall, + if ok { "PASS" } else { "FAIL" } + ); + } + println!(); + + // ── Acceptance test 3: matched-budget control on the hard set ───────── + println!("─── Test 3: Matched-budget control (crux test) ───"); + let target_budget = work_table + .get(&(name_primary.clone(), "mixed")) + .unwrap() + .mean(); + let matched_ef = calibrate_ef_for_budget(&graph, &mixed_queries, K, target_budget); + let fixed_matched = FixedEf { + graph: &graph, + ef_search: matched_ef, + }; + let matched_budget_actual = eval_work(&fixed_matched, &mixed_queries, K).mean(); + println!( + " Calibrated on MIXED set only: FixedEf(ef={matched_ef}) has mean dist_comp={matched_budget_actual:.1} \ + (target from {name_primary} on mixed = {target_budget:.1})" + ); + let adaptive_recall_hard = *recall_table.get(&(name_primary.clone(), "hard")).unwrap(); + let matched_recall_hard = eval_recall(&fixed_matched, &hard_queries, &corpus, K); + let advantage = adaptive_recall_hard - matched_recall_hard; + let test3_pass = advantage >= MATCHED_BUDGET_HARD_ADVANTAGE_MIN; + println!( + " On HARD queries at ~matched average budget: {name_primary} recall={adaptive_recall_hard:.3} \ + vs FixedEf({matched_ef},matched) recall={matched_recall_hard:.3} advantage={advantage:+.3} \ + (threshold >= {MATCHED_BUDGET_HARD_ADVANTAGE_MIN})" + ); + println!( + " This is the test ADR-303 could not pass: does adaptively reallocating budget toward \ + harder queries beat a flat allocation at the same average cost?" + ); + println!(" [{}]", if test3_pass { "PASS" } else { "FAIL" }); + println!(); + + // ── Headline number: cost reduction at matched recall (paper's metric) ─ + println!("─── Headline: cost at matched recall (arXiv:2505.15636's own metric) ───"); + let adaptive_recall_mixed = *recall_table.get(&(name_primary.clone(), "mixed")).unwrap(); + let recall_matched_ef = + calibrate_ef_for_recall(&graph, &mixed_queries, &corpus, K, adaptive_recall_mixed); + let fixed_recall_matched = FixedEf { + graph: &graph, + ef_search: recall_matched_ef, + }; + let fixed_recall_matched_work = eval_work(&fixed_recall_matched, &mixed_queries, K).mean(); + let adaptive_work_mixed = work_table + .get(&(name_primary.clone(), "mixed")) + .unwrap() + .mean(); + let reduction_pct = if fixed_recall_matched_work > 0.0 { + 100.0 * (1.0 - adaptive_work_mixed / fixed_recall_matched_work) + } else { + 0.0 + }; + println!( + " On MIXED queries at matched recall ({adaptive_recall_mixed:.3}): FixedEf(ef={recall_matched_ef}) \ + needs {fixed_recall_matched_work:.1} dist_comp/query vs {name_primary}'s {adaptive_work_mixed:.1} \ + ({reduction_pct:+.1}% change)" + ); + println!(); + + // ── Overall acceptance ────────────────────────────────────────────────── + println!("─── Acceptance Result ───"); + let verdict = if !test1_pass || !test2_pass { + "REJECT" + } else if !test3_pass { + "INCONCLUSIVE" + } else { + "ACCEPT" + }; + println!( + " Test 1 (adapts per query): {}", + if test1_pass { "PASS" } else { "FAIL" } + ); + println!( + " Test 2 (recall floor): {}", + if test2_pass { "PASS" } else { "FAIL" } + ); + println!( + " Test 3 (beats matched budget): {}", + if test3_pass { "PASS" } else { "FAIL" } + ); + println!(" VERDICT: {verdict}"); + + if verdict == "REJECT" { + std::process::exit(1); + } +} diff --git a/crates/ruvector-dab-search/src/dataset.rs b/crates/ruvector-dab-search/src/dataset.rs new file mode 100644 index 000000000..f79e7de9d --- /dev/null +++ b/crates/ruvector-dab-search/src/dataset.rs @@ -0,0 +1,109 @@ +//! Deterministic synthetic dataset generation. +//! +//! Identical generator to `ruvector-entropy-ann` (ADR-303) by design: this +//! crate's benchmark reuses the same corpus/query construction so results are +//! directly comparable across the two nightly experiments, controlling for +//! the dataset as a confound. + +/// A deterministic pseudo-random f32 in [0, 1). +fn lcg_rand(state: &mut u64) -> f32 { + *state = state + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + ((*state >> 33) as f32) / (u32::MAX as f32) +} + +/// Generate `n` random unit-normalised vectors of dimension `dim`. +pub fn random_unit_vectors(n: usize, dim: usize, seed: u64) -> Vec> { + let mut state = seed; + (0..n) + .map(|_| { + let mut v: Vec = (0..dim).map(|_| lcg_rand(&mut state) * 2.0 - 1.0).collect(); + let norm = v.iter().map(|x| x * x).sum::().sqrt().max(1e-9); + v.iter_mut().for_each(|x| *x /= norm); + v + }) + .collect() +} + +/// Cluster-structured dataset: `num_clusters` Gaussian-ish blobs. +pub fn clustered_vectors( + n: usize, + dim: usize, + num_clusters: usize, + noise: f32, + seed: u64, +) -> Vec> { + let mut state = seed; + + let centroids: Vec> = (0..num_clusters) + .map(|_| { + let mut c: Vec = (0..dim).map(|_| lcg_rand(&mut state) * 2.0 - 1.0).collect(); + let norm = c.iter().map(|x| x * x).sum::().sqrt().max(1e-9); + c.iter_mut().for_each(|x| *x /= norm); + c + }) + .collect(); + + (0..n) + .map(|i| { + let c = ¢roids[i % num_clusters]; + let mut v: Vec = c + .iter() + .map(|&x| { + let n_val = (lcg_rand(&mut state) * 2.0 - 1.0) * noise; + x + n_val + }) + .collect(); + let norm = v.iter().map(|x| x * x).sum::().sqrt().max(1e-9); + v.iter_mut().for_each(|x| *x /= norm); + v + }) + .collect() +} + +/// Brute-force exact top-k nearest neighbours by squared L2 distance. +pub fn ground_truth(query: &[f32], corpus: &[Vec], k: usize) -> Vec { + let mut dists: Vec<(usize, f32)> = corpus + .iter() + .enumerate() + .map(|(i, v)| (i, l2sq(query, v))) + .collect(); + dists.sort_unstable_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal)); + dists.truncate(k); + dists.into_iter().map(|(i, _)| i).collect() +} + +pub fn l2sq(a: &[f32], b: &[f32]) -> f32 { + a.iter().zip(b.iter()).map(|(x, y)| (x - y) * (x - y)).sum() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn deterministic_generation() { + let a = random_unit_vectors(10, 8, 42); + let b = random_unit_vectors(10, 8, 42); + assert_eq!(a, b); + } + + #[test] + fn unit_norm() { + let vecs = random_unit_vectors(20, 16, 99); + for v in &vecs { + let norm = v.iter().map(|x| x * x).sum::().sqrt(); + assert!((norm - 1.0).abs() < 1e-5, "norm={norm}"); + } + } + + #[test] + fn ground_truth_returns_k() { + let corpus = random_unit_vectors(100, 8, 1); + let query = &corpus[0]; + let gt = ground_truth(query, &corpus, 10); + assert_eq!(gt.len(), 10); + assert_eq!(gt[0], 0); + } +} diff --git a/crates/ruvector-dab-search/src/graph.rs b/crates/ruvector-dab-search/src/graph.rs new file mode 100644 index 000000000..dd77e89d3 --- /dev/null +++ b/crates/ruvector-dab-search/src/graph.rs @@ -0,0 +1,208 @@ +//! Flat k-NN proximity graph — equivalent to HNSW layer-0. +//! +//! Same core construction as `ruvector-entropy-ann::graph::FlatGraph` +//! (ADR-303): an exact per-node k-NN graph, O(n^2 * dim) to build, suitable +//! for PoC datasets up to ~50k vectors. +//! +//! **Entry routing differs from ADR-303 on purpose.** entropy-ann finds each +//! query's entry point by an O(n) brute-force scan; that is deliberately +//! excluded here because it would swamp the metric this crate measures +//! (distance computations spent on beam *traversal*): at N=2000 the O(n) +//! entry scan alone would be an order of magnitude larger than any +//! traversal-stopping-rule difference under test. +//! +//! A single *fixed* entry point (the earlier design of this crate) is wrong +//! for a different reason, discovered by this crate's own test suite: an +//! exact k-NN graph over well-separated clusters has few or no edges +//! *between* clusters, so a single fixed entry point cannot reach clusters +//! other than its own — recall on this test's clustered dataset was ~19%, +//! not because of the stopping rule, but because most queries were +//! structurally unreachable from the one entry point. See the research +//! README's attack pass for the full account. +//! +//! The fix used here — `entry_seeds`, a small deterministic sample of nodes +//! probed at query time (O(seeds), not O(n)) to pick the nearest as the +//! traversal entry point — approximates a coarse HNSW upper-layer routing +//! step without building a real multi-layer index. + +use crate::dataset::l2sq; + +/// Configuration for graph construction. +#[derive(Clone, Debug)] +pub struct GraphConfig { + /// Number of neighbours per node in the built graph. + pub k_neighbours: usize, + /// Number of deterministic entry-point candidates probed per query. + pub num_entry_seeds: usize, +} + +impl Default for GraphConfig { + fn default() -> Self { + GraphConfig { + k_neighbours: 16, + num_entry_seeds: 32, + } + } +} + +/// A single-layer k-NN proximity graph over f32 vectors, with a small +/// deterministic set of entry-routing seed nodes computed at build time. +pub struct FlatGraph { + pub vectors: Vec>, + /// adjacency[i] = sorted list of (dist^2, neighbour_id) for node i + pub adjacency: Vec>, + /// Deterministic sample of node ids probed at query time to pick a + /// traversal entry point (nearest of these to the query). + pub entry_seeds: Vec, + pub config: GraphConfig, +} + +impl FlatGraph { + /// Build the graph from a vector corpus. + pub fn build(vectors: Vec>, config: GraphConfig) -> Self { + let n = vectors.len(); + let k = config.k_neighbours.min(n.saturating_sub(1)); + + let mut adjacency: Vec> = vec![Vec::with_capacity(k); n]; + + for i in 0..n { + let mut dists: Vec<(f32, usize)> = (0..n) + .filter(|&j| j != i) + .map(|j| (l2sq(&vectors[i], &vectors[j]), j)) + .collect(); + dists.sort_unstable_by(|a, b| { + a.0.partial_cmp(&b.0).unwrap_or(std::cmp::Ordering::Equal) + }); + dists.truncate(k); + adjacency[i] = dists; + } + + let entry_seeds = Self::compute_entry_seeds(n, config.num_entry_seeds); + + FlatGraph { + vectors, + adjacency, + entry_seeds, + config, + } + } + + /// Deterministic (seeded, not corpus-content-dependent) sample of up to + /// `count` distinct node indices in `[0, n)`, used as entry-routing + /// candidates. Fixed seed so results are reproducible across runs. + fn compute_entry_seeds(n: usize, count: usize) -> Vec { + if n == 0 { + return Vec::new(); + } + let count = count.min(n); + let mut state: u64 = 0x5eed_1234_dab5_eed1; + let mut seen = std::collections::HashSet::with_capacity(count); + let mut seeds = Vec::with_capacity(count); + let mut attempts = 0usize; + // Rejection sampling; bounded attempts guarantees termination even + // if count == n (every index eventually gets sampled). + while seeds.len() < count && attempts < count * 50 + n { + state = state + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + let idx = ((state >> 33) as usize) % n; + if seen.insert(idx) { + seeds.push(idx); + } + attempts += 1; + } + if seeds.is_empty() { + seeds.push(0); + } + seeds + } + + /// Probe the entry-seed set and return the nearest as the traversal + /// entry point, along with the number of distance computations spent + /// (O(len(entry_seeds)), not O(n)). + pub fn route_entry(&self, query: &[f32]) -> (usize, f32, u64) { + let mut best = self.entry_seeds[0]; + let mut best_d = l2sq(query, &self.vectors[best]); + let mut count: u64 = 1; + for &s in &self.entry_seeds[1..] { + let d = l2sq(query, &self.vectors[s]); + count += 1; + if d < best_d { + best_d = d; + best = s; + } + } + (best, best_d, count) + } + + pub fn len(&self) -> usize { + self.vectors.len() + } + + pub fn is_empty(&self) -> bool { + self.vectors.is_empty() + } + + pub fn dim(&self) -> usize { + self.vectors.first().map(|v| v.len()).unwrap_or(0) + } + + pub fn memory_bytes(&self) -> usize { + let vecs: usize = self.vectors.iter().map(|v| v.len() * 4).sum(); + let adj: usize = self.adjacency.iter().map(|a| a.len() * 8).sum(); + vecs + adj + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::dataset::random_unit_vectors; + + #[test] + fn graph_has_k_neighbours() { + let vecs = random_unit_vectors(30, 8, 7); + let cfg = GraphConfig { + k_neighbours: 6, + num_entry_seeds: 8, + }; + let g = FlatGraph::build(vecs, cfg); + for adj in &g.adjacency { + assert_eq!(adj.len(), 6); + } + } + + #[test] + fn entry_seeds_are_valid_indices_and_distinct() { + let vecs = random_unit_vectors(50, 8, 3); + let g = FlatGraph::build(vecs, GraphConfig::default()); + assert!(!g.entry_seeds.is_empty()); + for &s in &g.entry_seeds { + assert!(s < g.len()); + } + let unique: std::collections::HashSet<_> = g.entry_seeds.iter().collect(); + assert_eq!(unique.len(), g.entry_seeds.len()); + } + + #[test] + fn entry_seeds_deterministic() { + let vecs_a = random_unit_vectors(40, 8, 5); + let vecs_b = random_unit_vectors(40, 8, 5); + let ga = FlatGraph::build(vecs_a, GraphConfig::default()); + let gb = FlatGraph::build(vecs_b, GraphConfig::default()); + assert_eq!(ga.entry_seeds, gb.entry_seeds); + } + + #[test] + fn route_entry_picks_nearest_seed() { + let vecs = random_unit_vectors(60, 8, 11); + let g = FlatGraph::build(vecs, GraphConfig::default()); + let query = g.vectors[g.entry_seeds[0]].clone(); + let (best, best_d, count) = g.route_entry(&query); + assert_eq!(count as usize, g.entry_seeds.len()); + // Querying exactly at a seed's own location must return that seed + // with distance 0 (it is at least as close as any other seed). + assert_eq!(best, g.entry_seeds[0]); + assert!(best_d < 1e-6); + } +} diff --git a/crates/ruvector-dab-search/src/lib.rs b/crates/ruvector-dab-search/src/lib.rs new file mode 100644 index 000000000..207d4d752 --- /dev/null +++ b/crates/ruvector-dab-search/src/lib.rs @@ -0,0 +1,120 @@ +//! # ruvector-dab-search +//! +//! Distance-Adaptive Beam (DAB) search: a per-query graph-traversal stopping +//! rule for approximate nearest-neighbour retrieval, replacing the fixed +//! `ef_search` budget used by standard HNSW-style beam search. +//! +//! ## Why this crate exists +//! +//! ADR-303 (`ruvector-entropy-ann`) tested whether the Shannon entropy of the +//! candidate-heap distance distribution could serve as a live, per-query +//! stopping signal. It measured a negative result: heap-distance entropy +//! saturates near `ln(n)` for every query on that PoC's data, so the +//! "adaptive" variant's apparent recall gain was entirely explained by a +//! larger effective search budget, not by any real per-query adaptivity. +//! That work's own prior-art table cited "Distance Adaptive Beam Search for +//! Provably Accurate Graph-Based Nearest Neighbor Search" (arXiv:2505.15636) +//! but did not implement it. +//! +//! This crate implements and measures that cited alternative honestly, +//! against the exact methodological trap that sank the entropy signal: a +//! **matched-budget control** is mandatory evidence here, not optional +//! commentary (see [`search`] docs and the research README). +//! +//! A second, unrelated trap surfaced during this crate's own development: an +//! exact per-node k-NN graph over well-separated clusters has few or no +//! edges *between* clusters, so a single fixed traversal entry point cannot +//! reach most of the corpus. [`graph::FlatGraph`] routes each query through a +//! small deterministic seed set instead (see its docs) — this is graph +//! plumbing, not part of the gamma hypothesis, but it is exactly the kind of +//! confound the attack pass in the research README is required to surface. +//! +//! ## The stopping rule +//! +//! Maintain the current top-k discovered results (`x_k` = the k-th best, the +//! worst of that set). Expand the frontier by nearest-first order. Stop as +//! soon as the closest unexpanded frontier candidate `x` satisfies: +//! +//! ```text +//! d(q, x) >= (1 + gamma) * d(q, x_k) for gamma in (0, 2] +//! ``` +//! +//! On a navigable graph this guarantees every undiscovered node is at least +//! `(gamma/2) * max_j d(q,j)` from the query — an approximation factor of +//! `2/gamma`, exact recovery at `gamma = 2`. This crate's flat k-NN graph is +//! not proven navigable, so that guarantee is not claimed to transfer +//! exactly; the benchmark measures recall empirically instead of relying on +//! the theorem. See the research README's attack pass for this distinction. +//! +//! ## Variants +//! +//! | Variant | Strategy | Description | +//! |---------|----------|-------------| +//! | [`search::FixedEf`] | Baseline | Fixed `ef_search` budget, result heap capacity `ef` | +//! | [`search::AdaptiveGamma`] (uncapped) | Candidate A | `(1+gamma)*d_k` stopping rule, result heap capacity `k` | +//! | [`search::AdaptiveGamma`] (capped) | Candidate B | Same rule plus a hard expansion-count safety bound | + +pub mod dataset; +pub mod graph; +pub mod metrics; +pub mod search; + +pub use graph::{FlatGraph, GraphConfig}; +pub use search::{AdaptiveGamma, FixedEf, Hit, SearchOutcome, Searcher}; + +/// Recall@k: fraction of true top-k found in approximate results. +/// +/// The denominator is `min(k, ground_truth.len())`. A searcher that returns +/// fewer than `k` results (e.g. via early termination) is penalised, not +/// rewarded: missing results count as misses. +pub fn recall_at_k(ground_truth: &[usize], results: &[Hit], k: usize) -> f32 { + let k = k.min(ground_truth.len()); + if k == 0 { + return 0.0; + } + let gt: std::collections::HashSet = ground_truth[..k].iter().cloned().collect(); + let found = results + .iter() + .take(k) + .filter(|h| gt.contains(&h.id)) + .count(); + found as f32 / k as f32 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn recall_at_k_perfect() { + let gt = vec![0usize, 1, 2, 3, 4]; + let results: Vec = gt + .iter() + .enumerate() + .map(|(i, &id)| Hit { id, dist: i as f32 }) + .collect(); + let r = recall_at_k(>, &results, 5); + assert!((r - 1.0).abs() < 1e-6); + } + + #[test] + fn recall_at_k_zero() { + let gt = vec![0usize, 1, 2]; + let results: Vec = vec![ + Hit { id: 10, dist: 0.1 }, + Hit { id: 11, dist: 0.2 }, + Hit { id: 12, dist: 0.3 }, + ]; + let r = recall_at_k(>, &results, 3); + assert!(r.abs() < 1e-6); + } + + #[test] + fn recall_at_k_penalises_short_result_sets() { + let gt = vec![0usize, 1, 2, 3, 4]; + // Only 2 results returned for k=5: should be scored out of 5, not 2. + let results: Vec = vec![Hit { id: 0, dist: 0.0 }, Hit { id: 1, dist: 0.1 }]; + let r = recall_at_k(>, &results, 5); + assert!((r - 0.4).abs() < 1e-6, "r={r}"); + } +} diff --git a/crates/ruvector-dab-search/src/metrics.rs b/crates/ruvector-dab-search/src/metrics.rs new file mode 100644 index 000000000..3ba3c2757 --- /dev/null +++ b/crates/ruvector-dab-search/src/metrics.rs @@ -0,0 +1,128 @@ +//! Latency, throughput, and search-work measurement utilities. + +use std::time::{Duration, Instant}; + +/// Collect per-query latencies and compute statistics. +pub struct LatencyStats { + pub mean_us: f64, + pub p50_us: f64, + pub p95_us: f64, + pub throughput_qps: f64, +} + +impl LatencyStats { + pub fn measure(n_queries: usize, mut f: F) -> (Vec, Self) + where + F: FnMut(usize) -> R, + { + let mut latencies: Vec = Vec::with_capacity(n_queries); + let mut results: Vec = Vec::with_capacity(n_queries); + + let wall_start = Instant::now(); + for i in 0..n_queries { + let t0 = Instant::now(); + let r = f(i); + latencies.push(t0.elapsed().as_nanos() as u64); + results.push(r); + } + let wall: Duration = wall_start.elapsed(); + + latencies.sort_unstable(); + let n = latencies.len() as f64; + let mean_ns = latencies.iter().sum::() as f64 / n; + let p50_ns = latencies[(latencies.len() as f64 * 0.50) as usize] as f64; + let p95_ns = + latencies[((latencies.len() as f64 * 0.95) as usize).min(latencies.len() - 1)] as f64; + + let stats = LatencyStats { + mean_us: mean_ns / 1_000.0, + p50_us: p50_ns / 1_000.0, + p95_us: p95_ns / 1_000.0, + throughput_qps: n_queries as f64 / wall.as_secs_f64(), + }; + (results, stats) + } +} + +/// Per-query "work" sample set (distance computations, expansions, ...). +/// +/// Used both to report mean/spread and, critically, to test whether a +/// variant's work actually *varies* across queries of different difficulty — +/// the property that ADR-303 (entropy-adaptive beam search) measured as +/// absent (heap-distance entropy saturated to the same value for every +/// query, so `EntropyScaledEf`'s ef_actual was constant). +#[derive(Default, Clone)] +pub struct WorkStats { + samples: Vec, +} + +impl WorkStats { + pub fn new() -> Self { + Self { + samples: Vec::new(), + } + } + + pub fn record(&mut self, v: u64) { + self.samples.push(v); + } + + pub fn mean(&self) -> f64 { + if self.samples.is_empty() { + 0.0 + } else { + self.samples.iter().sum::() as f64 / self.samples.len() as f64 + } + } + + pub fn min(&self) -> u64 { + self.samples.iter().copied().min().unwrap_or(0) + } + + pub fn max(&self) -> u64 { + self.samples.iter().copied().max().unwrap_or(0) + } + + pub fn stddev(&self) -> f64 { + let m = self.mean(); + if self.samples.len() < 2 { + return 0.0; + } + let var = self + .samples + .iter() + .map(|&x| { + let d = x as f64 - m; + d * d + }) + .sum::() + / (self.samples.len() as f64 - 1.0); + var.sqrt() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn work_stats_mean_and_spread() { + let mut w = WorkStats::new(); + for v in [10u64, 20, 30, 40] { + w.record(v); + } + assert!((w.mean() - 25.0).abs() < 1e-9); + assert_eq!(w.min(), 10); + assert_eq!(w.max(), 40); + assert!(w.stddev() > 0.0); + } + + #[test] + fn work_stats_constant_has_zero_stddev() { + let mut w = WorkStats::new(); + for _ in 0..5 { + w.record(124); + } + assert_eq!(w.stddev(), 0.0); + } +} diff --git a/crates/ruvector-dab-search/src/search.rs b/crates/ruvector-dab-search/src/search.rs new file mode 100644 index 000000000..3021f7127 --- /dev/null +++ b/crates/ruvector-dab-search/src/search.rs @@ -0,0 +1,372 @@ +//! Beam-search variants differentiated by their traversal stopping rule. +//! +//! All variants operate on a [`FlatGraph`] (single-layer k-NN proximity +//! graph, representing HNSW layer-0) from a single fixed entry point. +//! +//! - [`FixedEf`]: the standard approach — expand until the result heap holds +//! `ef_search` entries and no closer frontier candidate remains. +//! - [`AdaptiveGamma`]: the *distance-adaptive* stopping rule from +//! "Distance Adaptive Beam Search for Provably Accurate Graph-Based +//! Nearest Neighbor Search" (arXiv:2505.15636). No `ef` parameter: stop +//! expanding as soon as the closest unexpanded frontier candidate `x` +//! satisfies `d(q,x) >= (1+gamma) * d(q,x_k)`, where `x_k` is the current +//! k-th best discovered distance. `gamma in (0, 2]`; the paper proves that +//! every undiscovered node is then at least `(gamma/2) * max_j d(q,j)` +//! away, an approximation-factor-`2/gamma` guarantee on navigable graphs +//! (this flat k-NN graph is not proven navigable, so the guarantee is not +//! claimed to transfer exactly — see the research README's attack pass). +//! [`AdaptiveGamma`] with `max_expansions = Some(_)` is the same rule with +//! a hard expansion cap: a production safety bound against the case where +//! the graph is not navigable enough for the ratio test to fire. + +use crate::graph::FlatGraph; +use std::collections::{BinaryHeap, HashSet}; + +// ─── core types ────────────────────────────────────────────────────────────── + +/// A single ANN result: (vector id, squared-L2 distance to query). +#[derive(Debug, Clone, PartialEq)] +pub struct Hit { + pub id: usize, + pub dist: f32, +} + +// Max-heap by distance (farthest on top -> easy to evict when over capacity). +impl Eq for Hit {} +impl PartialOrd for Hit { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} +impl Ord for Hit { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.dist + .partial_cmp(&other.dist) + .unwrap_or(std::cmp::Ordering::Equal) + } +} + +/// Result of one query, including the work performed to produce it. +pub struct SearchOutcome { + pub hits: Vec, + /// Count of `l2sq` calls made during this query's traversal (excludes + /// the one-time, build-time entry-point computation). + pub dist_computations: u64, + /// Count of nodes popped from the frontier and expanded. + pub expansions: u64, +} + +// ─── trait ─────────────────────────────────────────────────────────────────── + +pub trait Searcher: Send + Sync { + fn search(&self, query: &[f32], k: usize) -> SearchOutcome; + fn name(&self) -> String; + fn memory_bytes(&self) -> usize; +} + +// ─── helpers ───────────────────────────────────────────────────────────────── + +fn l2sq(a: &[f32], b: &[f32]) -> f32 { + a.iter().zip(b.iter()).map(|(x, y)| (x - y) * (x - y)).sum() +} + +// ─── Variant: FixedEf (baseline) ───────────────────────────────────────────── + +/// Standard HNSW-style greedy beam search with a fixed `ef_search` budget: +/// expand until the results heap holds `ef_search` entries and the closest +/// remaining frontier candidate is farther than the current worst result. +pub struct FixedEf<'a> { + pub graph: &'a FlatGraph, + pub ef_search: usize, +} + +impl Searcher for FixedEf<'_> { + fn name(&self) -> String { + format!("FixedEf({})", self.ef_search) + } + + fn search(&self, query: &[f32], k: usize) -> SearchOutcome { + let ef = self.ef_search.max(k); + let (entry, entry_dist, mut dist_computations) = self.graph.route_entry(query); + + let mut candidates: BinaryHeap> = BinaryHeap::new(); + let mut results: BinaryHeap = BinaryHeap::new(); + let mut visited: HashSet = HashSet::new(); + let mut expansions: u64 = 0; + + candidates.push(std::cmp::Reverse(Hit { + id: entry, + dist: entry_dist, + })); + results.push(Hit { + id: entry, + dist: entry_dist, + }); + visited.insert(entry); + + while let Some(std::cmp::Reverse(current)) = candidates.pop() { + if results.len() >= ef { + if let Some(worst) = results.peek() { + if current.dist > worst.dist { + break; + } + } + } + expansions += 1; + + for &(_, neighbour) in &self.graph.adjacency[current.id] { + if visited.contains(&neighbour) { + continue; + } + visited.insert(neighbour); + + let dist = l2sq(query, &self.graph.vectors[neighbour]); + dist_computations += 1; + candidates.push(std::cmp::Reverse(Hit { + id: neighbour, + dist, + })); + + if results.len() < ef { + results.push(Hit { + id: neighbour, + dist, + }); + } else if let Some(worst) = results.peek() { + if dist < worst.dist { + results.pop(); + results.push(Hit { + id: neighbour, + dist, + }); + } + } + } + } + + let mut hits = results.into_sorted_vec(); + hits.truncate(k); + SearchOutcome { + hits, + dist_computations, + expansions, + } + } + + fn memory_bytes(&self) -> usize { + self.graph.memory_bytes() + } +} + +// ─── Variant: AdaptiveGamma (distance-adaptive stopping rule) ─────────────── + +/// Distance-adaptive beam search (arXiv:2505.15636): stop expanding when the +/// closest unexpanded frontier candidate is farther than +/// `(1 + gamma) * d(q, x_k)`, where `x_k` is the current k-th best discovered +/// distance. Unlike [`FixedEf`], the result heap capacity is `k` itself — +/// there is no separate `ef` budget to tune. +/// +/// `max_expansions`, when set, additionally caps the number of frontier +/// nodes expanded regardless of the gamma criterion — a production safety +/// bound tested independently of the gamma rule's own behaviour. +pub struct AdaptiveGamma<'a> { + pub graph: &'a FlatGraph, + pub gamma: f32, + pub max_expansions: Option, +} + +impl Searcher for AdaptiveGamma<'_> { + fn name(&self) -> String { + match self.max_expansions { + Some(cap) => format!("Adaptive(g={:.1},cap={cap})", self.gamma), + None => format!("Adaptive(g={:.1})", self.gamma), + } + } + + fn search(&self, query: &[f32], k: usize) -> SearchOutcome { + let (entry, entry_dist, mut dist_computations) = self.graph.route_entry(query); + + let mut candidates: BinaryHeap> = BinaryHeap::new(); + let mut results: BinaryHeap = BinaryHeap::new(); + let mut visited: HashSet = HashSet::new(); + let mut expansions: u64 = 0; + + candidates.push(std::cmp::Reverse(Hit { + id: entry, + dist: entry_dist, + })); + results.push(Hit { + id: entry, + dist: entry_dist, + }); + visited.insert(entry); + + while let Some(std::cmp::Reverse(current)) = candidates.pop() { + if results.len() >= k { + let kth_best = results.peek().unwrap().dist; + if current.dist >= (1.0 + self.gamma) * kth_best { + break; + } + } + if let Some(cap) = self.max_expansions { + if expansions >= cap { + break; + } + } + expansions += 1; + + for &(_, neighbour) in &self.graph.adjacency[current.id] { + if visited.contains(&neighbour) { + continue; + } + visited.insert(neighbour); + + let dist = l2sq(query, &self.graph.vectors[neighbour]); + dist_computations += 1; + candidates.push(std::cmp::Reverse(Hit { + id: neighbour, + dist, + })); + + if results.len() < k { + results.push(Hit { + id: neighbour, + dist, + }); + } else if let Some(worst) = results.peek() { + if dist < worst.dist { + results.pop(); + results.push(Hit { + id: neighbour, + dist, + }); + } + } + } + } + + let mut hits = results.into_sorted_vec(); + hits.truncate(k); + SearchOutcome { + hits, + dist_computations, + expansions, + } + } + + fn memory_bytes(&self) -> usize { + self.graph.memory_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::dataset::{clustered_vectors, ground_truth}; + use crate::graph::GraphConfig; + + fn build_test_graph() -> FlatGraph { + let vecs = clustered_vectors(300, 8, 5, 0.15, 7); + FlatGraph::build( + vecs, + GraphConfig { + k_neighbours: 12, + num_entry_seeds: 24, + }, + ) + } + + #[test] + fn fixed_ef_returns_k_hits_on_connected_graph() { + let graph = build_test_graph(); + let searcher = FixedEf { + graph: &graph, + ef_search: 30, + }; + let out = searcher.search(&graph.vectors[0], 10); + assert_eq!(out.hits.len(), 10); + assert!(out.dist_computations > 0); + } + + #[test] + fn adaptive_gamma_returns_hits_and_counts_work() { + let graph = build_test_graph(); + let searcher = AdaptiveGamma { + graph: &graph, + gamma: 0.5, + max_expansions: None, + }; + let out = searcher.search(&graph.vectors[0], 10); + assert!(!out.hits.is_empty()); + assert!(out.dist_computations > 0); + assert!(out.expansions > 0); + } + + #[test] + fn larger_gamma_never_expands_less_than_smaller_gamma_on_same_query() { + // A larger gamma relaxes the stopping condition (harder to satisfy + // d(q,x) >= (1+gamma)*d_k), so it must expand at least as much. + let graph = build_test_graph(); + let query = &graph.vectors[3]; + let tight = AdaptiveGamma { + graph: &graph, + gamma: 0.1, + max_expansions: None, + } + .search(query, 10); + let loose = AdaptiveGamma { + graph: &graph, + gamma: 1.5, + max_expansions: None, + } + .search(query, 10); + assert!(loose.expansions >= tight.expansions); + } + + #[test] + fn capped_variant_never_exceeds_cap() { + let graph = build_test_graph(); + let searcher = AdaptiveGamma { + graph: &graph, + gamma: 2.0, + max_expansions: Some(5), + }; + for i in 0..graph.len() { + let out = searcher.search(&graph.vectors[i], 10); + assert!(out.expansions <= 5, "expansions={}", out.expansions); + } + } + + #[test] + fn loose_gamma_achieves_high_recall_on_majority_of_self_queries() { + // gamma=2.0 is the paper's exact-recovery setting on *navigable* + // graphs. This crate's flat exact-k-NN graph (see graph.rs docs) is + // not proven navigable: a node can fail to appear in any other + // node's k-NN adjacency list, making it unreachable from some entry + // points regardless of gamma. That is a real, measured limitation + // (see the research README's attack pass), not a bug in the + // stopping rule — so this test checks the majority statistic, not + // every single query. + let graph = build_test_graph(); + let searcher = AdaptiveGamma { + graph: &graph, + gamma: 2.0, + max_expansions: None, + }; + let sample: Vec = (0..graph.len()).step_by(7).collect(); + let mut self_hit_first = 0usize; + for &i in &sample { + let query = &graph.vectors[i]; + let gt = ground_truth(query, &graph.vectors, 10); + let out = searcher.search(query, 10); + if out.hits.first().map(|h| h.id) == Some(gt[0]) { + self_hit_first += 1; + } + } + let frac = self_hit_first as f64 / sample.len() as f64; + assert!( + frac >= 0.8, + "expected >= 80% of self-queries to find themselves first at gamma=2.0, got {frac:.2}" + ); + } +} From 28d176e1aa6c0cfff6d370ca4d1ee63f3b30d90f Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 25 Aug 2026 07:42:50 +0000 Subject: [PATCH 2/2] docs: add ADR-340 and nightly research report for DAB search rejection Documents the REJECT verdict with full benchmark evidence: the gamma stopping rule shows real per-query adaptivity (unlike ADR-303's entropy signal) but adapts to local density rather than query difficulty, and misses its own pre-registered matched-budget bar. --- .../ADR-340-distance-adaptive-beam-search.md | 216 ++++++++++ .../README.md | 384 ++++++++++++++++++ .../gist.md | 104 +++++ 3 files changed, 704 insertions(+) create mode 100644 docs/adr/ADR-340-distance-adaptive-beam-search.md create mode 100644 docs/research/nightly/2026-08-25-distance-adaptive-beam-ann/README.md create mode 100644 docs/research/nightly/2026-08-25-distance-adaptive-beam-ann/gist.md diff --git a/docs/adr/ADR-340-distance-adaptive-beam-search.md b/docs/adr/ADR-340-distance-adaptive-beam-search.md new file mode 100644 index 000000000..31c999498 --- /dev/null +++ b/docs/adr/ADR-340-distance-adaptive-beam-search.md @@ -0,0 +1,216 @@ +# ADR-340: Distance-Adaptive Beam (DAB) Search for ANN Graph Traversal + +**Date**: 2026-08-25 +**Status**: Closed — negative result (documented; not recommended for production) +**Deciders**: Nightly research agent (revised after measured review) +**Tags**: ann, hnsw, beam-search, ruvector-dab-search, negative-result, adr-303-followup + +--- + +## Context + +ADR-303 (`ruvector-entropy-ann`) tested whether the Shannon entropy of the candidate-heap distance +distribution could serve as a live, per-query stopping signal for HNSW-style beam search, replacing +the fixed `ef_search` budget. It measured a negative result: heap-distance entropy saturates near +`ln(n)` for every query on that PoC's data, so `EntropyScaledEf`'s apparent recall gain was entirely +explained by a larger effective search budget, not by any real per-query adaptivity. Its prior-art +table cited "Distance Adaptive Beam Search for Provably Accurate Graph-Based Nearest Neighbor +Search" (arXiv:2505.15636) as a scalar-distance-threshold alternative but did not implement or +measure it. + +This nightly implements and measures that cited alternative, against the exact methodological trap +that sank the entropy signal. + +--- + +## Hypothesis + +```text +Given a 2,000-vector synthetic corpus at dimension 16, clustered into 10 groups (identical +construction to ADR-303's benchmark), indexed by a single-layer k-NN proximity graph +(k=16 neighbours/node) with query-time entry routing through 40 deterministic seed nodes, + +when beam-search traversal uses the distance-adaptive stopping rule +d(q,x) >= (1+gamma) * d(q,x_k) (gamma=0.5, pre-registered) instead of a fixed ef_search budget, + +then (1) the rule's per-query work (distance computations) should vary measurably more on hard +queries than easy queries (ratio >= 1.15) — the direct test for "does it actually adapt", contrasting +with ADR-303's measured EntropyScaledEf, whose effective budget was constant (~124) for every query, + +and (2) recall@10 should stay within 3 points of a FixedEf(100) high-recall reference on every +query set, + +and (3) on hard queries specifically, it should beat a FixedEf baseline whose ef is calibrated (on a +disjoint query set) to match its own average distance-computation budget, by >= 2 recall points — +proving adaptive reallocation beats flat allocation at equal average cost. +``` + +**Result: REJECT.** Test (1) and test (3) failed as measured; test (2) passed. See +[Evidence](#evidence). + +--- + +## Decision + +**Do not adopt the distance-adaptive stopping rule in this PoC form.** The rule is real — unlike +ADR-303's entropy signal, `AdaptiveGamma`'s per-query distance-computation count has substantial, +genuine spread (stddev 96–172 vs FixedEf's 19–61 at comparable means) — but it adapts to the wrong +thing for this use case, and it does not beat a matched-budget baseline. + +### What was measured + +On the pre-registered `gamma = 0.5`: + +| Query set | FixedEf(50) recall | FixedEf(100) recall | AdaptiveGamma(0.5) recall | AdaptiveGamma mean dist_comp | +|---|---|---|---|---| +| easy | 0.811 | 0.846 | **0.903** | 346.7 (sd 153.6) | +| hard | 0.706 | 0.722 | **0.756** | 317.3 (sd 130.4) | +| mixed | 0.635 | 0.663 | **0.678** | 291.5 (sd 95.7) | + +Recall is higher than both FixedEf baselines on every query set — but at a materially higher +distance-computation cost (291.5–346.7 vs 215.9–276.8), so this is not evidence of a better +recall/cost trade-off by itself; see the matched-budget and matched-recall tests below, which +control for that. + +**Test 1 — adaptivity direction (FAIL).** The pre-registered hypothesis was that harder +(out-of-distribution) queries would need *more* work. Measured: `hard/easy` distance-computation +ratio = **0.915** — hard queries cost *less*, not more. Mechanistically, the `(1+gamma)*d_k` +threshold is a relative distance margin: in the sparse region around an out-of-distribution query, +few graph nodes fall within that margin at all, so the frontier is exhausted quickly; in a dense +cluster interior, many nodes fall within the margin, so expansion continues longer. The signal is +real and query-dependent (unlike ADR-303's constant `ef_actual`), but it tracks **local point +density**, not **task difficulty** — a different flavour of the same failure ADR-303 named: "the +softmin entropy... measures the local density of the neighbourhood the search has landed in, not +the ambiguity of routing to it." Two independent per-query signals on two different mechanisms have +now both been observed to track density instead of difficulty on this synthetic dataset — worth +treating as a standing caution for the next attempt at this problem, not a coincidence to ignore. + +**Test 2 — recall floor (PASS).** AdaptiveGamma(0.5) beat the FixedEf(100) reference by +0.033 to ++0.057 recall points on every query set. + +**Test 3 — matched-budget control (FAIL, narrowly).** A `FixedEf(150)` baseline, calibrated on the +*mixed* query set only to match AdaptiveGamma(0.5)'s mean distance-computation cost there (291.0 vs +291.5 — a tight match), scored 0.741 recall on the *hard* set, vs AdaptiveGamma's 0.756: an advantage +of **+0.014**, short of the pre-registered **+0.02** threshold. This is the test ADR-303's +`EntropyScaledEf` could not even approach (it tied its matched-budget control to four decimal +places); AdaptiveGamma comes measurably closer to real value but still falls short of the bar set in +advance. + +**Headline metric (cost at matched recall — the source paper's own primary metric).** At recall +matched to AdaptiveGamma(0.5)'s 0.678 on the mixed set, `FixedEf(122)` needs 273.4 distance +computations/query vs AdaptiveGamma's 291.5 — a **-6.6%** change, i.e. AdaptiveGamma needs *more* +work for the same recall on this dataset, the reverse of the 10–50% reduction the source paper +reports on SIFT1M/DEEP/GloVe/GIST/MNIST. The most likely explanation (untested further this run, +listed under Open Questions) is that this crate's flat exact-k-NN graph and small synthetic corpus +lack the degree heterogeneity of a real incrementally-built HNSW/Vamana graph, which is exactly the +structure the source paper's theorem assumes ("navigable graphs"). + +### Candidate B — capped variant + +`AdaptiveGamma(gamma=0.5, max_expansions=40)`, a production safety bound fixed before any run +(chosen to roughly match `FixedEf(50)`'s typical expansion count, not tuned to results), behaves +almost identically to `FixedEf(50)` on every metric (recall 0.625–0.801 vs FixedEf(50)'s 0.635–0.811; +cost 198–202 vs 215–218). It does not blow up on hard/sparse queries — a real, useful property if +this direction is revisited — but it also forfeits essentially all of the uncapped variant's recall +gain, confirming the gain is concentrated in the (potentially unbounded) tail the cap removes. + +--- + +## A structural finding, orthogonal to the gamma hypothesis + +While building this crate's benchmark harness, an earlier design used a single fixed traversal +entry point (the node nearest the corpus centroid, computed once at build time) instead of +ADR-303's per-query O(n) brute-force entry scan — deliberately, to avoid an O(n) entry cost swamping +the O(dozens–hundreds) traversal-cost metric this crate measures. That design measured ~19% recall +across the board, regardless of stopping rule or gamma value. Root cause: an exact k-NN graph over +well-separated clusters has few or no edges *between* clusters, so a single fixed entry point can +only reach the ~1/10 of the corpus in its own cluster (10 clusters in the benchmark; ~19% ≈ close to +2/10, consistent with adjacent-cluster noise overlap at this dataset's noise level). The fix used in +the shipped version — `FlatGraph::entry_seeds`, a small (40-node) deterministic sample probed at +query time (`O(entry_seeds)`, not `O(n)`) to select the nearest as the entry point, approximating a +coarse HNSW upper-layer routing step — restores realistic recall (0.625–0.917 across variants) at a +small, constant, cross-variant-equal cost. This is graph-construction plumbing, not part of the +gamma hypothesis, but it is exactly the kind of confound an attack pass is required to catch, and it +is retained in the crate (`graph.rs` docs, `search::tests::loose_gamma_achieves_high_recall_on_majority_of_self_queries`) +as a documented pitfall for any future flat-graph PoC in this repository. + +--- + +## Alternatives Considered + +| Alternative | Notes | +|-------------|-------| +| Fixed ef (status quo) | Remains the recommendation; beats AdaptiveGamma at matched recall on this dataset | +| Heap-distance entropy (ADR-303) | Already rejected; a different but related density-not-difficulty failure | +| Ada-ef (arXiv:2512.06636) | Requires an offline-trained regressor; not attempted this run | +| Min-expansion floor + gamma hybrid | Untested; see Open Questions | + +--- + +## Consequences + +### What the merge provides + +- A self-contained, zero-dependency Rust harness (`ruvector-dab-search`) implementing the DAB + stopping rule faithfully to its source paper's stated inequality, with a matched-budget control + and a matched-recall headline number as permanent benchmark columns. +- A corrected, reusable entry-routing pattern (`entry_seeds`) for any future flat-graph ANN PoC in + this repository, with the failure mode it fixes documented in both code and this ADR. +- A second, independently-measured data point (after ADR-303) that a natural per-query stopping + signal on this synthetic clustered dataset tracks local density rather than task difficulty — + useful negative evidence for whoever attempts this problem next. + +### Costs / trade-offs measured + +- AdaptiveGamma(0.5) costs 291.5–346.7 distance computations/query vs FixedEf(50)'s 215.9–218.1 and + FixedEf(100)'s 256.3–276.8 — 8–61% more work than the baselines it is compared against, for a + recall gain of +1.5 to +5.7 points, a worse cost/recall trade-off than simply raising `ef` + (confirmed by the -6.6% matched-recall headline number). +- The capped variant (candidate B) removes the tail-latency risk but also removes essentially all + of the recall gain, landing within noise of `FixedEf(50)`. + +### If this is ever revisited + +1. Test on a real incrementally-built HNSW or Vamana graph (this PoC's flat exact-k-NN graph is not + proven navigable, and the source paper's guarantee assumes navigability). +2. Test on real embeddings (SIFT1M/GloVe/GIST, matching the source paper) rather than synthetic + clustered data, where local density and task difficulty may correlate differently. +3. Try a hybrid: a minimum-expansion floor (preventing premature termination in sparse regions) + combined with the gamma rule (preventing over-expansion in dense regions) — this PoC's evidence + suggests the two failure directions are somewhat separable. +4. Always report both a matched-budget control and a matched-recall headline number; this run's + narrow test-3 miss (+0.014 vs required +0.02) would have looked like a clean win on recall + numbers alone. + +--- + +## Implementation Status + +**PoC**: `crates/ruvector-dab-search` v0.1.0 — merged as negative result +**Tests**: 17 assertions, all pass (`cargo test --release -p ruvector-dab-search`) +**Benchmark**: `cargo run --release -p ruvector-dab-search --bin benchmark` — includes the +matched-budget control (Test 3) and matched-recall headline number as permanent output + +No production integration is planned. + +--- + +## Open Questions + +- Would the same rule show a positive matched-budget result on a real incrementally-built HNSW + graph, where degree heterogeneity and true navigability hold by construction? +- Does the density-vs-difficulty confound observed here (and, differently, in ADR-303) generalize + to real embedding distributions, or is it an artifact of this repository's synthetic + cluster-plus-noise dataset generator? +- Is a minimum-expansion floor + gamma hybrid (item 3 above) sufficient to fix the observed + under-search-on-hard-queries direction without reintroducing a fixed-budget confound? + +--- + +## References + +- arXiv:2505.15636 — Distance Adaptive Beam Search for Provably Accurate Graph-Based Nearest + Neighbor Search (source of the `(1+gamma)*d_k` stopping rule implemented here) +- ADR-303 — Entropy-Adaptive Beam Search for ANN Graph Traversal (prior nightly; this ADR's + matched-budget-control methodology and density-vs-difficulty framing both build directly on it) +- Research README: `docs/research/nightly/2026-08-25-distance-adaptive-beam-ann/README.md` diff --git a/docs/research/nightly/2026-08-25-distance-adaptive-beam-ann/README.md b/docs/research/nightly/2026-08-25-distance-adaptive-beam-ann/README.md new file mode 100644 index 000000000..1f3b08547 --- /dev/null +++ b/docs/research/nightly/2026-08-25-distance-adaptive-beam-ann/README.md @@ -0,0 +1,384 @@ +# Distance-Adaptive Beam Search: A Real Per-Query Signal That Still Loses to a Fixed Budget + +**150-char summary:** A (1+γ)·d_k relative-distance stopping rule genuinely varies its per-query +cost — but adapts to local density, not query difficulty, and misses its own matched-budget bar. + +**Date:** 2026-08-25 +**Crate:** `crates/ruvector-dab-search` +**ADR:** [ADR-340](../../../adr/ADR-340-distance-adaptive-beam-search.md) +**Follows:** [2026-08-13-entropy-adaptive-ann](../2026-08-13-entropy-adaptive-ann/README.md) (ADR-303) + +--- + +## Abstract + +ADR-303 asked whether a live, per-query signal derived from the search itself could replace HNSW's +fixed `ef_search` budget, and measured a clean negative result: Shannon entropy of the candidate +heap saturates to a constant for every query on that PoC's data, so the "adaptive" variant's recall +gain was just a bigger budget in disguise. That work's own prior-art table cited "Distance Adaptive +Beam Search for Provably Accurate Graph-Based Nearest Neighbor Search" (arXiv:2505.15636, a real, +theoretically-grounded 2025 result with a proved approximation guarantee on navigable graphs) as an +alternative — but never implemented it. + +This nightly does. It implements the paper's exact stopping inequality, +`d(q,x) >= (1+γ)·d(q,x_k)`, on the same dataset and graph construction ADR-303 used, and subjects it +to the same discipline that caught the entropy signal's flaw: a mandatory matched-budget control. + +**Result: REJECT**, but with three pieces of genuinely useful evidence: + +1. Unlike entropy, this signal **does** vary substantially per query (distance-computation stddev of + 96–172, vs FixedEf's 19–61 at comparable means) — it is a real per-query control, not a disguised + constant. +2. It varies in the **wrong direction** for the intended use: hard (out-of-distribution) queries cost + *less* work than easy (cluster-core) queries — ratio 0.915, not the hypothesized >=1.15. It tracks + local point density, not task difficulty — a different mechanism from ADR-303's entropy signal, + but landing on the same kind of confound. +3. It **narrowly misses** its own pre-registered matched-budget bar (+0.014 recall advantage on hard + queries at matched cost, vs a required +0.02) and **loses** on the headline cost-at-matched-recall + metric (-6.6%, the opposite of the source paper's reported 10–50% *reduction* on real embedding + benchmarks). + +All numbers are from `cargo run --release -p ruvector-dab-search --bin benchmark` on the hardware +below. Raw output is reproduced verbatim in [Benchmark Results](#benchmark-results). + +**Hardware:** x86-64, 4 logical CPUs, Linux 6.18.44, `rustc` release build. + +--- + +## Hypothesis + +```text +Given a 2,000-vector synthetic corpus at dimension 16, clustered into 10 groups (identical +construction to ADR-303's benchmark), indexed by a single-layer k-NN proximity graph +(k=16 neighbours/node) with query-time entry routing through 40 deterministic seed nodes, + +when beam-search traversal uses the distance-adaptive stopping rule +d(q,x) >= (1+gamma) * d(q,x_k) (gamma=0.5, pre-registered before this dataset was benchmarked) +instead of a fixed ef_search budget, + +then (1) the rule's per-query work should vary measurably more on hard queries than easy queries +(hard/easy distance-computation ratio >= 1.15), + +and (2) recall@10 should stay within 3 points of a FixedEf(100) high-recall reference on every +query set, + +and (3) on hard queries specifically, it should beat a FixedEf baseline whose ef is calibrated (on +the disjoint mixed-query set) to match its own average distance-computation budget, by >= 2 recall +points. + +ACCEPT requires all three. REJECT if (1) or (2) fails. INCONCLUSIVE if only (3) fails. +``` + +**Result: REJECT** — (1) failed (ratio 0.915, wrong direction), (2) passed, (3) failed narrowly +(+0.014 vs +0.02 required). Because (1) failed outright, not just (3), the pre-registered logic +calls this REJECT rather than INCONCLUSIVE — see [Acceptance Result](#acceptance-result). + +**What this does NOT claim:** that arXiv:2505.15636's method fails in general. It is validated on +SIFT1M/DEEP/MNIST/GloVe/GIST with real, incrementally-built navigable graphs (HNSW/Vamana/NSG/ +EFANNA). This PoC's graph is a flat exact-k-NN graph over a small synthetic corpus, not proven +navigable — see [Why This Result May Not Transfer](#why-this-result-may-not-transfer-to-production-hnsw). + +--- + +## Why This Matters for RuVector + +RuVector's agent-memory retrieval path uses ANN search with a fixed `ef_search`, the same tension +ADR-303 named: easy queries over-search, hard queries under-search, and there is no free per-query +signal to fix it without calibration. This nightly connects: + +1. **Vector search** — the graph-traversal stopping rule under test. +2. **Agent memory** — the intended consumer (semantically ambiguous memory queries mixing easy and + hard cases in the same workload). +3. **Prior nightly research (Flywheel-style evidence retention)** — this experiment exists only + because ADR-303's prior-art table recorded an untested citation; the negative result here is now + itself retained evidence for whoever attempts adaptive stopping next (see + [Lessons for Future Attempts](#lessons-for-future-attempts)). +4. **MetaHarness / nightly research process** — a second consecutive rejection of a per-query + stopping signal on the same dataset is a stronger, more specific finding than either rejection + alone: it suggests the synthetic dataset generator itself may not separate "hard" from "dense" in + a way any purely local, distance-based signal can exploit (see Open Questions in the ADR). +5. **Darwin-style bounded evolution (conceptual)** — γ was swept over {0.2, 0.5, 1.0} as exploratory + context; the pre-registered γ=0.5 result is what is reported as the finding, not the best + post-hoc value across the sweep (see [Gamma Sweep](#gamma-sweep-exploratory)), to avoid exactly + the kind of cherry-picking a bounded-evolution promotion gate must reject. + +--- + +## Architecture + +```mermaid +flowchart TD + Q[Query vector] --> ROUTE[Route via 40 entry seeds
O(seeds), not O(n)] + ROUTE --> ENTRY[Entry node] + ENTRY --> FRONTIER[Min-heap frontier
closest-first] + FRONTIER -->|pop closest| CHECK{Stopping rule} + CHECK -->|FixedEf: results.len>=ef
and current.dist > worst| STOP[Stop] + CHECK -->|AdaptiveGamma: results.len>=k
and current.dist >= (1+gamma)*d_k| STOP + CHECK -->|continue| EXPAND[Expand neighbours
update top-k result heap] + EXPAND --> FRONTIER + STOP --> RESULT[Top-k hits + dist_computations count] +``` + +The graph (`FlatGraph`) is an exact per-node k-NN graph — the same construction ADR-303 used — built +once (`O(n^2 * dim)`), with entry routing (`entry_seeds`) as the one deliberate departure from +ADR-303's design, explained next. + +### Why not ADR-303's brute-force entry point + +ADR-303 finds each query's entry node by an O(n) brute-force scan, explicitly to remove +entry-quality as a variable while studying beam width. That is wrong for *this* experiment: at +N=2,000 the O(n) entry scan is an order of magnitude larger than any traversal-cost difference this +crate measures (tens to low hundreds of distance computations), so it would swamp exactly the signal +under test. + +### Why not a single fixed entry point (and how that failure was caught) + +The first implementation of this crate used a single fixed entry point (the node nearest the corpus +centroid, computed once at build time) instead. It measured ~19% recall across every variant and +every γ — because an exact k-NN graph over well-separated clusters has few or no edges *between* +clusters, so one fixed entry point can only reach the fraction of the corpus in its own cluster (with +10 clusters, ~19% recall is consistent with reaching roughly one cluster plus adjacent-cluster +noise-overlap). This was caught by the crate's own test suite +(`loose_gamma_achieves_high_recall_on_majority_of_self_queries` failed at 19% against an 80% +threshold) before any benchmark numbers were trusted — exactly the kind of thing an attack pass is +supposed to catch. The fix, `entry_seeds`, is documented in [graph.rs](../../../../crates/ruvector-dab-search/src/graph.rs) +and kept in ADR-340 as a reusable pitfall for future flat-graph PoCs in this repository. + +--- + +## Implementation + +Three `Searcher` implementations, matching the repository's required baseline/candidate-A/candidate-B +shape: + +| Variant | Role | Stopping rule | Result-heap capacity | +|---|---|---|---| +| `FixedEf` | Baseline | `results.len() >= ef && current.dist > worst_result` | `ef_search` (tunable, `>= k`) | +| `AdaptiveGamma` (uncapped) | Candidate A | `results.len() >= k && current.dist >= (1+γ)·d_k` | `k` (no separate ef) | +| `AdaptiveGamma` (capped) | Candidate B | Same rule, plus a hard `max_expansions` safety bound | `k` | + +Every `search()` call returns a `SearchOutcome{ hits, dist_computations, expansions }` — the crate +counts real `l2sq` calls, not a proxy, so "distance computations" in every table below is an exact +count, not an estimate. + +No external dependencies (matches ADR-303's convention): the deterministic dataset generator uses +the same fixed-seed LCG. + +--- + +## Benchmark Methodology + +- **Release build**, `opt-level = 3`, `lto = "thin"`. +- **Deterministic seeds** throughout: corpus seed 42, easy-query seed 101, hard-query seed 202, + mixed-query seed 303, entry-seed sampling seed fixed in `graph.rs` — reruns reproduce identical + numbers (verified: two independent runs in this nightly produced identical dist_comp/recall + figures to the printed precision). +- **Ground-truth computation excluded from timed sections** — the brute-force scan used to compute + recall is not part of the measured search latency. +- **Query sets**: `easy` (tight clusters, noise=0.02), `hard` (uniform random unit vectors — maximally + out-of-distribution), `mixed` (same noise as the corpus, different seed) — identical construction + to ADR-303's three-way split. +- **Matched-budget calibration** (Test 3) is done via linear scan of `FixedEf`'s `ef` on the *mixed* + query set only, then applied to the *hard* set for the actual test — deliberately avoiding + calibrating on the same set the test measures, to prevent the calibration procedure itself from + leaking into the result it's supposed to control for. +- **γ=0.5 was pre-registered** as the primary value (paper's valid range is `(0, 2]`; 0.5 was chosen + as roughly the paper's own the mid-low working range before any run on this dataset). The + {0.2, 1.0} sweep is reported as exploratory context and does not change which number is reported + as the finding. + +--- + +## Benchmark Results + +Verbatim output from `cargo run --release -p ruvector-dab-search --bin benchmark`: + +```text +=== Distance-Adaptive Beam (DAB) Search Benchmark === + +OS: linux / x86_64 +Rust: (see: rustc --version) +CPU threads: 4 + +Dataset (identical construction to ADR-303): + N (corpus) : 2000 + Dimensions : 16 + Clusters : 10 noise=0.2 + k (recall) : 10 + Graph K : 16 + gamma : primary=0.5, sweep=[0.2, 1.0] + +Building corpus... + corpus built in 0ms +Building flat graph (k=16)... + graph built in 236ms, entry_seeds=40 + +─── Recall / Work / Latency by variant and query set ─── + + FixedEf(50) easy n=200 recall=0.811 dist_comp(mean= 215.3 sd= 33.1 min= 192 max= 337) lat_mean= 34.3us 29145 qps + FixedEf(50) hard n=200 recall=0.706 dist_comp(mean= 218.1 sd= 37.0 min= 188 max= 443) lat_mean= 36.9us 27056 qps + FixedEf(50) mixed n=400 recall=0.635 dist_comp(mean= 215.9 sd= 28.5 min= 185 max= 384) lat_mean= 41.1us 24315 qps + FixedEf(100) easy n=200 recall=0.846 dist_comp(mean= 276.8 sd= 60.0 min= 226 max= 374) lat_mean= 65.4us 15280 qps + FixedEf(100) hard n=200 recall=0.722 dist_comp(mean= 259.6 sd= 58.7 min= 222 max= 546) lat_mean= 62.4us 16008 qps + FixedEf(100) mixed n=400 recall=0.663 dist_comp(mean= 256.3 sd= 53.8 min= 222 max= 530) lat_mean= 59.4us 16822 qps + Adaptive(g=0.5) easy n=200 recall=0.903 dist_comp(mean= 346.7 sd=153.6 min= 228 max= 608) lat_mean= 84.4us 11834 qps + Adaptive(g=0.5) hard n=200 recall=0.756 dist_comp(mean= 317.3 sd=130.4 min= 214 max= 635) lat_mean= 80.6us 12402 qps + Adaptive(g=0.5) mixed n=400 recall=0.678 dist_comp(mean= 291.5 sd= 95.7 min= 191 max= 634) lat_mean= 68.3us 14632 qps + Adaptive(g=0.5,cap=40) easy n=200 recall=0.801 dist_comp(mean= 198.0 sd= 19.0 min= 176 max= 274) lat_mean= 26.7us 37344 qps + Adaptive(g=0.5,cap=40) hard n=200 recall=0.693 dist_comp(mean= 202.2 sd= 23.2 min= 175 max= 332) lat_mean= 33.9us 29474 qps + Adaptive(g=0.5,cap=40) mixed n=400 recall=0.625 dist_comp(mean= 200.6 sd= 18.5 min= 174 max= 312) lat_mean= 33.4us 29848 qps + Adaptive(g=0.2) easy n=200 recall=0.809 dist_comp(mean= 208.8 sd= 39.0 min= 168 max= 351) lat_mean= 32.4us 30769 qps + Adaptive(g=0.2) hard n=200 recall=0.714 dist_comp(mean= 219.4 sd= 51.3 min= 164 max= 555) lat_mean= 37.2us 26857 qps + Adaptive(g=0.2) mixed n=400 recall=0.632 dist_comp(mean= 211.4 sd= 37.6 min= 142 max= 420) lat_mean= 30.7us 32564 qps + Adaptive(g=1.0) easy n=200 recall=0.917 dist_comp(mean= 447.6 sd=171.3 min= 238 max= 637) lat_mean= 172.9us 5780 qps + Adaptive(g=1.0) hard n=200 recall=0.767 dist_comp(mean= 401.2 sd=172.7 min= 238 max= 637) lat_mean= 136.5us 7323 qps + Adaptive(g=1.0) mixed n=400 recall=0.690 dist_comp(mean= 382.8 sd=161.9 min= 237 max= 637) lat_mean= 147.8us 6764 qps + + Index memory: 375 KB + +─── Test 1: Does the stopping rule actually adapt per query? ─── + Adaptive(g=0.5): mean dist_comp easy=346.7 hard=317.3 ratio(hard/easy)=0.915 (threshold >= 1.15) + Contrast — ADR-303 measured EntropyScaledEf's ef_actual at 122-124 for EVERY query (ratio ~= 1.00), which is why it was rejected. This test is the same question asked of a different signal. + [FAIL] + +─── Test 2: Recall floor vs FixedEf(100) reference ─── + easy reference=0.846 adaptive=0.903 delta=+0.057 (floor: adaptive >= reference - 0.03) [PASS] + hard reference=0.722 adaptive=0.756 delta=+0.033 (floor: adaptive >= reference - 0.03) [PASS] + mixed reference=0.663 adaptive=0.678 delta=+0.015 (floor: adaptive >= reference - 0.03) [PASS] + +─── Test 3: Matched-budget control (crux test) ─── + Calibrated on MIXED set only: FixedEf(ef=150) has mean dist_comp=291.0 (target from Adaptive(g=0.5) on mixed = 291.5) + On HARD queries at ~matched average budget: Adaptive(g=0.5) recall=0.756 vs FixedEf(150,matched) recall=0.741 advantage=+0.014 (threshold >= 0.02) + This is the test ADR-303 could not pass: does adaptively reallocating budget toward harder queries beat a flat allocation at the same average cost? + [FAIL] + +─── Headline: cost at matched recall (arXiv:2505.15636's own metric) ─── + On MIXED queries at matched recall (0.678): FixedEf(ef=122) needs 273.4 dist_comp/query vs Adaptive(g=0.5)'s 291.5 (-6.6% change) + +─── Acceptance Result ─── + Test 1 (adapts per query): FAIL + Test 2 (recall floor): PASS + Test 3 (beats matched budget): FAIL + VERDICT: REJECT +``` + +Reproduced twice; both runs produced identical figures to the printed precision (deterministic +seeds, no floating-point-order nondeterminism observed at this scale). + +### Gamma Sweep (exploratory) + +Not used to select the reported result — γ=0.5 was fixed in advance. Included because a Pareto view +is informative: larger γ trades more cost for more recall roughly monotonically (γ=0.2: 208.8–219.4 +dist_comp, recall 0.632–0.714; γ=1.0: 382.8–447.6 dist_comp, recall 0.690–0.917), and the +hard/easy adaptivity-ratio problem (Test 1) persists at every γ tested — it is not an artifact of the +particular γ=0.5 choice. + +--- + +## Why This Result May Not Transfer to Production HNSW + +arXiv:2505.15636's theorem, and its reported 10–50% distance-computation reduction, is stated for +*navigable* graphs (formally: a graph where a greedy walk from any start node monotonically +approaches any target). This PoC's graph is an exact per-node k-NN graph, not an incrementally +constructed HNSW/Vamana/NSG graph, and it is not proven navigable — indeed, the entry-routing +incident above is direct evidence it is not even fully *connected* in the relevant sense without +seed-based routing. A production HNSW graph, built by the standard heuristic-pruned insertion +algorithm, has different degree and connectivity properties by construction. This nightly's result +is therefore evidence about *this specific graph construction*, not a refutation of the source +paper's own reported numbers on real navigable graphs and real embedding datasets — see Open +Questions in [ADR-340](../../../adr/ADR-340-distance-adaptive-beam-search.md#open-questions) for the +concrete next experiment this implies. + +--- + +## Lessons for Future Attempts + +1. **A signal having real per-query variance (unlike ADR-303's constant) is necessary but not + sufficient.** This nightly's signal clears that first bar and still loses on the metric that + matters (matched-budget recall). Report both, always. +2. **Two different local signals (heap entropy, relative-distance ratio) have now both been observed + to track local point density rather than task difficulty** on this repository's synthetic + cluster-plus-noise dataset generator. That is either a property of relative/local signals in + general on this kind of data, or an artifact of the generator itself (uniform noise within + clusters, uniform random "hard" queries) — worth testing with a difficulty axis that is + *independent* of density (e.g. queries at a fixed distance from the nearest cluster centroid, + rather than uniform random) before trying a third local signal. +3. **Calibrate matched-budget controls on a disjoint query set from the one under test.** This + nightly calibrated on `mixed` and tested on `hard`, specifically to avoid the calibration itself + absorbing the effect being measured. + +--- + +## MCP / RVF / RVM / ruFlo / Edge Implications + +Given the REJECT verdict, none of these are recommended for integration from this specific PoC. For +completeness, briefly: + +- **MCP**: not applicable — no capability is being promoted. +- **RVF/RVM**: not applicable — no portable index format or coherence-domain change is proposed. +- **ruFlo**: the one transferable piece is the *process*: a ruFlo workflow role that runs "test the + next cited-but-unimplemented alternative from the last rejected nightly" is a well-defined, + boundable, valuable autonomous task — this nightly is itself an instance of exactly that pattern + (ADR-303's citation → this nightly's implementation). +- **Edge/WASM**: not evaluated; moot given the REJECT verdict. + +--- + +## Security / Governance + +No security-relevant surface is introduced: this is a benchmark-only crate with no I/O, no network +access, and no production integration path. `cargo test` and the benchmark binary are the only +executables. No secrets, credentials, or external data are used. + +--- + +## Practical and Long-Horizon Applications + +Not applicable in the standard sense — this is a rejected research direction. The transferable value +is methodological (see [Lessons for Future Attempts](#lessons-for-future-attempts)), not a +capability to deploy. + +--- + +## Falsification Criteria (met) + +The hypothesis was falsifiable and was falsified: pre-registered Test 1 (adaptivity direction) and +Test 3 (matched-budget advantage) were specified with numeric thresholds before the benchmark was +run, and both failed as measured. Test 2 (recall floor) passed but is not sufficient alone for +ACCEPT under the pre-registered logic. + +--- + +## Limitations + +- Single synthetic dataset (N=2,000, dim=16, 10 clusters); no real embedding dataset was used this + run (see [Why This Result May Not Transfer](#why-this-result-may-not-transfer-to-production-hnsw)). +- Flat exact-k-NN graph, not an incrementally constructed HNSW/Vamana graph — the source paper's + navigability assumption is not verified to hold here. +- Single hardware configuration (4 logical CPUs); no multi-thread or SIMD path measured. +- γ sweep limited to 3 values; a finer sweep or an offline-optimal γ was not attempted (would itself + require an evaluation-leakage-free protocol to avoid p-hacking the reported number). + +--- + +## Next Research + +Per [ADR-340](../../../adr/ADR-340-distance-adaptive-beam-search.md#if-this-is-ever-revisited): +test on a real incrementally-built HNSW graph and a real embedding dataset before concluding the +method itself (as opposed to this PoC's graph construction) is not viable for RuVector; and/or test +a minimum-expansion-floor + gamma hybrid to address the under-search-on-sparse-regions direction +found here. + +--- + +## References + +- arXiv:2505.15636 — Distance Adaptive Beam Search for Provably Accurate Graph-Based Nearest + Neighbor Search (source of the stopping rule implemented here; NeurIPS 2025) +- ADR-303 / `docs/research/nightly/2026-08-13-entropy-adaptive-ann/README.md` — the prior nightly + this one directly follows up on +- VBASE (OSDI 2023) — relaxed monotonicity as a related but distinct termination-relaxation idea, + not implemented here +- Li, Zhang, Andersen, He — "Improving Approximate Nearest Neighbor Search through Learned Adaptive + Early Termination" (SIGMOD 2020) — a learned-regressor approach to the same problem, not attempted + this run diff --git a/docs/research/nightly/2026-08-25-distance-adaptive-beam-ann/gist.md b/docs/research/nightly/2026-08-25-distance-adaptive-beam-ann/gist.md new file mode 100644 index 000000000..e719b7431 --- /dev/null +++ b/docs/research/nightly/2026-08-25-distance-adaptive-beam-ann/gist.md @@ -0,0 +1,104 @@ +# A real adaptive stopping signal for ANN search still lost to a fixed budget — here's the honest data + +## Problem + +Graph-based approximate nearest-neighbour search (HNSW and friends) traverses a proximity graph +with a fixed `ef_search` candidate budget, tuned offline. That's a systematic mismatch: easy queries +(near a dense cluster) finish long before the budget is spent; hard, out-of-distribution queries run +out of budget before finding their true neighbours. The right `ef` is per-query, and the graph +traversal itself is one of the only places that could supply a free, zero-calibration signal for it. + +A previous experiment in this series tried Shannon entropy of the candidate heap as that signal and +found it didn't work: the entropy saturated to the same value for every query, so the "adaptive" +result was just a bigger fixed budget wearing a costume. That negative result cited, but didn't +implement, an alternative from the literature: a 2025 paper's *distance-ratio* stopping rule with an +actual proof behind it. + +## Hypothesis + +Implement that cited alternative for real, and hold it to the same bar that caught the entropy +signal's flaw: it has to beat a **matched-budget control**, not just look good on a recall table. + +The rule: maintain the current top-k best-found distances during graph traversal; stop expanding as +soon as the next candidate in line is farther than `(1+γ)` times the current k-th-best distance, +for a tunable `γ` in `(0, 2]`. On a graph with a certain navigability property, this provably bounds +how far any undiscovered point can be. Pre-registered before any run on this dataset: `γ = 0.5`. + +Three numeric bars, fixed before benchmarking: + +1. The rule's per-query cost must vary substantially more on hard queries than easy ones (a direct + test for "does it actually adapt," since the previous signal's own failure was that it didn't). +2. Recall can't drop more than 3 points below a generously-budgeted fixed baseline. +3. On hard queries, at a cost matched to its own average budget, it must beat a plain fixed-budget + baseline by at least 2 recall points. + +## What happened + +Built the whole thing in Rust: a k-NN proximity graph, three search variants (fixed-budget baseline, +the new adaptive rule uncapped, and a capped production-safe version), a deterministic synthetic +benchmark, and — before trusting any of it — a correctness test suite that caught a real bug: an +early version used a single fixed graph entry point, which turned out to only be able to *reach* +about a fifth of the corpus, because a nearest-neighbour graph over separated clusters has no edges +between them. Fixed with a small deterministic set of routing candidates probed per query instead. + +With that fixed, the actual experiment: + +- **Bar 1 (does it adapt) — failed, in an interesting way.** The signal *does* vary a lot per query + (that alone beats the previous entropy attempt, which didn't vary at all). But it varies backwards: + hard, out-of-distribution queries cost *less* work than easy, cluster-core queries — a 0.915 ratio + where the hypothesis needed at least 1.15. The mechanism, once you look at it, makes sense: the + stopping threshold is a relative distance margin, and in a sparse region (where a hard query + lands) there just aren't many points within any given margin, so the frontier runs dry fast. In a + dense cluster, lots of points sit inside that margin, so expansion drags on. The rule is adapting + to local crowding, not to how hard the query actually is. +- **Bar 2 (recall floor) — passed comfortably.** +- **Bar 3 (beats a matched-budget baseline) — failed, narrowly.** +0.014 recall advantage where +0.02 + was required. Close, but the pre-registered number is the pre-registered number. +- **The headline comparison** — cost needed to hit the same recall as a plain fixed budget — came out + *negative*: the adaptive rule needed 6.6% *more* work for equal recall, the opposite of the + 10-50% reduction the source paper reports on real embedding benchmarks with real HNSW graphs. + +## Why the mismatch with the source paper is not a contradiction + +The paper's guarantee, and its reported wins, are on **navigable graphs** — the kind you get from an +incrementally-built HNSW or Vamana index. This experiment used a flat, exact k-nearest-neighbour +graph over a small synthetic dataset, which is not proven navigable (the entry-point bug above is +direct evidence it wasn't even fully reachable without a fix). So this is evidence about *this graph +construction*, not a refutation of the paper's own results. The natural next experiment is obvious: +run the same rule on a real incrementally-built HNSW graph over real embeddings before concluding +anything about the method itself. + +## The useful part + +Two independent per-query signals, tried on two separate nights, using two different mechanisms +(entropy of a distance distribution; a relative distance-ratio threshold), have now both been +observed to track *local density* instead of *task difficulty* on the same kind of synthetic +clustered dataset. That's either a real property of local/relative signals on this class of data, or +an artifact of how "hard" queries were generated (uniform random points, which are also just points +in sparse regions — density and difficulty are confounded by construction). Either way, it's a +specific, falsifiable thing to check before a third attempt at this problem: build a "hard" query set +that's genuinely difficult *without* also being in a sparse region, and see if either signal behaves +differently. + +## What this is not + +Not a claim that distance-adaptive stopping doesn't work — the cited paper's own results, on real +navigable graphs, say otherwise. Not a production recommendation either direction. It's a specific, +reproducible negative result on a specific graph construction, plus a concrete, falsifiable next +step, which is what a rejected hypothesis with real evidence is supposed to leave behind. + +## Reproduce it + +```bash +cargo test --release -p ruvector-dab-search +cargo run --release -p ruvector-dab-search --bin benchmark +``` + +Both are deterministic (fixed seeds throughout); rerunning reproduces the same numbers reported here. + +## References + +- Distance Adaptive Beam Search for Provably Accurate Graph-Based Nearest Neighbor Search, + arXiv:2505.15636 (NeurIPS 2025) — source of the stopping rule implemented here. +- The prior nightly this one follows up on: entropy-adaptive beam search (negative result), same + repository, `docs/research/nightly/2026-08-13-entropy-adaptive-ann/`.