From ec954fba9728f531aedee70fa1903121b86c4fc3 Mon Sep 17 00:00:00 2001 From: OceanLi <122793010+ohdearquant@users.noreply.github.com> Date: Sun, 2 Aug 2026 13:14:44 -0400 Subject: [PATCH 1/3] feat(diskann): route f32 distance kernels through lattice-embed behind a feature Adds an off-by-default `lattice-simd` backend for `l2_squared` and `inner_product`, ahead of the existing wasm32 SIMD128, SimSIMD and scalar arms. The arms stay mutually exclusive and exhaustive, so exactly one compiles for any feature/target combination and default builds are unchanged. Also adds `backend_matches_scalar_reference`, a cross-dimension parity test against naive scalar references. The existing cross-dimension tests are gated on wasm32 + simd128, so no native backend was checked above dim 3. The new test is backend-independent and covers the SimSIMD, scalar and lattice paths alike. --- Cargo.lock | 21 ++++- crates/ruvector-diskann/Cargo.toml | 9 ++ crates/ruvector-diskann/src/distance.rs | 120 ++++++++++++++++++++++-- 3 files changed, 139 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 03d9322bc8..0c4a093577 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4938,6 +4938,24 @@ dependencies = [ "tracing", ] +[[package]] +name = "lattice-embed" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a8471670d8eb3dc5b52b7c977f1be449a4d39404ebd47bd084a1e922fa6002e" +dependencies = [ + "async-trait", + "blake3", + "chrono", + "lru 0.16.4", + "parking_lot 0.12.5", + "serde", + "serde_json", + "thiserror 2.0.18", + "tokio", + "tracing", +] + [[package]] name = "lattice-inference" version = "0.6.1" @@ -9203,7 +9221,7 @@ dependencies = [ "dashmap 6.2.1", "hf-hub", "hnsw_rs", - "lattice-embed", + "lattice-embed 0.6.1", "memmap2", "mockall", "ndarray 0.16.1", @@ -9407,6 +9425,7 @@ dependencies = [ "bincode 2.0.1", "bytemuck", "getrandom 0.2.17", + "lattice-embed 0.7.1", "memmap2", "parking_lot 0.12.5", "rand 0.8.6", diff --git a/crates/ruvector-diskann/Cargo.toml b/crates/ruvector-diskann/Cargo.toml index bf4adcb7f8..f2211bdd97 100644 --- a/crates/ruvector-diskann/Cargo.toml +++ b/crates/ruvector-diskann/Cargo.toml @@ -11,6 +11,10 @@ description = "DiskANN/Vamana — SSD-friendly approximate nearest neighbor sear default = [] gpu = [] # Feature flag for GPU acceleration (CUDA/Metal stubs) simd = ["simsimd"] +# Route the f32 distance kernels through lattice-embed rather than SimSIMD. +# Takes precedence over `simd` and over the wasm32 SIMD128 path when both are +# enabled, so exactly one backend compiles for any feature/target combination. +lattice-simd = ["lattice-embed"] # BET 1 (ADR-200): fixed-topology reuse + periodic rebuild under metric drift. reuse-under-drift = [] @@ -24,6 +28,11 @@ thiserror = { workspace = true } rand = { workspace = true } parking_lot = "0.12" bytemuck = { version = "1.14", features = ["derive"] } +# Kernels only: `native` is the feature that pulls the model stack, so with +# default features off this is a pure-Rust SIMD dependency with no tokenizer, +# no model loader, and no download path. Unlike `simsimd` below it is not +# target-gated, because it compiles for wasm32 as well. +lattice-embed = { version = "0.7.1", optional = true, default-features = false } [target.'cfg(target_arch = "wasm32")'.dependencies] # `rand` (used by PQ k-means training) pulls getrandom 0.2 transitively, which diff --git a/crates/ruvector-diskann/src/distance.rs b/crates/ruvector-diskann/src/distance.rs index 3c7dc34c2b..0786d142e3 100644 --- a/crates/ruvector-diskann/src/distance.rs +++ b/crates/ruvector-diskann/src/distance.rs @@ -194,19 +194,35 @@ impl FlatVectors { pub fn l2_squared(a: &[f32], b: &[f32]) -> f32 { assert_eq!(a.len(), b.len(), "distance vectors must have equal lengths"); - #[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] + #[cfg(feature = "lattice-simd")] + { + lattice_embed::simd::squared_euclidean_distance(a, b) + } + + #[cfg(all( + not(feature = "lattice-simd"), + target_arch = "wasm32", + target_feature = "simd128" + ))] { wasm_simd128_l2_squared(a, b) } - #[cfg(all(not(target_arch = "wasm32"), feature = "simd"))] + #[cfg(all( + not(feature = "lattice-simd"), + not(target_arch = "wasm32"), + feature = "simd" + ))] { simd_l2_squared(a, b) } - #[cfg(any( - all(target_arch = "wasm32", not(target_feature = "simd128")), - all(not(target_arch = "wasm32"), not(feature = "simd")) + #[cfg(all( + not(feature = "lattice-simd"), + any( + all(target_arch = "wasm32", not(target_feature = "simd128")), + all(not(target_arch = "wasm32"), not(feature = "simd")) + ) ))] { scalar_l2_squared(a, b) @@ -313,27 +329,51 @@ pub fn wasm_simd128_l2_squared(a: &[f32], b: &[f32]) -> f32 { pub fn inner_product(a: &[f32], b: &[f32]) -> f32 { assert_eq!(a.len(), b.len(), "distance vectors must have equal lengths"); - #[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] + #[cfg(feature = "lattice-simd")] + { + // Negated to match the other backends: this returns a distance for a + // min-heap, not a similarity. `SpatialSimilarity::inner` is a plain + // alias for `dot`, so both sides negate the same raw dot product. + -lattice_embed::simd::dot_product(a, b) + } + + #[cfg(all( + not(feature = "lattice-simd"), + target_arch = "wasm32", + target_feature = "simd128" + ))] { wasm_simd128_inner_product(a, b) } - #[cfg(all(not(target_arch = "wasm32"), feature = "simd"))] + #[cfg(all( + not(feature = "lattice-simd"), + not(target_arch = "wasm32"), + feature = "simd" + ))] { simsimd::SpatialSimilarity::inner(a, b) .map(|d| -(d as f32)) .unwrap_or_else(|| scalar_inner_product(a, b)) } - #[cfg(any( - all(target_arch = "wasm32", not(target_feature = "simd128")), - all(not(target_arch = "wasm32"), not(feature = "simd")) + #[cfg(all( + not(feature = "lattice-simd"), + any( + all(target_arch = "wasm32", not(target_feature = "simd128")), + all(not(target_arch = "wasm32"), not(feature = "simd")) + ) ))] { scalar_inner_product(a, b) } } +/// Retained under `lattice-simd` as the reference implementation the parity +/// test checks the compiled backend against, and as the fallback the other +/// backends still call. `scalar_l2_squared` is `pub` and so needs no such +/// annotation. +#[cfg_attr(feature = "lattice-simd", allow(dead_code))] #[inline] fn scalar_inner_product(a: &[f32], b: &[f32]) -> f32 { let mut s0 = 0.0f32; @@ -579,6 +619,66 @@ mod tests { assert!((inner_product(&a, &b) - (-32.0)).abs() < 1e-6); } + /// Checks whichever backend compiled in against naive scalar references, + /// across dimensions chosen to straddle 4/8/16-lane widths **and their + /// remainders** so tail handling is exercised. The references are naive + /// single-pass loops rather than `scalar_l2_squared` / `scalar_inner_product`, + /// which are themselves 4-accumulator implementations, so a reduction-order + /// bug in the shared shape cannot hide. + /// + /// Backend-independent by construction: it covers the SimSIMD path, the + /// lattice path, and the plain scalar path. The cross-dimension parity + /// tests that existed before this were gated on + /// `all(target_arch = "wasm32", target_feature = "simd128")`, so no native + /// backend was checked at any dimension wider than 3. + #[test] + fn backend_matches_scalar_reference() { + use rand::rngs::StdRng; + use rand::{Rng, SeedableRng}; + + fn naive_l2_squared(a: &[f32], b: &[f32]) -> f32 { + a.iter().zip(b).map(|(x, y)| (x - y) * (x - y)).sum() + } + + fn naive_inner_product(a: &[f32], b: &[f32]) -> f32 { + -a.iter().zip(b).map(|(x, y)| x * y).sum::() + } + + const DIMS: &[usize] = &[ + 0, 1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 384, 768, 1000, 1023, 1024, + ]; + + // Combined absolute+relative tolerance, numpy `allclose`-style. A flat + // absolute bound is not achievable: each backend uses a different + // reduction tree, f32 addition is not associative, so reordering shifts + // rounding by a few ULPs and that drift grows with accumulated + // magnitude. A real bug (wrong lane math, dropped remainder) produces + // relative error orders of magnitude above machine epsilon. + const ATOL: f32 = 1e-5; + const RTOL: f32 = 1e-5; + + let mut rng = StdRng::seed_from_u64(42); + for &dim in DIMS { + let a: Vec = (0..dim).map(|_| rng.gen_range(-10.0f32..10.0)).collect(); + let b: Vec = (0..dim).map(|_| rng.gen_range(-10.0f32..10.0)).collect(); + + for (op, got, want) in [ + ("l2_squared", l2_squared(&a, &b), naive_l2_squared(&a, &b)), + ( + "inner_product", + inner_product(&a, &b), + naive_inner_product(&a, &b), + ), + ] { + let bound = ATOL + RTOL * got.abs().max(want.abs()); + assert!( + (got - want).abs() <= bound, + "{op} dim={dim}: got={got} want={want} bound={bound}" + ); + } + } + } + #[test] fn test_flat_vectors() { let mut fv = FlatVectors::new(3); From 95814cf2ddc85686427966133b3ebc16cc6f2d83 Mon Sep 17 00:00:00 2001 From: OceanLi <122793010+ohdearquant@users.noreply.github.com> Date: Mon, 3 Aug 2026 15:17:15 -0400 Subject: [PATCH 2/3] test(diskann): pin the lattice distance backend selection The cross-dimension parity test checks numerical equivalence against a naive scalar oracle, which a lattice-simd build that silently fell back to the scalar/native kernel would still pass. Add a backend-selection seam (atomics set only from inside the lattice-simd dispatch arms, alongside the real kernel call) and a lattice-simd-gated test that fails if either arm's lattice call is replaced by a fallback route. Co-Authored-By: Claude Sonnet 5 --- crates/ruvector-diskann/src/distance.rs | 47 +++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/crates/ruvector-diskann/src/distance.rs b/crates/ruvector-diskann/src/distance.rs index 0786d142e3..1f1b261c5e 100644 --- a/crates/ruvector-diskann/src/distance.rs +++ b/crates/ruvector-diskann/src/distance.rs @@ -7,6 +7,18 @@ use crate::error::{DiskAnnError, Result}; use memmap2::Mmap; +/// Set from inside the `lattice-simd` dispatch arms, immediately alongside +/// the real kernel call, so that swapping the arm's body for a scalar/native +/// fallback (rather than genuinely calling `lattice_embed`) removes the +/// `store` along with it. Compiled only for `lattice-simd` test builds — zero +/// footprint elsewhere. See `lattice_backend_is_actually_invoked` below. +#[cfg(all(test, feature = "lattice-simd"))] +static LATTICE_L2_INVOKED: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); +#[cfg(all(test, feature = "lattice-simd"))] +static LATTICE_INNER_INVOKED: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + /// Backing storage for the flat vector slab. /// /// `Owned` is heap-resident — used while inserting/building, and by @@ -196,6 +208,8 @@ pub fn l2_squared(a: &[f32], b: &[f32]) -> f32 { #[cfg(feature = "lattice-simd")] { + #[cfg(test)] + LATTICE_L2_INVOKED.store(true, std::sync::atomic::Ordering::Relaxed); lattice_embed::simd::squared_euclidean_distance(a, b) } @@ -334,6 +348,8 @@ pub fn inner_product(a: &[f32], b: &[f32]) -> f32 { // Negated to match the other backends: this returns a distance for a // min-heap, not a similarity. `SpatialSimilarity::inner` is a plain // alias for `dot`, so both sides negate the same raw dot product. + #[cfg(test)] + LATTICE_INNER_INVOKED.store(true, std::sync::atomic::Ordering::Relaxed); -lattice_embed::simd::dot_product(a, b) } @@ -679,6 +695,37 @@ mod tests { } } + /// `backend_matches_scalar_reference` above only checks numerical parity + /// against a naive oracle; scalar and lattice implement the same + /// arithmetic, so a `lattice-simd` build that silently fell back to the + /// scalar/native kernel would still pass it. This pins actual backend + /// *selection*: it fails if either dispatch arm's lattice call is + /// replaced by a fallback route while the feature stays declared, since + /// `LATTICE_L2_INVOKED` / `LATTICE_INNER_INVOKED` are only set from + /// inside those exact arms, right next to the real call. + #[test] + #[cfg(feature = "lattice-simd")] + fn lattice_backend_is_actually_invoked() { + use std::sync::atomic::Ordering; + + LATTICE_L2_INVOKED.store(false, Ordering::Relaxed); + LATTICE_INNER_INVOKED.store(false, Ordering::Relaxed); + + let a = vec![1.0f32, 2.0, 3.0, 4.0, 5.0]; + let b = vec![6.0f32, 7.0, 8.0, 9.0, 10.0]; + let _ = l2_squared(&a, &b); + let _ = inner_product(&a, &b); + + assert!( + LATTICE_L2_INVOKED.load(Ordering::Relaxed), + "l2_squared did not route through the lattice-embed kernel" + ); + assert!( + LATTICE_INNER_INVOKED.load(Ordering::Relaxed), + "inner_product did not route through the lattice-embed kernel" + ); + } + #[test] fn test_flat_vectors() { let mut fv = FlatVectors::new(3); From a07b67371606fb9f6bc930a2147426739a19e9b4 Mon Sep 17 00:00:00 2001 From: OceanLi <122793010+ohdearquant@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:22:27 -0400 Subject: [PATCH 3/3] fix(diskann): make lattice-simd backend witness tamper-evident Move the lattice-embed dispatch calls for l2_squared and inner_product into dedicated wrapper functions (l2_lattice, inner_lattice) that are the sole callers of the lattice-embed kernels. The test witness that confirms backend selection is now set inside these wrappers, after the real kernel call returns, so a dispatch arm that swaps the wrapper call for a scalar/native fallback also loses the witness. Switch the witness flags from process-global atomics to thread-locals so a concurrently running sibling test cannot set them between this test's reset and its assert. Also correct the lattice-simd feature comment (the other backends' helper functions still compile; only the dispatch arm changes) and the lattice-embed dependency comment (blake3, a transitive dependency, still needs a C toolchain on AArch64). --- crates/ruvector-diskann/Cargo.toml | 9 ++- crates/ruvector-diskann/src/distance.rs | 97 +++++++++++++++---------- 2 files changed, 66 insertions(+), 40 deletions(-) diff --git a/crates/ruvector-diskann/Cargo.toml b/crates/ruvector-diskann/Cargo.toml index f2211bdd97..0ed9793389 100644 --- a/crates/ruvector-diskann/Cargo.toml +++ b/crates/ruvector-diskann/Cargo.toml @@ -13,7 +13,9 @@ gpu = [] # Feature flag for GPU acceleration (CUDA/Metal stubs) simd = ["simsimd"] # Route the f32 distance kernels through lattice-embed rather than SimSIMD. # Takes precedence over `simd` and over the wasm32 SIMD128 path when both are -# enabled, so exactly one backend compiles for any feature/target combination. +# enabled, so exactly one route is selected for any feature/target +# combination (the other backends' helper functions still compile; only the +# dispatch arm actually called changes). lattice-simd = ["lattice-embed"] # BET 1 (ADR-200): fixed-topology reuse + periodic rebuild under metric drift. reuse-under-drift = [] @@ -31,7 +33,10 @@ bytemuck = { version = "1.14", features = ["derive"] } # Kernels only: `native` is the feature that pulls the model stack, so with # default features off this is a pure-Rust SIMD dependency with no tokenizer, # no model loader, and no download path. Unlike `simsimd` below it is not -# target-gated, because it compiles for wasm32 as well. +# target-gated, because it compiles for wasm32 as well. The kernels themselves +# are pure Rust, but that does not make the feature's dependency tree +# C-toolchain-free: blake3 (a transitive dependency) needs a target C compiler +# on AArch64, where its build script compiles c/blake3_neon.c. lattice-embed = { version = "0.7.1", optional = true, default-features = false } [target.'cfg(target_arch = "wasm32")'.dependencies] diff --git a/crates/ruvector-diskann/src/distance.rs b/crates/ruvector-diskann/src/distance.rs index 1f1b261c5e..34f4565c3a 100644 --- a/crates/ruvector-diskann/src/distance.rs +++ b/crates/ruvector-diskann/src/distance.rs @@ -1,24 +1,12 @@ //! Distance computations with SIMD acceleration and optional GPU offload //! -//! Dispatch priority: GPU (if `gpu` feature) → SimSIMD (if `simd` feature, native -//! NEON/AVX2/AVX-512) → WASM SIMD128 (`wasm32` target with `simd128` -//! target-feature) → scalar +//! Dispatch priority: GPU (if `gpu` feature) → lattice-embed (if `lattice-simd` +//! feature) → SimSIMD (if `simd` feature, native NEON/AVX2/AVX-512) → WASM +//! SIMD128 (`wasm32` target with `simd128` target-feature) → scalar use crate::error::{DiskAnnError, Result}; use memmap2::Mmap; -/// Set from inside the `lattice-simd` dispatch arms, immediately alongside -/// the real kernel call, so that swapping the arm's body for a scalar/native -/// fallback (rather than genuinely calling `lattice_embed`) removes the -/// `store` along with it. Compiled only for `lattice-simd` test builds — zero -/// footprint elsewhere. See `lattice_backend_is_actually_invoked` below. -#[cfg(all(test, feature = "lattice-simd"))] -static LATTICE_L2_INVOKED: std::sync::atomic::AtomicBool = - std::sync::atomic::AtomicBool::new(false); -#[cfg(all(test, feature = "lattice-simd"))] -static LATTICE_INNER_INVOKED: std::sync::atomic::AtomicBool = - std::sync::atomic::AtomicBool::new(false); - /// Backing storage for the flat vector slab. /// /// `Owned` is heap-resident — used while inserting/building, and by @@ -201,6 +189,46 @@ impl FlatVectors { // Distance functions — auto-dispatch based on features // ============================================================================ +/// Set from inside [`l2_lattice`]/[`inner_lattice`] below, only after the +/// real kernel call returns — so a dispatch arm that swaps the wrapper call +/// for a scalar/native fallback bypasses the flag along with the kernel. +/// Thread-local (not a shared global) so a sibling test running on another +/// thread can't set the flag between this test's reset and its assert. +/// Compiled only for `lattice-simd` test builds — zero footprint elsewhere. +/// See `lattice_backend_is_actually_invoked` below. +#[cfg(all(test, feature = "lattice-simd"))] +thread_local! { + static LATTICE_L2_INVOKED: std::cell::Cell = const { std::cell::Cell::new(false) }; + static LATTICE_INNER_INVOKED: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +/// Sole caller of the `lattice-embed` L2 kernel — the dispatch arm in +/// [`l2_squared`] can only reach the kernel through this wrapper, so +/// replacing the arm's call expression with a fallback also removes the +/// witness store. +#[cfg(feature = "lattice-simd")] +#[inline] +fn l2_lattice(a: &[f32], b: &[f32]) -> f32 { + let result = lattice_embed::simd::squared_euclidean_distance(a, b); + #[cfg(test)] + LATTICE_L2_INVOKED.with(|invoked| invoked.set(true)); + result +} + +/// Sole caller of the `lattice-embed` dot-product kernel — see +/// [`l2_lattice`] for why the store lives here rather than inline in the +/// dispatch arm. Negated to match the other backends: this returns a +/// distance for a min-heap, not a similarity. `SpatialSimilarity::inner` is a +/// plain alias for `dot`, so both sides negate the same raw dot product. +#[cfg(feature = "lattice-simd")] +#[inline] +fn inner_lattice(a: &[f32], b: &[f32]) -> f32 { + let result = -lattice_embed::simd::dot_product(a, b); + #[cfg(test)] + LATTICE_INNER_INVOKED.with(|invoked| invoked.set(true)); + result +} + /// L2 squared distance — dispatches to best available implementation #[inline] pub fn l2_squared(a: &[f32], b: &[f32]) -> f32 { @@ -208,9 +236,7 @@ pub fn l2_squared(a: &[f32], b: &[f32]) -> f32 { #[cfg(feature = "lattice-simd")] { - #[cfg(test)] - LATTICE_L2_INVOKED.store(true, std::sync::atomic::Ordering::Relaxed); - lattice_embed::simd::squared_euclidean_distance(a, b) + l2_lattice(a, b) } #[cfg(all( @@ -345,12 +371,7 @@ pub fn inner_product(a: &[f32], b: &[f32]) -> f32 { #[cfg(feature = "lattice-simd")] { - // Negated to match the other backends: this returns a distance for a - // min-heap, not a similarity. `SpatialSimilarity::inner` is a plain - // alias for `dot`, so both sides negate the same raw dot product. - #[cfg(test)] - LATTICE_INNER_INVOKED.store(true, std::sync::atomic::Ordering::Relaxed); - -lattice_embed::simd::dot_product(a, b) + inner_lattice(a, b) } #[cfg(all( @@ -385,10 +406,10 @@ pub fn inner_product(a: &[f32], b: &[f32]) -> f32 { } } -/// Retained under `lattice-simd` as the reference implementation the parity -/// test checks the compiled backend against, and as the fallback the other -/// backends still call. `scalar_l2_squared` is `pub` and so needs no such -/// annotation. +/// Retained under `lattice-simd` as the fallback the `simd` backend still +/// calls if SimSIMD returns `None`. The parity test's reference is the local +/// `naive_inner_product`, not this function. `scalar_l2_squared` is `pub` and +/// so needs no such annotation. #[cfg_attr(feature = "lattice-simd", allow(dead_code))] #[inline] fn scalar_inner_product(a: &[f32], b: &[f32]) -> f32 { @@ -699,17 +720,17 @@ mod tests { /// against a naive oracle; scalar and lattice implement the same /// arithmetic, so a `lattice-simd` build that silently fell back to the /// scalar/native kernel would still pass it. This pins actual backend - /// *selection*: it fails if either dispatch arm's lattice call is - /// replaced by a fallback route while the feature stays declared, since - /// `LATTICE_L2_INVOKED` / `LATTICE_INNER_INVOKED` are only set from - /// inside those exact arms, right next to the real call. + /// *selection*: it fails if either dispatch arm's call to + /// [`l2_lattice`]/[`inner_lattice`] is replaced by a fallback route while + /// the feature stays declared, since the witness flags are only set + /// inside those wrappers, after the real kernel call returns. Thread-local + /// storage means a sibling test running concurrently on another thread + /// can't set these flags between this test's reset and its assert. #[test] #[cfg(feature = "lattice-simd")] fn lattice_backend_is_actually_invoked() { - use std::sync::atomic::Ordering; - - LATTICE_L2_INVOKED.store(false, Ordering::Relaxed); - LATTICE_INNER_INVOKED.store(false, Ordering::Relaxed); + LATTICE_L2_INVOKED.with(|invoked| invoked.set(false)); + LATTICE_INNER_INVOKED.with(|invoked| invoked.set(false)); let a = vec![1.0f32, 2.0, 3.0, 4.0, 5.0]; let b = vec![6.0f32, 7.0, 8.0, 9.0, 10.0]; @@ -717,11 +738,11 @@ mod tests { let _ = inner_product(&a, &b); assert!( - LATTICE_L2_INVOKED.load(Ordering::Relaxed), + LATTICE_L2_INVOKED.with(|invoked| invoked.get()), "l2_squared did not route through the lattice-embed kernel" ); assert!( - LATTICE_INNER_INVOKED.load(Ordering::Relaxed), + LATTICE_INNER_INVOKED.with(|invoked| invoked.get()), "inner_product did not route through the lattice-embed kernel" ); }