From e2bf968e9bd4f3657bbf1d57484db09237f15a19 Mon Sep 17 00:00:00 2001 From: OceanLi <122793010+ohdearquant@users.noreply.github.com> Date: Mon, 17 Aug 2026 02:06:31 -0400 Subject: [PATCH] feat(storage): add complete embedding-space identity --- CHANGELOG.md | 6 + crates/khive-pack-moodboard/src/handlers.rs | 56 ++- crates/khive-pack-moodboard/src/model.rs | 271 +++++++++++- crates/khive-runtime/README.md | 5 + crates/khive-runtime/docs/design.md | 17 + crates/khive-runtime/src/lib.rs | 4 +- crates/khive-runtime/src/runtime.rs | 396 +++++++++++------- crates/khive-storage/README.md | 14 + .../docs/api/embedding-space-identity.md | 67 +++ crates/khive-storage/docs/design.md | 58 ++- crates/khive-storage/src/embedding_space.rs | 226 ++++++++++ crates/khive-storage/src/lib.rs | 7 +- .../tests/embedding_space_identity.rs | 111 +++++ ...ADR-148-moodboard-visual-retrieval-pack.md | 32 +- ...ADR-156-named-vector-restart-durability.md | 6 +- scripts/ci.sh | 1 + tests/test_moodboard_descriptor_golden.py | 67 +++ 17 files changed, 1133 insertions(+), 211 deletions(-) create mode 100644 crates/khive-storage/docs/api/embedding-space-identity.md create mode 100644 crates/khive-storage/src/embedding_space.rs create mode 100644 crates/khive-storage/tests/embedding_space_identity.rs create mode 100644 tests/test_moodboard_descriptor_golden.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b5ae66a48..4d9015996 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- `khive_storage::EmbeddingSpaceIdentity`, a validated immutable physical-vector + fence derived from a protocol-owned fingerprint and dimensions (ADR-160 D6). - ADR-149 Moodboard pairwise preference learning: actor-attributed randomized serve/judgment events, deterministic grouped logistic BCE training, temperature/tie calibration, and BlobStore-backed `lattice-fann` model @@ -26,6 +28,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Pack-owned vector consumers replace `khive_runtime::NamedVectorIdentity` and + `vectors_for_named_identity` with `khive_storage::EmbeddingSpaceIdentity` and + `vectors_for_embedding_space`; the source-breaking replacement intentionally + has no arbitrary-key compatibility alias (ADR-160 D6). - `resolve_project_actor_id` (khive-runtime) now returns `ConfigError::ExplicitConfigMissing` when the explicit `--config`/`KHIVE_CONFIG` path does not exist, instead of resolving to diff --git a/crates/khive-pack-moodboard/src/handlers.rs b/crates/khive-pack-moodboard/src/handlers.rs index 61ab36221..9ec75a775 100644 --- a/crates/khive-pack-moodboard/src/handlers.rs +++ b/crates/khive-pack-moodboard/src/handlers.rs @@ -22,7 +22,7 @@ use khive_storage::blob::ContentRef; use khive_storage::types::{ SqlStatement, SqlValue, VectorIndexKind, VectorSearchHit, VectorSearchRequest, }; -use khive_storage::{BlobStore, Entity, NewAttachment, VectorStore}; +use khive_storage::{BlobStore, EmbeddingSpaceIdentity, Entity, NewAttachment, VectorStore}; use khive_types::SubstrateKind; use crate::model::{validate_embedding, DescriptorIdentity, LoadedVisionModel, VisionModelState}; @@ -66,6 +66,7 @@ pub(crate) async fn handle_ingest( // this preserves the no-blob-side-effect identity fence without holding // the preprocessing memory permit across Qwen construction. let model = pack.model_state().get().await?; + let embedding_identity = model.embedding_identity().clone(); let descriptor = model.descriptor().clone(); let preprocessing_permit = pack.model_state().acquire_preprocessing_permit().await?; let raw = decode_image_base64(encoded_image)?; @@ -93,7 +94,15 @@ pub(crate) async fn handle_ingest( &descriptor, ) .await?; - index_embedding(pack.runtime(), token, &descriptor, asset.id, &embedding).await?; + index_embedding_with_identity( + pack.runtime(), + token, + &embedding_identity, + &descriptor, + asset.id, + &embedding, + ) + .await?; Ok(json!({ "asset_id": asset.id.to_string(), @@ -123,6 +132,7 @@ pub(crate) async fn handle_search( let prepared = prepare_source_raster(pack, &core, &content_ref).await?; let model = pack.model_state().get().await?; + let embedding_identity = model.embedding_identity().clone(); let descriptor = model.descriptor().clone(); let embedding = infer_prepared( pack.model_state(), @@ -131,9 +141,10 @@ pub(crate) async fn handle_search( &descriptor, ) .await?; - let raw_hits = search_embedding( + let raw_hits = search_embedding_with_identity( pack.runtime(), token, + &embedding_identity, &descriptor, &embedding, candidate_limit(top_k), @@ -720,10 +731,9 @@ async fn infer_prepared( async fn exact_store( runtime: &KhiveRuntime, token: &NamespaceToken, - descriptor: &DescriptorIdentity, + identity: &EmbeddingSpaceIdentity, ) -> Result, RuntimeError> { - let identity = descriptor.vector_identity()?; - let store = runtime.vectors_for_named_identity(token, &identity).await?; + let store = runtime.vectors_for_embedding_space(token, identity).await?; let info = store.info().await?; if info.index_kind != VectorIndexKind::SqliteVec { return Err(RuntimeError::Unconfigured(format!( @@ -734,15 +744,16 @@ async fn exact_store( Ok(store) } -async fn index_embedding( +async fn index_embedding_with_identity( runtime: &KhiveRuntime, token: &NamespaceToken, + identity: &EmbeddingSpaceIdentity, descriptor: &DescriptorIdentity, asset_id: Uuid, embedding: &[f32], ) -> Result<(), RuntimeError> { validate_embedding(embedding, descriptor)?; - let store = exact_store(runtime, token, descriptor).await?; + let store = exact_store(runtime, token, identity).await?; store .insert_exact_only( asset_id, @@ -755,15 +766,16 @@ async fn index_embedding( Ok(()) } -async fn search_embedding( +async fn search_embedding_with_identity( runtime: &KhiveRuntime, token: &NamespaceToken, + identity: &EmbeddingSpaceIdentity, descriptor: &DescriptorIdentity, embedding: &[f32], top_k: u32, ) -> Result, RuntimeError> { validate_embedding(embedding, descriptor)?; - let store = exact_store(runtime, token, descriptor).await?; + let store = exact_store(runtime, token, identity).await?; let namespaces: BTreeSet = token .visible_namespaces() .iter() @@ -788,6 +800,30 @@ async fn search_embedding( Ok(merge_namespace_hits(namespace_hits, top_k)) } +#[cfg(test)] +async fn index_embedding( + runtime: &KhiveRuntime, + token: &NamespaceToken, + descriptor: &DescriptorIdentity, + asset_id: Uuid, + embedding: &[f32], +) -> Result<(), RuntimeError> { + let identity = descriptor.vector_identity()?; + index_embedding_with_identity(runtime, token, &identity, descriptor, asset_id, embedding).await +} + +#[cfg(test)] +async fn search_embedding( + runtime: &KhiveRuntime, + token: &NamespaceToken, + descriptor: &DescriptorIdentity, + embedding: &[f32], + top_k: u32, +) -> Result, RuntimeError> { + let identity = descriptor.vector_identity()?; + search_embedding_with_identity(runtime, token, &identity, descriptor, embedding, top_k).await +} + fn merge_namespace_hits( namespace_hits: Vec>, top_k: u32, diff --git a/crates/khive-pack-moodboard/src/model.rs b/crates/khive-pack-moodboard/src/model.rs index f8c0839c2..18a2857e2 100644 --- a/crates/khive-pack-moodboard/src/model.rs +++ b/crates/khive-pack-moodboard/src/model.rs @@ -8,7 +8,8 @@ use serde::Serialize; use sha2::{Digest, Sha256}; use tokio::sync::{watch, OwnedSemaphorePermit, Semaphore}; -use khive_runtime::{NamedVectorIdentity, RuntimeError}; +use khive_runtime::RuntimeError; +use khive_storage::EmbeddingSpaceIdentity; pub(crate) const MODEL_NAME: &str = "qwen3.5-vlm-pooled-visual"; pub(crate) const PROMPT: &str = @@ -103,9 +104,12 @@ impl DescriptorIdentity { dimensions, normalization: "l2", }; - let canonical = canonical_json_bytes(&core)?; - let fingerprint = sha256_hex(&canonical); - let model_key = format!("moodboard_{fingerprint}_{dimensions}"); + Self::from_core(core) + } + + fn from_core(core: DescriptorCore) -> Result { + let (identity, fingerprint) = embedding_identity_for_core(&core)?; + let model_key = identity.space_key().to_string(); Ok(Self { schema_version: core.schema_version, model_key, @@ -122,12 +126,37 @@ impl DescriptorIdentity { }) } - pub(crate) fn vector_identity(&self) -> Result { - NamedVectorIdentity::new( - self.model_key.clone(), - self.model_name.to_string(), - self.dimensions, - ) + fn core(&self) -> DescriptorCore { + DescriptorCore { + schema_version: self.schema_version, + model_name: self.model_name, + model_revision: self.model_revision.clone(), + checkpoint_sha256: self.checkpoint_sha256.clone(), + inference: self.inference.clone(), + preprocessing: self.preprocessing.clone(), + prompt: self.prompt.clone(), + pooling: self.pooling, + dimensions: self.dimensions, + normalization: self.normalization, + } + } + + pub(crate) fn vector_identity(&self) -> Result { + let (identity, fingerprint) = embedding_identity_for_core(&self.core())?; + if self.fingerprint != fingerprint { + return Err(RuntimeError::InvalidInput(format!( + "moodboard descriptor fingerprint {:?} does not match its canonical identity {fingerprint:?}", + self.fingerprint + ))); + } + if self.model_key != identity.space_key().as_str() { + return Err(RuntimeError::InvalidInput(format!( + "moodboard descriptor model_key {:?} does not match its derived embedding space {:?}", + self.model_key, + identity.space_key().as_str() + ))); + } + Ok(identity) } #[cfg(test)] @@ -137,9 +166,35 @@ impl DescriptorIdentity { } } +fn embedding_identity_for_core( + core: &DescriptorCore, +) -> Result<(EmbeddingSpaceIdentity, String), RuntimeError> { + let canonical = canonical_json_bytes(core)?; + let fingerprint_bytes = sha256_digest(&canonical); + let fingerprint = lowercase_hex(&fingerprint_bytes); + let dimensions = u32::try_from(core.dimensions).map_err(|_| { + RuntimeError::InvalidInput(format!( + "moodboard descriptor dimensions exceed u32: {}", + core.dimensions + )) + })?; + let identity = EmbeddingSpaceIdentity::new( + "moodboard", + core.schema_version, + fingerprint_bytes, + core.model_name, + dimensions, + ) + .map_err(|error| { + RuntimeError::InvalidInput(format!("invalid moodboard embedding identity: {error}")) + })?; + Ok((identity, fingerprint)) +} + pub(crate) struct LoadedVisionModel { model: VisionEmbeddingModel, descriptor: DescriptorIdentity, + embedding_identity: EmbeddingSpaceIdentity, // Lattice may retain memory maps into the checkpoint. Keep the private, // attested snapshot alive until the model itself is dropped. _checkpoint: Arc, @@ -150,6 +205,10 @@ impl LoadedVisionModel { &self.descriptor } + pub(crate) fn embedding_identity(&self) -> &EmbeddingSpaceIdentity { + &self.embedding_identity + } + fn embed_with_prompt(&self, image_png: &[u8], prompt: &str) -> Result, RuntimeError> { self.model .embed_image(image_png, prompt, PoolingStrategy::MeanVisualTokens) @@ -475,9 +534,12 @@ fn load_prepared_checkpoint( } Ok(model) })?; + let descriptor = checkpoint.descriptor.clone(); + let embedding_identity = descriptor.vector_identity()?; Ok(LoadedVisionModel { model, - descriptor: checkpoint.descriptor.clone(), + descriptor, + embedding_identity, _checkpoint: checkpoint, }) } @@ -1113,11 +1175,13 @@ fn metadata_is_checkpoint_link(metadata: &std::fs::Metadata) -> bool { } fn required_env(name: &str) -> Result { + const MAX_VALUE_BYTES: usize = 4096; + let value = std::env::var(name) .map_err(|_| RuntimeError::Unconfigured(format!("{name} must be set for moodboard")))?; - if value.trim().is_empty() || value.trim() != value { + if value.is_empty() || value.len() > MAX_VALUE_BYTES || value.trim() != value { return Err(RuntimeError::Unconfigured(format!( - "{name} must be non-empty with no surrounding whitespace" + "{name} must be 1..={MAX_VALUE_BYTES} bytes with no surrounding whitespace" ))); } Ok(value) @@ -1162,12 +1226,21 @@ fn validate_sha256(name: &str, value: &str) -> Result<(), RuntimeError> { } fn sha256_hex(bytes: &[u8]) -> String { - let digest = Sha256::digest(bytes); - let mut out = String::with_capacity(64); - for byte in digest { - write!(&mut out, "{byte:02x}").expect("writing to String cannot fail"); + lowercase_hex(&sha256_digest(bytes)) +} + +fn sha256_digest(bytes: &[u8]) -> [u8; 32] { + Sha256::digest(bytes).into() +} + +fn lowercase_hex(bytes: &[u8]) -> String { + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut output = String::with_capacity(bytes.len() * 2); + for &byte in bytes { + output.push(HEX[(byte >> 4) as usize] as char); + output.push(HEX[(byte & 0x0f) as usize] as char); } - out + output } fn canonical_json_bytes(value: &T) -> Result, RuntimeError> { @@ -1231,6 +1304,9 @@ mod tests { use std::sync::atomic::{AtomicUsize, Ordering}; use image::{DynamicImage, ImageFormat, Rgb, RgbImage}; + use khive_runtime::KhiveRuntime; + use khive_storage::types::{SqlStatement, SqlValue}; + use khive_types::Namespace; use super::*; @@ -1242,7 +1318,17 @@ mod tests { assert_eq!(left.fingerprint.len(), 64); assert_eq!(left.model_key, format!("moodboard_{}_4", left.fingerprint)); - let value = serde_json::to_value(left).unwrap(); + let identity = left.vector_identity().expect("shared identity"); + assert_eq!(identity.space_key().as_str(), left.model_key); + assert_eq!(identity.protocol().as_str(), SCHEMA_VERSION); + assert_eq!(identity.model_name(), MODEL_NAME); + assert_eq!(identity.dimensions().get(), 4); + assert_eq!( + identity.fingerprint(), + &sha256_digest(&canonical_json_bytes(&left.core()).unwrap()) + ); + + let value = serde_json::to_value(&left).unwrap(); let object = value.as_object().unwrap(); assert_eq!(object.len(), 12); assert_eq!(value["schema_version"], SCHEMA_VERSION); @@ -1252,7 +1338,94 @@ mod tests { } #[test] - fn descriptor_fingerprint_matches_cross_language_golden() { + fn every_visual_protocol_field_mutation_changes_the_embedding_space() { + let baseline = DescriptorIdentity::fixture(4).core(); + let baseline_identity = embedding_identity_for_core(&baseline) + .expect("baseline identity") + .0; + let baseline_key = baseline_identity.space_key().clone(); + let baseline_fingerprint = *baseline_identity.fingerprint(); + + macro_rules! assert_mutation_changes_key { + ($name:literal, $mutate:expr) => {{ + let mut mutated = baseline.clone(); + let mutate: fn(&mut DescriptorCore) = $mutate; + mutate(&mut mutated); + let mutated_identity = embedding_identity_for_core(&mutated).expect($name).0; + let mutated_key = mutated_identity.space_key().clone(); + assert_ne!( + baseline_fingerprint, + *mutated_identity.fingerprint(), + "{} must change the fingerprint", + $name + ); + assert_ne!(baseline_key, mutated_key, "{} must change the key", $name); + }}; + } + + assert_mutation_changes_key!("schema_version", |core| { + core.schema_version = "moodboard.visual-descriptor.v2"; + }); + assert_mutation_changes_key!("model_name", |core| { + core.model_name = "different-visual-model"; + }); + assert_mutation_changes_key!("model_revision", |core| { + core.model_revision.push_str("-changed"); + }); + assert_mutation_changes_key!("checkpoint_sha256", |core| { + core.checkpoint_sha256 = "b".repeat(64); + }); + assert_mutation_changes_key!("inference.provider", |core| { + core.inference.provider = "different-provider"; + }); + assert_mutation_changes_key!("inference.version", |core| { + core.inference.version = "0.9.1"; + }); + assert_mutation_changes_key!("preprocessing.revision", |core| { + core.preprocessing.revision = "different-preprocessing"; + }); + assert_mutation_changes_key!("preprocessing.max_side", |core| { + core.preprocessing.max_side += 1; + }); + assert_mutation_changes_key!("preprocessing.alignment", |core| { + core.preprocessing.alignment /= 2; + }); + assert_mutation_changes_key!("preprocessing.matte_rgb", |core| { + core.preprocessing.matte_rgb[0] ^= 1; + }); + assert_mutation_changes_key!("preprocessing.resample", |core| { + core.preprocessing.resample = "nearest"; + }); + assert_mutation_changes_key!("prompt.revision", |core| { + core.prompt.revision = "different-prompt"; + }); + assert_mutation_changes_key!("prompt.sha256", |core| { + core.prompt.sha256 = "b".repeat(64); + }); + assert_mutation_changes_key!("pooling", |core| { + core.pooling = "cls"; + }); + assert_mutation_changes_key!("dimensions", |core| { + core.dimensions += 1; + }); + assert_mutation_changes_key!("normalization", |core| { + core.normalization = "none"; + }); + } + + #[test] + fn descriptor_rejects_stored_fingerprint_or_key_drift() { + let mut bad_fingerprint = DescriptorIdentity::fixture(4); + bad_fingerprint.fingerprint = "0".repeat(64); + assert!(bad_fingerprint.vector_identity().is_err()); + + let mut bad_key = DescriptorIdentity::fixture(4); + bad_key.model_key.push_str("_other"); + assert!(bad_key.vector_identity().is_err()); + } + + #[tokio::test] + async fn descriptor_fingerprint_response_and_table_match_cross_language_golden() { let core = DescriptorCore { schema_version: SCHEMA_VERSION, model_name: MODEL_NAME, @@ -1277,7 +1450,25 @@ mod tests { dimensions: 4, normalization: "l2", }; - let fingerprint = sha256_hex(&canonical_json_bytes(&core).unwrap()); + let descriptor = DescriptorIdentity::from_core(core.clone()).unwrap(); + let canonical = canonical_json_bytes(&core).unwrap(); + assert_eq!( + canonical, + concat!( + "{\"checkpoint_sha256\":\"1111111111111111111111111111111111111111111111111111111111111111\",", + "\"dimensions\":4,", + "\"inference\":{\"provider\":\"lattice-embed\",\"version\":\"0.9.0\"},", + "\"model_name\":\"qwen3.5-vlm-pooled-visual\",", + "\"model_revision\":\"weights-r1\",", + "\"normalization\":\"l2\",", + "\"pooling\":\"mean_visual_tokens\",", + "\"preprocessing\":{\"alignment\":32,\"matte_rgb\":[128,128,128],\"max_side\":448,\"resample\":\"lanczos3\",\"revision\":\"moodboard-qwen35-srgb-pad32-max448-v1\"},", + "\"prompt\":{\"revision\":\"moodboard-style-retrieval-v1\",\"sha256\":\"2222222222222222222222222222222222222222222222222222222222222222\"},", + "\"schema_version\":\"moodboard.visual-descriptor.v1\"}" + ) + .as_bytes() + ); + let fingerprint = sha256_hex(&canonical); assert_eq!( fingerprint, "b57fb3cf43da387cde12425e6d7d442af269ba37ecabfbe4c975cb80abdf56e5" @@ -1286,6 +1477,44 @@ mod tests { format!("moodboard_{fingerprint}_4"), "moodboard_b57fb3cf43da387cde12425e6d7d442af269ba37ecabfbe4c975cb80abdf56e5_4" ); + assert_eq!( + serde_json::to_string(&descriptor).unwrap(), + concat!( + "{\"schema_version\":\"moodboard.visual-descriptor.v1\",", + "\"model_key\":\"moodboard_b57fb3cf43da387cde12425e6d7d442af269ba37ecabfbe4c975cb80abdf56e5_4\",", + "\"model_name\":\"qwen3.5-vlm-pooled-visual\",", + "\"model_revision\":\"weights-r1\",", + "\"checkpoint_sha256\":\"1111111111111111111111111111111111111111111111111111111111111111\",", + "\"inference\":{\"provider\":\"lattice-embed\",\"version\":\"0.9.0\"},", + "\"preprocessing\":{\"revision\":\"moodboard-qwen35-srgb-pad32-max448-v1\",\"max_side\":448,\"alignment\":32,\"matte_rgb\":[128,128,128],\"resample\":\"lanczos3\"},", + "\"prompt\":{\"revision\":\"moodboard-style-retrieval-v1\",\"sha256\":\"2222222222222222222222222222222222222222222222222222222222222222\"},", + "\"pooling\":\"mean_visual_tokens\",\"dimensions\":4,\"normalization\":\"l2\",", + "\"fingerprint\":\"b57fb3cf43da387cde12425e6d7d442af269ba37ecabfbe4c975cb80abdf56e5\"}" + ) + ); + + let runtime = KhiveRuntime::memory().expect("memory runtime"); + let token = runtime.authorize(Namespace::local()).expect("authorize"); + let identity = descriptor.vector_identity().expect("shared identity"); + runtime + .vectors_for_embedding_space(&token, &identity) + .await + .expect("open exact golden space"); + let expected_table = format!("vec_{}", descriptor.model_key); + let mut reader = runtime.sql().reader().await.expect("sql reader"); + let stored_table = reader + .query_scalar(SqlStatement { + sql: "SELECT name FROM sqlite_schema WHERE type = 'table' AND name = ?1" + .to_string(), + params: vec![SqlValue::Text(expected_table.clone())], + label: Some("moodboard_golden_embedding_table".to_string()), + }) + .await + .expect("query golden table"); + match stored_table { + Some(SqlValue::Text(table)) => assert_eq!(table, expected_table), + other => panic!("unexpected golden table query result: {other:?}"), + } let production_prompt = DescriptorIdentity::build("weights-r1".to_string(), "1".repeat(64), 4).unwrap(); diff --git a/crates/khive-runtime/README.md b/crates/khive-runtime/README.md index 64b7d72c4..c21e0183c 100644 --- a/crates/khive-runtime/README.md +++ b/crates/khive-runtime/README.md @@ -16,6 +16,11 @@ verb-dispatch machinery that lets packs (`kg`, `gtd`, `memory`, …) extend the - **Role-keyed attachments** — main-backend-only record metadata over `ContentRef`, atomic entity-plus-role publication, compatibility `content_ref` projection, and transactional hard-delete cleanup +- **Complete pack-owned embedding-space binding** — consumers of + `vectors_for_embedding_space` pass the immutable + `khive_storage::EmbeddingSpaceIdentity`; that seam derives no table from a + display model name and verifies existing geometry/model metadata. The text + provider cutover remains ADR-160 Phase 7 - **`VerbRegistry` / `VerbRegistryBuilder`** — registers packs (`PackRuntime` impls), an authorization `Gate`, an actor identity, and dispatches verbs by name - **`PackRuntime` trait** — the object-safe runtime counterpart to `khive-types::Pack`; diff --git a/crates/khive-runtime/docs/design.md b/crates/khive-runtime/docs/design.md index d263dbf51..b8bbee454 100644 --- a/crates/khive-runtime/docs/design.md +++ b/crates/khive-runtime/docs/design.md @@ -37,6 +37,23 @@ maintenance. Production whole-buffer reads route through `BlobHydrator`; the unbounded raw read surface was removed in ADR-160 Phase 3 +### Complete Embedding-Space Identity (ADR-160 D6) + +- Pack-owned vector consumers construct `khive_storage::EmbeddingSpaceIdentity` + from a protocol-owned fingerprint and pass it to + `vectors_for_embedding_space`; runtime no longer exposes an arbitrary named + vector key constructor +- The derived complete key selects the physical table. Runtime verifies the + actual sqlite-vec geometry and stored model metadata before returning a + table-bound handle seeded with the token namespace as its default +- Namespace is row/query scope, never part of identity. Unchanged identity + reopens the same table after restart; a changed fingerprint or dimensions + selects an isolated table +- The Phase-6 bridge records pack-owned identities in the current registry by + full space key. Text-provider registration, lineage schema, vector/ANN/cache + keys, rebuild, and atomic serving cutover remain the indivisible Phase-7 + program + ### Role-Keyed Attachments and V21 Cutover (ADR-121, ADR-160 D4) - Phase 4a separately ships the transactional-GC compatibility gate without diff --git a/crates/khive-runtime/src/lib.rs b/crates/khive-runtime/src/lib.rs index a150ee0df..a3fe6db50 100644 --- a/crates/khive-runtime/src/lib.rs +++ b/crates/khive-runtime/src/lib.rs @@ -130,8 +130,8 @@ pub use retrieval::{SearchHit, SearchSource}; pub use runtime::{ assert_captured_db_anchor_consistent, assert_db_anchor_consistent, expand_tilde, parse_pack_list, resolve_db_anchor, resolve_project_actor_id, runtime_config_from_khive_config, - BackendId, EntityTypeValidatorFn, KhiveRuntime, NamedVectorIdentity, NamespaceToken, - NoteMutationHookFn, NoteWriteValidatorFn, RuntimeConfig, + BackendId, EntityTypeValidatorFn, KhiveRuntime, NamespaceToken, NoteMutationHookFn, + NoteWriteValidatorFn, RuntimeConfig, }; pub use secret_gate::SecretMatch; pub use validation::{ diff --git a/crates/khive-runtime/src/runtime.rs b/crates/khive-runtime/src/runtime.rs index d8118c767..f70e32c06 100644 --- a/crates/khive-runtime/src/runtime.rs +++ b/crates/khive-runtime/src/runtime.rs @@ -12,7 +12,8 @@ use khive_gate::AllowAllGate; use khive_gate::GateRequest; use khive_storage::types::{SqlStatement, SqlValue}; use khive_storage::{ - AttachmentStore, EntityStore, Event, EventStore, GraphStore, NoteStore, SqlAccess, VectorStore, + AttachmentStore, EmbeddingSpaceIdentity, EntityStore, Event, EventStore, GraphStore, NoteStore, + SqlAccess, VectorStore, }; use khive_types::{EdgeEndpointRule, EventKind, Namespace, SubstrateKind}; use lattice_embed::{EmbeddingModel, EmbeddingService}; @@ -64,76 +65,6 @@ pub type NoteWriteValidatorFn = Arc< + Sync, >; -/// Immutable identity for a non-text vector store owned by a pack consumer. -/// -/// This does not register an [`crate::EmbedderProvider`]. It gives a pack that -/// performs its own governed inference a narrow path to a namespace-scoped -/// Khive vector table while keeping model-key and dimension validation at the -/// runtime boundary. -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct NamedVectorIdentity { - model_key: String, - model_name: String, - dimensions: usize, -} - -impl NamedVectorIdentity { - const MAX_MODEL_KEY_BYTES: usize = 128; - const MAX_MODEL_NAME_BYTES: usize = 512; - - /// Validate and construct a named vector identity. - pub fn new( - model_key: impl Into, - model_name: impl Into, - dimensions: usize, - ) -> RuntimeResult { - let model_key = model_key.into(); - let model_name = model_name.into(); - if model_key.is_empty() - || model_key.len() > Self::MAX_MODEL_KEY_BYTES - || !model_key - .chars() - .all(|c| c.is_ascii_alphanumeric() || c == '_') - { - return Err(RuntimeError::InvalidInput(format!( - "named vector model_key must be 1..={} bytes of ASCII alphanumeric/underscore", - Self::MAX_MODEL_KEY_BYTES - ))); - } - if model_name.trim().is_empty() - || model_name.trim() != model_name - || model_name.len() > Self::MAX_MODEL_NAME_BYTES - { - return Err(RuntimeError::InvalidInput(format!( - "named vector model_name must be 1..={} bytes with no surrounding whitespace", - Self::MAX_MODEL_NAME_BYTES - ))); - } - if !(1..=8192).contains(&dimensions) { - return Err(RuntimeError::InvalidInput(format!( - "named vector dimensions must be in 1..=8192, got {dimensions}" - ))); - } - Ok(Self { - model_key, - model_name, - dimensions, - }) - } - - pub fn model_key(&self) -> &str { - &self.model_key - } - - pub fn model_name(&self) -> &str { - &self.model_name - } - - pub fn dimensions(&self) -> usize { - self.dimensions - } -} - pub use crate::config::{ assert_captured_db_anchor_consistent, assert_db_anchor_consistent, expand_tilde, parse_pack_list, resolve_db_anchor, resolve_project_actor_id, runtime_config_from_khive_config, @@ -631,26 +562,27 @@ impl KhiveRuntime { )?) } - /// Get a namespace-scoped vector store for a pack-owned immutable identity. + /// Get a table-bound vector store using the token namespace as its default. /// - /// The table key is syntactically validated by [`NamedVectorIdentity`]. This - /// accessor additionally verifies the table's actual sqlite-vec dimension - /// declaration and every persisted `embedding_model` value before returning - /// the store, so reusing one key for incompatible descriptor geometry or - /// semantics fails before a caller can replace rows. - pub async fn vectors_for_named_identity( + /// [`EmbeddingSpaceIdentity`] derives the physical key from the governed + /// fingerprint and dimensions. This accessor additionally verifies the + /// table's actual sqlite-vec dimension declaration and every persisted + /// `embedding_model` value before returning the store, so reusing one key + /// for incompatible descriptor semantics fails before a caller can replace + /// rows. + pub async fn vectors_for_embedding_space( &self, token: &NamespaceToken, - identity: &NamedVectorIdentity, + identity: &EmbeddingSpaceIdentity, ) -> RuntimeResult> { let store = self.backend.vectors_for_namespace( - identity.model_key(), + identity.space_key().as_str(), identity.model_name(), - identity.dimensions(), + identity.dimensions().get() as usize, token.namespace().as_str(), )?; - let table = format!("vec_{}", identity.model_key()); + let table = format!("vec_{}", identity.space_key()); let mut reader = self.sql().reader().await?; let dimension_row = reader .query_row(SqlStatement { @@ -677,11 +609,11 @@ impl KhiveRuntime { "named vector table {table} has no parseable embedding dimension" )) })?; - if declared_dimensions != identity.dimensions() { + if declared_dimensions != identity.dimensions().get() as usize { return Err(RuntimeError::InvalidInput(format!( - "named vector model_key {:?} is already bound to {declared_dimensions} dimensions, expected {}", - identity.model_key(), - identity.dimensions() + "embedding space key {:?} is already bound to {declared_dimensions} dimensions, expected {}", + identity.space_key().as_str(), + identity.dimensions().get() ))); } @@ -705,8 +637,8 @@ impl KhiveRuntime { }; if stored != identity.model_name() { return Err(RuntimeError::InvalidInput(format!( - "named vector model_key {:?} already contains model {stored:?}, cannot bind it to {:?}", - identity.model_key(), + "embedding space key {:?} already contains model {stored:?}, cannot bind it to {:?}", + identity.space_key().as_str(), identity.model_name() ))); } @@ -714,10 +646,10 @@ impl KhiveRuntime { self.backend .register_embedding_model( - identity.model_key(), + identity.space_key().as_str(), identity.model_name(), - identity.model_key(), - identity.dimensions() as u32, + identity.space_key().as_str(), + identity.dimensions().get(), ) .map_err(|error| { if matches!( @@ -726,8 +658,8 @@ impl KhiveRuntime { if code.code == rusqlite::ErrorCode::ConstraintViolation ) { RuntimeError::InvalidInput(format!( - "named vector model_key {:?} is already bound to a different active model identity", - identity.model_key() + "embedding space key {:?} is already bound to a different active model identity", + identity.space_key().as_str() )) } else { RuntimeError::Sqlite(error) @@ -1557,6 +1489,7 @@ fn vector_dimensions_from_ddl(ddl: &str) -> Option { mod tests { use super::*; use khive_gate::GateRef; + use khive_storage::EmbeddingSpaceIdentity; use serial_test::serial; fn test_blob_hydrator() -> (tempfile::TempDir, Arc) { @@ -1578,6 +1511,30 @@ mod tests { assert!(rt.config().db_path.is_none()); } + #[tokio::test] + async fn complete_embedding_identity_selects_and_registers_the_physical_space() { + let runtime = KhiveRuntime::memory().expect("memory runtime"); + let token = runtime.authorize(Namespace::local()).expect("authorize"); + let identity = + EmbeddingSpaceIdentity::new("visual", "test.visual.v1", [0x11; 32], "visual-model", 4) + .expect("identity"); + + runtime + .vectors_for_embedding_space(&token, &identity) + .await + .expect("open vector space"); + + let records = runtime + .list_embedding_models(Some(identity.space_key().as_str())) + .await + .expect("registry rows"); + assert_eq!(records.len(), 1); + assert_eq!(records[0].engine_name, identity.space_key().as_str()); + assert_eq!(records[0].model_id, identity.model_name()); + assert_eq!(records[0].key_version, identity.space_key().as_str()); + assert_eq!(records[0].dimensions, identity.dimensions().get()); + } + #[test] fn installed_blob_hydrator_is_shared_by_clone_and_core_handles() { let main_backend = Arc::new(StorageBackend::memory().expect("main backend")); @@ -2783,67 +2740,89 @@ mod tests { assert!(no_match.is_empty()); } - #[test] - fn named_vector_identity_rejects_ambiguous_or_unsafe_values() { - assert!(NamedVectorIdentity::new("", "model", 4).is_err()); - assert!(NamedVectorIdentity::new("bad-key", "model", 4).is_err()); - assert!(NamedVectorIdentity::new("valid_key", " model", 4).is_err()); - assert!(NamedVectorIdentity::new("valid_key", "model", 0).is_err()); - assert!(NamedVectorIdentity::new("valid_key", "model", 8193).is_err()); - assert!(NamedVectorIdentity::new("k".repeat(128), "m".repeat(512), 4).is_ok()); - assert!(NamedVectorIdentity::new("k".repeat(129), "model", 4).is_err()); - assert!(NamedVectorIdentity::new("valid_key", "m".repeat(513), 4).is_err()); - assert_eq!( - NamedVectorIdentity::new("valid_key", "model", 4) - .expect("valid identity") - .dimensions(), - 4 - ); + fn test_embedding_identity( + prefix: &str, + fingerprint_byte: u8, + model_name: &str, + dimensions: u32, + ) -> EmbeddingSpaceIdentity { + EmbeddingSpaceIdentity::new( + prefix, + "test.embedding-space.v1", + [fingerprint_byte; 32], + model_name, + dimensions, + ) + .expect("valid test identity") } #[tokio::test] - async fn named_vector_store_rejects_dimension_or_model_key_rebinding() { + async fn embedding_space_rejects_model_rebinding_but_dimensions_select_a_new_key() { let rt = KhiveRuntime::memory().expect("memory runtime"); let token = rt.authorize(Namespace::local()).expect("authorize"); - let original = NamedVectorIdentity::new("visual_contract", "model-a", 4).unwrap(); - rt.vectors_for_named_identity(&token, &original) + let original = test_embedding_identity("visual", 0x11, "model-a", 4); + rt.vectors_for_embedding_space(&token, &original) .await - .expect("create named vector store"); + .expect("create embedding space"); let registered = rt - .list_embedding_models(Some("visual_contract")) + .list_embedding_models(Some(original.space_key().as_str())) .await .expect("list model registry"); assert!(registered.iter().any(|record| { record.model_id == "model-a" - && record.key_version == "visual_contract" + && record.key_version == original.space_key().as_str() && record.dimensions == 4 })); - let wrong_dimensions = NamedVectorIdentity::new("visual_contract", "model-a", 5).unwrap(); - let Err(dimension_error) = rt - .vectors_for_named_identity(&token, &wrong_dimensions) + + let different_dimensions = test_embedding_identity("visual", 0x11, "model-a", 5); + assert_ne!( + original.space_key(), + different_dimensions.space_key(), + "geometry is part of the derived physical key" + ); + rt.vectors_for_embedding_space(&token, &different_dimensions) .await - else { - panic!("same key cannot change dimensions"); - }; - assert!(dimension_error.to_string().contains("dimensions")); + .expect("different dimensions select an isolated space"); - let wrong_model = NamedVectorIdentity::new("visual_contract", "model-b", 4).unwrap(); - let Err(model_error) = rt.vectors_for_named_identity(&token, &wrong_model).await else { + let wrong_model = test_embedding_identity("visual", 0x11, "model-b", 4); + let Err(model_error) = rt.vectors_for_embedding_space(&token, &wrong_model).await else { panic!("same key cannot change model identity"); }; assert!(model_error.to_string().contains("already bound")); } #[tokio::test] - async fn concurrent_named_vector_first_bind_has_one_immutable_winner() { + async fn embedding_space_rejects_a_preexisting_table_with_wrong_geometry() { let rt = KhiveRuntime::memory().expect("memory runtime"); let token = rt.authorize(Namespace::local()).expect("authorize"); - let first = NamedVectorIdentity::new("visual_race", "model-a", 4).unwrap(); - let second = NamedVectorIdentity::new("visual_race", "model-b", 4).unwrap(); + let identity = test_embedding_identity("geometry", 0x12, "model-a", 4); + + rt.backend + .vectors_for_namespace( + identity.space_key().as_str(), + identity.model_name(), + 5, + token.namespace().as_str(), + ) + .expect("plant incompatible table"); + + let error = match rt.vectors_for_embedding_space(&token, &identity).await { + Ok(_) => panic!("wrong table geometry must fail closed"), + Err(error) => error, + }; + assert!(error.to_string().contains("dimensions")); + } + + #[tokio::test] + async fn concurrent_embedding_space_first_bind_has_one_immutable_winner() { + let rt = KhiveRuntime::memory().expect("memory runtime"); + let token = rt.authorize(Namespace::local()).expect("authorize"); + let first = test_embedding_identity("visual_race", 0x22, "model-a", 4); + let second = test_embedding_identity("visual_race", 0x22, "model-b", 4); let (first_result, second_result) = tokio::join!( - rt.vectors_for_named_identity(&token, &first), - rt.vectors_for_named_identity(&token, &second), + rt.vectors_for_embedding_space(&token, &first), + rt.vectors_for_embedding_space(&token, &second), ); assert_ne!( first_result.is_ok(), @@ -2856,17 +2835,17 @@ mod tests { } else { (&second, &first) }; - rt.vectors_for_named_identity(&token, winner) + rt.vectors_for_embedding_space(&token, winner) .await .expect("winning identity remains idempotent"); - let error = match rt.vectors_for_named_identity(&token, loser).await { + let error = match rt.vectors_for_embedding_space(&token, loser).await { Ok(_) => panic!("losing identity cannot rebind the empty table"), Err(error) => error, }; assert!(error.to_string().contains("already bound")); let registered = rt - .list_embedding_models(Some("visual_race")) + .list_embedding_models(Some(winner.space_key().as_str())) .await .expect("list race registry"); assert_eq!(registered.len(), 1); @@ -2874,31 +2853,166 @@ mod tests { } #[tokio::test] - async fn named_vector_registry_keeps_immutable_revisions_active_together() { + async fn embedding_space_registry_keeps_immutable_revisions_active_together() { let rt = KhiveRuntime::memory().expect("memory runtime"); let token = rt.authorize(Namespace::local()).expect("authorize"); - let first = NamedVectorIdentity::new("visual_revision_a", "visual-model", 4).unwrap(); - let second = NamedVectorIdentity::new("visual_revision_b", "visual-model", 4).unwrap(); + let first = test_embedding_identity("visual", 0x33, "visual-model", 4); + let second = test_embedding_identity("visual", 0x44, "visual-model", 4); - rt.vectors_for_named_identity(&token, &first) + let first_store = rt + .vectors_for_embedding_space(&token, &first) .await .expect("open first immutable space"); - rt.vectors_for_named_identity(&token, &second) + let second_store = rt + .vectors_for_embedding_space(&token, &second) .await .expect("open second immutable space"); + let subject_id = uuid::Uuid::new_v4(); + first_store + .insert_exact_only( + subject_id, + SubstrateKind::Entity, + Namespace::LOCAL, + "entity.body", + vec![vec![1.0, 0.0, 0.0, 0.0]], + ) + .await + .expect("insert first revision"); + assert!(first_store + .batch_exists(&[subject_id], Namespace::LOCAL) + .await + .expect("first existence") + .contains(&subject_id)); + assert!(!second_store + .batch_exists(&[subject_id], Namespace::LOCAL) + .await + .expect("second existence") + .contains(&subject_id)); + let registered = rt.list_embedding_models(None).await.expect("list registry"); assert!(registered.iter().any(|record| { - record.engine_name == "visual_revision_a" + record.engine_name == first.space_key().as_str() && record.model_id == "visual-model" - && record.key_version == "visual_revision_a" + && record.key_version == first.space_key().as_str() && record.status == "active" })); assert!(registered.iter().any(|record| { - record.engine_name == "visual_revision_b" + record.engine_name == second.space_key().as_str() && record.model_id == "visual-model" - && record.key_version == "visual_revision_b" + && record.key_version == second.space_key().as_str() && record.status == "active" })); } + + #[tokio::test] + async fn namespace_changes_do_not_change_the_embedding_space_identity() { + let rt = KhiveRuntime::memory().expect("memory runtime"); + let local = rt.authorize(Namespace::local()).expect("local token"); + let project = rt + .authorize(Namespace::parse("project").expect("namespace")) + .expect("project token"); + let identity = test_embedding_identity("namespace", 0x55, "visual-model", 4); + + let local_store = rt + .vectors_for_embedding_space(&local, &identity) + .await + .expect("local store"); + let project_store = rt + .vectors_for_embedding_space(&project, &identity) + .await + .expect("project store"); + + let subject_id = uuid::Uuid::new_v4(); + local_store + .insert_exact_only( + subject_id, + SubstrateKind::Entity, + Namespace::LOCAL, + "entity.body", + vec![vec![1.0, 0.0, 0.0, 0.0]], + ) + .await + .expect("insert local vector"); + assert!(local_store + .batch_exists(&[subject_id], Namespace::LOCAL) + .await + .expect("local visibility") + .contains(&subject_id)); + assert!(!project_store + .batch_exists(&[subject_id], "project") + .await + .expect("project visibility") + .contains(&subject_id)); + + let registered = rt + .list_embedding_models(Some(identity.space_key().as_str())) + .await + .expect("registry rows"); + assert_eq!(registered.len(), 1); + } + + #[tokio::test] + async fn unchanged_embedding_identity_reopens_the_same_space_after_restart() { + let directory = tempfile::tempdir().expect("database directory"); + let mut config = RuntimeConfig::no_embeddings(); + config.db_path = Some(directory.path().join("embedding-space.db")); + let identity = test_embedding_identity("restart", 0x66, "visual-model", 4); + let subject_id = uuid::Uuid::new_v4(); + + let first = KhiveRuntime::new(config.clone()).expect("first runtime"); + let first_token = first.authorize(Namespace::local()).expect("first token"); + let first_store = first + .vectors_for_embedding_space(&first_token, &identity) + .await + .expect("first store"); + first_store + .insert_exact_only( + subject_id, + SubstrateKind::Entity, + Namespace::LOCAL, + "entity.body", + vec![vec![1.0, 0.0, 0.0, 0.0]], + ) + .await + .expect("insert vector"); + let request = khive_storage::types::VectorSearchRequest { + query_vectors: vec![vec![1.0, 0.0, 0.0, 0.0]], + top_k: 1, + namespace: Some(Namespace::LOCAL.to_string()), + kind: Some(SubstrateKind::Entity), + embedding_model: Some(identity.model_name().to_string()), + filter: None, + backend_hints: None, + }; + let before_restart = first_store + .search(request.clone()) + .await + .expect("search before restart"); + assert_eq!(before_restart.len(), 1); + drop(first_store); + drop(first); + + let restarted = KhiveRuntime::new(config).expect("restarted runtime"); + let restarted_token = restarted + .authorize(Namespace::local()) + .expect("restarted token"); + let reopened = restarted + .vectors_for_embedding_space(&restarted_token, &identity) + .await + .expect("reopen unchanged space"); + assert!(reopened + .batch_exists(&[subject_id], Namespace::LOCAL) + .await + .expect("reopened existence") + .contains(&subject_id)); + let after_restart = reopened + .search(request) + .await + .expect("search after restart"); + assert_eq!(after_restart.len(), 1); + assert_eq!(after_restart[0].subject_id, before_restart[0].subject_id); + assert_eq!(after_restart[0].score, before_restart[0].score); + assert_eq!(after_restart[0].rank, before_restart[0].rank); + } } diff --git a/crates/khive-storage/README.md b/crates/khive-storage/README.md index 520601670..cebe5f1aa 100644 --- a/crates/khive-storage/README.md +++ b/crates/khive-storage/README.md @@ -8,6 +8,11 @@ A concrete backend (`khive-db`'s SQLite implementation, for example) implements these traits; the runtime and every pack depend only on this crate, never on a specific backend. +The crate also owns backend-neutral value contracts shared by storage and +runtime. [`EmbeddingSpaceIdentity`](docs/api/embedding-space-identity.md) +derives one immutable physical vector key from a protocol-owned fingerprint and +geometry; it is not an additional capability trait. + ## Capability traits | Trait | Surface | @@ -21,6 +26,15 @@ specific backend. | `BlobStore` | content-addressed CRUD and bounded, digest-verified whole-object reads | | `SparseStore` | sparse (BM25-style) vector storage | +## Embedding-space identity + +`EmbeddingSpaceIdentity` validates the key prefix, governed protocol, owner +fingerprint, model label, and dimensions, then derives +`{prefix}_{lowercase_hex(fingerprint)}_{dimensions}`. The physical key has no +unchecked public constructor. Protocol owners remain responsible for defining +and golden-testing every vector-affecting field in their fingerprint preimage; +the shared type does not infer model semantics. + Every method returns `StorageResult = Result`. `StorageError` variants (`NotFound`, `AlreadyExists`, `Conflict`, `InvalidInput`, `Unsupported`, `Pool`, `Timeout`, `Transaction`, …) are tagged diff --git a/crates/khive-storage/docs/api/embedding-space-identity.md b/crates/khive-storage/docs/api/embedding-space-identity.md new file mode 100644 index 000000000..01701d8a7 --- /dev/null +++ b/crates/khive-storage/docs/api/embedding-space-identity.md @@ -0,0 +1,67 @@ +# Embedding-space identity + +`EmbeddingSpaceIdentity` is the backend-neutral, immutable fence for one +physical vector space. It is a value contract, not a storage capability and +not a generic canonical-JSON library. + +## Construction + +```rust +use khive_storage::EmbeddingSpaceIdentity; + +let identity = EmbeddingSpaceIdentity::new( + "moodboard", + "moodboard.visual-descriptor.v1", + [0xab; 32], + "qwen3.5-vlm-pooled-visual", + 1024, +)?; + +assert_eq!( + identity.space_key().as_str(), + "moodboard_abababababababababababababababababababababababababababababababab_1024" +); +# Ok::<(), khive_storage::EmbeddingSpaceIdentityError>(()) +``` + +The constructor validates: + +- a non-empty ASCII-alphanumeric/underscore key prefix; +- a 1–128-byte protocol from `[A-Za-z0-9._-]`; +- a 32-byte owner-supplied fingerprint; +- a non-empty model label of at most 512 bytes with no surrounding whitespace; and +- dimensions in `1..=8192`. + +It derives the only physical key as +`{prefix}_{lowercase_hex(fingerprint)}_{dimensions}` and rejects a result over +128 bytes. `EmbeddingSpaceKey` has no public unchecked constructor or +deserializer, so callers cannot supply a table key independently from the +fingerprint and geometry. + +## Owner responsibility + +The model or protocol owner defines and golden-tests the fingerprint preimage. +Every input that can change emitted vectors belongs in that document, including +the protocol identifier itself. The shared type deliberately does not guess +whether a checkpoint, tokenizer, prompt, transform, pooling rule, provider +revision, adapter, or normalization mode matters, and it does not prepend or +hash the protocol a second time. + +The model label is descriptive metadata. It does not select storage. Namespace +is likewise excluded from the identity: namespace remains a row/query scope +inside the same physical space. + +## Runtime binding + +Pack-owned consumers pass the complete value to +`KhiveRuntime::vectors_for_embedding_space`. The returned `VectorStore` handle +is bound to the derived table and uses the token namespace as its default. +Individual operations remain responsible for carrying only an authorized write +namespace or visible read scope. Runtime verifies the existing table geometry +and stored model metadata before allowing use. + +ADR-160 Phase 6 introduces this value and migrates moodboard without changing +its canonical descriptor bytes or table key. Text-provider registration, +registry lineage, vector-row columns, ANN logs, caches, snapshots, rebuild, and +atomic source cutover remain one coordinated Phase 7 change; Phase 6 does not +partially widen those identities. diff --git a/crates/khive-storage/docs/design.md b/crates/khive-storage/docs/design.md index 0955d2f20..2b78bd8ed 100644 --- a/crates/khive-storage/docs/design.md +++ b/crates/khive-storage/docs/design.md @@ -1,7 +1,8 @@ # khive-storage Design -Function-specific technical reference docs (error taxonomy, blob store, attachments, -transaction registry) live in [`docs/api/`](api/). This document covers +Function-specific technical reference docs (error taxonomy, blob store, +attachments, embedding-space identity, transaction registry) live in +[`docs/api/`](api/). This document covers design rationale and ADR compliance. ## Scope @@ -56,6 +57,15 @@ Non-default gather options return `StorageError::Unsupported` on backends that d not override the method. Term-level document-frequency statistics are exposed via `term_stats`, also optional (`Unsupported` by default). +### [ADR-160: Shared Pack Infrastructure](../../../docs/adr/ADR-160-shared-pack-infrastructure.md) + +`EmbeddingSpaceIdentity` is the immutable backend-neutral fence for a physical +vector space. It stores the governed protocol and fingerprint, display model +label, and validated dimensions while deriving the only usable +`EmbeddingSpaceKey`. It owns no model-specific canonicalization policy and +contains no namespace. Phase 6 migrates the pack-owned moodboard space; the +text-provider registry and ANN persistence cutover remain atomic Phase 7 work. + ### [ADR-041: Event Provenance Projection — Hybrid Log + Graph Edges](../../../docs/adr/ADR-041-event-provenance-projection.md) / [ADR-044: Vector Store Extensions — Capabilities, Metadata Filter, Batched Search, Update, Orphan Sweep](../../../docs/adr/ADR-044-vector-store-extensions.md) `VectorStoreCapabilities` is returned by `VectorStore::capabilities()` and @@ -77,29 +87,31 @@ Key design constraints: ## Modules -| Module | Purpose | -| ------------------------------------------- | --------------------------------------------------------------------------- | -| [`src/attachment.rs`](../src/attachment.rs) | role-keyed attachment types and `AttachmentStore` | -| [`src/blob.rs`](../src/blob.rs) | `ContentRef`, bounded read contract, and `BlobStore` | -| [`src/capability.rs`](../src/capability.rs) | `StorageCapability` enum | -| [`src/entity.rs`](../src/entity.rs) | `Entity`, `EntityFilter`, `EntityStore` | -| [`src/error.rs`](../src/error.rs) | `StorageError` | -| [`src/event.rs`](../src/event.rs) | `Event`, `EventFilter`, `EventStore` | -| [`src/graph.rs`](../src/graph.rs) | `GraphStore` | -| [`src/note.rs`](../src/note.rs) | `Note`, `NoteFilter`, `NoteStore` | -| [`src/sparse.rs`](../src/sparse.rs) | `SparseStore` | -| [`src/sql.rs`](../src/sql.rs) | `SqlAccess`, `SqlReader`, `SqlWriter`, `AtomicUnitOp` | -| [`src/text.rs`](../src/text.rs) | `TextSearch` | -| [`src/types/`](../src/types/) | Shared types split by domain (vector, text, graph, sparse, sql, pagination) | -| [`src/vectors.rs`](../src/vectors.rs) | `VectorStore` | +| Module | Purpose | +| ----------------------------------------------------- | --------------------------------------------------------------------------- | +| [`src/attachment.rs`](../src/attachment.rs) | role-keyed attachment types and `AttachmentStore` | +| [`src/blob.rs`](../src/blob.rs) | `ContentRef`, bounded read contract, and `BlobStore` | +| [`src/capability.rs`](../src/capability.rs) | `StorageCapability` enum | +| [`src/entity.rs`](../src/entity.rs) | `Entity`, `EntityFilter`, `EntityStore` | +| [`src/embedding_space.rs`](../src/embedding_space.rs) | complete immutable vector-space identity and derived key | +| [`src/error.rs`](../src/error.rs) | `StorageError` | +| [`src/event.rs`](../src/event.rs) | `Event`, `EventFilter`, `EventStore` | +| [`src/graph.rs`](../src/graph.rs) | `GraphStore` | +| [`src/note.rs`](../src/note.rs) | `Note`, `NoteFilter`, `NoteStore` | +| [`src/sparse.rs`](../src/sparse.rs) | `SparseStore` | +| [`src/sql.rs`](../src/sql.rs) | `SqlAccess`, `SqlReader`, `SqlWriter`, `AtomicUnitOp` | +| [`src/text.rs`](../src/text.rs) | `TextSearch` | +| [`src/types/`](../src/types/) | Shared types split by domain (vector, text, graph, sparse, sql, pagination) | +| [`src/vectors.rs`](../src/vectors.rs) | `VectorStore` | ## Tests -| Path | Coverage | -| ----------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------ | -| [`tests/attachment_contract.rs`](../tests/attachment_contract.rs) | attachment validation and stable substrate wire values | -| [`tests/compliance.rs`](../tests/compliance.rs) | Validate() invariant tests for `VectorSearchRequest`, `SparseVector`, `EdgeFilter`; vector filter compliance suite | -| [`tests/vectors.rs`](../tests/vectors.rs) | `VectorStore` default-impl behavior: capabilities, batch, update, orphan sweep | +| Path | Coverage | +| --------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------ | +| [`tests/attachment_contract.rs`](../tests/attachment_contract.rs) | attachment validation and stable substrate wire values | +| [`tests/compliance.rs`](../tests/compliance.rs) | Validate() invariant tests for `VectorSearchRequest`, `SparseVector`, `EdgeFilter`; vector filter compliance suite | +| [`tests/embedding_space_identity.rs`](../tests/embedding_space_identity.rs) | identity derivation, typed validation boundaries, key ceiling | +| [`tests/vectors.rs`](../tests/vectors.rs) | `VectorStore` default-impl behavior: capabilities, batch, update, orphan sweep | ## Invariants @@ -107,6 +119,8 @@ Key design constraints: - `SparseVector`: indices and values must be equal length, indices strictly increasing, all values finite. - `VectorSearchRequest`: query_vectors non-empty, top_k > 0, all values finite. +- `EmbeddingSpaceIdentity`: physical key is derived from prefix, fingerprint, + and dimensions; callers cannot inject an independent key. - `EdgeFilter`: weight bounds must be finite and min <= max. - Attachment roles are non-empty and contain no control characters; optional sizes fit SQLite's signed integer envelope. diff --git a/crates/khive-storage/src/embedding_space.rs b/crates/khive-storage/src/embedding_space.rs new file mode 100644 index 000000000..e1c37bb4d --- /dev/null +++ b/crates/khive-storage/src/embedding_space.rs @@ -0,0 +1,226 @@ +//! Backend-neutral identity for one immutable physical embedding space. + +use std::fmt; +use std::num::NonZeroU32; + +const MAX_SPACE_KEY_BYTES: usize = 128; +const MAX_PROTOCOL_BYTES: usize = 128; +const MAX_MODEL_NAME_BYTES: usize = 512; +const MAX_DIMENSIONS: u32 = 8192; + +/// Validation failure while constructing an [`EmbeddingSpaceIdentity`]. +#[derive(Clone, Debug, Eq, PartialEq, thiserror::Error)] +#[non_exhaustive] +pub enum EmbeddingSpaceIdentityError { + /// The caller-supplied prefix is empty or contains a non-key character. + #[error("embedding space key prefix must be non-empty ASCII alphanumeric/underscore")] + InvalidKeyPrefix, + /// The owner protocol is empty, overlong, or contains a disallowed byte. + #[error("embedding protocol must be 1..=128 bytes from [A-Za-z0-9._-]")] + InvalidProtocol, + /// The display label is empty, overlong, or has surrounding whitespace. + #[error("embedding model name must be 1..=512 bytes with no surrounding whitespace")] + InvalidModelName, + /// The vector geometry is outside the portable supported range. + #[error("embedding dimensions must be in 1..=8192, got {dimensions}")] + InvalidDimensions { dimensions: u32 }, + /// The derived physical key exceeds the portable key limit. + #[error("derived embedding space key must be at most {max_bytes} bytes, got {actual_bytes}")] + DerivedKeyTooLong { + actual_bytes: usize, + max_bytes: usize, + }, +} + +/// Validated physical key derived from an embedding fingerprint and geometry. +/// +/// There is intentionally no public constructor: callers construct a complete +/// [`EmbeddingSpaceIdentity`], which derives this key from the fingerprint and +/// dimensions instead of accepting an independently supplied table name. +/// +/// ```compile_fail +/// use khive_storage::EmbeddingSpaceKey; +/// +/// let _unchecked = EmbeddingSpaceKey("caller_selected_table".to_string()); +/// ``` +#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct EmbeddingSpaceKey(String); + +impl EmbeddingSpaceKey { + /// Borrow the canonical ASCII key. + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl AsRef for EmbeddingSpaceKey { + fn as_ref(&self) -> &str { + self.as_str() + } +} + +impl fmt::Display for EmbeddingSpaceKey { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(self.as_str()) + } +} + +/// Validated owner and canonicalization revision for an embedding identity. +#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct EmbeddingProtocol(String); + +impl EmbeddingProtocol { + /// Borrow the governed protocol identifier. + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl AsRef for EmbeddingProtocol { + fn as_ref(&self) -> &str { + self.as_str() + } +} + +impl fmt::Display for EmbeddingProtocol { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(self.as_str()) + } +} + +/// Immutable, complete fence for one physical vector space. +/// +/// The protocol owner decides which vector-affecting fields form the supplied +/// fingerprint and golden-tests that preimage. This shared type validates the +/// closed envelope and derives the physical key; it does not infer or +/// canonicalize model-specific identity fields. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct EmbeddingSpaceIdentity { + space_key: EmbeddingSpaceKey, + protocol: EmbeddingProtocol, + fingerprint: [u8; 32], + model_name: String, + dimensions: NonZeroU32, +} + +impl EmbeddingSpaceIdentity { + /// Validate a complete identity and derive its physical space key. + /// + /// `key_prefix` must be non-empty ASCII alphanumeric/underscore. The + /// derived key is `{key_prefix}_{lowercase_hex(fingerprint)}_{dimensions}` + /// and may contain at most 128 bytes. `protocol` identifies the owner and + /// canonicalization revision and is restricted to 1..=128 bytes from + /// `[A-Za-z0-9._-]`. `model_name` is a display label, not a storage key. + pub fn new( + key_prefix: &str, + protocol: &str, + fingerprint: [u8; 32], + model_name: &str, + dimensions: u32, + ) -> Result { + if key_prefix.is_empty() { + return Err(EmbeddingSpaceIdentityError::InvalidKeyPrefix); + } + + let derived_key_bytes = key_prefix + .len() + .saturating_add(1 + 64 + 1 + decimal_digits(dimensions)); + if derived_key_bytes > MAX_SPACE_KEY_BYTES { + return Err(EmbeddingSpaceIdentityError::DerivedKeyTooLong { + actual_bytes: derived_key_bytes, + max_bytes: MAX_SPACE_KEY_BYTES, + }); + } + if !key_prefix + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') + { + return Err(EmbeddingSpaceIdentityError::InvalidKeyPrefix); + } + + if protocol.is_empty() + || protocol.len() > MAX_PROTOCOL_BYTES + || !protocol + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-')) + { + return Err(EmbeddingSpaceIdentityError::InvalidProtocol); + } + + if model_name.is_empty() + || model_name.len() > MAX_MODEL_NAME_BYTES + || model_name.trim() != model_name + { + return Err(EmbeddingSpaceIdentityError::InvalidModelName); + } + + let dimensions = NonZeroU32::new(dimensions) + .filter(|value| value.get() <= MAX_DIMENSIONS) + .ok_or(EmbeddingSpaceIdentityError::InvalidDimensions { dimensions })?; + + let space_key = format!( + "{key_prefix}_{}_{}", + lowercase_hex(&fingerprint), + dimensions.get() + ); + debug_assert_eq!(space_key.len(), derived_key_bytes); + + Ok(Self { + space_key: EmbeddingSpaceKey(space_key), + protocol: EmbeddingProtocol(protocol.to_string()), + fingerprint, + model_name: model_name.to_string(), + dimensions, + }) + } + + /// Return the complete derived physical key. + pub fn space_key(&self) -> &EmbeddingSpaceKey { + &self.space_key + } + + /// Return the governed identity protocol. + pub fn protocol(&self) -> &EmbeddingProtocol { + &self.protocol + } + + /// Return the owner's canonical 32-byte fingerprint. + pub fn fingerprint(&self) -> &[u8; 32] { + &self.fingerprint + } + + /// Return the display model label. + pub fn model_name(&self) -> &str { + &self.model_name + } + + /// Return the validated vector dimensions. + pub fn dimensions(&self) -> NonZeroU32 { + self.dimensions + } +} + +fn decimal_digits(value: u32) -> usize { + match value { + 0..=9 => 1, + 10..=99 => 2, + 100..=999 => 3, + 1_000..=9_999 => 4, + 10_000..=99_999 => 5, + 100_000..=999_999 => 6, + 1_000_000..=9_999_999 => 7, + 10_000_000..=99_999_999 => 8, + 100_000_000..=999_999_999 => 9, + _ => 10, + } +} + +fn lowercase_hex(bytes: &[u8]) -> String { + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut output = String::with_capacity(bytes.len() * 2); + for &byte in bytes { + output.push(HEX[(byte >> 4) as usize] as char); + output.push(HEX[(byte & 0x0f) as usize] as char); + } + output +} diff --git a/crates/khive-storage/src/lib.rs b/crates/khive-storage/src/lib.rs index 97c970512..4262d1671 100644 --- a/crates/khive-storage/src/lib.rs +++ b/crates/khive-storage/src/lib.rs @@ -1,11 +1,13 @@ //! Storage capability traits: `SqlAccess`, `VectorStore`, `TextSearch`, //! `GraphStore`, `NoteStore`, `EntityStore`, `EventStore`, `SparseStore`, -//! `BlobStore`, and `AttachmentStore`. +//! `BlobStore`, and `AttachmentStore`. Backend-neutral value contracts also +//! include the immutable [`EmbeddingSpaceIdentity`] physical-vector fence. pub mod agent; pub mod attachment; pub mod blob; pub mod capability; +pub mod embedding_space; pub mod entity; pub mod error; pub mod event; @@ -27,6 +29,9 @@ pub use blob::{ BlobOrphanSweepConfig, BlobOrphanSweepResult, BlobStore, ContentRef, MAX_BLOB_WHOLE_BYTES, }; pub use capability::StorageCapability; +pub use embedding_space::{ + EmbeddingProtocol, EmbeddingSpaceIdentity, EmbeddingSpaceIdentityError, EmbeddingSpaceKey, +}; pub use entity::{Entity, EntityFilter, EntityStore}; pub use error::{StorageError, WriterTaskRequestState}; diff --git a/crates/khive-storage/tests/embedding_space_identity.rs b/crates/khive-storage/tests/embedding_space_identity.rs new file mode 100644 index 000000000..43676d312 --- /dev/null +++ b/crates/khive-storage/tests/embedding_space_identity.rs @@ -0,0 +1,111 @@ +use khive_storage::{EmbeddingSpaceIdentity, EmbeddingSpaceIdentityError}; + +const FINGERPRINT: [u8; 32] = [0xab; 32]; + +#[test] +fn derives_the_complete_physical_key_and_retains_identity_fields() { + let identity = EmbeddingSpaceIdentity::new( + "moodboard", + "moodboard.visual-descriptor.v1", + FINGERPRINT, + "qwen3.5-vlm-pooled-visual", + 1024, + ) + .expect("valid identity"); + + assert_eq!( + identity.space_key().as_str(), + "moodboard_abababababababababababababababababababababababababababababababab_1024" + ); + assert_eq!( + identity.protocol().as_str(), + "moodboard.visual-descriptor.v1" + ); + assert_eq!(identity.fingerprint(), &FINGERPRINT); + assert_eq!(identity.model_name(), "qwen3.5-vlm-pooled-visual"); + assert_eq!(identity.dimensions().get(), 1024); +} + +#[test] +fn validates_every_caller_owned_field_and_the_derived_key_bound() { + let valid = |prefix: String, protocol: String, model: String, dimensions| { + EmbeddingSpaceIdentity::new(&prefix, &protocol, FINGERPRINT, &model, dimensions) + }; + + assert!(valid("p".repeat(58), "p".repeat(128), "m".repeat(512), 8192).is_ok()); + assert!(valid( + "space".to_string(), + "owner.v1".to_string(), + "model".to_string(), + 1 + ) + .is_ok()); + + for bad_prefix in [String::new(), "bad-key".to_string(), "p".repeat(59)] { + assert!(valid( + bad_prefix, + "owner.v1".to_string(), + "model".to_string(), + 8192 + ) + .is_err()); + } + + for bad_protocol in [String::new(), "p".repeat(129), "owner/v1".to_string()] { + assert!(valid("space".to_string(), bad_protocol, "model".to_string(), 4).is_err()); + } + + for bad_model in [String::new(), " model".to_string(), "m".repeat(513)] { + assert!(valid("space".to_string(), "owner.v1".to_string(), bad_model, 4).is_err()); + } + + for bad_dimensions in [0, 8193] { + assert!(valid( + "space".to_string(), + "owner.v1".to_string(), + "model".to_string(), + bad_dimensions + ) + .is_err()); + } +} + +#[test] +fn validation_failures_have_stable_typed_categories() { + let construct = |prefix: &str, protocol: &str, model: &str, dimensions| { + EmbeddingSpaceIdentity::new(prefix, protocol, FINGERPRINT, model, dimensions) + }; + + assert_eq!( + construct("bad-key", "owner.v1", "model", 4), + Err(EmbeddingSpaceIdentityError::InvalidKeyPrefix) + ); + assert_eq!( + construct("space", "owner/v1", "model", 4), + Err(EmbeddingSpaceIdentityError::InvalidProtocol) + ); + assert_eq!( + construct("space", "owner.v1", " model", 4), + Err(EmbeddingSpaceIdentityError::InvalidModelName) + ); + assert_eq!( + construct("space", "owner.v1", "model", 0), + Err(EmbeddingSpaceIdentityError::InvalidDimensions { dimensions: 0 }) + ); + assert_eq!( + construct(&"p".repeat(59), "owner.v1", "model", 8192), + Err(EmbeddingSpaceIdentityError::DerivedKeyTooLong { + actual_bytes: 129, + max_bytes: 128, + }) + ); + + let oversized_borrowed_prefix = "p".repeat(1024 * 1024); + assert_eq!( + construct(&oversized_borrowed_prefix, "owner.v1", "model", 4), + Err(EmbeddingSpaceIdentityError::DerivedKeyTooLong { + actual_bytes: 1024 * 1024 + 67, + max_bytes: 128, + }) + ); +} diff --git a/docs/adr/ADR-148-moodboard-visual-retrieval-pack.md b/docs/adr/ADR-148-moodboard-visual-retrieval-pack.md index b469a9547..2eaaca180 100644 --- a/docs/adr/ADR-148-moodboard-visual-retrieval-pack.md +++ b/docs/adr/ADR-148-moodboard-visual-retrieval-pack.md @@ -13,9 +13,9 @@ materialization, and checkpoint seams on acceptance. Graphic-media curation needs a durable asset identity, a model-identity-bound visual descriptor, and similarity retrieval without pretending that one embedding is a complete measure of aesthetic coherence. Khive already owns the required persistence boundaries: -artifact entities, `BlobStore` content-addressed bytes, token-scoped vector stores, and the -single MCP `request` surface. Lattice already exposes local Qwen3.5 vision-language pooled -embedding inference. +artifact entities, `BlobStore` content-addressed bytes, scope-filtered vector operations, and +the single MCP `request` surface. Lattice already exposes local Qwen3.5 vision-language +pooled embedding inference. The missing layer is composition. Registering a vision checkpoint as a text `EmbedderProvider` is incorrect: entity and note creation fans text into every registered @@ -144,15 +144,23 @@ The runtime adds two consumer seams rather than exposing its backend: `BlobStore`, verifies every `ContentRef` before mutation, and commits the entity plus roles in one transaction before the existing FTS/vector compensation path; compensation hard-deletes the entity and attachment rows together. -2. `vectors_for_named_identity(token, &NamedVectorIdentity)` returns a token-scoped vector store. - `NamedVectorIdentity::new` rejects an empty/unsafe or over-128-byte model key, an empty or - over-512-byte model name, zero dimension, and dimensions above 8192. The accessor validates the actual vec table's declared dimension - and any stored `embedding_model` value before returning it. After validation it registers the - identity in ADR-043's `_embedding_models` lineage with collision-safe engine name `model_key`, - model id `model_name`, key version `model_key`, and the validated dimension. A provider-wide - engine name such as `lattice-embed` would incorrectly make immutable descriptor revisions - contend for ADR-043's one-active-model slot. Pack-owned visual spaces therefore remain visible - to `engine list`, and old/new descriptor revisions can remain active together. +2. `vectors_for_embedding_space(token, &EmbeddingSpaceIdentity)` returns a table-bound vector + store seeded with the token namespace as its default; moodboard continues to pass only the + authorized write namespace or visible read scopes on each operation. Moodboard supplies prefix + `moodboard`, protocol + `moodboard.visual-descriptor.v1`, the existing canonical SHA-256 bytes, model label, and + dimensions. The shared constructor derives the exact existing + `moodboard_{fingerprint}_{dimensions}` key; callers cannot provide a physical key independently. + It rejects an invalid prefix/protocol, an empty or over-512-byte model name, zero dimension, + dimensions above 8192, or a derived key over 128 bytes. The accessor validates the actual vec + table's declared dimension and any stored `embedding_model` value before returning it. During + ADR-160 Phase 6 it bridges the identity into ADR-043's current registry with collision-safe + engine name `space_key`, model id `model_name`, key version `space_key`, and the validated + dimension. A provider-wide engine name such as `lattice-embed` would incorrectly make immutable + descriptor revisions contend for ADR-043's one-active-model slot. Pack-owned visual spaces + therefore remain visible to `engine list`, and old/new descriptor revisions can remain active + together. ADR-160 Phase 7 replaces that transitional registry shape atomically with complete + identity and lineage fields. The vision model is never registered as a text `EmbedderProvider`. diff --git a/docs/adr/ADR-156-named-vector-restart-durability.md b/docs/adr/ADR-156-named-vector-restart-durability.md index 3bb03a84f..96fcd279f 100644 --- a/docs/adr/ADR-156-named-vector-restart-durability.md +++ b/docs/adr/ADR-156-named-vector-restart-durability.md @@ -106,6 +106,8 @@ restart semantics for exact-search named vector spaces only. - `crates/khive-db/src/stores/vectors.rs` — one instance per `vec_{model_key}` table; namespace and model predicates applied before rank projection -- `crates/khive-pack-moodboard/src/model.rs` — descriptor identity fingerprint and - `NamedVectorIdentity` derivation +- `crates/khive-storage/src/embedding_space.rs` — validated complete identity and derived physical + key +- `crates/khive-pack-moodboard/src/model.rs` — descriptor protocol fingerprint and + `EmbeddingSpaceIdentity` construction - `crates/khive-pack-moodboard/src/handlers.rs` — result materialization and orphan skip diff --git a/scripts/ci.sh b/scripts/ci.sh index 59d86ca06..af6666988 100755 --- a/scripts/ci.sh +++ b/scripts/ci.sh @@ -305,6 +305,7 @@ phase_deno_tests() { phase_smoke_tests() { echo "=== Smoke Test ===" python3 "$SCRIPT_DIR/../tests/test_documented_verb_counts.py" + python3 "$SCRIPT_DIR/../tests/test_moodboard_descriptor_golden.py" python3 "$SCRIPT_DIR/../tests/smoke_test.py" python3 "$SCRIPT_DIR/../tests/smoke_brain.py" python3 "$SCRIPT_DIR/../tests/smoke_comm.py" diff --git a/tests/test_moodboard_descriptor_golden.py b/tests/test_moodboard_descriptor_golden.py new file mode 100644 index 000000000..0c3e560ed --- /dev/null +++ b/tests/test_moodboard_descriptor_golden.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 + +import hashlib +import json +import unittest + + +EXPECTED_CANONICAL = ( + b'{"checkpoint_sha256":"1111111111111111111111111111111111111111111111111111111111111111",' + b'"dimensions":4,' + b'"inference":{"provider":"lattice-embed","version":"0.9.0"},' + b'"model_name":"qwen3.5-vlm-pooled-visual",' + b'"model_revision":"weights-r1",' + b'"normalization":"l2",' + b'"pooling":"mean_visual_tokens",' + b'"preprocessing":{"alignment":32,"matte_rgb":[128,128,128],"max_side":448,' + b'"resample":"lanczos3","revision":"moodboard-qwen35-srgb-pad32-max448-v1"},' + b'"prompt":{"revision":"moodboard-style-retrieval-v1",' + b'"sha256":"2222222222222222222222222222222222222222222222222222222222222222"},' + b'"schema_version":"moodboard.visual-descriptor.v1"}' +) +EXPECTED_FINGERPRINT = ( + "b57fb3cf43da387cde12425e6d7d442af269ba37ecabfbe4c975cb80abdf56e5" +) +EXPECTED_SPACE_KEY = f"moodboard_{EXPECTED_FINGERPRINT}_4" + + +class MoodboardDescriptorGoldenTest(unittest.TestCase): + def test_python_canonicalization_matches_rust_golden(self) -> None: + core = { + "schema_version": "moodboard.visual-descriptor.v1", + "model_name": "qwen3.5-vlm-pooled-visual", + "model_revision": "weights-r1", + "checkpoint_sha256": "1" * 64, + "inference": {"provider": "lattice-embed", "version": "0.9.0"}, + "preprocessing": { + "revision": "moodboard-qwen35-srgb-pad32-max448-v1", + "max_side": 448, + "alignment": 32, + "matte_rgb": [128, 128, 128], + "resample": "lanczos3", + }, + "prompt": { + "revision": "moodboard-style-retrieval-v1", + "sha256": "2" * 64, + }, + "pooling": "mean_visual_tokens", + "dimensions": 4, + "normalization": "l2", + } + canonical = json.dumps( + core, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + + self.assertEqual(canonical, EXPECTED_CANONICAL) + self.assertEqual(hashlib.sha256(canonical).hexdigest(), EXPECTED_FINGERPRINT) + self.assertEqual( + f"moodboard_{hashlib.sha256(canonical).hexdigest()}_{core['dimensions']}", + EXPECTED_SPACE_KEY, + ) + + +if __name__ == "__main__": + unittest.main()