From 137721dcdc1147f860ea66d09ca9ac4e0f3fa95f Mon Sep 17 00:00:00 2001 From: Ruben van der Linde Date: Mon, 25 May 2026 08:08:56 +0200 Subject: [PATCH] retrofit: reverse-spec vector-embeddings capability (5 REQs / mint) --- .../Handlers/EmbeddingGeneratorHandler.php | 10 + .../Handlers/VectorStatsHandler.php | 8 + .../Handlers/VectorStorageHandler.php | 12 + .../ObjectVectorizationStrategy.php | 14 + .../VectorizationStrategyInterface.php | 8 + .../Vectorization/VectorEmbeddings.php | 20 ++ lib/Service/VectorizationService.php | 20 ++ .../design.md | 82 ++++++ .../proposal.md | 62 +++++ .../specs/vector-embeddings/spec.md | 263 ++++++++++++++++++ .../tasks.md | 20 ++ 11 files changed, 519 insertions(+) create mode 100644 openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/design.md create mode 100644 openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/proposal.md create mode 100644 openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/specs/vector-embeddings/spec.md create mode 100644 openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md diff --git a/lib/Service/Vectorization/Handlers/EmbeddingGeneratorHandler.php b/lib/Service/Vectorization/Handlers/EmbeddingGeneratorHandler.php index 0d8116a30..dd5d36d6d 100644 --- a/lib/Service/Vectorization/Handlers/EmbeddingGeneratorHandler.php +++ b/lib/Service/Vectorization/Handlers/EmbeddingGeneratorHandler.php @@ -82,6 +82,8 @@ public function __construct( * @throws \Exception If configuration is invalid or generator cannot be created * * @SuppressWarnings(PHPMD.CyclomaticComplexity) Multiple provider configurations require separate conditions + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-1 */ public function getGenerator(array $config): EmbeddingGeneratorInterface { @@ -131,6 +133,8 @@ public function getGenerator(array $config): EmbeddingGeneratorInterface * @return int Default dimensions * * @psalm-return 384|1536|3072 + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-1 */ public function getDefaultDimensions(string $model): int { @@ -148,6 +152,8 @@ public function getDefaultDimensions(string $model): int * |OpenAI3LargeEmbeddingGenerator Generator instance * * @throws \Exception If model is not supported + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-1 */ private function createOpenAIGenerator( string $model, @@ -185,6 +191,8 @@ private function createOpenAIGenerator( * @throws \Exception If model is not supported * * @SuppressWarnings(PHPMD.ExcessiveMethodLength) Anonymous class requires complete implementation + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-1 */ private function createFireworksGenerator(string $model, array $config): object { @@ -357,6 +365,8 @@ public function embedDocuments(array $documents): array * @param array $config Configuration array with base_url * * @return OllamaEmbeddingGenerator Generator instance + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-1 */ private function createOllamaGenerator(string $model, array $config): OllamaEmbeddingGenerator { diff --git a/lib/Service/Vectorization/Handlers/VectorStatsHandler.php b/lib/Service/Vectorization/Handlers/VectorStatsHandler.php index 225ab5baa..cc10cba77 100644 --- a/lib/Service/Vectorization/Handlers/VectorStatsHandler.php +++ b/lib/Service/Vectorization/Handlers/VectorStatsHandler.php @@ -67,6 +67,8 @@ public function __construct( * @psalm-return array{total_vectors: int, by_type: array, * by_model: array, object_vectors?: int, file_vectors?: int, * source?: 'solr'|'solr_error'|'solr_unavailable'} + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-5 */ public function getStats(string $backend='php'): array { @@ -97,6 +99,8 @@ public function getStats(string $backend='php'): array * * @psalm-return array{total_vectors: int, by_type: array, * by_model: array, object_vectors: int, file_vectors: int} + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-5 */ private function getStatsFromDatabase(): array { @@ -157,6 +161,8 @@ private function getStatsFromDatabase(): array * } * * @SuppressWarnings(PHPMD.CyclomaticComplexity) Multi-collection stats gathering requires multiple conditions + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-5 */ private function getStatsFromSolr(): array { @@ -265,6 +271,8 @@ private function getStatsFromSolr(): array * @return array{count: int, by_model: array} Count and breakdown by model * * @SuppressWarnings(PHPMD.CyclomaticComplexity) Facet processing requires multiple conditions + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-5 */ private function countVectorsInCollection( string $collection, diff --git a/lib/Service/Vectorization/Handlers/VectorStorageHandler.php b/lib/Service/Vectorization/Handlers/VectorStorageHandler.php index ead33ed7e..486be40d4 100644 --- a/lib/Service/Vectorization/Handlers/VectorStorageHandler.php +++ b/lib/Service/Vectorization/Handlers/VectorStorageHandler.php @@ -81,6 +81,8 @@ public function __construct( * @throws \Exception If storage fails * * @SuppressWarnings(PHPMD.ExcessiveParameterList) Required for flexible vector storage options + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-2 */ public function storeVector( string $entityType, @@ -172,6 +174,8 @@ public function storeVector( * * @SuppressWarnings(PHPMD.ExcessiveParameterList) Required for flexible vector storage options * @SuppressWarnings(PHPMD.CyclomaticComplexity) Multiple storage conditions and error handling + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-2 */ private function storeVectorInDatabase( string $entityType, @@ -285,6 +289,8 @@ private function storeVectorInDatabase( * @SuppressWarnings(PHPMD.CyclomaticComplexity) Multiple Solr storage conditions * @SuppressWarnings(PHPMD.NPathComplexity) Multiple storage paths with error handling * @SuppressWarnings(PHPMD.ExcessiveMethodLength) Comprehensive Solr vector storage with atomic updates + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-2 */ private function storeVectorInSolr( string $entityType, @@ -414,6 +420,8 @@ private function storeVectorInSolr( * * @SuppressWarnings(PHPMD.CyclomaticComplexity) Collection resolution requires multiple conditions * @SuppressWarnings(PHPMD.NPathComplexity) Multiple collection determination paths + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-2 */ private function getSolrCollectionForEntityType(string $entityType): ?string { @@ -464,6 +472,8 @@ private function getSolrCollectionForEntityType(string $entityType): ?string * Get the configured Solr vector field name * * @return string Solr vector field name (default: '_embedding_') + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-2 */ private function getSolrVectorField(): string { @@ -493,6 +503,8 @@ private function getSolrVectorField(): string * @param string $text Text to sanitize * * @return string Sanitized text safe for UTF-8 storage + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-2 */ private function sanitizeText(string $text): string { diff --git a/lib/Service/Vectorization/Strategies/ObjectVectorizationStrategy.php b/lib/Service/Vectorization/Strategies/ObjectVectorizationStrategy.php index 03bcebd6b..3701d3f61 100644 --- a/lib/Service/Vectorization/Strategies/ObjectVectorizationStrategy.php +++ b/lib/Service/Vectorization/Strategies/ObjectVectorizationStrategy.php @@ -87,6 +87,8 @@ public function __construct( * @return \OCA\OpenRegister\Db\ObjectEntity[] * * @psalm-return list<\OCA\OpenRegister\Db\ObjectEntity> + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-4 */ public function fetchEntities(array $options): array { @@ -144,6 +146,8 @@ public function fetchEntities(array $options): array * @return (int|string)[][] Array with single item containing serialized object * * @psalm-return list{array{text: string, index: 0}} + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-4 */ public function extractVectorizationItems($entity): array { @@ -196,6 +200,8 @@ public function extractVectorizationItems($entity): array * * @SuppressWarnings(PHPMD.CyclomaticComplexity) Complex metadata extraction with multiple fallbacks * @SuppressWarnings(PHPMD.NPathComplexity) Multiple field extraction paths + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-4 */ public function prepareVectorMetadata($entity, array $item): array { @@ -270,6 +276,8 @@ public function prepareVectorMetadata($entity, array $item): array * @param array $objectData Object data * * @return array Array of @self keys + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-4 */ private function extractSelfKeys(array $objectData): array { @@ -292,6 +300,8 @@ private function extractSelfKeys(array $objectData): array * * @SuppressWarnings(PHPMD.CyclomaticComplexity) Multiple field type checks required * @SuppressWarnings(PHPMD.NPathComplexity) Multiple field validation paths + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-4 */ private function extractFirstStringField(array $objectData): ?string { @@ -324,6 +334,8 @@ private function extractFirstStringField(array $objectData): ?string * @param mixed $entity ObjectEntity * * @return string|int Object ID + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-4 */ public function getEntityIdentifier($entity) { @@ -343,6 +355,8 @@ public function getEntityIdentifier($entity) * @param array $config Vectorization configuration * * @return false|string Serialized text + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-4 */ private function serializeObject(array $object, array $config): string|false { diff --git a/lib/Service/Vectorization/Strategies/VectorizationStrategyInterface.php b/lib/Service/Vectorization/Strategies/VectorizationStrategyInterface.php index 383e5ea1a..24b08a75a 100644 --- a/lib/Service/Vectorization/Strategies/VectorizationStrategyInterface.php +++ b/lib/Service/Vectorization/Strategies/VectorizationStrategyInterface.php @@ -43,6 +43,8 @@ interface VectorizationStrategyInterface * @param array $options Strategy-specific options * * @return array Array of entities to vectorize + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-3 */ public function fetchEntities(array $options): array; @@ -60,6 +62,8 @@ public function fetchEntities(array $options): array; * @param mixed $entity Entity to extract items from * * @return array Array of items, each with 'text' and other data + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-3 */ public function extractVectorizationItems($entity): array; @@ -78,6 +82,8 @@ public function extractVectorizationItems($entity): array; * @param array $item Vectorization item (from extractVectorizationItems) * * @return array Metadata for vector storage + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-3 */ public function prepareVectorMetadata($entity, array $item): array; @@ -87,6 +93,8 @@ public function prepareVectorMetadata($entity, array $item): array; * @param mixed $entity Entity * * @return string|int Identifier + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-3 */ public function getEntityIdentifier($entity); }//end interface diff --git a/lib/Service/Vectorization/VectorEmbeddings.php b/lib/Service/Vectorization/VectorEmbeddings.php index 79ab934a5..dabe1a575 100644 --- a/lib/Service/Vectorization/VectorEmbeddings.php +++ b/lib/Service/Vectorization/VectorEmbeddings.php @@ -87,6 +87,8 @@ public function __construct( * @throws \Exception If embedding generation fails * * @psalm-return array{embedding: array, model: string, dimensions: int<0, max>} + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-1 */ public function generateEmbedding(string $text, ?string $provider=null): array { @@ -151,6 +153,8 @@ public function generateEmbedding(string $text, ?string $provider=null): array * @psalm-return array * * @SuppressWarnings(PHPMD.CyclomaticComplexity) Complex config validation logic + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-1 */ public function generateEmbeddingWithCustomConfig(string $text, array $config): array { @@ -228,6 +232,8 @@ public function generateEmbeddingWithCustomConfig(string $text, array $config): * @param string $testText Optional test text to embed * * @return array + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-1 */ public function testEmbedding(string $provider, array $config, string $testText='Test.'): array { @@ -303,6 +309,8 @@ public function testEmbedding(string $provider, array $config, string $testText= * @return array, model: string, dimensions: int}> Array of embeddings * * @throws \Exception If batch embedding generation fails + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-1 */ public function generateBatchEmbeddings(array $texts, ?string $provider=null): array { @@ -399,6 +407,8 @@ public function generateBatchEmbeddings(array $texts, ?string $provider=null): a * @return int The ID of the inserted vector * * @throws \Exception If storage fails + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-2 */ public function storeVector( string $entityType, @@ -515,6 +525,8 @@ public function hybridSearch( * @psalm-return array{total_vectors: int, by_type: array, * by_model: array, object_vectors?: int, file_vectors?: int, * source?: 'solr'|'solr_error'|'solr_unavailable'} + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-5 */ public function getVectorStats(): array { @@ -535,6 +547,8 @@ public function getVectorStats(): array * @SuppressWarnings(PHPMD.CyclomaticComplexity) Complex model comparison logic * @SuppressWarnings(PHPMD.NPathComplexity) Multiple database queries and conditions * @SuppressWarnings(PHPMD.ExcessiveMethodLength) Thorough model mismatch detection + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-5 */ public function checkEmbeddingModelMismatch(): array { @@ -647,6 +661,8 @@ public function checkEmbeddingModelMismatch(): array * @return (bool|int|string)[] * * @psalm-return array{success: bool, error?: string, message: string, deleted?: int} + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-5 */ public function clearAllEmbeddings(): array { @@ -704,6 +720,8 @@ public function clearAllEmbeddings(): array * Get the configured vector search backend * * @return string Vector search backend ('php', 'database', or 'solr') + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-2 */ private function getVectorSearchBackend(): string { @@ -728,6 +746,8 @@ private function getVectorSearchBackend(): string * api_key: string|null, base_url: string|null} Configuration * * @SuppressWarnings(PHPMD.CyclomaticComplexity) Provider-specific configuration mapping + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-1 */ private function getEmbeddingConfig(?string $provider=null): array { diff --git a/lib/Service/VectorizationService.php b/lib/Service/VectorizationService.php index b0a569ea5..deffb2801 100644 --- a/lib/Service/VectorizationService.php +++ b/lib/Service/VectorizationService.php @@ -92,6 +92,8 @@ public function __construct( * @param VectorizationStrategyInterface $strategy Strategy implementation * * @return void + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-3 */ public function registerStrategy(string $entityType, VectorizationStrategyInterface $strategy): void { @@ -123,6 +125,8 @@ public function registerStrategy(string $entityType, VectorizationStrategyInterf * @SuppressWarnings(PHPMD.CyclomaticComplexity) Complex batch processing with error handling * @SuppressWarnings(PHPMD.NPathComplexity) Multiple processing paths with exceptions * @SuppressWarnings(PHPMD.ExcessiveMethodLength) Comprehensive batch processing with progress tracking + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-3 */ public function vectorizeBatch(string $entityType, array $options=[]): array { @@ -263,6 +267,8 @@ public function vectorizeBatch(string $entityType, array $options=[]): array * @SuppressWarnings(PHPMD.CyclomaticComplexity) Complex batch vs serial processing logic * @SuppressWarnings(PHPMD.NPathComplexity) Multiple embedding and error handling paths * @SuppressWarnings(PHPMD.ExcessiveMethodLength) Comprehensive entity vectorization with error handling + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-3 */ private function vectorizeEntity($entity, VectorizationStrategyInterface $strategy, array $options): array { @@ -382,6 +388,8 @@ private function vectorizeEntity($entity, VectorizationStrategyInterface $strate * @param VectorizationStrategyInterface $strategy Strategy * * @return void + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-3 */ private function storeVector($entity, array $item, array $embeddingData, VectorizationStrategyInterface $strategy): void { @@ -408,6 +416,8 @@ private function storeVector($entity, array $item, array $embeddingData, Vectori * @return VectorizationStrategyInterface * * @throws \Exception If strategy not registered + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-3 */ private function getStrategy(string $entityType): VectorizationStrategyInterface { @@ -439,6 +449,8 @@ private function getStrategy(string $entityType): VectorizationStrategyInterface * @throws \Exception If embedding generation fails * * @psalm-return array{embedding: array, model: string, dimensions: int<0, max>} + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-1 */ public function generateEmbedding(string $text, ?string $provider=null): array { @@ -505,6 +517,8 @@ public function hybridSearch( * Delegates to VectorEmbeddings. * * @return array Vector statistics with totals and breakdowns by type and model. + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-5 */ public function getVectorStats(): array { @@ -526,6 +540,8 @@ public function getVectorStats(): array * data?: array{provider: string, model: 'unknown'|mixed, * vectorLength: int<0, max>, sampleValues: array, * testText: string}} + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-1 */ public function testEmbedding(string $provider, array $config, string $testText='Test.'): array { @@ -543,6 +559,8 @@ public function testEmbedding(string $provider, array $config, string $testText= * message?: string, current_model?: mixed, * existing_models?: list{0?: mixed,...}, total_vectors?: int, * null_model_count?: int, mismatched_models?: list} + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-5 */ public function checkEmbeddingModelMismatch(): array { @@ -557,6 +575,8 @@ public function checkEmbeddingModelMismatch(): array * @return (bool|int|string)[] Deletion results * * @psalm-return array{success: bool, error?: string, message: string, deleted?: int} + * + * @spec openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md#task-5 */ public function clearAllEmbeddings(): array { diff --git a/openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/design.md b/openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/design.md new file mode 100644 index 000000000..ed3c807cf --- /dev/null +++ b/openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/design.md @@ -0,0 +1,82 @@ +# Design — Retrofit Phase 4c new capability `vector-embeddings` + +**Retrofit change. Tasks describe retroactive annotation, not new implementation work.** + +## Context + +`/opsx-coverage-scan` on 2026-05-24 placed `lib/Service/VectorizationService.php` and the six files under `lib/Service/Vectorization/` into Phase 4c — Bucket 2c, "code exists, no spec to point at." The architect plan called for minting a single new `vector-embeddings` capability spanning all 48 methods. This document records the design decisions that produced the five-REQ shape. + +## Why one capability, not two + +The architect's batch was titled `vector-embeddings-and-semantic-search`. After reading the code, the two halves diverge along a clear interface line: + +- **Embedding generation + storage + batch + stats** — owned by this cap. Public entry through `VectorizationService` facade. +- **Vector-query execution (KNN / cosine / RRF / hybrid merge)** — owned by `search/spec.md` REQ-002 (PR #1791). Public entry through `FileSearchController` and `VectorSearchHandler`. + +Lumping both under one cap would have duplicated PR #1791's REQ-002. Splitting along the existing search-spec boundary keeps each cap testable in isolation and avoids the double-spec problem flagged in the batch JSON notes. + +The four facade query methods (`VectorizationService::semanticSearch`, `::hybridSearch`, `VectorEmbeddings::semanticSearch`, `::hybridSearch`) are pure delegation glue: they call `generateEmbedding()` (in scope here, REQ-001) and then `VectorSearchHandler->semanticSearch(...)` (out of scope, owned by search). Annotating them dual-owned is fragile — once #1791 lands, a tiny follow-up can `@spec`-pointer them at search REQ-002. + +## REQ granularity calls + +Five REQs map 1:1 to the five behavioral domains identified in the proposal table: + +- **REQ-001 — generation** is independent of where vectors land. The provider matrix (OpenAI / Fireworks / Ollama), the LLPhant integration, and the cached-generator pattern are testable purely by invoking `generateEmbedding('hello')` and inspecting the result shape. Splitting into "default" vs "custom config" sub-REQs was rejected because both code paths exit the same `EmbeddingGeneratorHandler::getGenerator()` chokepoint — the only difference is whether config came from `SettingsService` or the caller's payload. +- **REQ-002 — storage** is independent of how the vector was generated. The Postgres `openregister_vectors` table layout, the Solr atomic-update payload shape, and the UTF-8 sanitiser are testable by handing the storage handler a precomputed embedding array. Grouping storage with generation would have produced a 30-bullet REQ. +- **REQ-003 — batch orchestration** is independent of which strategy is registered. The orchestrator's contract (loop, error-capture, aggregated counts, serial-vs-parallel branch) is testable with a mock `VectorizationStrategyInterface`. It needs its own REQ because it does NOT call back into REQ-001 or REQ-002 directly — it delegates through the strategy. +- **REQ-004 — object strategy** is the only concrete strategy implementation visible in this tree. Its title-extraction fallback chain (4-level lookup) and its `@self`/`_register` metadata-folding are observable in production today and worth pinning. File strategy is `future-pass:next`. +- **REQ-005 — stats + rotation** is independent of generation and storage. Stats read from whatever backend stored the vectors; rotation detection compares `SettingsService` config to distinct `embedding_model` values seen in `openregister_vectors`. It bundles in `clearAllEmbeddings` because all three are admin-surface utility operations that share the "scan the table, summarise" pattern. + +## Why we did not extend chat-ai + +`chat-ai/spec.md` REQ-001 says the LLM pipeline "retrieves relevant context from registered objects and Nextcloud files using the active agent's RAG configuration." That sentence references RAG at the *consumer* level — what the chat pipeline does with the embeddings, not how the embeddings get there. Extending chat-ai to cover embedding generation / storage / batch / stats would have: + +- Coupled a backend infrastructure cap to a frontend user-flow cap (chat-ai is about agents, conversations, messages, feedback). +- Forced the chat-ai spec to grow scenarios about provider testing, Solr atomic updates, and table-scan model-mismatch detection that no chat-ai consumer ever sees. +- Made the chat-ai cap fail-on-implementation-change every time we swap embedding backends. + +The shared boundary stays clean: chat-ai consumes vector search through `VectorSearchHandler` (owned by search), and search consumes embedding generation through `VectorEmbeddings::generateEmbedding` (owned by this new cap). + +## Why we did not extend search + +PR #1791's `search` spec is *also* retrofit and is still pre-merge. We considered folding the 42 vector-embedding methods into search to keep search-related territory in one place, but rejected because: + +- Search REQ-002 explicitly scopes itself to "vector search" and the KNN / cosine / RRF math. It is not about *producing* embeddings. +- The 42 methods in this cap include a strategy interface, a multi-provider factory, a Postgres BLOB writer, a Solr atomic-update path, a UTF-8 sanitiser, a model-rotation auditor, and a JSON-serialiser. Half of those have nothing to do with search at all (e.g. `clearAllEmbeddings`, `testEmbedding`, `getDefaultDimensions`). +- Search's BackendInterface model is for indexing implementation (Solr / Elasticsearch) — not for orchestration of multi-provider LLM API clients. + +Two narrow capabilities is more honest than one swollen one. + +## Drop list rationale + +Six methods deliberately not annotated: + +| Method | Reason | +|--------|--------| +| `VectorizationService::semanticSearch` | Facade delegation to `VectorSearchHandler` — owned by `search/spec.md` REQ-002 (PR #1791). | +| `VectorizationService::hybridSearch` | Same as above. | +| `VectorEmbeddings::semanticSearch` | Same as above. | +| `VectorEmbeddings::hybridSearch` | Same as above. | + +These four are listed as `future-pass:next` in `tasks.md` for a follow-up annotation once search merges. + +## Notes-section drift policy + +This retrofit run is *annotation-only*. The Notes section in the spec flags six observed-but-suspicious behaviors: + +1. `openregister_vectors.embedding` is PHP-`serialize()`'d into a BLOB, not stored in `pgvector` or `JSONB`. This means it cannot be used for native Postgres KNN — every similarity calc has to deserialise into PHP. Search PR #1791's KNN path goes through Solr for this reason. (Already flagged in PR #1791 notes too.) +2. `EmbeddingGeneratorHandler::createFireworksGenerator` builds a curl-based anonymous LLPhant adapter rather than wrapping LLPhant's OpenAI client (Fireworks is OpenAI-API-compatible). It works, but it ignores LLPhant's retry / timeout config. +3. `EmbeddingGeneratorHandler::getDefaultDimensions` falls back to `1536` for unknown models, which is the OpenAI ada-002 size — silently wrong for any non-OpenAI provider. Callers that supply their own dimensions via storage args avoid this; the fallback only bites the `generateEmbedding()` return shape. +4. `VectorStorageHandler::storeVectorInSolr` parameters `$totalChunks`, `$chunkText`, `$metadata` are accepted but never sent to Solr — only `$entityId`, `$embedding`, `$model`, `$dimensions`, `$chunkIndex` make it into the atomic-update payload. The Postgres path uses all of them. +5. `VectorStorageHandler::storeVectorInSolr` returns a `crc32` of the document ID cast to `int`, while the DB path returns the autoincrement `id`. Callers that compare the two IDs across backends would see false negatives. +6. `ObjectVectorizationStrategy::serializeObject` always returns `json_encode($object, JSON_PRETTY_PRINT)` — the `includeMetadata` / `includeRelations` / `maxNestingDepth` config knobs are read from settings but never applied. Self-documented as `TODO: Implement configurable serialization`. + +None of these are fixed here. Future tightening belongs in proper `add-*` changes. + +## Scope guard + +No production code changes. No "while we're in here" refactors. If a method's observable behavior looks buggy (and several do, see above), the Notes section flags it; the spec does not silently re-describe it as the correct behavior. + +## Source + +`/tmp/or-scan/rspec-newcap-vector-embeddings-and-semantic-search.json` (48 methods, 7 files). Retrofit playbook: `.github/docs/claude/retrofit.md`. Sibling reference: PR #1791 (`search`), `chat-ai/spec.md` REQ-001 RAG sentence. diff --git a/openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/proposal.md b/openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/proposal.md new file mode 100644 index 000000000..b508bd30e --- /dev/null +++ b/openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/proposal.md @@ -0,0 +1,62 @@ +# Retrofit — Phase 4c new capability `vector-embeddings` + +## Why + +The 2026-05-24 coverage scan identified `lib/Service/VectorizationService.php` and the six files under `lib/Service/Vectorization/` (48 methods total) as Bucket 2c — code exists, but no capability spec describes embedding generation, vector storage, batch vectorisation, or vector statistics. `chat-ai/spec.md` references RAG and embeddings at the consumer level (REQ-001 mentions "retrieves relevant context from registered objects and Nextcloud files using the active agent's RAG configuration") but it does not specify how embeddings are produced, where vectors live, how they are indexed, or how a model rotation is detected. The `search` capability (PR #1791 — not merged on this branch) describes the *query* layer (KNN / cosine / RRF in `VectorSearchHandler`), not the *generation* + *storage* + *batch* layer. + +This change mints a new `vector-embeddings` capability to close that gap. + +## What the cluster actually contains + +Seven files, 48 methods, decomposing into five coherent behavioral domains: + +| Domain | Files / methods | +| --------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 1. Multi-provider embedding generation | `Handlers/EmbeddingGeneratorHandler.php`: `getGenerator`, `getDefaultDimensions`, `createOpenAIGenerator`, `createFireworksGenerator`, `createOllamaGenerator`. `Vectorization/VectorEmbeddings.php`: `generateEmbedding`, `generateEmbeddingWithCustomConfig`, `generateBatchEmbeddings`, `testEmbedding`, `getEmbeddingConfig`. | +| 2. Vector storage backend routing | `Handlers/VectorStorageHandler.php`: `storeVector`, `storeVectorInDatabase`, `storeVectorInSolr`, `getSolrCollectionForEntityType`, `getSolrVectorField`, `sanitizeText`. `Vectorization/VectorEmbeddings.php`: `storeVector`, `getVectorSearchBackend`. | +| 3. Strategy-driven batch vectorisation | `VectorizationService.php`: `registerStrategy`, `vectorizeBatch`, `vectorizeEntity`, `storeVector`, `getStrategy`. `Strategies/VectorizationStrategyInterface.php`: `fetchEntities`, `extractVectorizationItems`, `prepareVectorMetadata`, `getEntityIdentifier`. | +| 4. Object vectorisation strategy | `Strategies/ObjectVectorizationStrategy.php`: `fetchEntities`, `extractVectorizationItems`, `prepareVectorMetadata`, `extractSelfKeys`, `extractFirstStringField`, `getEntityIdentifier`, `serializeObject`. | +| 5. Vector statistics + model rotation | `Handlers/VectorStatsHandler.php`: `getStats`, `getStatsFromDatabase`, `getStatsFromSolr`, `countVectorsInCollection`. `Vectorization/VectorEmbeddings.php`: `getVectorStats`, `checkEmbeddingModelMismatch`, `clearAllEmbeddings`. `VectorizationService.php` facade: `getVectorStats`, `testEmbedding`, `checkEmbeddingModelMismatch`, `clearAllEmbeddings`. | + +## Approach + +One new capability: `vector-embeddings`, with 5 REQs (one per domain). Each REQ describes observed behavior at the public-API boundary (`VectorizationService` facade) and dives into the handler/strategy detail in scenarios. + +The facade methods on `VectorizationService` and `VectorEmbeddings` that **delegate** to `VectorSearchHandler::semanticSearch` / `::hybridSearch` (i.e. `VectorizationService::semanticSearch`, `::hybridSearch`, `VectorEmbeddings::semanticSearch`, `::hybridSearch`) are intentionally **NOT** specced here — the query-execution layer is owned by `search/spec.md` REQ-002 (PR #1791). The four facade entrypoints route a query through `generateEmbedding()` (in scope) then hand off to `VectorSearchHandler` (out of scope), so annotating them here would create a double-spec. + +For each REQ, scenarios describe observed behavior on the current implementation (not aspirational). The Notes section flags inconsistencies (serialized blob format, hardcoded fallback dimensions, missing pgvector / Postgres-native path, etc.). + +## Affected code units + +**vector-embeddings — 42 method annotations across 7 files:** + +REQ-001 — multi-provider embedding generation: +- `lib/Service/VectorizationService.php::generateEmbedding`, `::testEmbedding` +- `lib/Service/Vectorization/VectorEmbeddings.php::generateEmbedding`, `::generateEmbeddingWithCustomConfig`, `::generateBatchEmbeddings`, `::testEmbedding`, `::getEmbeddingConfig` +- `lib/Service/Vectorization/Handlers/EmbeddingGeneratorHandler.php::getGenerator`, `::getDefaultDimensions`, `::createOpenAIGenerator`, `::createFireworksGenerator`, `::createOllamaGenerator` + +REQ-002 — vector storage backend routing: +- `lib/Service/Vectorization/VectorEmbeddings.php::storeVector`, `::getVectorSearchBackend` +- `lib/Service/Vectorization/Handlers/VectorStorageHandler.php::storeVector`, `::storeVectorInDatabase`, `::storeVectorInSolr`, `::getSolrCollectionForEntityType`, `::getSolrVectorField`, `::sanitizeText` + +REQ-003 — strategy-driven batch vectorisation: +- `lib/Service/VectorizationService.php::registerStrategy`, `::vectorizeBatch`, `::vectorizeEntity`, `::storeVector`, `::getStrategy` +- `lib/Service/Vectorization/Strategies/VectorizationStrategyInterface.php::fetchEntities`, `::extractVectorizationItems`, `::prepareVectorMetadata`, `::getEntityIdentifier` + +REQ-004 — object vectorisation strategy: +- `lib/Service/Vectorization/Strategies/ObjectVectorizationStrategy.php::fetchEntities`, `::extractVectorizationItems`, `::prepareVectorMetadata`, `::extractSelfKeys`, `::extractFirstStringField`, `::getEntityIdentifier`, `::serializeObject` + +REQ-005 — vector statistics + model rotation: +- `lib/Service/VectorizationService.php::getVectorStats`, `::checkEmbeddingModelMismatch`, `::clearAllEmbeddings` +- `lib/Service/Vectorization/VectorEmbeddings.php::getVectorStats`, `::checkEmbeddingModelMismatch`, `::clearAllEmbeddings` +- `lib/Service/Vectorization/Handlers/VectorStatsHandler.php::getStats`, `::getStatsFromDatabase`, `::getStatsFromSolr`, `::countVectorsInCollection` + +## Out of scope + +- **Vector *query* execution.** `VectorSearchHandler::semanticSearch` / `::hybridSearch` are owned by `search/spec.md` REQ-002 (PR #1791). The four facade entrypoints in `VectorizationService` / `VectorEmbeddings` that delegate to those handler methods are deliberately **DROP**ed from this run so we do not double-spec the KNN / cosine / RRF contract. Once #1791 merges, those four facade methods can be `@spec`-annotated against `search/spec.md` REQ-002 in a tiny follow-up. +- **Settings UI / config storage.** `SettingsService::getLLMSettingsOnly` and the admin LLM configuration view live in `chat-ai` (settings shape) and are not respecced here. +- **Solr index lifecycle.** Collection creation, schema upload, and Solr connectivity health are owned by the index/search capability and are not re-described here. This cap consumes Solr through `IndexService->getBackend()` only. +- **File vectorisation strategy.** No `FileVectorizationStrategy.php` exists yet in this tree — only `ObjectVectorizationStrategy` is observable. The `Strategies/` directory and `VectorizationStrategyInterface` document the extension point, but file-specific behavior is left for a future retrofit run when the file strategy code lands. +- Any reshaping or "fixing" of observed behavior. Drift (PHP-`serialize()`'d blobs, hardcoded fallback dimensions, missing pgvector path, asymmetric Solr/DB chunk handling) is flagged in the Notes section, not silently corrected. + +Source: `/tmp/or-scan/rspec-newcap-vector-embeddings-and-semantic-search.json` (48 methods, 7 files). Retrofit playbook: `.github/docs/claude/retrofit.md`. Sibling reference: PR #1791 (`search`) and `chat-ai/spec.md`. diff --git a/openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/specs/vector-embeddings/spec.md b/openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/specs/vector-embeddings/spec.md new file mode 100644 index 000000000..cef14f68b --- /dev/null +++ b/openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/specs/vector-embeddings/spec.md @@ -0,0 +1,263 @@ +--- +retrofit: true +--- + +# Vector Embeddings Specification + +**Status**: implemented (retrofit — code already exists) +**Scope**: openregister + +## ADDED Requirements + +### Requirement: The system MUST generate text embeddings through a pluggable multi-provider backend (OpenAI / Fireworks AI / Ollama) + +The `VectorizationService::generateEmbedding(string $text, ?string $provider = null)` entrypoint MUST resolve the active embedding configuration from `SettingsService::getLLMSettingsOnly()` when `$provider` is `null`, otherwise honour the explicit override. The configured provider MUST be one of `openai`, `fireworks`, or `ollama`; any other provider name MUST throw `Unsupported embedding provider: {provider}` from `EmbeddingGeneratorHandler::getGenerator()`. The handler MUST cache generator instances keyed by `"{provider}_{model}"` and reuse them across calls. + +Each provider has a distinct construction path: OpenAI MUST route through one of `OpenAIADA002EmbeddingGenerator`, `OpenAI3SmallEmbeddingGenerator`, or `OpenAI3LargeEmbeddingGenerator` based on the model name; unknown OpenAI models MUST throw `Unsupported OpenAI model: {model}`. Fireworks MUST be wrapped in an anonymous LLPhant-compatible adapter that posts directly to `{base_url}/embeddings` with `Authorization: Bearer {api_key}` and returns `data[0].embedding`. Ollama MUST instantiate `OllamaEmbeddingGenerator` with `url = rtrim(base_url, '/') + '/api/'`. + +`generateEmbedding()` MUST return `['embedding' => array, 'model' => string, 'dimensions' => int]`. The dimensions value MUST come from `EmbeddingGeneratorHandler::getDefaultDimensions()` which maps `text-embedding-ada-002` and `text-embedding-3-small` to 1536, `text-embedding-3-large` to 3072, `ollama-default` to 384, and any unknown model to a fallback of 1536. + +Batch generation (`generateBatchEmbeddings(array $texts)`) MUST iterate texts serially (no provider-side batching is attempted) and MUST tolerate per-text failures: a failed call MUST yield `['embedding' => null, 'dimensions' => 0, 'error' => string]` for that index without aborting the batch. The aggregate result MUST be returned in input order. + +The admin-facing test affordance (`testEmbedding(string $provider, array $config, string $testText = 'Test.')`) MUST validate the supplied config (`provider` and `model` required; `api_key` required for `openai` and `fireworks`), generate one embedding, and return `['success' => bool, 'message' => string, 'data' => ['provider', 'model', 'vectorLength', 'sampleValues' => first 5 floats, 'testText']]`. On failure the response MUST be `['success' => false, 'error' => string, 'message' => "Failed to generate embedding: ..."]` — the call MUST NOT throw. + +#### Scenario: Default provider resolved from settings on simple generate call +- **GIVEN** `SettingsService::getLLMSettingsOnly()` returns `embeddingProvider: "openai"` and `openaiConfig: {model: "text-embedding-3-small", apiKey: "sk-..."}` +- **WHEN** `VectorizationService::generateEmbedding('hello')` is invoked with no provider override +- **THEN** `EmbeddingGeneratorHandler::createOpenAIGenerator('text-embedding-3-small', ...)` MUST be invoked +- **AND** the cached generator MUST be reused on the next call with the same provider/model pair +- **AND** the response MUST be `['embedding' => array, 'model' => 'text-embedding-3-small', 'dimensions' => 1536]` + +#### Scenario: Unsupported provider throws on getGenerator +- **GIVEN** a caller invokes `generateEmbedding('hello', 'cohere')` +- **WHEN** `EmbeddingGeneratorHandler::getGenerator(['provider' => 'cohere', ...])` runs +- **THEN** an `Exception` with message `Unsupported embedding provider: cohere` MUST be thrown +- **AND** no entry MUST be written to `$generatorCache` + +#### Scenario: Custom-config test rejects missing API key for OpenAI / Fireworks +- **GIVEN** the admin LLM settings page POSTs `{provider: "openai", model: "text-embedding-3-large", apiKey: ""}` to the test endpoint +- **WHEN** `VectorEmbeddings::generateEmbeddingWithCustomConfig(...)` validates the payload +- **THEN** an `Exception` with message `API key is required for openai` MUST be thrown +- **AND** the wrapping `testEmbedding()` MUST catch it and return `['success' => false, 'error' => 'API key is required for openai', 'message' => 'Failed to generate embedding: ...']` + +#### Scenario: Batch generation tolerates per-text failure +- **GIVEN** three texts `['ok-1', 'ok-2', 'ok-3']` are passed to `generateBatchEmbeddings` +- **AND** the second call to `EmbeddingGeneratorInterface::embedText` throws an HTTP-500 error +- **WHEN** the batch loop completes +- **THEN** the result MUST be an array of length 3 in input order +- **AND** index 1 MUST be `['embedding' => null, 'model' => string, 'dimensions' => 0, 'error' => string]` +- **AND** indices 0 and 2 MUST be successful `['embedding' => array, 'model' => string, 'dimensions' => int]` entries + +#### Scenario: Fireworks adapter posts directly without LLPhant client wrapping +- **GIVEN** the configured provider is `fireworks` with `model: "thenlper/gte-base"` and `base_url: "https://api.fireworks.ai/inference/v1"` +- **WHEN** `EmbeddingGeneratorHandler::createFireworksGenerator(...)` is invoked +- **THEN** an anonymous class implementing `EmbeddingGeneratorInterface` MUST be returned +- **AND** calling `embedText('hello')` on that instance MUST issue a `POST https://api.fireworks.ai/inference/v1/embeddings` with `Authorization: Bearer {api_key}` and body `{model: "thenlper/gte-base", input: "hello"}` +- **AND** the embedding length for `thenlper/gte-base` MUST be reported as 768 by `getEmbeddingLength()` + +### Requirement: The system MUST route vector storage to a configured backend (Postgres BLOB or Solr atomic update) with UTF-8 sanitisation + +`VectorEmbeddings::storeVector(...)` MUST resolve the backend by reading `llmSettings.vectorConfig.backend` from `SettingsService::getLLMSettingsOnly()`, defaulting to `'php'` on missing/error. The backend value MUST then be passed to `VectorStorageHandler::storeVector(..., string $backend)` which MUST dispatch to either `storeVectorInSolr` (when backend === `'solr'`) or `storeVectorInDatabase` (default). + +**Database path** (`storeVectorInDatabase`) MUST insert a row into `openregister_vectors` with: `entity_type`, `entity_id`, `chunk_index`, `total_chunks`, `chunk_text` (after sanitisation), `embedding` (PHP-`serialize()`'d to a BLOB via `PDO::PARAM_LOB`), `embedding_model`, `embedding_dimensions`, `metadata` (JSON-encoded when non-empty), and `created_at` / `updated_at` set to `date('Y-m-d H:i:s')`. The autoincrement `id` MUST be returned. Sanitisation (`sanitizeText`) MUST: (1) `mb_convert_encoding($text, 'UTF-8', 'UTF-8')`, (2) strip `\x00-\x08\x0B\x0C\x0E-\x1F\x7F` control characters, (3) `iconv('UTF-8', 'UTF-8//IGNORE', $text)`, (4) collapse whitespace via `preg_replace('/\s+/u', ' ', $text)`, (5) `trim()`. + +**Solr path** (`storeVectorInSolr`) MUST resolve the target collection by entity type: `file` / `files` MUST use `settings.solr.fileCollection`; anything else MUST use `settings.solr.objectCollection` (or fall back to `settings.solr.collection`). A missing collection MUST throw `Solr collection not configured for entity type: {entityType}`. The Solr vector field name MUST come from `settings.llm.vectorConfig.solrField`, defaulting to `'_embedding_'`. The document ID MUST be `entity_id` for objects and `"{entity_id}_chunk_{chunk_index}"` for files. The Solr atomic update payload MUST be `[{id, {vectorField}: {set: embedding}, _embedding_model_: {set: model}, _embedding_dim_: {set: dimensions}, self_updated: {set: gmdate('Y-m-d\TH:i:s\Z')}}]`. The backend instance MUST be an instance of `SolrBackend` (cast from `IndexService::getBackend()`) and MUST report `isAvailable() === true`; otherwise the call MUST throw. On Solr response with `responseHeader.status !== 0` (or missing), the call MUST throw `Solr atomic update failed: {responseJson}`. The Solr path MUST return the string document ID; `VectorStorageHandler::storeVector` then MUST return `crc32(documentId)` as the int facade value. + +Any failure in either path MUST be re-wrapped as `Vector storage failed: {message}` (or `Solr vector storage failed: ...`) before being thrown. + +#### Scenario: Default backend is PHP (database) when settings unavailable +- **GIVEN** `SettingsService::getLLMSettingsOnly()` throws an exception +- **WHEN** `VectorEmbeddings::getVectorSearchBackend()` is invoked +- **THEN** a warning MUST be logged `Failed to get vector search backend, defaulting to PHP` +- **AND** the resolved backend MUST be `'php'` +- **AND** `VectorStorageHandler::storeVectorInDatabase(...)` MUST be the path taken + +#### Scenario: Database insert serialises embedding as a PHP BLOB +- **GIVEN** an embedding `[0.1, 0.2, 0.3]` for object `'abc-uuid'` with model `text-embedding-3-small` +- **WHEN** `storeVectorInDatabase(...)` runs +- **THEN** the inserted row's `embedding` column MUST contain `serialize([0.1, 0.2, 0.3])` bound as `PDO::PARAM_LOB` +- **AND** the inserted row's `embedding_dimensions` MUST equal 3 +- **AND** the function MUST return the autoincrement `id` of the new row + +#### Scenario: Solr file-type document ID embeds chunk index +- **GIVEN** a file vector with `entity_id = 'file-uuid'`, `chunk_index = 2`, `entityType = 'file'` +- **WHEN** `storeVectorInSolr(...)` builds the atomic-update document +- **THEN** the document `id` MUST equal `"file-uuid_chunk_2"` +- **AND** the payload MUST include `_embedding_dim_: {set: dimensions}` and `self_updated: {set: gmdate('Y-m-d\TH:i:s\Z')}` +- **AND** `VectorStorageHandler::storeVector` MUST return `crc32("file-uuid_chunk_2")` to the caller + +#### Scenario: Solr unavailable triggers re-wrap +- **GIVEN** backend is `'solr'` and `IndexService::getBackend()->isAvailable()` returns `false` +- **WHEN** `storeVectorInSolr(...)` is invoked +- **THEN** an `Exception` with message `Solr vector storage failed: Solr service is not available` MUST be thrown +- **AND** the original message MUST be wrapped in the outer `storeVector` catch as `Vector storage failed: Solr vector storage failed: Solr service is not available` + +#### Scenario: sanitizeText strips control chars and normalises whitespace +- **GIVEN** input `"hello\x00world\t\t\nfoo"` +- **WHEN** `VectorStorageHandler::sanitizeText(...)` is invoked +- **THEN** the output MUST equal `"helloworld foo"` (NUL byte removed, tab+newline collapsed to single space, trimmed) + +### Requirement: The system MUST orchestrate batch vectorisation through a pluggable strategy interface with per-entity error capture + +`VectorizationService` MUST expose `registerStrategy(string $entityType, VectorizationStrategyInterface $strategy)` to bind a strategy to an entity-type identifier (e.g. `'object'`, `'file'`). `vectorizeBatch(string $entityType, array $options = [])` MUST resolve the strategy via `getStrategy()`, MUST throw `No vectorization strategy registered for entity type: {entityType}` when none is registered, and MUST invoke `strategy->fetchEntities($options)` to obtain the work list. + +When `fetchEntities` returns `[]`, the call MUST short-circuit with `['success' => true, 'message' => 'No entities found to vectorize', 'entity_type', 'total_entities' => 0, 'total_items' => 0, 'vectorized' => 0, 'failed' => 0]`. Otherwise the orchestrator MUST iterate entities, calling `vectorizeEntity()` for each, and MUST accumulate per-entity errors into a top-level `errors` array without aborting the batch on a single failure. + +`vectorizeEntity()` MUST first call `strategy->extractVectorizationItems($entity)`. When items is `[]`, the entity MUST be skipped (returning zero counts). For non-empty items, the orchestrator MUST select serial vs parallel mode based on `options.mode` (default `'serial'`) and `options.batch_size` (default 50): parallel MUST batch into chunks of `batch_size` and call `VectorEmbeddings::generateBatchEmbeddings()`, while serial MUST call `generateEmbedding()` per item. + +After embedding generation, successful vectors MUST be passed to `storeVector()` which MUST delegate to `strategy->prepareVectorMetadata($entity, $item)` to assemble the storage payload (`entity_type`, `entity_id`, `chunk_index`, `total_chunks`, `chunk_text`, `additional_metadata`) before calling `VectorEmbeddings::storeVector(...)`. Per-item failures (null embedding or thrown exception) MUST be recorded in the `errors` array as `['entity_id' => ..., 'item_index' => ..., 'error' => ...]` and incremented in the `failed` counter. + +The aggregate response MUST be `['success' => true, 'message' => "Batch vectorization completed: {V} vectorized, {F} failed", 'entity_type', 'total_entities' => count(entities), 'total_items', 'vectorized', 'failed', 'errors']`. Top-level orchestration failures (e.g. strategy fetch throws) MUST be re-thrown after error logging — they MUST NOT be swallowed. + +`VectorizationStrategyInterface` MUST define the four-method contract: `fetchEntities(array $options): array`, `extractVectorizationItems(mixed $entity): array`, `prepareVectorMetadata(mixed $entity, array $item): array`, `getEntityIdentifier(mixed $entity): string|int`. The metadata return MUST carry `entity_type`, `entity_id`, optional `chunk_index` (default 0), optional `total_chunks` (default 1), optional `chunk_text`, optional `additional_metadata`. + +#### Scenario: Unregistered entity type throws on vectorizeBatch +- **GIVEN** no strategy has been registered for `'image'` +- **WHEN** `VectorizationService::vectorizeBatch('image', [])` is invoked +- **THEN** an `Exception` with message `No vectorization strategy registered for entity type: image` MUST be thrown +- **AND** the exception MUST be re-thrown after error logging — the orchestrator MUST NOT return a normal result + +#### Scenario: Empty fetch returns success with zero counts +- **GIVEN** a registered strategy whose `fetchEntities([])` returns `[]` +- **WHEN** `vectorizeBatch('object', [])` is invoked +- **THEN** the response MUST be `['success' => true, 'message' => 'No entities found to vectorize', 'entity_type' => 'object', 'total_entities' => 0, 'total_items' => 0, 'vectorized' => 0, 'failed' => 0]` +- **AND** no calls to `strategy->extractVectorizationItems` MUST be made + +#### Scenario: Per-entity failure does not abort the batch +- **GIVEN** three entities are returned by `fetchEntities` +- **AND** `vectorizeEntity` for the second entity throws (strategy `extractVectorizationItems` raises) +- **WHEN** the orchestrator loop completes +- **THEN** the result MUST report `total_entities: 3` +- **AND** the `errors` array MUST contain exactly one entry for the failing entity: `['entity_id' => , 'error' => ]` +- **AND** entities 1 and 3 MUST have contributed normally to `vectorized` / `failed` / `total_items` + +#### Scenario: Parallel mode batches via generateBatchEmbeddings; serial mode loops generateEmbedding +- **GIVEN** an entity that yields 10 items, with `options = ['mode' => 'parallel', 'batch_size' => 4]` +- **WHEN** `vectorizeEntity()` processes the items +- **THEN** `VectorEmbeddings::generateBatchEmbeddings()` MUST be invoked exactly three times (chunks of 4, 4, 2) +- **AND** `VectorEmbeddings::generateEmbedding()` MUST NOT be called on this path +- **AND** with `options = ['mode' => 'serial']` the inverse MUST hold (10 calls to `generateEmbedding`, 0 to `generateBatchEmbeddings`) + +#### Scenario: Per-item null embedding records error and increments failed +- **GIVEN** parallel mode returns an embedding result where index 2 has `embedding => null` +- **WHEN** the orchestrator inspects the result +- **THEN** `failed` MUST be incremented for that item +- **AND** an entry `['entity_id', 'item_index' => 2, 'error' => ]` MUST be appended to `errors` +- **AND** `storeVector` MUST NOT be called for that item + +### Requirement: The system MUST vectorise OpenRegister objects via a strategy that JSON-serialises content and folds register/schema/uuid metadata + +`ObjectVectorizationStrategy` MUST implement `VectorizationStrategyInterface`. `fetchEntities($options)` MUST call `ObjectService::searchObjects(query: ['_limit' => $options['batch_size'] ?? 25, '_source' => 'database'], _rbac: false, _multitenancy: false, ids: null, uses: null, views: $options['views'] ?? null)` and MUST return the resulting list as an array (or `[]` when the return is not an array). + +`extractVectorizationItems($entity)` MUST always produce exactly one item per object: `[['text' => json_encode($objectData, JSON_PRETTY_PRINT), 'index' => 0]]`. `$objectData` MUST come from `$entity->jsonSerialize()` (or `$entity` itself when already an array). `SettingsService::getObjectSettingsOnly()` MUST be consulted for serialisation config (`includeMetadata`, `includeRelations`, `maxNestingDepth`), but the current implementation MUST emit those values in debug logs and then ignore them (the actual encoding is unconditional `json_encode(..., JSON_PRETTY_PRINT)`). + +`prepareVectorMetadata($entity, $item)` MUST extract a title using the fallback chain: (1) `objectData['title']`, (2) `['name']`, (3) `['_name']`, (4) `['summary']`, (5) `extractFirstStringField()` — the first non-`_`/non-`@`-prefixed key whose value is a string of length 1–99 and whose key is not in the skip set `[id, uuid, description, Beschrijving, beschrijving, content, text]`, (6) terminal fallback `"Object #{$objectId}"`. Description MUST be extracted via the chain: `description`, `_description`, `Beschrijving`, `beschrijving`, `summary`, `_summary`, default `""`. + +Register / schema / UUID / URI MUST be folded from multiple possible locations into the metadata: `register` from `_register` → `register` → `@self.register`; `schema` from `_schema` → `schema` → `@self.schema`; `uuid` from `uuid` → `_uuid` → `@self.id`; `uri` from `uri` → `_uri` → `@self.uri`. The returned metadata MUST be: + +``` +['entity_type' => 'object', + 'entity_id' => (string) $objectId, // 'unknown' when id is missing + 'chunk_index' => 0, + 'total_chunks' => 1, + 'chunk_text' => substr($item['text'], 0, 500), + 'additional_metadata' => [ + 'object_id', 'object_title', 'title', 'name', 'description', + 'register', 'register_id', 'schema', 'schema_id', 'uuid', 'uri' + ]] +``` + +`title` and `object_title` and `name` MUST all carry the same resolved title value. `register` and `register_id` MUST carry the same value (likewise `schema` / `schema_id`). The `chunk_text` field MUST be truncated to 500 chars of the serialised JSON. + +`getEntityIdentifier($entity)` MUST return `objectData['id']` when set, else the string `'unknown'`. + +#### Scenario: Single item produced per object regardless of size +- **GIVEN** an `ObjectEntity` whose `jsonSerialize()` returns a 50 KB object +- **WHEN** `extractVectorizationItems($entity)` is invoked +- **THEN** the result MUST be an array of length exactly 1 +- **AND** that item MUST be `['text' => , 'index' => 0]` +- **AND** the JSON MUST include all top-level keys present in the source object + +#### Scenario: Title fallback chain reaches `extractFirstStringField` when standard keys missing +- **GIVEN** an object `{label: "Resident #42", description: "Some long text", other: 12345}` +- **WHEN** `prepareVectorMetadata` extracts the title +- **THEN** the title MUST be `"Resident #42"` (matched via `extractFirstStringField` because `title`, `name`, `_name`, and `summary` are all missing, and `label` is the first eligible non-skip key) +- **AND** the title MUST NOT be `12345` (integer is rejected) and MUST NOT be `"Some long text"` (description is in the skip set) + +#### Scenario: Register/schema folded from multiple legacy locations +- **GIVEN** an object whose top-level lacks `register`/`schema` but has `@self: {register: "9", schema: "12", id: "abc-uuid"}` +- **WHEN** `prepareVectorMetadata` runs +- **THEN** the returned `additional_metadata` MUST have `register: "9"`, `register_id: "9"`, `schema: "12"`, `schema_id: "12"`, `uuid: "abc-uuid"` +- **AND** the same values MUST be present even if the object instead used `_register: "9"`, `_schema: "12"`, `_uuid: "abc-uuid"` + +#### Scenario: Missing id falls back to entity_id 'unknown' +- **GIVEN** an object whose `jsonSerialize()` returns no `id` key +- **WHEN** `getEntityIdentifier($entity)` is invoked +- **THEN** the result MUST be the string `'unknown'` +- **AND** the metadata `entity_id` MUST also be `'unknown'` +- **AND** the metadata `object_title` MUST be `"Object #unknown"` + +#### Scenario: chunk_text is truncated to 500 chars +- **GIVEN** a serialised object JSON of length 12 000 chars +- **WHEN** `prepareVectorMetadata` populates `chunk_text` +- **THEN** the resulting `chunk_text` MUST be exactly the first 500 chars of `$item['text']` + +### Requirement: The system MUST expose vector statistics and surface embedding-model rotation needs + +`VectorizationService::getVectorStats()` MUST resolve the configured backend (same logic as REQ-002), then delegate to `VectorStatsHandler::getStats(string $backend)`. On `'solr'` the handler MUST query the configured object/file collections via `_embedding_:* facet=true facet.field=_embedding_model_` and aggregate counts; on any other backend it MUST run three queryBuilder calls on `openregister_vectors` (total via `COUNT(*)`, breakdown via `GROUP BY entity_type`, breakdown via `GROUP BY embedding_model`). The returned shape MUST be `['total_vectors', 'by_type' => array, 'by_model' => array, 'object_vectors' => int, 'file_vectors' => int]`. Solr responses MUST additionally include `'source' => 'solr' | 'solr_error' | 'solr_unavailable'`. On any exception the handler MUST return `['total_vectors' => 0, 'by_type' => [], 'by_model' => []]` rather than throw. + +`VectorEmbeddings::checkEmbeddingModelMismatch()` MUST compare the currently-configured embedding model (from `SettingsService::getLLMSettingsOnly()`, mapped by provider: `openaiConfig.model`, `ollamaConfig.model`, `fireworksConfig.embeddingModel`) against the distinct `embedding_model` values found in `openregister_vectors`. The response MUST be one of: + +- `['has_vectors' => false, 'mismatch' => false, 'message' => 'No embedding model configured']` — when no current model is configured. +- `['has_vectors' => false, 'mismatch' => false, 'current_model' => string, 'message' => 'No vectors exist yet']` — when the table is empty. +- `['has_vectors' => true, 'mismatch' => bool, 'current_model', 'existing_models' => array, 'total_vectors' => int, 'null_model_count' => int, 'mismatched_models' => array, 'message' => string]` — when vectors exist. `mismatch` MUST be `true` if any stored model differs from current OR if `null_model_count > 0`. + +The user-facing `message` MUST be one of three strings depending on state: `Multiple embedding models detected. Consider re-embedding all vectors with a single model.` (any drift), `{N} vectors have no model information.` (only NULL models), `All vectors use the same embedding model.` (no drift). On exception the response MUST be `['has_vectors' => false, 'mismatch' => false, 'error' => string]`. + +`VectorEmbeddings::clearAllEmbeddings()` MUST count the rows in `openregister_vectors`, return `['success' => true, 'deleted' => 0, 'message' => 'No vectors to delete']` when empty, otherwise `DELETE FROM openregister_vectors` and return `['success' => true, 'deleted' => int, 'message' => "Deleted {N} vectors successfully"]`. On exception the response MUST be `['success' => false, 'error' => string, 'message' => 'Failed to clear embeddings: ...']` — the call MUST NOT throw. + +#### Scenario: Database stats return total + by-type + by-model breakdown +- **GIVEN** `openregister_vectors` contains 100 rows: 70 with `entity_type='object'`, 30 with `entity_type='file'`, split across models `text-embedding-3-small` (60) and `nomic-embed-text` (40) +- **WHEN** `VectorizationService::getVectorStats()` is invoked with backend = `'php'` +- **THEN** the response MUST be `['total_vectors' => 100, 'by_type' => ['object' => 70, 'file' => 30], 'by_model' => ['text-embedding-3-small' => 60, 'nomic-embed-text' => 40], 'object_vectors' => 70, 'file_vectors' => 30]` +- **AND** no `'source'` key MUST be present (that key is Solr-only) + +#### Scenario: Solr stats include `source` discriminator +- **GIVEN** backend = `'solr'` and `IndexService::getBackend()->isAvailable() === false` +- **WHEN** `getVectorStats()` is invoked +- **THEN** the response MUST include `'source' => 'solr_unavailable'` and all numeric fields zeroed +- **AND** on a successful Solr call the response MUST include `'source' => 'solr'` +- **AND** on a thrown error inside Solr query path the response MUST include `'source' => 'solr_error'` + +#### Scenario: Model mismatch flagged when stored vectors use a different model than current settings +- **GIVEN** current `openaiConfig.model` is `text-embedding-3-large` and the vectors table holds 50 rows with `embedding_model = text-embedding-ada-002` +- **WHEN** `checkEmbeddingModelMismatch()` is invoked +- **THEN** the response MUST be `['has_vectors' => true, 'mismatch' => true, 'current_model' => 'text-embedding-3-large', 'existing_models' => ['text-embedding-ada-002'], 'total_vectors' => 50, 'null_model_count' => 0, 'mismatched_models' => ['text-embedding-ada-002'], 'message' => 'Multiple embedding models detected. Consider re-embedding all vectors with a single model.']` + +#### Scenario: NULL embedding_model counted separately and surfaces re-embed need +- **GIVEN** the vectors table holds 30 rows with `embedding_model = text-embedding-3-large` (matching current settings) and 5 rows with `embedding_model IS NULL` +- **WHEN** `checkEmbeddingModelMismatch()` is invoked +- **THEN** the response MUST include `mismatch: true` and `null_model_count: 5` and `mismatched_models: []` +- **AND** the message MUST be `5 vectors have no model information.` + +#### Scenario: clearAllEmbeddings on empty table is a no-op +- **GIVEN** `openregister_vectors` is empty +- **WHEN** `clearAllEmbeddings()` is invoked +- **THEN** the response MUST be `['success' => true, 'deleted' => 0, 'message' => 'No vectors to delete']` +- **AND** no `DELETE` statement MUST be issued + +#### Scenario: clearAllEmbeddings deletes all rows and returns count +- **GIVEN** the table holds 123 rows +- **WHEN** `clearAllEmbeddings()` is invoked +- **THEN** the table MUST be emptied via `DELETE FROM openregister_vectors` +- **AND** the response MUST be `['success' => true, 'deleted' => 123, 'message' => 'Deleted 123 vectors successfully']` + +## Notes + +- `openregister_vectors.embedding` is PHP-`serialize()`'d, not stored in `pgvector` or `JSONB`. This means native Postgres KNN is impossible — every similarity calc must deserialise into PHP. The KNN path goes through Solr for this reason. (Also flagged in PR #1791 search-spec notes.) +- `EmbeddingGeneratorHandler::getDefaultDimensions` falls back to `1536` (OpenAI ada-002 size) for unknown models, which is silently wrong for any non-OpenAI provider. Storage callers that pass dimensions explicitly avoid this; the fallback only bites the `generateEmbedding()` return-shape. +- `EmbeddingGeneratorHandler::createFireworksGenerator` builds a curl-based anonymous LLPhant adapter rather than wrapping LLPhant's OpenAI client (Fireworks is OpenAI-API-compatible). It works, but it ignores LLPhant's retry / timeout config. +- `VectorStorageHandler::storeVectorInSolr` parameters `$totalChunks`, `$chunkText`, `$metadata` are accepted but never sent to Solr — only `$entityId`, `$embedding`, `$model`, `$dimensions`, `$chunkIndex` make it into the atomic-update payload. The Postgres path uses all of them. Asymmetric storage between backends. +- `VectorStorageHandler::storeVectorInSolr` returns the Solr document ID as a string, which `storeVector` then casts to `int` via `crc32`. The DB path returns the autoincrement integer `id` directly. Callers comparing the two across backends would see false negatives. +- `ObjectVectorizationStrategy::serializeObject` reads `includeMetadata` / `includeRelations` / `maxNestingDepth` from settings and emits them in debug logs but always returns `json_encode($object, JSON_PRETTY_PRINT)`. Self-documented as `TODO: Implement configurable serialization`. Not fixed here. +- `VectorizationStrategyInterface::getEntityIdentifier` declares no return type on the interface signature, only in the implementation's docblock. Implementations are free to return `string|int`; the orchestrator does not coerce. diff --git a/openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md b/openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md new file mode 100644 index 000000000..fa713a5cd --- /dev/null +++ b/openspec/changes/retrofit-2026-05-24-newcap-vector-embeddings/tasks.md @@ -0,0 +1,20 @@ +# Tasks + +All tasks are marked `[x]` because the code already exists. This is a retrofit — tasks describe retroactive annotation, not new implementation work. + +## vector-embeddings + +- [x] task-1: vector-embeddings#REQ-001 — Multi-provider embedding generation (OpenAI / Fireworks AI / Ollama) with a cached generator factory, model-to-dimensions fallback, and a custom-config validation path used for the admin "test embedding" affordance (retroactive annotation: `VectorizationService::generateEmbedding`, `VectorizationService::testEmbedding`, `VectorEmbeddings::generateEmbedding`, `VectorEmbeddings::generateEmbeddingWithCustomConfig`, `VectorEmbeddings::generateBatchEmbeddings`, `VectorEmbeddings::testEmbedding`, `VectorEmbeddings::getEmbeddingConfig`, `EmbeddingGeneratorHandler::getGenerator`, `EmbeddingGeneratorHandler::getDefaultDimensions`, `EmbeddingGeneratorHandler::createOpenAIGenerator`, `EmbeddingGeneratorHandler::createFireworksGenerator`, `EmbeddingGeneratorHandler::createOllamaGenerator`) + +- [x] task-2: vector-embeddings#REQ-002 — Backend-routed vector storage (Postgres `openregister_vectors` PHP-serialised blob OR Solr atomic update on object/file collection) with UTF-8 sanitisation and entity-type-aware document IDs (retroactive annotation: `VectorEmbeddings::storeVector`, `VectorEmbeddings::getVectorSearchBackend`, `VectorStorageHandler::storeVector`, `VectorStorageHandler::storeVectorInDatabase`, `VectorStorageHandler::storeVectorInSolr`, `VectorStorageHandler::getSolrCollectionForEntityType`, `VectorStorageHandler::getSolrVectorField`, `VectorStorageHandler::sanitizeText`) + +- [x] task-3: vector-embeddings#REQ-003 — Strategy-driven batch vectorisation orchestrator with pluggable `VectorizationStrategyInterface`, serial / parallel modes, per-entity error capture, aggregated counts (retroactive annotation: `VectorizationService::registerStrategy`, `VectorizationService::vectorizeBatch`, `VectorizationService::vectorizeEntity`, `VectorizationService::storeVector`, `VectorizationService::getStrategy`, `VectorizationStrategyInterface::fetchEntities`, `VectorizationStrategyInterface::extractVectorizationItems`, `VectorizationStrategyInterface::prepareVectorMetadata`, `VectorizationStrategyInterface::getEntityIdentifier`) + +- [x] task-4: vector-embeddings#REQ-004 — Object vectorisation strategy: JSON-serialise object to text, multi-field title/description fallback (`title` → `name` → `@self.name` → first short string field), pass register/schema/uuid/uri through to vector metadata for downstream filtering (retroactive annotation: `ObjectVectorizationStrategy::fetchEntities`, `ObjectVectorizationStrategy::extractVectorizationItems`, `ObjectVectorizationStrategy::prepareVectorMetadata`, `ObjectVectorizationStrategy::extractSelfKeys`, `ObjectVectorizationStrategy::extractFirstStringField`, `ObjectVectorizationStrategy::getEntityIdentifier`, `ObjectVectorizationStrategy::serializeObject`) + +- [x] task-5: vector-embeddings#REQ-005 — Vector statistics and model-rotation detection: total + by-type + by-model counts (DB groupBy or Solr facet), current-vs-stored model mismatch detection that surfaces re-embedding need, clear-all utility (retroactive annotation: `VectorizationService::getVectorStats`, `VectorizationService::checkEmbeddingModelMismatch`, `VectorizationService::clearAllEmbeddings`, `VectorEmbeddings::getVectorStats`, `VectorEmbeddings::checkEmbeddingModelMismatch`, `VectorEmbeddings::clearAllEmbeddings`, `VectorStatsHandler::getStats`, `VectorStatsHandler::getStatsFromDatabase`, `VectorStatsHandler::getStatsFromSolr`, `VectorStatsHandler::countVectorsInCollection`) + +## Deferred (`future-pass:next`) + +- 4 facade query methods (`VectorizationService::semanticSearch` / `::hybridSearch`, `VectorEmbeddings::semanticSearch` / `::hybridSearch`) — `@spec`-annotate against `search/spec.md` REQ-002 once PR #1791 lands on this branch. They generate the query embedding (covered here by REQ-001) and then delegate execution to `VectorSearchHandler`, which the search spec already owns. +- File vectorisation strategy — once a `FileVectorizationStrategy` class lands in `lib/Service/Vectorization/Strategies/`, file-specific chunk handling, total_chunks > 1, and `chunk_index` behaviour can be added as a sixth REQ (or extended into REQ-004).