From 156385ab80bb2fa44ae2031b970470b4b85f6e9c Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 22 Jul 2026 15:50:21 +1200 Subject: [PATCH] feat: round-trip vectorsdb collection dimension through migrations A restored vectorsdb collection lost its dimension: the Collection resource never carried it, so archives dropped it at write time, and the destination both created the per-database metadata collection from the generic databases schema (which lacks the attribute) and wrote collection documents without it. Restored collections reported dimension null and creating additional collections on a restored vectorsdb database failed structure validation. Collection now carries a nullable dimension (serialized and exported from the Appwrite source), and the destination accepts an optional collectionStructures map keyed by database type so vectorsdb databases get their own metadata schema. Types with an entry get dimension written on entity import; archives written before dimension was serialized get a placeholder that syncVectorDimension corrects from the vector column size when the embeddings field is imported. DocumentsDB/VectorsDB fromArray also stop warning on entity lines whose nested database object predates the database DSN key. Co-Authored-By: Claude Fable 5 --- src/Migration/Destinations/Appwrite.php | 74 +++++++++++++++++-- .../Resources/Database/Collection.php | 33 ++++++++- .../Resources/Database/DocumentsDB.php | 4 +- .../Resources/Database/VectorsDB.php | 4 +- src/Migration/Sources/Appwrite.php | 1 + .../Unit/Resources/CollectionTest.php | 58 +++++++++++++++ 6 files changed, 162 insertions(+), 12 deletions(-) create mode 100644 tests/Migration/Unit/Resources/CollectionTest.php diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index e1b31eb3..f6c89809 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -50,6 +50,7 @@ use Utopia\Migration\Resources\Auth\Team; use Utopia\Migration\Resources\Auth\User; use Utopia\Migration\Resources\Database\Attribute; +use Utopia\Migration\Resources\Database\Collection as CollectionResource; use Utopia\Migration\Resources\Database\Column; use Utopia\Migration\Resources\Database\Database; use Utopia\Migration\Resources\Database\Index; @@ -189,6 +190,7 @@ class Appwrite extends Destination * @param array> $collectionStructure * @param OnDuplicate $onDuplicate Behavior when a row with an existing $id is encountered. * @param (callable(Database $resource): string)|null $getDatabaseDSN Resolver for the destination's `_databases.database` value. Pass when the destination project's DSN differs from the source's, so the destination row carries its own DSN instead of inheriting the source's. + * @param array>> $collectionStructures Per-database-type metadata collection structures (e.g. `['vectorsdb' => ...]`), used instead of $collectionStructure when the imported database's type has an entry. Types with an entry also get type-specific metadata written (e.g. vectorsdb collection `dimension`). */ public function __construct( string $project, @@ -201,6 +203,7 @@ public function __construct( protected string $projectInternalId, protected OnDuplicate $onDuplicate = OnDuplicate::Fail, ?callable $getDatabaseDSN = null, + protected array $collectionStructures = [], ) { $this->projectId = $project; $this->endpoint = $endpoint; @@ -718,14 +721,16 @@ protected function createDatabase(Database $resource): bool // collection, so we skip the lookup entirely. if ($isFailed && $this->dbForProject->getCollection($this->databaseCollectionId($existing))->isEmpty()) { try { + $structure = $this->collectionStructureFor($resource); + $columns = \array_map( fn ($attr) => new UtopiaDocument($attr), - $this->collectionStructure['attributes'] + $structure['attributes'] ); $indexes = \array_map( fn ($index) => new UtopiaDocument($index), - $this->collectionStructure['indexes'] + $structure['indexes'] ); $this->dbForProject->createCollection( @@ -777,14 +782,16 @@ protected function createDatabase(Database $resource): bool $resource->setSequence($database->getSequence()); try { + $structure = $this->collectionStructureFor($resource); + $columns = \array_map( fn ($attr) => new UtopiaDocument($attr), - $this->collectionStructure['attributes'] + $structure['attributes'] ); $indexes = \array_map( fn ($index) => new UtopiaDocument($index), - $this->collectionStructure['indexes'] + $structure['indexes'] ); $this->dbForProject->createCollection( @@ -889,7 +896,7 @@ protected function createEntity(Table $resource): bool '$permissions' => Permission::aggregate($resource->getPermissions()), 'documentSecurity' => $resource->getRowSecurity(), '$updatedAt' => $updatedAt, - ]) + ] + $this->entityTypeMetadata($resource)) ); $resource->setSequence($existing->getSequence()); return true; @@ -912,7 +919,7 @@ protected function createEntity(Table $resource): bool 'search' => implode(' ', [$resource->getId(), $resource->getTableName()]), '$createdAt' => $createdAt, '$updatedAt' => $updatedAt, - ])); + ] + $this->entityTypeMetadata($resource))); $resource->setSequence($table->getSequence()); @@ -1281,9 +1288,64 @@ protected function createField(Column|Attribute $resource): bool $this->processedTwoWayPairs[$twoWayPairKey] = true; } + $this->syncVectorDimension($resource, $type, $database, $table); + return true; } + /** + * @return array> + */ + private function collectionStructureFor(Database $resource): array + { + return $this->collectionStructures[$resource->getType()] ?? $this->collectionStructure; + } + + /** + * Type-specific metadata for an imported entity's collection document. VectorsDB collections + * carry a required `dimension`; archives written before it was serialized have none, so a + * placeholder satisfies the schema until {@see self::syncVectorDimension()} corrects it from + * the vector column's size. + * + * @return array + */ + private function entityTypeMetadata(Table $resource): array + { + if ( + $resource instanceof CollectionResource + && $resource->getDatabase()->getType() === Resource::TYPE_DATABASE_VECTORSDB + && isset($this->collectionStructures[Resource::TYPE_DATABASE_VECTORSDB]) + ) { + return ['dimension' => $resource->getDimension() ?? 0]; + } + + return []; + } + + /** + * @throws \Throwable + */ + private function syncVectorDimension(Column|Attribute $resource, string $type, UtopiaDocument $database, UtopiaDocument $table): void + { + if ( + $type !== UtopiaDatabase::VAR_VECTOR + || $resource->getTable()->getDatabase()->getType() !== Resource::TYPE_DATABASE_VECTORSDB + || !isset($this->collectionStructures[Resource::TYPE_DATABASE_VECTORSDB]) + ) { + return; + } + + if ((int) $table->getAttribute('dimension', 0) === $resource->getSize()) { + return; + } + + $this->dbForProject->updateDocument( + $this->databaseCollectionId($database), + $table->getId(), + new UtopiaDocument(['dimension' => $resource->getSize()]) + ); + } + /** * @throws Exception * @throws \Throwable diff --git a/src/Migration/Resources/Database/Collection.php b/src/Migration/Resources/Database/Collection.php index e927daf9..25dedf93 100644 --- a/src/Migration/Resources/Database/Collection.php +++ b/src/Migration/Resources/Database/Collection.php @@ -6,6 +6,8 @@ class Collection extends Table { + protected ?int $dimension = null; + public static function getName(): string { return Resource::TYPE_COLLECTION; @@ -25,7 +27,8 @@ public static function getName(): string * permissions: ?array, * createdAt: string, * updatedAt: string, - * enabled: bool + * enabled: bool, + * dimension?: ?int * } $array */ public static function fromArray(array $array): self @@ -36,7 +39,7 @@ public static function fromArray(array $array): self default => Database::fromArray($array['database']) }; - return new self( + $collection = new self( $database, name: $array['name'], id: $array['id'], @@ -46,5 +49,31 @@ public static function fromArray(array $array): self updatedAt: $array['updatedAt'] ?? '', enabled: $array['enabled'] ?? true, ); + + $collection->setDimension($array['dimension'] ?? null); + + return $collection; + } + + public function getDimension(): ?int + { + return $this->dimension; + } + + public function setDimension(?int $dimension): self + { + $this->dimension = $dimension; + + return $this; + } + + /** + * @return array + */ + public function jsonSerialize(): array + { + return array_merge(parent::jsonSerialize(), [ + 'dimension' => $this->dimension, + ]); } } diff --git a/src/Migration/Resources/Database/DocumentsDB.php b/src/Migration/Resources/Database/DocumentsDB.php index 76afacb8..82f1e2ed 100644 --- a/src/Migration/Resources/Database/DocumentsDB.php +++ b/src/Migration/Resources/Database/DocumentsDB.php @@ -19,7 +19,7 @@ public static function getName(): string * updatedAt: string, * enabled: bool, * originalId: string|null, - * database: string, + * database?: string, * status: string|null * } $array */ @@ -33,7 +33,7 @@ public static function fromArray(array $array): self enabled: $array['enabled'] ?? true, originalId: $array['originalId'] ?? '', type: $array['type'] ?? Resource::TYPE_DATABASE_DOCUMENTSDB, - database: $array['database'], + database: $array['database'] ?? '', databaseStatus: $array['status'] ?? null ); } diff --git a/src/Migration/Resources/Database/VectorsDB.php b/src/Migration/Resources/Database/VectorsDB.php index f751c321..1a9dfd05 100644 --- a/src/Migration/Resources/Database/VectorsDB.php +++ b/src/Migration/Resources/Database/VectorsDB.php @@ -19,7 +19,7 @@ public static function getName(): string * updatedAt: string, * enabled: bool, * originalId: string|null, - * database: string, + * database?: string, * type: string, * status: string|null * } $array @@ -34,7 +34,7 @@ public static function fromArray(array $array): self enabled: $array['enabled'] ?? true, originalId: $array['originalId'] ?? '', type: $array['type'] ?? Resource::TYPE_DATABASE_VECTORSDB, - database: $array['database'], + database: $array['database'] ?? '', databaseStatus: $array['status'] ?? null ); } diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 25996248..d057e5d0 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -1230,6 +1230,7 @@ private function exportEntities(string $databaseName, int $batchSize): void 'createdAt' => $table['$createdAt'], 'updatedAt' => $table['$updatedAt'], 'enabled' => $table['enabled'] ?? true, + 'dimension' => $table['dimension'] ?? null, 'database' => [ 'id' => $database->getId(), 'name' => $database->getDatabaseName(), diff --git a/tests/Migration/Unit/Resources/CollectionTest.php b/tests/Migration/Unit/Resources/CollectionTest.php new file mode 100644 index 00000000..f89ed938 --- /dev/null +++ b/tests/Migration/Unit/Resources/CollectionTest.php @@ -0,0 +1,58 @@ + 'vectors', + 'name' => 'Vectors', + 'type' => Resource::TYPE_DATABASE_VECTORSDB, + ]; + + public function testDimensionRoundTrip(): void + { + $collection = Collection::fromArray([ + 'database' => self::DATABASE, + 'id' => 'embeddings-store', + 'name' => 'Embeddings Store', + 'rowSecurity' => true, + 'permissions' => [], + 'createdAt' => '2026-01-01T00:00:00.000+00:00', + 'updatedAt' => '2026-01-01T00:00:00.000+00:00', + 'enabled' => true, + 'dimension' => 1536, + ]); + + $this->assertInstanceOf(VectorsDB::class, $collection->getDatabase()); + $this->assertSame(1536, $collection->getDimension()); + + $serialized = $collection->jsonSerialize(); + $this->assertSame(1536, $serialized['dimension']); + + $rehydrated = Collection::fromArray(\json_decode(\json_encode($collection), true)); + $this->assertSame(1536, $rehydrated->getDimension()); + } + + public function testDimensionDefaultsToNull(): void + { + $collection = Collection::fromArray([ + 'database' => self::DATABASE, + 'id' => 'embeddings-store', + 'name' => 'Embeddings Store', + 'rowSecurity' => true, + 'permissions' => [], + 'createdAt' => '', + 'updatedAt' => '', + 'enabled' => true, + ]); + + $this->assertNull($collection->getDimension()); + $this->assertNull($collection->jsonSerialize()['dimension']); + } +}