Skip to content

Round-trip vectorsdb collection dimension through migrations #210

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
abnegate merged 1 commit into from
Jul 22, 2026
Merged

Round-trip vectorsdb collection dimension through migrations #210

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 68 additions & 6 deletions src/Migration/Destinations/Appwrite.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
use Utopia\Migration\Resources\Auth\Team;
use Utopia\Migration\Resources\Auth\User;
use Utopia\Migration\Resources\Database\Attribute;
use Utopia\Migration\Resources\Database\Collection as CollectionResource;
use Utopia\Migration\Resources\Database\Column;
use Utopia\Migration\Resources\Database\Database;
use Utopia\Migration\Resources\Database\Index;
Expand Down Expand Up @@ -189,6 +190,7 @@ class Appwrite extends Destination
* @param array<array<string, mixed>> $collectionStructure
* @param OnDuplicate $onDuplicate Behavior when a row with an existing $id is encountered.
* @param (callable(Database $resource): string)|null $getDatabaseDSN Resolver for the destination's `_databases.database` value. Pass when the destination project's DSN differs from the source's, so the destination row carries its own DSN instead of inheriting the source's.
* @param array<string, array<array<string, mixed>>> $collectionStructures Per-database-type metadata collection structures (e.g. `['vectorsdb' => ...]`), used instead of $collectionStructure when the imported database's type has an entry. Types with an entry also get type-specific metadata written (e.g. vectorsdb collection `dimension`).
*/
public function __construct(
string $project,
Expand All @@ -201,6 +203,7 @@ public function __construct(
protected string $projectInternalId,
protected OnDuplicate $onDuplicate = OnDuplicate::Fail,
?callable $getDatabaseDSN = null,
protected array $collectionStructures = [],
) {
$this->projectId = $project;
$this->endpoint = $endpoint;
Expand Down Expand Up @@ -718,14 +721,16 @@ protected function createDatabase(Database $resource): bool
// collection, so we skip the lookup entirely.
if ($isFailed && $this->dbForProject->getCollection($this->databaseCollectionId($existing))->isEmpty()) {
try {
$structure = $this->collectionStructureFor($resource);

$columns = \array_map(
fn ($attr) => new UtopiaDocument($attr),
$this->collectionStructure['attributes']
$structure['attributes']
);

$indexes = \array_map(
fn ($index) => new UtopiaDocument($index),
$this->collectionStructure['indexes']
$structure['indexes']
);

$this->dbForProject->createCollection(
Expand Down Expand Up @@ -777,14 +782,16 @@ protected function createDatabase(Database $resource): bool
$resource->setSequence($database->getSequence());

try {
$structure = $this->collectionStructureFor($resource);

$columns = \array_map(
fn ($attr) => new UtopiaDocument($attr),
$this->collectionStructure['attributes']
$structure['attributes']
);

$indexes = \array_map(
fn ($index) => new UtopiaDocument($index),
$this->collectionStructure['indexes']
$structure['indexes']
);

$this->dbForProject->createCollection(
Expand Down Expand Up @@ -889,7 +896,7 @@ protected function createEntity(Table $resource): bool
'$permissions' => Permission::aggregate($resource->getPermissions()),
'documentSecurity' => $resource->getRowSecurity(),
'$updatedAt' => $updatedAt,
])
] + $this->entityTypeMetadata($resource))
);
$resource->setSequence($existing->getSequence());
return true;
Expand All @@ -912,7 +919,7 @@ protected function createEntity(Table $resource): bool
'search' => implode(' ', [$resource->getId(), $resource->getTableName()]),
'$createdAt' => $createdAt,
'$updatedAt' => $updatedAt,
]));
] + $this->entityTypeMetadata($resource)));

$resource->setSequence($table->getSequence());

Expand Down Expand Up @@ -1281,9 +1288,64 @@ protected function createField(Column|Attribute $resource): bool
$this->processedTwoWayPairs[$twoWayPairKey] = true;
}

$this->syncVectorDimension($resource, $type, $database, $table);

return true;
}

/**
* @return array<array<string, mixed>>
*/
private function collectionStructureFor(Database $resource): array
{
return $this->collectionStructures[$resource->getType()] ?? $this->collectionStructure;
}

/**
* Type-specific metadata for an imported entity's collection document. VectorsDB collections
* carry a required `dimension`; archives written before it was serialized have none, so a
* placeholder satisfies the schema until {@see self::syncVectorDimension()} corrects it from
* the vector column's size.
*
* @return array<string, mixed>
*/
private function entityTypeMetadata(Table $resource): array
{
if (
$resource instanceof CollectionResource
&& $resource->getDatabase()->getType() === Resource::TYPE_DATABASE_VECTORSDB
&& isset($this->collectionStructures[Resource::TYPE_DATABASE_VECTORSDB])
) {
return ['dimension' => $resource->getDimension() ?? 0];
}

return [];
}

/**
* @throws \Throwable
*/
private function syncVectorDimension(Column|Attribute $resource, string $type, UtopiaDocument $database, UtopiaDocument $table): void
{
if (
$type !== UtopiaDatabase::VAR_VECTOR
|| $resource->getTable()->getDatabase()->getType() !== Resource::TYPE_DATABASE_VECTORSDB

@greptile-apps greptile-apps Bot Jul 22, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Vector Dimension Overwritten

When a vectorsdb collection has more than one vector attribute, this sync runs for each one and writes the collection-level dimension from the current attribute size. A later non-embeddings vector field with a different size can overwrite the restored collection metadata, so new inserts or collection operations validate against the wrong dimension.

Knowledge Base Used:

Prompt To Fix With AI 
This is a comment left during a code review.
Path: src/Migration/Destinations/Appwrite.php
Line: 1332

Comment:
**Vector Dimension Overwritten**

When a vectorsdb collection has more than one vector attribute, this sync runs for each one and writes the collection-level `dimension` from the current attribute size. A later non-`embeddings` vector field with a different size can overwrite the restored collection metadata, so new inserts or collection operations validate against the wrong dimension.

**Knowledge Base Used:**
- [Destinations](https://app.greptile.com/appwrite/-/custom-context/knowledge-base/utopia-php/migration/-/docs/destinations.md)
- [Resource Model Classes](https://app.greptile.com/appwrite/-/custom-context/knowledge-base/utopia-php/migration/-/docs/resources.md)

How can I resolve this? If you propose a fix, please make it concise.

Fix in Claude Code Fix in Codex

@abnegate abnegate Jul 22, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed: syncVectorDimension now only writes when the collection's dimension is still the placeholder (0), so it fills in the value for pre-dimension archives but never overwrites an archived dimension — a second vector column import is a no-op.

|| !isset($this->collectionStructures[Resource::TYPE_DATABASE_VECTORSDB])
) {
return;
}

if ((int) $table->getAttribute('dimension', 0) === $resource->getSize()) {
return;
}

$this->dbForProject->updateDocument(
$this->databaseCollectionId($database),
$table->getId(),
new UtopiaDocument(['dimension' => $resource->getSize()])
);
}

/**
* @throws Exception
* @throws \Throwable
Expand Down
33 changes: 31 additions & 2 deletions src/Migration/Resources/Database/Collection.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@

class Collection extends Table
{
protected ?int $dimension = null;

public static function getName(): string
{
return Resource::TYPE_COLLECTION;
Expand All @@ -25,7 +27,8 @@ public static function getName(): string
* permissions: ?array<string>,
* createdAt: string,
* updatedAt: string,
* enabled: bool
* enabled: bool,
* dimension?: ?int
* } $array
*/
public static function fromArray(array $array): self
Expand All @@ -36,7 +39,7 @@ public static function fromArray(array $array): self
default => Database::fromArray($array['database'])
};

return new self(
$collection = new self(
$database,
name: $array['name'],
id: $array['id'],
Expand All @@ -46,5 +49,31 @@ public static function fromArray(array $array): self
updatedAt: $array['updatedAt'] ?? '',
enabled: $array['enabled'] ?? true,
);

$collection->setDimension($array['dimension'] ?? null);

return $collection;
}

public function getDimension(): ?int
{
return $this->dimension;
}

public function setDimension(?int $dimension): self
{
$this->dimension = $dimension;

return $this;
}

/**
* @return array<string, mixed>
*/
public function jsonSerialize(): array
{
return array_merge(parent::jsonSerialize(), [
'dimension' => $this->dimension,

@greptile-apps greptile-apps Bot Jul 22, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Null Dimension Changes Archives

Collection::jsonSerialize() now writes dimension: null for every collection that does not set a dimension, including non-vectors collections and older archive data. Any strict archive reader or downstream schema that only accepts the previous collection shape can reject otherwise valid non-vectors collection exports because the field is present instead of omitted.

Knowledge Base Used:

Prompt To Fix With AI 
This is a comment left during a code review.
Path: src/Migration/Resources/Database/Collection.php
Line: 76

Comment:
**Null Dimension Changes Archives**

`Collection::jsonSerialize()` now writes `dimension: null` for every collection that does not set a dimension, including non-vectors collections and older archive data. Any strict archive reader or downstream schema that only accepts the previous collection shape can reject otherwise valid non-vectors collection exports because the field is present instead of omitted.

**Knowledge Base Used:**
- [Resource Model Classes](https://app.greptile.com/appwrite/-/custom-context/knowledge-base/utopia-php/migration/-/docs/resources.md)
- [Migration Sources](https://app.greptile.com/appwrite/-/custom-context/knowledge-base/utopia-php/migration/-/docs/sources.md)

How can I resolve this? If you propose a fix, please make it concise.

Fix in Claude Code Fix in Codex

@abnegate abnegate Jul 22, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed: jsonSerialize now omits dimension when unset, so documentsdb and other non-vector collection lines keep their previous shape.

]);
}
}
4 changes: 2 additions & 2 deletions src/Migration/Resources/Database/DocumentsDB.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ public static function getName(): string
* updatedAt: string,
* enabled: bool,
* originalId: string|null,
* database: string,
* database?: string,
* status: string|null
* } $array
*/
Expand All @@ -33,7 +33,7 @@ public static function fromArray(array $array): self
enabled: $array['enabled'] ?? true,
originalId: $array['originalId'] ?? '',
type: $array['type'] ?? Resource::TYPE_DATABASE_DOCUMENTSDB,
database: $array['database'],
database: $array['database'] ?? '',
databaseStatus: $array['status'] ?? null
);
}
Expand Down
4 changes: 2 additions & 2 deletions src/Migration/Resources/Database/VectorsDB.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ public static function getName(): string
* updatedAt: string,
* enabled: bool,
* originalId: string|null,
* database: string,
* database?: string,
* type: string,
* status: string|null
* } $array
Expand All @@ -34,7 +34,7 @@ public static function fromArray(array $array): self
enabled: $array['enabled'] ?? true,
originalId: $array['originalId'] ?? '',
type: $array['type'] ?? Resource::TYPE_DATABASE_VECTORSDB,
database: $array['database'],
database: $array['database'] ?? '',
databaseStatus: $array['status'] ?? null
);
}
Expand Down
1 change: 1 addition & 0 deletions src/Migration/Sources/Appwrite.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1230,6 +1230,7 @@ private function exportEntities(string $databaseName, int $batchSize): void
'createdAt' => $table['$createdAt'],
'updatedAt' => $table['$updatedAt'],
'enabled' => $table['enabled'] ?? true,
'dimension' => $table['dimension'] ?? null,
'database' => [
'id' => $database->getId(),
'name' => $database->getDatabaseName(),
Expand Down
58 changes: 58 additions & 0 deletions tests/Migration/Unit/Resources/CollectionTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
<?php

namespace Utopia\Tests\Unit\Resources;

use PHPUnit\Framework\TestCase;
use Utopia\Migration\Resource;
use Utopia\Migration\Resources\Database\Collection;
use Utopia\Migration\Resources\Database\VectorsDB;

class CollectionTest extends TestCase
{
private const DATABASE = [
'id' => 'vectors',
'name' => 'Vectors',
'type' => Resource::TYPE_DATABASE_VECTORSDB,
];

public function testDimensionRoundTrip(): void
{
$collection = Collection::fromArray([
'database' => self::DATABASE,
'id' => 'embeddings-store',
'name' => 'Embeddings Store',
'rowSecurity' => true,
'permissions' => [],
'createdAt' => '2026-01-01T00:00:00.000+00:00',
'updatedAt' => '2026-01-01T00:00:00.000+00:00',
'enabled' => true,
'dimension' => 1536,
]);

$this->assertInstanceOf(VectorsDB::class, $collection->getDatabase());
$this->assertSame(1536, $collection->getDimension());

$serialized = $collection->jsonSerialize();
$this->assertSame(1536, $serialized['dimension']);

$rehydrated = Collection::fromArray(\json_decode(\json_encode($collection), true));
$this->assertSame(1536, $rehydrated->getDimension());
}

public function testDimensionDefaultsToNull(): void
{
$collection = Collection::fromArray([
'database' => self::DATABASE,
'id' => 'embeddings-store',
'name' => 'Embeddings Store',
'rowSecurity' => true,
'permissions' => [],
'createdAt' => '',
'updatedAt' => '',
'enabled' => true,
]);

$this->assertNull($collection->getDimension());
$this->assertNull($collection->jsonSerialize()['dimension']);
}
}
Loading