fix: Metadata record optimization

package.json

@@ -22,6 +22,7 @@
     "tool:cleanupSessions": "tsx tools/cleanupSessions.ts",
     "tool:pruneWebhookLogs": "tsx tools/pruneWebhookLogs.ts",
     "tool:seed-mirror": "tsx tools/seedMirror.ts",
+    "tool:verifyRecordSharing": "tsx tools/verifyRecordSharing.ts",
     "cli": "tsx src/cli/cli.ts",
     "test": "vitest run",
     "test:watch": "vitest",

src/api/agent.ts

@@ -7,6 +7,7 @@ import {
   getLatestReadyVersion,
   hasOrgAccess,
   loadVersionSchemas,
+  recordsVersionId,
 } from '../lib/version-helpers.server.js'
 
 const DEFAULT_SCHEMA_SLUG = 'update'
@@ -119,7 +120,7 @@ export async function agentPage(c: Context) {
         schema.recordObjects,
         eq(schema.versionRecords.recordHash, schema.recordObjects.hash),
       )
-      .where(eq(schema.versionRecords.versionId, latest.id))
+      .where(eq(schema.versionRecords.versionId, recordsVersionId(latest)))
       .limit(3)
     examples = rows.map((r) => ({ id: r.recordId, type: r.type, data: r.data }))
   }

src/api/ark.ts

@@ -15,6 +15,7 @@ import {
   filterTypeSchema,
   getPrivateFields,
   parseSemver,
+  recordsVersionId,
 } from '../lib/version-helpers.server.js'
 import { type AuthEnv } from './auth.server.js'
 
@@ -96,6 +97,7 @@ export async function resolve(c: Context<AuthEnv>) {
     appId: string | null
     actorId: string | null
     createdAt: Date
+    recordsFromVersionId: number | null
   } | null = null
 
   if (version !== undefined) {
@@ -110,6 +112,7 @@ export async function resolve(c: Context<AuthEnv>) {
         appId: schema.versions.appId,
         actorId: schema.versions.actorId,
         createdAt: schema.versions.createdAt,
+        recordsFromVersionId: schema.versions.recordsFromVersionId,
       })
       .from(schema.versions)
       .where(
@@ -133,6 +136,7 @@ export async function resolve(c: Context<AuthEnv>) {
         appId: schema.versions.appId,
         actorId: schema.versions.actorId,
         createdAt: schema.versions.createdAt,
+        recordsFromVersionId: schema.versions.recordsFromVersionId,
       })
       .from(schema.versions)
       .where(
@@ -183,7 +187,7 @@ export async function resolve(c: Context<AuthEnv>) {
         // were denormalized this scanned every version_records row for the
         // version just to resolve one ARK.
         and(
-          eq(schema.versionRecords.versionId, versionRow.id),
+          eq(schema.versionRecords.versionId, recordsVersionId(versionRow)),
           eq(schema.versionRecords.recordId, recordId),
           eq(schema.versionRecords.type, recordType),
         ),

src/api/collections.ts

@@ -19,6 +19,7 @@ import {
   getPrivateTypes,
   hasOrgAccess,
   loadVersionSchemas,
+  recordsVersionId,
 } from '../lib/version-helpers.server.js'
 import { type AuthEnv } from './auth.server.js'
 import { fullPrincipalUserId, requireAuth, requireUnscopedKey } from './auth.server.js'
@@ -410,7 +411,7 @@ const app = new Hono<AuthEnv>()
             count: sql<number>`count(*)::int`,
           })
           .from(schema.versionRecords)
-          .where(eq(schema.versionRecords.versionId, latestVersion.id))
+          .where(eq(schema.versionRecords.versionId, recordsVersionId(latestVersion)))
           .groupBy(schema.versionRecords.type)
         typeCounts = rows.map((r) => ({ type: r.type, count: r.count }))
       }
@@ -946,7 +947,7 @@ const app = new Hono<AuthEnv>()
       const types = await db
         .selectDistinct({ type: schema.versionRecords.type })
         .from(schema.versionRecords)
-        .where(eq(schema.versionRecords.versionId, version.id))
+        .where(eq(schema.versionRecords.versionId, recordsVersionId(version)))
 
       // tar needs each entry's byte length in its header, so an entry cannot be
       // written from an unbounded stream. Previously every record of a type was
@@ -1006,7 +1007,7 @@ const app = new Hono<AuthEnv>()
           // Walk the (version_id, type, record_id) index; record_objects is
           // joined only to pick up the body for the rows on this page.
           const conditions = [
-            eq(schema.versionRecords.versionId, version.id),
+            eq(schema.versionRecords.versionId, recordsVersionId(version)),
             eq(schema.versionRecords.type, type),
           ]
           // Non-owners never see records flagged private (per-version flag).
@@ -1212,7 +1213,7 @@ const app = new Hono<AuthEnv>()
           .from(schema.versionRecords)
           .where(
             and(
-              eq(schema.versionRecords.versionId, latestVersion.id),
+              eq(schema.versionRecords.versionId, recordsVersionId(latestVersion)),
               eq(schema.versionRecords.private, true),
             ),
           )
@@ -1221,7 +1222,7 @@ const app = new Hono<AuthEnv>()
           .from(schema.versionRecords)
           .where(
             and(
-              eq(schema.versionRecords.versionId, latestVersion.id),
+              eq(schema.versionRecords.versionId, recordsVersionId(latestVersion)),
               sql`${schema.versionRecords.publicRecordHash} IS NOT NULL`,
             ),
           )
@@ -1277,11 +1278,15 @@ const app = new Hono<AuthEnv>()
 
         // Copy the record set server-side. A fork of a multi-million-record
         // collection has no reason to round-trip every row through the app.
+        //
+        // A real copy, not a shared pointer: the fork is an independent
+        // collection, and sharing across collections would make the source's
+        // rows undeletable for as long as any fork existed.
         await tx.execute(sql`
           INSERT INTO version_records (version_id, record_hash, public_record_hash, record_id, type, private)
           SELECT ${newVersion!.id}, record_hash, public_record_hash, record_id, type, private
           FROM version_records
-          WHERE version_id = ${latestVersion.id}
+          WHERE version_id = ${recordsVersionId(latestVersion)}
         `)
 
         const sourceFiles = await tx

src/api/files.ts

@@ -91,6 +91,11 @@ async function isFilePubliclyAccessible(
       data: schema.recordObjects.data,
     })
     .from(schema.versionRecords)
+    // Ownership-only join: this asks whether the file is referenced anywhere in
+    // THIS collection, and a version sharing another's rows is in the same
+    // collection as the version that owns them. The `versionId` it selects is
+    // used to load that version's schemas for privacy, and a metadata patch has
+    // the same schema set as its base, so the filtering is unchanged too.
     .innerJoin(schema.versions, eq(schema.versionRecords.versionId, schema.versions.id))
     .innerJoin(
       schema.recordObjects,

src/api/negotiate.ts

@@ -10,6 +10,7 @@ import {
   canonicalize,
   checkSchemaBounds,
   deriveSemver,
+  describeError,
   filterRecordData,
   filterTypeSchema,
   findExtraFields,
@@ -21,6 +22,7 @@ import {
   hasOrgAccess,
   loadVersionSchemas,
   parseSemver,
+  recordsVersionId,
   resolveCollection,
   type SchemaEntry,
   stripToSchema,
@@ -132,37 +134,6 @@ async function ingestManifestEntries(
   return deduped.filter((r) => !existingRecordSet.has(r.hash)).map((r) => r.hash)
 }
 
-/**
- * Flatten an error and its `cause` chain into one string.
- *
- * Drizzle puts the SQL in `message` and the underlying Postgres error — the part
- * that says *why* — in `cause`. Recording only `message` yields "Failed query:
- * SELECT …" with no reason, which is unactionable for whoever is reading a
- * failed session hours later.
- */
-function describeError(err: unknown): string {
-  const parts: string[] = []
-  let current: unknown = err
-  for (let depth = 0; current && depth < 5; depth++) {
-    if (current instanceof Error) {
-      parts.push(current.message)
-      // Postgres errors carry the useful specifics outside `message`.
-      const pg = current as { code?: string; detail?: string; hint?: string }
-      const extra = [
-        pg.code && `code ${pg.code}`,
-        pg.detail && `detail: ${pg.detail}`,
-        pg.hint && `hint: ${pg.hint}`,
-      ].filter(Boolean)
-      if (extra.length > 0) parts.push(extra.join(', '))
-      current = current.cause
-    } else {
-      parts.push(String(current))
-      break
-    }
-  }
-  return parts.join(' | ')
-}
-
 /** Mirrors `c.json(body, status)` so the finalize body reads unchanged. */
 const reply = (body: unknown, status: ContentfulStatusCode = 200) => ({ status, body })
 
@@ -1041,7 +1012,7 @@ app.post(
         ? sql`NOT m.submitted`
         : sql`NOT m.submitted AND NOT EXISTS (
               SELECT 1 FROM version_records vr
-              WHERE vr.version_id = ${latest.id} AND vr.record_hash = m.hash
+              WHERE vr.version_id = ${recordsVersionId(latest)} AND vr.record_hash = m.hash
             )`
 
       const WALK_BATCH = 5000
@@ -1294,13 +1265,13 @@ app.post(
         SELECT
           (SELECT count(DISTINCT coalesce(final_hash, hash)) FROM negotiate_session_manifest
              WHERE session_id = ${sessionId}) AS new_count,
-          (SELECT count(*) FROM version_records WHERE version_id = ${latest.id}) AS old_count,
+          (SELECT count(*) FROM version_records WHERE version_id = ${recordsVersionId(latest)}) AS old_count,
           EXISTS (
             SELECT 1 FROM negotiate_session_manifest m
             WHERE m.session_id = ${sessionId}
               AND NOT EXISTS (
                 SELECT 1 FROM version_records vr
-                WHERE vr.version_id = ${latest.id}
+                WHERE vr.version_id = ${recordsVersionId(latest)}
                   AND vr.record_hash = coalesce(m.final_hash, m.hash)
               )
           ) AS has_new

src/api/query.ts

@@ -10,6 +10,7 @@ import {
   getPrivateTypes,
   hasOrgAccess,
   parseSemver,
+  recordsVersionId,
   type SchemaEntry,
 } from '../lib/version-helpers.server.js'
 import { type AuthEnv, fullPrincipalUserId } from './auth.server.js'
@@ -108,6 +109,7 @@ async function getOrBuildSqlite(
       id: schema.versions.id,
       semver: schema.versions.semver,
       recordCount: schema.versions.recordCount,
+      recordsFromVersionId: schema.versions.recordsFromVersionId,
     })
     .from(schema.versions)
     .where(
@@ -163,7 +165,7 @@ async function getOrBuildSqlite(
 
   // Load records (excluding private types and private records for non-owners).
   // Record-level privacy is the per-version version_records.private flag.
-  const recordConditions = [eq(schema.versionRecords.versionId, version.id)]
+  const recordConditions = [eq(schema.versionRecords.versionId, recordsVersionId(version))]
   if (!ownerAccess) {
     recordConditions.push(eq(schema.versionRecords.private, false))
     for (const pt of privateTypes) {

src/api/records.ts

@@ -1,4 +1,4 @@
-import { and, eq, inArray, sql } from 'drizzle-orm'
+import { and, eq, inArray, or, sql } from 'drizzle-orm'
 import { Hono } from 'hono'
 import { openApi } from 'hono-zod-openapi'
 import { streamText } from 'hono/streaming'
@@ -36,6 +36,9 @@ async function resolvePublicHashes(hashes: string[]): Promise<Map<string, Public
       schema.recordObjects,
       eq(schema.versionRecords.recordHash, schema.recordObjects.hash),
     )
+    // Keyed off the version that owns the row. A metadata patch copies its base's
+    // schema set verbatim, so the binding this resolves is the same one either
+    // way — the digests would not match otherwise.
     .innerJoin(
       schema.versionSchemas,
       and(
@@ -79,6 +82,10 @@ async function resolveRecordAccess(
     const memberRows = await db
       .select({ hash: schema.versionRecords.recordHash })
       .from(schema.versionRecords)
+      // Ownership-only join, deliberately: access is decided by the COLLECTION,
+      // and the version that owns a row is always in the same collection as any
+      // version sharing it. Resolving through the pointer here would widen the
+      // join for no change in answer.
       .innerJoin(schema.versions, eq(schema.versionRecords.versionId, schema.versions.id))
       .innerJoin(schema.collections, eq(schema.versions.collectionId, schema.collections.id))
       .innerJoin(
@@ -106,6 +113,7 @@ async function resolveRecordAccess(
       schema.recordObjects,
       eq(schema.versionRecords.recordHash, schema.recordObjects.hash),
     )
+    // Ownership-only, as above: same collection, same answer.
     .innerJoin(schema.versions, eq(schema.versionRecords.versionId, schema.versions.id))
     .innerJoin(
       schema.collections,
@@ -205,7 +213,17 @@ const app = new Hono<AuthEnv>()
           versionCreatedAt: schema.versions.createdAt,
         })
         .from(schema.versionRecords)
-        .innerJoin(schema.versions, eq(schema.versionRecords.versionId, schema.versions.id))
+        // A version contains this record if it owns the row OR shares the rows of
+        // the version that does. Matching only on ownership would drop every
+        // metadata-only patch version from the provenance list, even though the
+        // record is just as much a part of it.
+        .innerJoin(
+          schema.versions,
+          or(
+            eq(schema.versionRecords.versionId, schema.versions.id),
+            eq(schema.versionRecords.versionId, schema.versions.recordsFromVersionId),
+          ),
+        )
         .innerJoin(schema.collections, eq(schema.versions.collectionId, schema.collections.id))
         .innerJoin(
           schema.organization,