From 98cf80d5a15105933c2f0a68976580db30fb214a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 22 May 2026 18:54:39 +0000 Subject: [PATCH 01/11] feat: project root sql fields Agent-Logs-Url: https://github.com/effect-app/libs/sessions/f324ecdc-fbb7-4471-9947-241bf85170ff Co-authored-by: patroza <42661+patroza@users.noreply.github.com> --- .../src/Model/Repository/internal/internal.ts | 4 + packages/infra/src/Store/SQL.ts | 187 ++++++++++++++---- packages/infra/src/Store/SQL/Pg.ts | 114 +++++++++-- packages/infra/src/Store/SQL/query.ts | 88 ++++++++- packages/infra/src/Store/rootLevelFields.ts | 74 +++++++ packages/infra/src/Store/service.ts | 4 + .../infra/test/sql-root-level-columns.test.ts | 133 +++++++++++++ packages/infra/test/sql-store.test.ts | 90 +++++++++ 8 files changed, 640 insertions(+), 54 deletions(-) create mode 100644 packages/infra/src/Store/rootLevelFields.ts create mode 100644 packages/infra/test/sql-root-level-columns.test.ts diff --git a/packages/infra/src/Model/Repository/internal/internal.ts b/packages/infra/src/Model/Repository/internal/internal.ts index 6ba7a9345..5eed29628 100644 --- a/packages/infra/src/Model/Repository/internal/internal.ts +++ b/packages/infra/src/Model/Repository/internal/internal.ts @@ -21,6 +21,7 @@ import * as Unify from "effect/Unify" import { setupRequestContextFromCurrent } from "../../../api/setupRequest.js" import { type FilterArgs, type PersistenceModelType, type StoreConfig, StoreMaker } from "../../../Store.js" import { getContextMap } from "../../../Store/ContextMapContainer.js" +import { makeRootLevelFieldColumns } from "../../../Store/rootLevelFields.js" import type { FieldValues } from "../../filter/types.js" import * as Q from "../../query.js" import type { Repository } from "../service.js" @@ -595,6 +596,8 @@ export function makeStore() { mapTo: (e: E, etag: string | undefined) => Encoded, idKey: IdKey ) => { + const rootLevelFieldColumns = makeRootLevelFieldColumns(schema, idKey) + function makeStore( makeInitial?: Effect.Effect, config?: Omit, "partitionValue"> & { @@ -634,6 +637,7 @@ export function makeStore() { : undefined, { ...config, + rootLevelFieldColumns, partitionValue: config?.partitionValue ?? ((_) => "primary") /*(isIntegrationEvent(r) ? r.companyId : r.id*/ } diff --git a/packages/infra/src/Store/SQL.ts b/packages/infra/src/Store/SQL.ts index 4cded7139..5412a01a3 100644 --- a/packages/infra/src/Store/SQL.ts +++ b/packages/infra/src/Store/SQL.ts @@ -15,8 +15,18 @@ import type { FieldValues } from "../Model/filter/types.js" import type { ComputedProjectionIrExpression } from "../Model/query.js" import { annotateDb, type DbSystem } from "../otel.js" import { storeId } from "./Memory.js" +import type { RootLevelFieldColumn } from "./rootLevelFields.js" import { type FilterArgs, type PersistenceModelType, type StorageConfig, type Store, type StoreConfig, StoreMaker } from "./service.js" -import { buildWhereSQLQuery, logQuery, type SQLDialect, sqliteDialect } from "./SQL/query.js" +import { + buildWhereSQLQuery, + logQuery, + normalizeProjectedColumnValue, + projectedColumnBackfillExpr, + projectedColumnSqlType, + quoteIdentifier, + type SQLDialect, + sqliteDialect +} from "./SQL/query.js" import { makeETag } from "./utils.js" export type WithNsTransactionFn = (effect: Effect.Effect) => Effect.Effect @@ -27,23 +37,41 @@ export class WithNsTransaction /** @internal */ export const parseRow = ( - row: { id: string; _etag: string | null; data: string }, + row: { id: string; _etag: string | null; data: string } & Record, idKey: PropertyKey, - defaultValues: Partial + defaultValues: Partial, + rootLevelFieldColumns: readonly RootLevelFieldColumn[] = [] ): PersistenceModelType => { const data = (typeof row.data === "string" ? JSON.parse(row.data) : row.data) as object - return { ...defaultValues, ...data, [idKey]: row.id, _etag: row._etag ?? undefined } as PersistenceModelType + const projectedFields = rootLevelFieldColumns.reduce>((acc, column) => { + const value = row[column.columnName] + if (value !== null && value !== undefined) { + acc[column.key] = normalizeProjectedColumnValue(column, value) + } + return acc + }, {}) + return { + ...defaultValues, + ...data, + ...projectedFields, + [idKey]: row.id, + _etag: row._etag ?? undefined + } as PersistenceModelType } const parseSelectRow = ( row: Record, - idKey: PropertyKey + idKey: PropertyKey, + rootLevelFieldColumns: readonly RootLevelFieldColumn[] = [] ): any => { const result: Record = {} + const projectedFields = new Map(rootLevelFieldColumns.map((column) => [column.key, column] as const)) for (const [key, value] of Object.entries(row)) { if (key === "id") { result[idKey as string] = value result["id"] = value + } else if (projectedFields.has(key)) { + result[key] = normalizeProjectedColumnValue(projectedFields.get(key)!, value) } else if (typeof value === "string") { try { result[key] = JSON.parse(value) @@ -57,8 +85,16 @@ const parseSelectRow = ( return result } +const serializeProjectedValue = ( + system: DbSystem, + column: RootLevelFieldColumn, + value: unknown +) => column.kind === "boolean" && system === "sqlite" && value !== null && value !== undefined + ? value === true ? 1 : 0 + : value + function makeSQLStoreInt(system: DbSystem, dialect: SQLDialect, jsonColumnType: string) { - return Effect.fnUntraced(function*({ prefix }: StorageConfig) { + return Effect.fnUntraced(function*({ prefix, rootLevelFieldsWhenAvailable: rootLevelFieldsWhenAvailableDefault }: StorageConfig) { const sql = yield* SqlClient.SqlClient return { make: Effect.fnUntraced(function*( @@ -70,6 +106,12 @@ function makeSQLStoreInt(system: DbSystem, dialect: SQLDialect, jsonColumnType: type PM = PersistenceModelType const tableName = `${prefix}${name}` const defaultValues = config?.defaultValues ?? {} + const rootLevelFieldColumns = config?.rootLevelFieldColumns ?? [] + const useRootLevelFields = config?.rootLevelFieldsWhenAvailable ?? rootLevelFieldsWhenAvailableDefault ?? false + const activeRootLevelFieldColumns = useRootLevelFields ? rootLevelFieldColumns : [] + const selectColumnsSql = activeRootLevelFieldColumns.length > 0 + ? `, ${activeRootLevelFieldColumns.map((column) => quoteIdentifier(column.columnName)).join(", ")}` + : "" const resolveNamespace = !config?.allowNamespace ? Effect.succeed("primary") @@ -80,6 +122,26 @@ function makeSQLStoreInt(system: DbSystem, dialect: SQLDialect, jsonColumnType: return namespace })) + const exec = (query: string, params?: readonly unknown[]) => sql.unsafe(query, params as any).pipe(Effect.orDie) + + const ensureRootLevelFieldColumns = Effect.fnUntraced(function*() { + if (activeRootLevelFieldColumns.length === 0) { + return + } + const existing = yield* exec(`PRAGMA table_info(${JSON.stringify(tableName)})`) as Array<{ name: string }> + const existingColumns = new Set(existing.map((column) => column.name)) + for (const column of activeRootLevelFieldColumns) { + if (!existingColumns.has(column.columnName)) { + yield* exec( + `ALTER TABLE "${tableName}" ADD COLUMN ${quoteIdentifier(column.columnName)} ${projectedColumnSqlType(dialect, column.kind)}` + ) + } + yield* exec( + `UPDATE "${tableName}" SET ${quoteIdentifier(column.columnName)} = ${projectedColumnBackfillExpr(dialect, column)} WHERE ${quoteIdentifier(column.columnName)} IS NULL` + ) + } + }) + const ensureTable = sql .unsafe( `CREATE TABLE IF NOT EXISTS "${tableName}" (id TEXT NOT NULL, _namespace TEXT NOT NULL DEFAULT 'primary', _etag TEXT, data ${jsonColumnType} NOT NULL, PRIMARY KEY (id, _namespace))` @@ -90,6 +152,7 @@ function makeSQLStoreInt(system: DbSystem, dialect: SQLDialect, jsonColumnType: `CREATE TABLE IF NOT EXISTS "_migrations" (id TEXT NOT NULL, version TEXT NOT NULL, PRIMARY KEY (id, version))` ) ), + Effect.andThen(ensureRootLevelFieldColumns()), Effect.orDie, Effect.asVoid ) @@ -99,17 +162,21 @@ function makeSQLStoreInt(system: DbSystem, dialect: SQLDialect, jsonColumnType: const id = newE[idKey] as string const { _etag, [idKey]: _id, ...rest } = newE as any const data = JSON.stringify(rest) - return { id, _etag: newE._etag!, data, item: newE } + const rootLevelFieldValues = activeRootLevelFieldColumns.map((column) => + serializeProjectedValue(system, column, rest[column.key]) + ) + return { id, _etag: newE._etag!, data, item: newE, rootLevelFieldValues } } - const exec = (query: string, params?: readonly unknown[]) => sql.unsafe(query, params as any).pipe(Effect.orDie) - const setInternal = Effect.fnUntraced(function*(e: PM, ns: string) { const row = toRow(e) if (e._etag) { + const projectedSetSql = activeRootLevelFieldColumns.length > 0 + ? `, ${activeRootLevelFieldColumns.map((column) => `${quoteIdentifier(column.columnName)} = ?`).join(", ")}` + : "" yield* exec( - `UPDATE "${tableName}" SET _etag = ?, data = ? WHERE id = ? AND _etag = ? AND _namespace = ?`, - [row._etag, row.data, row.id, e._etag, ns] + `UPDATE "${tableName}" SET _etag = ?, data = ?${projectedSetSql} WHERE id = ? AND _etag = ? AND _namespace = ?`, + [row._etag, row.data, ...row.rootLevelFieldValues, row.id, e._etag, ns] ) const existing = yield* exec( `SELECT _etag FROM "${tableName}" WHERE id = ? AND _namespace = ?`, @@ -135,9 +202,15 @@ function makeSQLStoreInt(system: DbSystem, dialect: SQLDialect, jsonColumnType: }) } } else { + const projectedColumnsSql = activeRootLevelFieldColumns.length > 0 + ? `, ${activeRootLevelFieldColumns.map((column) => quoteIdentifier(column.columnName)).join(", ")}` + : "" + const projectedPlaceholders = activeRootLevelFieldColumns.map(() => "?").join(", ") yield* exec( - `INSERT INTO "${tableName}" (id, _namespace, _etag, data) VALUES (?, ?, ?, ?)`, - [row.id, ns, row._etag, row.data] + `INSERT INTO "${tableName}" (id, _namespace, _etag, data${projectedColumnsSql}) VALUES (?, ?, ?, ?${ + projectedPlaceholders.length > 0 ? `, ${projectedPlaceholders}` : "" + })`, + [row.id, ns, row._etag, row.data, ...row.rootLevelFieldValues] ) } return row.item @@ -183,10 +256,12 @@ function makeSQLStoreInt(system: DbSystem, dialect: SQLDialect, jsonColumnType: all: resolveNamespace.pipe( Effect.flatMap((ns) => { - const sqlText = `SELECT id, _etag, data FROM "${tableName}" WHERE _namespace = ?` + const sqlText = `SELECT id, _etag, data${selectColumnsSql} FROM "${tableName}" WHERE _namespace = ?` return exec(sqlText, [ns]) .pipe( - Effect.map((rows) => (rows as any[]).map((r) => parseRow(r, idKey, defaultValues))), + Effect.map((rows) => + (rows as any[]).map((r) => parseRow(r, idKey, defaultValues, activeRootLevelFieldColumns)) + ), annotateDb({ operation: "all", system, @@ -202,13 +277,13 @@ function makeSQLStoreInt(system: DbSystem, dialect: SQLDialect, jsonColumnType: find: (id) => resolveNamespace.pipe( Effect.flatMap((ns) => { - const sqlText = `SELECT id, _etag, data FROM "${tableName}" WHERE id = ? AND _namespace = ?` + const sqlText = `SELECT id, _etag, data${selectColumnsSql} FROM "${tableName}" WHERE id = ? AND _namespace = ?` return exec(sqlText, [id, ns]) .pipe( Effect.map((rows) => { const row = (rows as any[])[0] return row - ? Option.some(parseRow(row, idKey, defaultValues)) + ? Option.some(parseRow(row, idKey, defaultValues, activeRootLevelFieldColumns)) : Option.none() }), annotateDb({ @@ -258,7 +333,8 @@ function makeSQLStoreInt(system: DbSystem, dialect: SQLDialect, jsonColumnType: .skip, f .limit, - ns + ns, + activeRootLevelFieldColumns ) }) .pipe( @@ -270,7 +346,7 @@ function makeSQLStoreInt(system: DbSystem, dialect: SQLDialect, jsonColumnType: Effect.map((rows) => { if (f.select) { return (rows as any[]).map((r) => { - const selected = parseSelectRow(r, idKey) + const selected = parseSelectRow(r, idKey, activeRootLevelFieldColumns) return { ...Struct.pick( defaultValues as any, @@ -280,7 +356,9 @@ function makeSQLStoreInt(system: DbSystem, dialect: SQLDialect, jsonColumnType: } as M }) } - return (rows as any[]).map((r) => parseRow(r, idKey, defaultValues) as any as M) + return (rows as any[]).map((r) => + parseRow(r, idKey, defaultValues, activeRootLevelFieldColumns) as any as M + ) }) ) ), @@ -382,7 +460,7 @@ type WithNsSqlFn = ( function makeSQLiteStorePerNs( withNsSql: WithNsSqlFn, - { prefix }: StorageConfig + { prefix, rootLevelFieldsWhenAvailable: rootLevelFieldsWhenAvailableDefault }: StorageConfig ) { return { make: Effect.fnUntraced(function*( @@ -394,6 +472,12 @@ function makeSQLiteStorePerNs( type PM = PersistenceModelType const tableName = `${prefix}${name}` const defaultValues = config?.defaultValues ?? {} + const rootLevelFieldColumns = config?.rootLevelFieldColumns ?? [] + const useRootLevelFields = config?.rootLevelFieldsWhenAvailable ?? rootLevelFieldsWhenAvailableDefault ?? false + const activeRootLevelFieldColumns = useRootLevelFields ? rootLevelFieldColumns : [] + const selectColumnsSql = activeRootLevelFieldColumns.length > 0 + ? `, ${activeRootLevelFieldColumns.map((column) => quoteIdentifier(column.columnName)).join(", ")}` + : "" const resolveNamespace = !config?.allowNamespace ? Effect.succeed("primary") @@ -409,7 +493,10 @@ function makeSQLiteStorePerNs( const id = newE[idKey] as string const { _etag, [idKey]: _id, ...rest } = newE as any const data = JSON.stringify(rest) - return { id, _etag: newE._etag!, data, item: newE } + const rootLevelFieldValues = activeRootLevelFieldColumns.map((column) => + serializeProjectedValue("sqlite", column, rest[column.key]) + ) + return { id, _etag: newE._etag!, data, item: newE, rootLevelFieldValues } } const exec = (ns: string, query: string, params?: readonly unknown[]) => @@ -427,6 +514,25 @@ function makeSQLiteStorePerNs( `CREATE TABLE IF NOT EXISTS "_migrations" (id TEXT NOT NULL, version TEXT NOT NULL, PRIMARY KEY (id, version))` ) ), + Effect.andThen(Effect.gen(function*() { + if (activeRootLevelFieldColumns.length === 0) { + return + } + const existing = yield* exec(ns, `PRAGMA table_info(${JSON.stringify(tableName)})`) as Array<{ name: string }> + const existingColumns = new Set(existing.map((column) => column.name)) + for (const column of activeRootLevelFieldColumns) { + if (!existingColumns.has(column.columnName)) { + yield* exec( + ns, + `ALTER TABLE "${tableName}" ADD COLUMN ${quoteIdentifier(column.columnName)} ${projectedColumnSqlType(sqliteDialect, column.kind)}` + ) + } + yield* exec( + ns, + `UPDATE "${tableName}" SET ${quoteIdentifier(column.columnName)} = ${projectedColumnBackfillExpr(sqliteDialect, column)} WHERE ${quoteIdentifier(column.columnName)} IS NULL` + ) + } + })), Effect.orDie, Effect.asVoid )) @@ -434,10 +540,13 @@ function makeSQLiteStorePerNs( const setInternal = Effect.fnUntraced(function*(e: PM, ns: string) { const row = toRow(e) if (e._etag) { + const projectedSetSql = activeRootLevelFieldColumns.length > 0 + ? `, ${activeRootLevelFieldColumns.map((column) => `${quoteIdentifier(column.columnName)} = ?`).join(", ")}` + : "" yield* exec( ns, - `UPDATE "${tableName}" SET _etag = ?, data = ? WHERE id = ? AND _etag = ?`, - [row._etag, row.data, row.id, e._etag] + `UPDATE "${tableName}" SET _etag = ?, data = ?${projectedSetSql} WHERE id = ? AND _etag = ?`, + [row._etag, row.data, ...row.rootLevelFieldValues, row.id, e._etag] ) const existing = yield* exec( ns, @@ -464,10 +573,16 @@ function makeSQLiteStorePerNs( }) } } else { + const projectedColumnsSql = activeRootLevelFieldColumns.length > 0 + ? `, ${activeRootLevelFieldColumns.map((column) => quoteIdentifier(column.columnName)).join(", ")}` + : "" + const projectedPlaceholders = activeRootLevelFieldColumns.map(() => "?").join(", ") yield* exec( ns, - `INSERT INTO "${tableName}" (id, _etag, data) VALUES (?, ?, ?)`, - [row.id, row._etag, row.data] + `INSERT INTO "${tableName}" (id, _etag, data${projectedColumnsSql}) VALUES (?, ?, ?${ + projectedPlaceholders.length > 0 ? `, ${projectedPlaceholders}` : "" + })`, + [row.id, row._etag, row.data, ...row.rootLevelFieldValues] ) } return row.item @@ -516,10 +631,12 @@ function makeSQLiteStorePerNs( seedNamespace: (ns) => seedNamespace(ns), all: resolveNamespace.pipe(Effect.flatMap((ns) => { - const sqlText = `SELECT id, _etag, data FROM "${tableName}"` + const sqlText = `SELECT id, _etag, data${selectColumnsSql} FROM "${tableName}"` return exec(ns, sqlText) .pipe( - Effect.map((rows) => (rows as any[]).map((r) => parseRow(r, idKey, defaultValues))), + Effect.map((rows) => + (rows as any[]).map((r) => parseRow(r, idKey, defaultValues, activeRootLevelFieldColumns)) + ), annotateDb({ operation: "all", system: "sqlite", @@ -534,13 +651,13 @@ function makeSQLiteStorePerNs( find: (id) => resolveNamespace.pipe( Effect.flatMap((ns) => { - const sqlText = `SELECT id, _etag, data FROM "${tableName}" WHERE id = ?` + const sqlText = `SELECT id, _etag, data${selectColumnsSql} FROM "${tableName}" WHERE id = ?` return exec(ns, sqlText, [id]) .pipe( Effect.map((rows) => { const row = (rows as any[])[0] return row - ? Option.some(parseRow(row, idKey, defaultValues)) + ? Option.some(parseRow(row, idKey, defaultValues, activeRootLevelFieldColumns)) : Option.none() }), annotateDb({ @@ -589,7 +706,9 @@ function makeSQLiteStorePerNs( f .skip, f - .limit + .limit, + undefined, + activeRootLevelFieldColumns ) ) .pipe( @@ -601,7 +720,7 @@ function makeSQLiteStorePerNs( Effect.map((rows) => { if (f.select) { return (rows as any[]).map((r) => { - const selected = parseSelectRow(r, idKey) + const selected = parseSelectRow(r, idKey, activeRootLevelFieldColumns) return { ...Struct.pick( defaultValues as any, @@ -611,7 +730,9 @@ function makeSQLiteStorePerNs( } as M }) } - return (rows as any[]).map((r) => parseRow(r, idKey, defaultValues) as any as M) + return (rows as any[]).map((r) => + parseRow(r, idKey, defaultValues, activeRootLevelFieldColumns) as any as M + ) }) ) ), diff --git a/packages/infra/src/Store/SQL/Pg.ts b/packages/infra/src/Store/SQL/Pg.ts index 2313ae20c..88d8fcce2 100644 --- a/packages/infra/src/Store/SQL/Pg.ts +++ b/packages/infra/src/Store/SQL/Pg.ts @@ -12,29 +12,56 @@ import type { FieldValues } from "../../Model/filter/types.js" import type { ComputedProjectionIrExpression } from "../../Model/query.js" import { annotateDb } from "../../otel.js" import { storeId } from "../Memory.js" +import type { RootLevelFieldColumn } from "../rootLevelFields.js" import { type FilterArgs, type PersistenceModelType, type StorageConfig, type Store, type StoreConfig, StoreMaker } from "../service.js" import { makeETag } from "../utils.js" -import { buildWhereSQLQuery, logQuery, pgDialect } from "./query.js" +import { + buildWhereSQLQuery, + logQuery, + normalizeProjectedColumnValue, + projectedColumnBackfillExpr, + projectedColumnSqlType, + quoteIdentifier, + pgDialect +} from "./query.js" const parseRow = ( - row: { id: string; _etag: string | null; data: unknown }, + row: { id: string; _etag: string | null; data: unknown } & Record, idKey: PropertyKey, - defaultValues: Partial + defaultValues: Partial, + rootLevelFieldColumns: readonly RootLevelFieldColumn[] = [] ): PersistenceModelType => { const data = (typeof row.data === "string" ? JSON.parse(row.data) : row.data) as object - return { ...defaultValues, ...data, [idKey]: row.id, _etag: row._etag ?? undefined } as PersistenceModelType + const projectedFields = rootLevelFieldColumns.reduce>((acc, column) => { + const value = row[column.columnName] + if (value !== null && value !== undefined) { + acc[column.key] = normalizeProjectedColumnValue(column, value) + } + return acc + }, {}) + return { + ...defaultValues, + ...data, + ...projectedFields, + [idKey]: row.id, + _etag: row._etag ?? undefined + } as PersistenceModelType } const parseSelectRow = ( row: Record, idKey: PropertyKey, - defaultValues: Record + defaultValues: Record, + rootLevelFieldColumns: readonly RootLevelFieldColumn[] = [] ): any => { const result: Record = { ...defaultValues } + const projectedFields = new Map(rootLevelFieldColumns.map((column) => [column.key, column] as const)) for (const [key, value] of Object.entries(row)) { if (key === "id") { result[idKey as string] = value result["id"] = value + } else if (projectedFields.has(key)) { + result[key] = normalizeProjectedColumnValue(projectedFields.get(key)!, value) } else { result[key] = value } @@ -42,7 +69,7 @@ const parseSelectRow = ( return result } -const makePgStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { +const makePgStore = Effect.fnUntraced(function*({ prefix, rootLevelFieldsWhenAvailable: rootLevelFieldsWhenAvailableDefault }: StorageConfig) { const sql = yield* SqlClient.SqlClient return { make: Effect.fnUntraced(function*( @@ -54,6 +81,12 @@ const makePgStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { type PM = PersistenceModelType const tableName = `${prefix}${name}` const defaultValues = config?.defaultValues ?? {} + const rootLevelFieldColumns = config?.rootLevelFieldColumns ?? [] + const useRootLevelFields = config?.rootLevelFieldsWhenAvailable ?? rootLevelFieldsWhenAvailableDefault ?? false + const activeRootLevelFieldColumns = useRootLevelFields ? rootLevelFieldColumns : [] + const selectColumnsSql = activeRootLevelFieldColumns.length > 0 + ? `, ${activeRootLevelFieldColumns.map((column) => quoteIdentifier(column.columnName)).join(", ")}` + : "" const resolveNamespace = !config?.allowNamespace ? Effect.succeed("primary") @@ -64,6 +97,8 @@ const makePgStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { return namespace })) + const exec = (query: string, params?: readonly unknown[]) => sql.unsafe(query, params as any).pipe(Effect.orDie) + const ensureTable = sql .unsafe( `CREATE TABLE IF NOT EXISTS "${tableName}" (id TEXT NOT NULL, _namespace TEXT NOT NULL DEFAULT 'primary', _etag TEXT, data JSONB NOT NULL, PRIMARY KEY (id, _namespace))` @@ -74,6 +109,22 @@ const makePgStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { `CREATE TABLE IF NOT EXISTS "_migrations" (id TEXT NOT NULL, version TEXT NOT NULL, PRIMARY KEY (id, version))` ) ), + Effect.andThen( + Effect.forEach( + activeRootLevelFieldColumns, + (column) => + exec( + `ALTER TABLE "${tableName}" ADD COLUMN IF NOT EXISTS ${quoteIdentifier(column.columnName)} ${projectedColumnSqlType(pgDialect, column.kind)}` + ).pipe( + Effect.andThen( + exec( + `UPDATE "${tableName}" SET ${quoteIdentifier(column.columnName)} = ${projectedColumnBackfillExpr(pgDialect, column)} WHERE ${quoteIdentifier(column.columnName)} IS NULL` + ) + ) + ), + { discard: true } + ) + ), Effect.orDie, Effect.asVoid ) @@ -83,17 +134,26 @@ const makePgStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { const id = newE[idKey] as string const { _etag, [idKey]: _id, ...rest } = newE as any const data = JSON.stringify(rest) - return { id, _etag: newE._etag!, data, item: newE } + const rootLevelFieldValues = activeRootLevelFieldColumns.map((column) => rest[column.key]) + return { id, _etag: newE._etag!, data, item: newE, rootLevelFieldValues } } - const exec = (query: string, params?: readonly unknown[]) => sql.unsafe(query, params as any).pipe(Effect.orDie) - const setInternal = Effect.fnUntraced(function*(e: PM, ns: string) { const row = toRow(e) if (e._etag) { + const projectedSetSql = activeRootLevelFieldColumns.length > 0 + ? `, ${ + activeRootLevelFieldColumns + .map((column, index) => `${quoteIdentifier(column.columnName)} = $${index + 3}`) + .join(", ") + }` + : "" + const idParam = row.rootLevelFieldValues.length + 3 + const etagParam = row.rootLevelFieldValues.length + 4 + const namespaceParam = row.rootLevelFieldValues.length + 5 yield* exec( - `UPDATE "${tableName}" SET _etag = $1, data = $2 WHERE id = $3 AND _etag = $4 AND _namespace = $5`, - [row._etag, row.data, row.id, e._etag, ns] + `UPDATE "${tableName}" SET _etag = $1, data = $2${projectedSetSql} WHERE id = $${idParam} AND _etag = $${etagParam} AND _namespace = $${namespaceParam}`, + [row._etag, row.data, ...row.rootLevelFieldValues, row.id, e._etag, ns] ) const existing = yield* exec( `SELECT _etag FROM "${tableName}" WHERE id = $1 AND _namespace = $2`, @@ -119,9 +179,17 @@ const makePgStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { }) } } else { + const projectedColumnsSql = activeRootLevelFieldColumns.length > 0 + ? `, ${activeRootLevelFieldColumns.map((column) => quoteIdentifier(column.columnName)).join(", ")}` + : "" + const projectedValuesSql = activeRootLevelFieldColumns + .map((_, index) => `$${index + 5}`) + .join(", ") yield* exec( - `INSERT INTO "${tableName}" (id, _namespace, _etag, data) VALUES ($1, $2, $3, $4)`, - [row.id, ns, row._etag, row.data] + `INSERT INTO "${tableName}" (id, _namespace, _etag, data${projectedColumnsSql}) VALUES ($1, $2, $3, $4${ + projectedValuesSql.length > 0 ? `, ${projectedValuesSql}` : "" + })`, + [row.id, ns, row._etag, row.data, ...row.rootLevelFieldValues] ) } return row.item @@ -167,10 +235,12 @@ const makePgStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { all: resolveNamespace.pipe( Effect.flatMap((ns) => { - const sqlText = `SELECT id, _etag, data FROM "${tableName}" WHERE _namespace = $1` + const sqlText = `SELECT id, _etag, data${selectColumnsSql} FROM "${tableName}" WHERE _namespace = $1` return exec(sqlText, [ns]) .pipe( - Effect.map((rows) => (rows as any[]).map((r) => parseRow(r, idKey, defaultValues))), + Effect.map((rows) => + (rows as any[]).map((r) => parseRow(r, idKey, defaultValues, activeRootLevelFieldColumns)) + ), annotateDb({ operation: "all", system: "postgresql", @@ -186,13 +256,13 @@ const makePgStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { find: (id) => resolveNamespace.pipe(Effect .flatMap((ns) => { - const sqlText = `SELECT id, _etag, data FROM "${tableName}" WHERE id = $1 AND _namespace = $2` + const sqlText = `SELECT id, _etag, data${selectColumnsSql} FROM "${tableName}" WHERE id = $1 AND _namespace = $2` return exec(sqlText, [id, ns]) .pipe( Effect.map((rows) => { const row = (rows as any[])[0] return row - ? Option.some(parseRow(row, idKey, defaultValues)) + ? Option.some(parseRow(row, idKey, defaultValues, activeRootLevelFieldColumns)) : Option.none() }), annotateDb({ @@ -233,7 +303,9 @@ const makePgStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { | undefined, f.order, f.skip, - f.limit + f.limit, + undefined, + activeRootLevelFieldColumns ) const nsPlaceholder = pgDialect.placeholder(q.params.length + 1) const hasWhere = q.sql.includes("WHERE") @@ -253,7 +325,7 @@ const makePgStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { Effect.map((rows) => { if (f.select) { return (rows as any[]).map((r) => { - const selected = parseSelectRow(r, idKey, {}) + const selected = parseSelectRow(r, idKey, {}, activeRootLevelFieldColumns) return { ...Struct.pick( defaultValues as any, @@ -263,7 +335,9 @@ const makePgStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) { } as M }) } - return (rows as any[]).map((r) => parseRow(r, idKey, defaultValues) as any as M) + return (rows as any[]).map((r) => + parseRow(r, idKey, defaultValues, activeRootLevelFieldColumns) as any as M + ) }) ) ), diff --git a/packages/infra/src/Store/SQL/query.ts b/packages/infra/src/Store/SQL/query.ts index 3f4def0aa..718e481f2 100644 --- a/packages/infra/src/Store/SQL/query.ts +++ b/packages/infra/src/Store/SQL/query.ts @@ -5,6 +5,7 @@ import { assertUnreachable } from "effect-app/utils" import { InfraLogger } from "../../logger.js" import type { FilterR, FilterResult } from "../../Model/filter/filterApi.js" import type { AggregateIrExpression, ComputedProjectionIrExpression, ComputedProjectionMathIrExpression } from "../../Model/query.js" +import type { RootLevelFieldColumn, RootLevelFieldColumnKind } from "../rootLevelFields.js" import { isRelationCheck } from "../codeFilter.js" export interface SQLDialect { @@ -143,6 +144,75 @@ export function logQuery(q: { sql: string; params: unknown[] }) { })) } +export const quoteIdentifier = (value: string) => `"${value.replaceAll("\"", "\"\"")}"` + +const projectedColumnJsonFallbackExpr = ( + dialect: SQLDialect, + column: RootLevelFieldColumn +) => { + const expr = dialect.jsonExtract(column.key) + switch (column.kind) { + case "string": + return expr + case "number": + return dialect.jsonColumnType === "JSON" ? expr : `(${expr})::double precision` + case "boolean": + return dialect.jsonColumnType === "JSON" ? expr : `(${expr})::boolean` + default: + return assertUnreachable(column.kind) + } +} + +export const projectedColumnFieldExpr = ( + dialect: SQLDialect, + column: RootLevelFieldColumn +) => `COALESCE(${quoteIdentifier(column.columnName)}, ${projectedColumnJsonFallbackExpr(dialect, column)})` + +export const projectedColumnSelectExpr = ( + dialect: SQLDialect, + column: RootLevelFieldColumn +) => + column.kind === "boolean" && dialect.jsonColumnType === "JSON" + ? `CASE ${projectedColumnFieldExpr(dialect, column)} WHEN 1 THEN 'true' WHEN 0 THEN 'false' ELSE 'null' END` + : projectedColumnFieldExpr(dialect, column) + +export const projectedColumnSqlType = ( + dialect: SQLDialect, + kind: RootLevelFieldColumnKind +) => { + switch (kind) { + case "string": + return "TEXT" + case "number": + return dialect.jsonColumnType === "JSON" ? "REAL" : "DOUBLE PRECISION" + case "boolean": + return dialect.jsonColumnType === "JSON" ? "INTEGER" : "BOOLEAN" + default: + return assertUnreachable(kind) + } +} + +export const projectedColumnBackfillExpr = ( + dialect: SQLDialect, + column: RootLevelFieldColumn +) => projectedColumnJsonFallbackExpr(dialect, column) + +export const normalizeProjectedColumnValue = ( + column: RootLevelFieldColumn, + value: unknown +) => { + if (column.kind === "boolean") { + if (typeof value === "number") { + return value !== 0 + } + if (typeof value === "string") { + if (value === "true") return true + if (value === "false") return false + } + } + return value +} + const dottedToJsonPath = (path: string) => path .split(".") @@ -175,10 +245,12 @@ export function buildWhereSQLQuery( order?: NonEmptyReadonlyArray<{ key: string; direction: "ASC" | "DESC" }>, skip?: number, limit?: number, - namespace?: string + namespace?: string, + rootLevelFieldColumns: readonly RootLevelFieldColumn[] = [] ) { const params: unknown[] = [] let paramIndex = 1 + const projectedColumns = new Map(rootLevelFieldColumns.map((column) => [column.key, column] as const)) const addParam = (value: unknown): string => { params.push(dialect.serializeScalar(value)) @@ -199,6 +271,16 @@ export function buildWhereSQLQuery( const arrPath = dottedToJsonPath(path.slice(0, -".length".length)) return dialect.arrayLength(arrPath) } + if (!relation && !path.includes(".")) { + const projected = projectedColumns.get(path) + if (projected) { + const expr = projectedColumnFieldExpr(dialect, projected) + if (path in defaultValues) { + return `COALESCE(${expr}, ${addParam(defaultValues[path])})` + } + return expr + } + } const jsonPath = dottedToJsonPath(path) const expr = dialect.jsonExtract(jsonPath) const topKey = path.split(".")[0] @@ -533,6 +615,10 @@ export function buildWhereSQLQuery( if (typeof s === "string") { if (s === idKey || s === "id") return `id` if (s === "_etag") return `_etag` + const projected = projectedColumns.get(s) + if (projected) { + return `${projectedColumnSelectExpr(dialect, projected)} AS "${s}"` + } return `${dialect.jsonExtractJson(s)} AS "${s}"` } if ("computed" in s) { diff --git a/packages/infra/src/Store/rootLevelFields.ts b/packages/infra/src/Store/rootLevelFields.ts new file mode 100644 index 000000000..47f390e3c --- /dev/null +++ b/packages/infra/src/Store/rootLevelFields.ts @@ -0,0 +1,74 @@ +import type * as S from "effect-app/Schema" +import * as SchemaAST from "effect/SchemaAST" + +export type RootLevelFieldColumnKind = "string" | "number" | "boolean" + +export interface RootLevelFieldColumn { + readonly key: string + readonly columnName: string + readonly kind: RootLevelFieldColumnKind +} + +const walkTransformation = (ast: SchemaAST.AST): SchemaAST.AST => { + if (ast._tag === "Declaration" && ast.typeParameters.length > 0) { + return walkTransformation(ast.typeParameters[0]!) + } + return ast +} + +const getScalarKind = (ast: SchemaAST.AST): RootLevelFieldColumnKind | undefined => { + const encoded = walkTransformation(SchemaAST.toEncoded(ast)) + switch (encoded._tag) { + case "String": + return "string" + case "Number": + return "number" + case "Boolean": + return "boolean" + case "Literal": + switch (typeof encoded.literal) { + case "string": + return "string" + case "number": + return "number" + case "boolean": + return "boolean" + default: + return undefined + } + case "Union": { + const scalarKinds = encoded.types + .flatMap((type) => { + const kind = getScalarKind(type) + return kind === undefined ? [] : [kind] + }) + const distinctKinds = [...new Set(scalarKinds)] + return distinctKinds.length === 1 ? distinctKinds[0] : undefined + } + default: + return undefined + } +} + +const makeColumnName = (key: string) => `__root_${key}` + +export const makeRootLevelFieldColumns = ( + schema: S.Schema, + idKey: PropertyKey +): readonly RootLevelFieldColumn[] => { + const encoded = walkTransformation(SchemaAST.toEncoded(schema.ast)) + if (!SchemaAST.isObjects(encoded)) { + return [] + } + + return encoded.propertySignatures.flatMap((property) => { + const key = String(property.name) + if (key === String(idKey) || key === "id" || key === "_etag") { + return [] + } + const kind = getScalarKind(property.type) + return kind === undefined + ? [] + : [{ key, kind, columnName: makeColumnName(key) }] + }) +} diff --git a/packages/infra/src/Store/service.ts b/packages/infra/src/Store/service.ts index 49f4d0858..bdec69c22 100644 --- a/packages/infra/src/Store/service.ts +++ b/packages/infra/src/Store/service.ts @@ -11,9 +11,12 @@ import type { FilterResult } from "../Model/filter/filterApi.js" import type { FieldValues } from "../Model/filter/types.js" import type { FieldPath } from "../Model/filter/types/path/index.js" import type { AggregateIrExpression, ComputedProjectionIrExpression, RawQuery } from "../Model/query.js" +import type { RootLevelFieldColumn } from "./rootLevelFields.js" export interface StoreConfig { partitionValue: (e?: E) => string + rootLevelFieldsWhenAvailable?: boolean + rootLevelFieldColumns?: readonly RootLevelFieldColumn[] /** * Primarily used for testing, creating namespaces in the database to separate data e.g to run multiple tests in isolation within the same database. * Memory/Disk use separate store instances per namespace. CosmosDB uses namespace-prefixed partition keys. SQL uses a `_namespace` column. @@ -230,4 +233,5 @@ export interface StorageConfig { url: Redacted.Redacted prefix: string dbName: string + rootLevelFieldsWhenAvailable?: boolean } diff --git a/packages/infra/test/sql-root-level-columns.test.ts b/packages/infra/test/sql-root-level-columns.test.ts new file mode 100644 index 000000000..9f913d239 --- /dev/null +++ b/packages/infra/test/sql-root-level-columns.test.ts @@ -0,0 +1,133 @@ +import { SqliteClient } from "@effect/sql-sqlite-node" +import * as Effect from "effect-app/Effect" +import * as Layer from "effect-app/Layer" +import * as S from "effect-app/Schema" +import { SqlClient } from "effect/unstable/sql" +import * as Redacted from "effect/Redacted" +import { describe, expect, it } from "vitest" +import { setupRequestContextFromCurrent } from "../src/api/setupRequest.js" +import { where } from "../src/Model/query.js" +import { makeRepo } from "../src/Model/Repository.js" +import { RepositoryRegistryLive } from "../src/Model/Repository/Registry.js" +import { SQLiteStoreLayer } from "../src/Store/SQL.js" + +class ProjectedItem extends S.Class("ProjectedItem")({ + id: S.String, + name: S.String, + age: S.Number, + active: S.Boolean, + createdAt: S.Date, + meta: S.Struct({ city: S.String }) +}) {} + +const tableName = "test_projectedItems" + +const makeStoreLayer = (rootLevelFieldsWhenAvailable?: boolean) => + Layer + .merge( + SQLiteStoreLayer({ + url: Redacted.make("sqlite://"), + prefix: "test_", + dbName: "test", + rootLevelFieldsWhenAvailable + }), + RepositoryRegistryLive + ) + .pipe(Layer.provide(SqliteClient.layer({ filename: ":memory:" }))) + +describe("SQLite root-level projected columns", () => { + it("creates and writes projected root-level columns when enabled per repository", async () => { + const createdAt = new Date("2024-01-02T03:04:05.000Z") + const result = await Effect + .gen(function*() { + const repo = yield* makeRepo("projectedItem", ProjectedItem, { + config: { rootLevelFieldsWhenAvailable: true } + }) + + yield* repo.saveAndPublish([{ + id: "1", + name: "Alice", + age: 30, + active: true, + createdAt, + meta: { city: "Oslo" } + }]) + + const sql = yield* SqlClient.SqlClient + const columns = yield* sql.unsafe(`PRAGMA table_info(${JSON.stringify(tableName)})`) as Array<{ name: string }> + const row = (yield* sql.unsafe( + `SELECT id, "__root_name", "__root_age", "__root_active", "__root_createdAt", data FROM "${tableName}"` + ) as any[])[0] + + return { columns, row } + }) + .pipe(setupRequestContextFromCurrent("sql root-level columns")) + .pipe(Effect.provide(makeStoreLayer())) + .pipe(Effect.runPromise) + + const columnNames = result.columns.map((column) => column.name) + expect(columnNames).toEqual(expect.arrayContaining([ + "__root_name", + "__root_age", + "__root_active", + "__root_createdAt" + ])) + expect(columnNames).not.toContain("__root_meta") + expect(result.row.__root_name).toBe("Alice") + expect(result.row.__root_age).toBe(30) + expect(result.row.__root_active).toBe(1) + expect(result.row.__root_createdAt).toBe(createdAt.toJSON()) + expect(JSON.parse(result.row.data)).toMatchObject({ + name: "Alice", + age: 30, + active: true, + meta: { city: "Oslo" } + }) + }) + + it("allows repository config to disable an adapter-level default", async () => { + const columnNames = await Effect + .gen(function*() { + yield* makeRepo("projectedItem", ProjectedItem, { + config: { rootLevelFieldsWhenAvailable: false } + }) + const sql = yield* SqlClient.SqlClient + const columns = yield* sql.unsafe(`PRAGMA table_info(${JSON.stringify(tableName)})`) as Array<{ name: string }> + return columns.map((column) => column.name) + }) + .pipe(setupRequestContextFromCurrent("sql root-level columns override")) + .pipe(Effect.provide(makeStoreLayer(true))) + .pipe(Effect.runPromise) + + expect(columnNames).not.toContain("__root_name") + expect(columnNames).not.toContain("__root_age") + }) + + it("queries legacy rows through JSON fallback when projected columns are null", async () => { + const rows = await Effect + .gen(function*() { + const repo = yield* makeRepo("projectedItem", ProjectedItem, { + config: { rootLevelFieldsWhenAvailable: true } + }) + const sql = yield* SqlClient.SqlClient + yield* sql.unsafe( + `INSERT INTO "${tableName}" (id, _namespace, _etag, data) VALUES (?, ?, ?, ?)`, + ["legacy", "primary", "etag1", JSON.stringify({ + name: "Legacy", + age: 42, + active: true, + createdAt: "2024-01-02T03:04:05.000Z", + meta: { city: "Oslo" } + })] + ) + return yield* repo.query(where("age", "gt", 40)) + }) + .pipe(setupRequestContextFromCurrent("sql root-level columns legacy")) + .pipe(Effect.provide(makeStoreLayer())) + .pipe(Effect.runPromise) + + expect(rows).toHaveLength(1) + expect(rows[0]!.id).toBe("legacy") + expect(rows[0]!.age).toBe(42) + }) +}) diff --git a/packages/infra/test/sql-store.test.ts b/packages/infra/test/sql-store.test.ts index 433f7373e..d9b79c3e8 100644 --- a/packages/infra/test/sql-store.test.ts +++ b/packages/infra/test/sql-store.test.ts @@ -2,6 +2,7 @@ import type Sqlite from "better-sqlite3" import BetterSqlite from "better-sqlite3" import { describe, expect, it } from "vitest" +import type { RootLevelFieldColumn } from "../src/Store/rootLevelFields.js" import { parseRow } from "../src/Store/SQL.js" import { buildWhereSQLQuery, pgDialect, sqliteDialect } from "../src/Store/SQL/query.js" import { makeETag } from "../src/Store/utils.js" @@ -9,6 +10,12 @@ import { makeETag } from "../src/Store/utils.js" const query = (db: Sqlite.Database, sql: string, params: unknown[] = []) => db.prepare(sql).all(...params as any[]) as any[] +const projectedRootLevelFields: readonly RootLevelFieldColumn[] = [ + { key: "name", columnName: "__root_name", kind: "string" }, + { key: "age", columnName: "__root_age", kind: "number" }, + { key: "flag", columnName: "__root_flag", kind: "boolean" } +] + // --- Query builder unit tests --- describe("SQL query builder (SQLite dialect)", () => { @@ -48,6 +55,23 @@ describe("SQL query builder (SQLite dialect)", () => { expect(result.params).toContain(18) }) + it("uses projected root columns with JSON fallback for root-level filters", () => { + const result = buildWhereSQLQuery( + sqliteDialect, + "id", + [{ t: "where", path: "age", op: "gt", value: 18 as any }], + "users", + {}, + undefined, + undefined, + undefined, + undefined, + undefined, + projectedRootLevelFields + ) + expect(result.sql).toContain(`COALESCE("__root_age", json_extract(data, '$.age')) > ?`) + }) + it("where or", () => { const result = buildWhereSQLQuery( sqliteDialect, @@ -711,6 +735,23 @@ describe("SQL query builder (PostgreSQL dialect)", () => { ) expect(result.sql).toContain(`COALESCE(jsonb_agg(DISTINCT _items->>'articleId'), '[]'::jsonb)`) }) + + it("uses projected root columns with JSON fallback for pg filters", () => { + const result = buildWhereSQLQuery( + pgDialect, + "id", + [{ t: "where", path: "flag", op: "eq", value: true as any }], + "users", + {}, + undefined, + undefined, + undefined, + undefined, + undefined, + projectedRootLevelFields + ) + expect(result.sql).toContain(`COALESCE("__root_flag", (data->>'flag')::boolean) = $1`) + }) }) // --- Integration tests with in-memory SQLite (direct, no Effect SQL client) --- @@ -811,6 +852,38 @@ describe("SQL Store (SQLite integration)", () => { expect(query(db, q3.sql, q3.params).length).toBe(2) })) + it("projected root columns still fall back to JSON data for legacy rows", () => + withDb((db) => { + db.exec( + `CREATE TABLE IF NOT EXISTS "test_projected" (id TEXT PRIMARY KEY, _etag TEXT, data JSON NOT NULL, "__root_name" TEXT, "__root_age" REAL, "__root_flag" INTEGER)` + ) + db + .prepare(`INSERT INTO "test_projected" (id, _etag, data) VALUES (?, ?, ?)`) + .run("1", "etag1", JSON.stringify({ name: "Alice", age: 30, flag: true })) + db + .prepare( + `INSERT INTO "test_projected" (id, _etag, data, "__root_name", "__root_age", "__root_flag") VALUES (?, ?, ?, ?, ?, ?)` + ) + .run("2", "etag2", JSON.stringify({ name: "Bob", age: 10, flag: false }), "Bob", 10, 0) + + const q = buildWhereSQLQuery( + sqliteDialect, + "id", + [{ t: "where", path: "age", op: "gt", value: 20 as any }], + "test_projected", + {}, + undefined, + undefined, + undefined, + undefined, + undefined, + projectedRootLevelFields + ) + const rows = query(db, q.sql, q.params) + expect(rows).toHaveLength(1) + expect((rows[0] as any).id).toBe("1") + })) + it("queries work when data does not contain id", () => withDb((db) => { db.exec( @@ -1708,4 +1781,21 @@ describe("parseRow reconstructs full object from row", () => { expect(reconstructed.tags).toEqual(["admin"]) expect(reconstructed._etag).toBe(newE._etag) }) + + it("prefers projected root-level columns when available", () => { + const result: any = parseRow( + { + id: "1", + _etag: "e1", + data: JSON.stringify({ name: "stale", flag: false }), + __root_name: "Alice", + __root_flag: 1 + }, + "id", + {}, + projectedRootLevelFields + ) + expect(result.name).toBe("Alice") + expect(result.flag).toBe(true) + }) }) From a51d7ede552a2fabe44fe889c159b8dbebc9667b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 22 May 2026 18:58:01 +0000 Subject: [PATCH 02/11] test: cover root sql column projection Agent-Logs-Url: https://github.com/effect-app/libs/sessions/f324ecdc-fbb7-4471-9947-241bf85170ff Co-authored-by: patroza <42661+patroza@users.noreply.github.com> --- .changeset/few-oranges-project.md | 5 + packages/infra/src/Store/SQL.ts | 700 +++++++++--------- packages/infra/src/Store/SQL/Pg.ts | 647 ++++++++-------- packages/infra/src/Store/SQL/query.ts | 2 +- packages/infra/src/Store/rootLevelFields.ts | 3 +- .../infra/test/sql-root-level-columns.test.ts | 133 ---- packages/infra/test/sql-store.test.ts | 26 +- 7 files changed, 716 insertions(+), 800 deletions(-) create mode 100644 .changeset/few-oranges-project.md delete mode 100644 packages/infra/test/sql-root-level-columns.test.ts diff --git a/.changeset/few-oranges-project.md b/.changeset/few-oranges-project.md new file mode 100644 index 000000000..4056f72ff --- /dev/null +++ b/.changeset/few-oranges-project.md @@ -0,0 +1,5 @@ +--- +"@effect-app/infra": minor +--- + +Add optional SQL root-level projected columns for repository-backed SQLite and PostgreSQL stores. diff --git a/packages/infra/src/Store/SQL.ts b/packages/infra/src/Store/SQL.ts index 5412a01a3..7644857bc 100644 --- a/packages/infra/src/Store/SQL.ts +++ b/packages/infra/src/Store/SQL.ts @@ -17,16 +17,7 @@ import { annotateDb, type DbSystem } from "../otel.js" import { storeId } from "./Memory.js" import type { RootLevelFieldColumn } from "./rootLevelFields.js" import { type FilterArgs, type PersistenceModelType, type StorageConfig, type Store, type StoreConfig, StoreMaker } from "./service.js" -import { - buildWhereSQLQuery, - logQuery, - normalizeProjectedColumnValue, - projectedColumnBackfillExpr, - projectedColumnSqlType, - quoteIdentifier, - type SQLDialect, - sqliteDialect -} from "./SQL/query.js" +import { buildWhereSQLQuery, logQuery, normalizeProjectedColumnValue, projectedColumnBackfillExpr, projectedColumnSqlType, quoteIdentifier, type SQLDialect, sqliteDialect } from "./SQL/query.js" import { makeETag } from "./utils.js" export type WithNsTransactionFn = (effect: Effect.Effect) => Effect.Effect @@ -89,368 +80,388 @@ const serializeProjectedValue = ( system: DbSystem, column: RootLevelFieldColumn, value: unknown -) => column.kind === "boolean" && system === "sqlite" && value !== null && value !== undefined - ? value === true ? 1 : 0 - : value +) => + column.kind === "boolean" && system === "sqlite" && value !== null && value !== undefined + ? value === true ? 1 : 0 + : value function makeSQLStoreInt(system: DbSystem, dialect: SQLDialect, jsonColumnType: string) { - return Effect.fnUntraced(function*({ prefix, rootLevelFieldsWhenAvailable: rootLevelFieldsWhenAvailableDefault }: StorageConfig) { - const sql = yield* SqlClient.SqlClient - return { - make: Effect.fnUntraced(function*( - name: string, - idKey: IdKey, - seed?: Effect.Effect, E, R>, - config?: StoreConfig - ) { - type PM = PersistenceModelType - const tableName = `${prefix}${name}` - const defaultValues = config?.defaultValues ?? {} - const rootLevelFieldColumns = config?.rootLevelFieldColumns ?? [] - const useRootLevelFields = config?.rootLevelFieldsWhenAvailable ?? rootLevelFieldsWhenAvailableDefault ?? false - const activeRootLevelFieldColumns = useRootLevelFields ? rootLevelFieldColumns : [] - const selectColumnsSql = activeRootLevelFieldColumns.length > 0 - ? `, ${activeRootLevelFieldColumns.map((column) => quoteIdentifier(column.columnName)).join(", ")}` - : "" - - const resolveNamespace = !config?.allowNamespace - ? Effect.succeed("primary") - : storeId.pipe(Effect.map((namespace) => { - if (namespace !== "primary" && !config.allowNamespace!(namespace)) { - throw new Error(`Namespace ${namespace} not allowed!`) - } - return namespace - })) + return Effect.fnUntraced( + function*({ prefix, rootLevelFieldsWhenAvailable: rootLevelFieldsWhenAvailableDefault }: StorageConfig) { + const sql = yield* SqlClient.SqlClient + return { + make: Effect.fnUntraced( + function*( + name: string, + idKey: IdKey, + seed?: Effect.Effect, E, R>, + config?: StoreConfig + ) { + type PM = PersistenceModelType + const tableName = `${prefix}${name}` + const defaultValues = config?.defaultValues ?? {} + const rootLevelFieldColumns = config?.rootLevelFieldColumns ?? [] + const useRootLevelFields = config?.rootLevelFieldsWhenAvailable + ?? rootLevelFieldsWhenAvailableDefault + ?? false + const activeRootLevelFieldColumns = useRootLevelFields ? rootLevelFieldColumns : [] + const selectColumnsSql = activeRootLevelFieldColumns.length > 0 + ? `, ${activeRootLevelFieldColumns.map((column) => quoteIdentifier(column.columnName)).join(", ")}` + : "" - const exec = (query: string, params?: readonly unknown[]) => sql.unsafe(query, params as any).pipe(Effect.orDie) + const resolveNamespace = !config?.allowNamespace + ? Effect.succeed("primary") + : storeId.pipe(Effect.map((namespace) => { + if (namespace !== "primary" && !config.allowNamespace!(namespace)) { + throw new Error(`Namespace ${namespace} not allowed!`) + } + return namespace + })) - const ensureRootLevelFieldColumns = Effect.fnUntraced(function*() { - if (activeRootLevelFieldColumns.length === 0) { - return - } - const existing = yield* exec(`PRAGMA table_info(${JSON.stringify(tableName)})`) as Array<{ name: string }> - const existingColumns = new Set(existing.map((column) => column.name)) - for (const column of activeRootLevelFieldColumns) { - if (!existingColumns.has(column.columnName)) { - yield* exec( - `ALTER TABLE "${tableName}" ADD COLUMN ${quoteIdentifier(column.columnName)} ${projectedColumnSqlType(dialect, column.kind)}` + const exec = (query: string, params?: readonly unknown[]) => + sql.unsafe(query, params as any).pipe(Effect.orDie) + + const ensureRootLevelFieldColumns = Effect.gen(function*() { + if (activeRootLevelFieldColumns.length === 0) { + return + } + const existing = (yield* exec(`PRAGMA table_info(${JSON.stringify(tableName)})`)) as Array< + { name: string } + > + const existingColumns = new Set(existing.map((column) => column.name)) + for (const column of activeRootLevelFieldColumns) { + if (!existingColumns.has(column.columnName)) { + yield* exec( + `ALTER TABLE "${tableName}" ADD COLUMN ${quoteIdentifier(column.columnName)} ${ + projectedColumnSqlType(dialect, column.kind) + }` + ) + } + yield* exec( + `UPDATE "${tableName}" SET ${quoteIdentifier(column.columnName)} = ${ + projectedColumnBackfillExpr(dialect, column) + } WHERE ${quoteIdentifier(column.columnName)} IS NULL` + ) + } + }) + + const ensureTable = sql + .unsafe( + `CREATE TABLE IF NOT EXISTS "${tableName}" (id TEXT NOT NULL, _namespace TEXT NOT NULL DEFAULT 'primary', _etag TEXT, data ${jsonColumnType} NOT NULL, PRIMARY KEY (id, _namespace))` + ) + .pipe( + Effect.andThen( + sql.unsafe( + `CREATE TABLE IF NOT EXISTS "_migrations" (id TEXT NOT NULL, version TEXT NOT NULL, PRIMARY KEY (id, version))` + ) + ), + Effect.andThen(ensureRootLevelFieldColumns), + Effect.orDie, + Effect.asVoid ) - } - yield* exec( - `UPDATE "${tableName}" SET ${quoteIdentifier(column.columnName)} = ${projectedColumnBackfillExpr(dialect, column)} WHERE ${quoteIdentifier(column.columnName)} IS NULL` - ) - } - }) - const ensureTable = sql - .unsafe( - `CREATE TABLE IF NOT EXISTS "${tableName}" (id TEXT NOT NULL, _namespace TEXT NOT NULL DEFAULT 'primary', _etag TEXT, data ${jsonColumnType} NOT NULL, PRIMARY KEY (id, _namespace))` - ) - .pipe( - Effect.andThen( - sql.unsafe( - `CREATE TABLE IF NOT EXISTS "_migrations" (id TEXT NOT NULL, version TEXT NOT NULL, PRIMARY KEY (id, version))` + const toRow = (e: PM) => { + const newE = makeETag(e) + const id = newE[idKey] as string + const { _etag, [idKey]: _id, ...rest } = newE as any + const data = JSON.stringify(rest) + const rootLevelFieldValues = activeRootLevelFieldColumns.map((column) => + serializeProjectedValue(system, column, rest[column.key]) ) - ), - Effect.andThen(ensureRootLevelFieldColumns()), - Effect.orDie, - Effect.asVoid - ) + return { id, _etag: newE._etag!, data, item: newE, rootLevelFieldValues } + } - const toRow = (e: PM) => { - const newE = makeETag(e) - const id = newE[idKey] as string - const { _etag, [idKey]: _id, ...rest } = newE as any - const data = JSON.stringify(rest) - const rootLevelFieldValues = activeRootLevelFieldColumns.map((column) => - serializeProjectedValue(system, column, rest[column.key]) - ) - return { id, _etag: newE._etag!, data, item: newE, rootLevelFieldValues } - } + const setInternal = Effect.fnUntraced(function*(e: PM, ns: string) { + const row = toRow(e) + if (e._etag) { + const projectedSetSql = activeRootLevelFieldColumns.length > 0 + ? `, ${ + activeRootLevelFieldColumns.map((column) => `${quoteIdentifier(column.columnName)} = ?`).join(", ") + }` + : "" + yield* exec( + `UPDATE "${tableName}" SET _etag = ?, data = ?${projectedSetSql} WHERE id = ? AND _etag = ? AND _namespace = ?`, + [row._etag, row.data, ...row.rootLevelFieldValues, row.id, e._etag, ns] + ) + const existing = yield* exec( + `SELECT _etag FROM "${tableName}" WHERE id = ? AND _namespace = ?`, + [row.id, ns] + ) + const current = (existing as any[])[0] + if (!current || current._etag !== row._etag) { + if (current) { + return yield* new OptimisticConcurrencyException({ + type: name, + id: row.id, + current: current._etag, + found: e._etag, + code: 412 + }) + } + return yield* new OptimisticConcurrencyException({ + type: name, + id: row.id, + current: "", + found: e._etag, + code: 404 + }) + } + } else { + const projectedColumnsSql = activeRootLevelFieldColumns.length > 0 + ? `, ${activeRootLevelFieldColumns.map((column) => quoteIdentifier(column.columnName)).join(", ")}` + : "" + const projectedPlaceholders = activeRootLevelFieldColumns.map(() => "?").join(", ") + yield* exec( + `INSERT INTO "${tableName}" (id, _namespace, _etag, data${projectedColumnsSql}) VALUES (?, ?, ?, ?${ + projectedPlaceholders.length > 0 ? `, ${projectedPlaceholders}` : "" + })`, + [row.id, ns, row._etag, row.data, ...row.rootLevelFieldValues] + ) + } + return row.item + }) - const setInternal = Effect.fnUntraced(function*(e: PM, ns: string) { - const row = toRow(e) - if (e._etag) { - const projectedSetSql = activeRootLevelFieldColumns.length > 0 - ? `, ${activeRootLevelFieldColumns.map((column) => `${quoteIdentifier(column.columnName)} = ?`).join(", ")}` - : "" - yield* exec( - `UPDATE "${tableName}" SET _etag = ?, data = ?${projectedSetSql} WHERE id = ? AND _etag = ? AND _namespace = ?`, - [row._etag, row.data, ...row.rootLevelFieldValues, row.id, e._etag, ns] - ) - const existing = yield* exec( - `SELECT _etag FROM "${tableName}" WHERE id = ? AND _namespace = ?`, - [row.id, ns] - ) - const current = (existing as any[])[0] - if (!current || current._etag !== row._etag) { - if (current) { - return yield* new OptimisticConcurrencyException({ - type: name, - id: row.id, - current: current._etag, - found: e._etag, - code: 412 - }) + const bulkSetInternal = (items: NonEmptyReadonlyArray, ns: string) => + sql + .withTransaction(Effect.forEach(items, (e) => setInternal(e, ns))) + .pipe( + Effect.orDie, + Effect.map((_) => _ as unknown as NonEmptyReadonlyArray) + ) + + const ctx = yield* Effect.context() + const seedCache = new Map>() + const makeSeedEffect = Effect.fnUntraced(function*(ns: string) { + yield* ensureTable + if (!seed) return + const existing = yield* exec( + `SELECT id FROM "_migrations" WHERE id = ? AND version = ?`, + [`${tableName}::${ns}`, tableName] + ) + if ((existing as any[]).length > 0) return + yield* InfraLogger.logInfo(`Seeding data for ${name} (namespace: ${ns})`) + const items = yield* seed.pipe(Effect.provide(ctx), Effect.orDie) + const ne = toNonEmptyArray([...items]) + if (Option.isSome(ne)) yield* bulkSetInternal(ne.value, ns) + yield* exec( + `INSERT INTO "_migrations" (id, version) VALUES (?, ?)`, + [`${tableName}::${ns}`, tableName] + ) + }) + const seedNamespace = (ns: string) => { + const cached = seedCache.get(ns) + if (cached) { + return cached } - return yield* new OptimisticConcurrencyException({ - type: name, - id: row.id, - current: "", - found: e._etag, - code: 404 - }) + const next = Effect.cached(Effect.uninterruptible(makeSeedEffect(ns))).pipe(Effect.runSync) + seedCache.set(ns, next) + return next } - } else { - const projectedColumnsSql = activeRootLevelFieldColumns.length > 0 - ? `, ${activeRootLevelFieldColumns.map((column) => quoteIdentifier(column.columnName)).join(", ")}` - : "" - const projectedPlaceholders = activeRootLevelFieldColumns.map(() => "?").join(", ") - yield* exec( - `INSERT INTO "${tableName}" (id, _namespace, _etag, data${projectedColumnsSql}) VALUES (?, ?, ?, ?${ - projectedPlaceholders.length > 0 ? `, ${projectedPlaceholders}` : "" - })`, - [row.id, ns, row._etag, row.data, ...row.rootLevelFieldValues] - ) - } - return row.item - }) - - const bulkSetInternal = (items: NonEmptyReadonlyArray, ns: string) => - sql - .withTransaction(Effect.forEach(items, (e) => setInternal(e, ns))) - .pipe( - Effect.orDie, - Effect.map((_) => _ as unknown as NonEmptyReadonlyArray) - ) + const s: Store = { + seedNamespace: (ns) => seedNamespace(ns), - const ctx = yield* Effect.context() - const seedCache = new Map>() - const makeSeedEffect = Effect.fnUntraced(function*(ns: string) { - yield* ensureTable - if (!seed) return - const existing = yield* exec( - `SELECT id FROM "_migrations" WHERE id = ? AND version = ?`, - [`${tableName}::${ns}`, tableName] - ) - if ((existing as any[]).length > 0) return - yield* InfraLogger.logInfo(`Seeding data for ${name} (namespace: ${ns})`) - const items = yield* seed.pipe(Effect.provide(ctx), Effect.orDie) - const ne = toNonEmptyArray([...items]) - if (Option.isSome(ne)) yield* bulkSetInternal(ne.value, ns) - yield* exec( - `INSERT INTO "_migrations" (id, version) VALUES (?, ?)`, - [`${tableName}::${ns}`, tableName] - ) - }) - const seedNamespace = (ns: string) => { - let cached = seedCache.get(ns) - if (!cached) { - cached = Effect.cached(Effect.uninterruptible(makeSeedEffect(ns))).pipe(Effect.runSync) - seedCache.set(ns, cached) - } - return cached - } - const s: Store = { - seedNamespace: (ns) => seedNamespace(ns), + all: resolveNamespace.pipe( + Effect.flatMap((ns) => { + const sqlText = `SELECT id, _etag, data${selectColumnsSql} FROM "${tableName}" WHERE _namespace = ?` + return exec(sqlText, [ns]) + .pipe( + Effect.map((rows) => + (rows as any[]).map((r) => + parseRow(r, idKey, defaultValues, activeRootLevelFieldColumns) + ) + ), + annotateDb({ + operation: "all", + system, + collection: tableName, + namespace: ns, + entity: name, + query: sqlText + }) + ) + }) + ), - all: resolveNamespace.pipe( - Effect.flatMap((ns) => { - const sqlText = `SELECT id, _etag, data${selectColumnsSql} FROM "${tableName}" WHERE _namespace = ?` - return exec(sqlText, [ns]) - .pipe( - Effect.map((rows) => - (rows as any[]).map((r) => parseRow(r, idKey, defaultValues, activeRootLevelFieldColumns)) - ), - annotateDb({ - operation: "all", - system, - collection: tableName, - namespace: ns, - entity: name, - query: sqlText + find: (id) => + resolveNamespace.pipe( + Effect.flatMap((ns) => { + const sqlText = + `SELECT id, _etag, data${selectColumnsSql} FROM "${tableName}" WHERE id = ? AND _namespace = ?` + return exec(sqlText, [id, ns]) + .pipe( + Effect.map((rows) => { + const row = (rows as any[])[0] + return row + ? Option.some(parseRow(row, idKey, defaultValues, activeRootLevelFieldColumns)) + : Option.none() + }), + annotateDb({ + operation: "find", + system, + collection: tableName, + namespace: ns, + entity: name, + query: sqlText, + extra: { "app.entity.id": id } + }) + ) }) - ) - }) - ), + ), + + filter: (f: FilterArgs) => { + const filter = f + .filter + type M = U extends undefined ? Encoded + : Pick + return resolveNamespace + .pipe(Effect + .flatMap((ns) => + Effect + .sync(() => { + return buildWhereSQLQuery( + dialect, + idKey, + filter ? [{ t: "where-scope", result: filter, relation: "some" }] : [], + tableName, + defaultValues, + f + .select as + | NonEmptyReadonlyArray< + string | { + key: string + subKeys: readonly string[] + } | { + key: string + computed: ComputedProjectionIrExpression + } + > + | undefined, + f + .order, + f + .skip, + f + .limit, + ns, + activeRootLevelFieldColumns + ) + }) + .pipe( + Effect + .tap((q) => logQuery(q)), + Effect.tap((q) => Effect.annotateCurrentSpan({ "db.query.text": q.sql })), + Effect.flatMap((q) => + exec(q.sql, q.params).pipe( + Effect.map((rows) => { + if (f.select) { + return (rows as any[]).map((r) => { + const selected = parseSelectRow(r, idKey, activeRootLevelFieldColumns) + return { + ...Struct.pick( + defaultValues as any, + f.select!.filter((_) => typeof _ === "string") as never[] + ), + ...selected + } as M + }) + } + return (rows as any[]).map((r) => + parseRow(r, idKey, defaultValues, activeRootLevelFieldColumns) as any as M + ) + }) + ) + ), + annotateDb({ + operation: "filter", + system, + collection: tableName, + namespace: ns, + entity: name + }) + ) + )) + }, - find: (id) => - resolveNamespace.pipe( - Effect.flatMap((ns) => { - const sqlText = `SELECT id, _etag, data${selectColumnsSql} FROM "${tableName}" WHERE id = ? AND _namespace = ?` - return exec(sqlText, [id, ns]) - .pipe( - Effect.map((rows) => { - const row = (rows as any[])[0] - return row - ? Option.some(parseRow(row, idKey, defaultValues, activeRootLevelFieldColumns)) - : Option.none() - }), + set: (e) => + resolveNamespace.pipe(Effect.flatMap((ns) => + setInternal(e, ns).pipe( annotateDb({ - operation: "find", + operation: "set", system, collection: tableName, namespace: ns, entity: name, - query: sqlText, - extra: { "app.entity.id": id } + extra: { "app.entity.id": e[idKey] } }) ) - }) - ), - - filter: (f: FilterArgs) => { - const filter = f - .filter - type M = U extends undefined ? Encoded - : Pick - return resolveNamespace - .pipe(Effect - .flatMap((ns) => - Effect - .sync(() => { - return buildWhereSQLQuery( - dialect, - idKey, - filter ? [{ t: "where-scope", result: filter, relation: "some" }] : [], - tableName, - defaultValues, - f - .select as - | NonEmptyReadonlyArray< - string | { - key: string - subKeys: readonly string[] - } | { - key: string - computed: ComputedProjectionIrExpression - } - > - | undefined, - f - .order, - f - .skip, - f - .limit, - ns, - activeRootLevelFieldColumns - ) + )), + + batchSet: (items) => + resolveNamespace.pipe(Effect.flatMap((ns) => + bulkSetInternal(items, ns).pipe( + annotateDb({ + operation: "batchSet", + system, + collection: tableName, + namespace: ns, + entity: name + }) + ) + )), + + bulkSet: (items) => + resolveNamespace.pipe(Effect.flatMap((ns) => + bulkSetInternal(items, ns).pipe( + annotateDb({ + operation: "bulkSet", + system, + collection: tableName, + namespace: ns, + entity: name }) + ) + )), + + batchRemove: (ids) => { + const placeholders = ids.map(() => "?").join(", ") + return resolveNamespace.pipe(Effect.flatMap((ns) => { + const sqlText = `DELETE FROM "${tableName}" WHERE id IN (${placeholders}) AND _namespace = ?` + return exec(sqlText, [...ids, ns]) .pipe( - Effect - .tap((q) => logQuery(q)), - Effect.tap((q) => Effect.annotateCurrentSpan({ "db.query.text": q.sql })), - Effect.flatMap((q) => - exec(q.sql, q.params).pipe( - Effect.map((rows) => { - if (f.select) { - return (rows as any[]).map((r) => { - const selected = parseSelectRow(r, idKey, activeRootLevelFieldColumns) - return { - ...Struct.pick( - defaultValues as any, - f.select!.filter((_) => typeof _ === "string") as never[] - ), - ...selected - } as M - }) - } - return (rows as any[]).map((r) => - parseRow(r, idKey, defaultValues, activeRootLevelFieldColumns) as any as M - ) - }) - ) - ), + Effect.asVoid, annotateDb({ - operation: "filter", + operation: "batchRemove", system, collection: tableName, namespace: ns, - entity: name + entity: name, + query: sqlText }) ) - )) - }, - - set: (e) => - resolveNamespace.pipe(Effect.flatMap((ns) => - setInternal(e, ns).pipe( - annotateDb({ - operation: "set", - system, - collection: tableName, - namespace: ns, - entity: name, - extra: { "app.entity.id": e[idKey] } - }) - ) - )), - - batchSet: (items) => - resolveNamespace.pipe(Effect.flatMap((ns) => - bulkSetInternal(items, ns).pipe( - annotateDb({ - operation: "batchSet", - system, - collection: tableName, - namespace: ns, - entity: name - }) - ) - )), - - bulkSet: (items) => - resolveNamespace.pipe(Effect.flatMap((ns) => - bulkSetInternal(items, ns).pipe( - annotateDb({ - operation: "bulkSet", - system, - collection: tableName, - namespace: ns, - entity: name - }) - ) - )), + })) + }, - batchRemove: (ids) => { - const placeholders = ids.map(() => "?").join(", ") - return resolveNamespace.pipe(Effect.flatMap((ns) => { - const sqlText = `DELETE FROM "${tableName}" WHERE id IN (${placeholders}) AND _namespace = ?` - return exec(sqlText, [...ids, ns]) - .pipe( - Effect.asVoid, + queryRaw: (query) => + s.all.pipe( + Effect.map(query.memory), annotateDb({ - operation: "batchRemove", + operation: "queryRaw", system, collection: tableName, - namespace: ns, - entity: name, - query: sqlText + entity: name }) ) - })) - }, - - queryRaw: (query) => - s.all.pipe( - Effect.map(query.memory), - annotateDb({ - operation: "queryRaw", - system, - collection: tableName, - entity: name - }) - ) - } + } - // Eagerly seed primary namespace on initialization - yield* seedNamespace("primary") + // Eagerly seed primary namespace on initialization + yield* seedNamespace("primary") - return s - }) + return s + } + ) + } } - }) + ) } type WithNsSqlFn = ( @@ -518,18 +529,24 @@ function makeSQLiteStorePerNs( if (activeRootLevelFieldColumns.length === 0) { return } - const existing = yield* exec(ns, `PRAGMA table_info(${JSON.stringify(tableName)})`) as Array<{ name: string }> + const existing = (yield* exec(ns, `PRAGMA table_info(${JSON.stringify(tableName)})`)) as Array< + { name: string } + > const existingColumns = new Set(existing.map((column) => column.name)) for (const column of activeRootLevelFieldColumns) { if (!existingColumns.has(column.columnName)) { yield* exec( ns, - `ALTER TABLE "${tableName}" ADD COLUMN ${quoteIdentifier(column.columnName)} ${projectedColumnSqlType(sqliteDialect, column.kind)}` + `ALTER TABLE "${tableName}" ADD COLUMN ${quoteIdentifier(column.columnName)} ${ + projectedColumnSqlType(sqliteDialect, column.kind) + }` ) } yield* exec( ns, - `UPDATE "${tableName}" SET ${quoteIdentifier(column.columnName)} = ${projectedColumnBackfillExpr(sqliteDialect, column)} WHERE ${quoteIdentifier(column.columnName)} IS NULL` + `UPDATE "${tableName}" SET ${quoteIdentifier(column.columnName)} = ${ + projectedColumnBackfillExpr(sqliteDialect, column) + } WHERE ${quoteIdentifier(column.columnName)} IS NULL` ) } })), @@ -619,12 +636,13 @@ function makeSQLiteStorePerNs( ) }) const seedNamespace = (ns: string) => { - let cached = seedCache.get(ns) - if (!cached) { - cached = Effect.cached(Effect.uninterruptible(makeSeedEffect(ns))).pipe(Effect.runSync) - seedCache.set(ns, cached) + const cached = seedCache.get(ns) + if (cached) { + return cached } - return cached + const next = Effect.cached(Effect.uninterruptible(makeSeedEffect(ns))).pipe(Effect.runSync) + seedCache.set(ns, next) + return next } const s: Store = { diff --git a/packages/infra/src/Store/SQL/Pg.ts b/packages/infra/src/Store/SQL/Pg.ts index 88d8fcce2..8de59d092 100644 --- a/packages/infra/src/Store/SQL/Pg.ts +++ b/packages/infra/src/Store/SQL/Pg.ts @@ -15,15 +15,7 @@ import { storeId } from "../Memory.js" import type { RootLevelFieldColumn } from "../rootLevelFields.js" import { type FilterArgs, type PersistenceModelType, type StorageConfig, type Store, type StoreConfig, StoreMaker } from "../service.js" import { makeETag } from "../utils.js" -import { - buildWhereSQLQuery, - logQuery, - normalizeProjectedColumnValue, - projectedColumnBackfillExpr, - projectedColumnSqlType, - quoteIdentifier, - pgDialect -} from "./query.js" +import { buildWhereSQLQuery, logQuery, normalizeProjectedColumnValue, pgDialect, projectedColumnBackfillExpr, projectedColumnSqlType, quoteIdentifier } from "./query.js" const parseRow = ( row: { id: string; _etag: string | null; data: unknown } & Record, @@ -69,368 +61,377 @@ const parseSelectRow = ( return result } -const makePgStore = Effect.fnUntraced(function*({ prefix, rootLevelFieldsWhenAvailable: rootLevelFieldsWhenAvailableDefault }: StorageConfig) { - const sql = yield* SqlClient.SqlClient - return { - make: Effect.fnUntraced(function*( - name: string, - idKey: IdKey, - seed?: Effect.Effect, E, R>, - config?: StoreConfig - ) { - type PM = PersistenceModelType - const tableName = `${prefix}${name}` - const defaultValues = config?.defaultValues ?? {} - const rootLevelFieldColumns = config?.rootLevelFieldColumns ?? [] - const useRootLevelFields = config?.rootLevelFieldsWhenAvailable ?? rootLevelFieldsWhenAvailableDefault ?? false - const activeRootLevelFieldColumns = useRootLevelFields ? rootLevelFieldColumns : [] - const selectColumnsSql = activeRootLevelFieldColumns.length > 0 - ? `, ${activeRootLevelFieldColumns.map((column) => quoteIdentifier(column.columnName)).join(", ")}` - : "" +const makePgStore = Effect.fnUntraced( + function*({ prefix, rootLevelFieldsWhenAvailable: rootLevelFieldsWhenAvailableDefault }: StorageConfig) { + const sql = yield* SqlClient.SqlClient + return { + make: Effect.fnUntraced(function*( + name: string, + idKey: IdKey, + seed?: Effect.Effect, E, R>, + config?: StoreConfig + ) { + type PM = PersistenceModelType + const tableName = `${prefix}${name}` + const defaultValues = config?.defaultValues ?? {} + const rootLevelFieldColumns = config?.rootLevelFieldColumns ?? [] + const useRootLevelFields = config?.rootLevelFieldsWhenAvailable ?? rootLevelFieldsWhenAvailableDefault ?? false + const activeRootLevelFieldColumns = useRootLevelFields ? rootLevelFieldColumns : [] + const selectColumnsSql = activeRootLevelFieldColumns.length > 0 + ? `, ${activeRootLevelFieldColumns.map((column) => quoteIdentifier(column.columnName)).join(", ")}` + : "" - const resolveNamespace = !config?.allowNamespace - ? Effect.succeed("primary") - : storeId.pipe(Effect.map((namespace) => { - if (namespace !== "primary" && !config.allowNamespace!(namespace)) { - throw new Error(`Namespace ${namespace} not allowed!`) - } - return namespace - })) + const resolveNamespace = !config?.allowNamespace + ? Effect.succeed("primary") + : storeId.pipe(Effect.map((namespace) => { + if (namespace !== "primary" && !config.allowNamespace!(namespace)) { + throw new Error(`Namespace ${namespace} not allowed!`) + } + return namespace + })) - const exec = (query: string, params?: readonly unknown[]) => sql.unsafe(query, params as any).pipe(Effect.orDie) + const exec = (query: string, params?: readonly unknown[]) => sql.unsafe(query, params as any).pipe(Effect.orDie) - const ensureTable = sql - .unsafe( - `CREATE TABLE IF NOT EXISTS "${tableName}" (id TEXT NOT NULL, _namespace TEXT NOT NULL DEFAULT 'primary', _etag TEXT, data JSONB NOT NULL, PRIMARY KEY (id, _namespace))` - ) - .pipe( - Effect.andThen( - sql.unsafe( - `CREATE TABLE IF NOT EXISTS "_migrations" (id TEXT NOT NULL, version TEXT NOT NULL, PRIMARY KEY (id, version))` - ) - ), - Effect.andThen( - Effect.forEach( - activeRootLevelFieldColumns, - (column) => - exec( - `ALTER TABLE "${tableName}" ADD COLUMN IF NOT EXISTS ${quoteIdentifier(column.columnName)} ${projectedColumnSqlType(pgDialect, column.kind)}` - ).pipe( - Effect.andThen( - exec( - `UPDATE "${tableName}" SET ${quoteIdentifier(column.columnName)} = ${projectedColumnBackfillExpr(pgDialect, column)} WHERE ${quoteIdentifier(column.columnName)} IS NULL` - ) + const ensureTable = sql + .unsafe( + `CREATE TABLE IF NOT EXISTS "${tableName}" (id TEXT NOT NULL, _namespace TEXT NOT NULL DEFAULT 'primary', _etag TEXT, data JSONB NOT NULL, PRIMARY KEY (id, _namespace))` + ) + .pipe( + Effect.andThen( + sql.unsafe( + `CREATE TABLE IF NOT EXISTS "_migrations" (id TEXT NOT NULL, version TEXT NOT NULL, PRIMARY KEY (id, version))` + ) + ), + Effect.andThen( + Effect.forEach( + activeRootLevelFieldColumns, + (column) => + exec( + `ALTER TABLE "${tableName}" ADD COLUMN IF NOT EXISTS ${quoteIdentifier(column.columnName)} ${ + projectedColumnSqlType(pgDialect, column.kind) + }` ) - ), - { discard: true } - ) - ), - Effect.orDie, - Effect.asVoid - ) + .pipe( + Effect.andThen( + exec( + `UPDATE "${tableName}" SET ${quoteIdentifier(column.columnName)} = ${ + projectedColumnBackfillExpr(pgDialect, column) + } WHERE ${quoteIdentifier(column.columnName)} IS NULL` + ) + ) + ), + { discard: true } + ) + ), + Effect.orDie, + Effect.asVoid + ) - const toRow = (e: PM) => { - const newE = makeETag(e) - const id = newE[idKey] as string - const { _etag, [idKey]: _id, ...rest } = newE as any - const data = JSON.stringify(rest) - const rootLevelFieldValues = activeRootLevelFieldColumns.map((column) => rest[column.key]) - return { id, _etag: newE._etag!, data, item: newE, rootLevelFieldValues } - } + const toRow = (e: PM) => { + const newE = makeETag(e) + const id = newE[idKey] as string + const { _etag, [idKey]: _id, ...rest } = newE as any + const data = JSON.stringify(rest) + const rootLevelFieldValues = activeRootLevelFieldColumns.map((column) => rest[column.key]) + return { id, _etag: newE._etag!, data, item: newE, rootLevelFieldValues } + } - const setInternal = Effect.fnUntraced(function*(e: PM, ns: string) { - const row = toRow(e) - if (e._etag) { - const projectedSetSql = activeRootLevelFieldColumns.length > 0 - ? `, ${ - activeRootLevelFieldColumns - .map((column, index) => `${quoteIdentifier(column.columnName)} = $${index + 3}`) - .join(", ") - }` - : "" - const idParam = row.rootLevelFieldValues.length + 3 - const etagParam = row.rootLevelFieldValues.length + 4 - const namespaceParam = row.rootLevelFieldValues.length + 5 - yield* exec( - `UPDATE "${tableName}" SET _etag = $1, data = $2${projectedSetSql} WHERE id = $${idParam} AND _etag = $${etagParam} AND _namespace = $${namespaceParam}`, - [row._etag, row.data, ...row.rootLevelFieldValues, row.id, e._etag, ns] - ) - const existing = yield* exec( - `SELECT _etag FROM "${tableName}" WHERE id = $1 AND _namespace = $2`, - [row.id, ns] - ) - const current = (existing as any[])[0] - if (!current || current._etag !== row._etag) { - if (current) { + const setInternal = Effect.fnUntraced(function*(e: PM, ns: string) { + const row = toRow(e) + if (e._etag) { + const projectedSetSql = activeRootLevelFieldColumns.length > 0 + ? `, ${ + activeRootLevelFieldColumns + .map((column, index) => `${quoteIdentifier(column.columnName)} = $${index + 3}`) + .join(", ") + }` + : "" + const idParam = row.rootLevelFieldValues.length + 3 + const etagParam = row.rootLevelFieldValues.length + 4 + const namespaceParam = row.rootLevelFieldValues.length + 5 + yield* exec( + `UPDATE "${tableName}" SET _etag = $1, data = $2${projectedSetSql} WHERE id = $${idParam} AND _etag = $${etagParam} AND _namespace = $${namespaceParam}`, + [row._etag, row.data, ...row.rootLevelFieldValues, row.id, e._etag, ns] + ) + const existing = yield* exec( + `SELECT _etag FROM "${tableName}" WHERE id = $1 AND _namespace = $2`, + [row.id, ns] + ) + const current = (existing as any[])[0] + if (!current || current._etag !== row._etag) { + if (current) { + return yield* new OptimisticConcurrencyException({ + type: name, + id: row.id, + current: current._etag, + found: e._etag, + code: 412 + }) + } return yield* new OptimisticConcurrencyException({ type: name, id: row.id, - current: current._etag, + current: "", found: e._etag, - code: 412 + code: 404 }) } - return yield* new OptimisticConcurrencyException({ - type: name, - id: row.id, - current: "", - found: e._etag, - code: 404 - }) + } else { + const projectedColumnsSql = activeRootLevelFieldColumns.length > 0 + ? `, ${activeRootLevelFieldColumns.map((column) => quoteIdentifier(column.columnName)).join(", ")}` + : "" + const projectedValuesSql = activeRootLevelFieldColumns + .map((_, index) => `$${index + 5}`) + .join(", ") + yield* exec( + `INSERT INTO "${tableName}" (id, _namespace, _etag, data${projectedColumnsSql}) VALUES ($1, $2, $3, $4${ + projectedValuesSql.length > 0 ? `, ${projectedValuesSql}` : "" + })`, + [row.id, ns, row._etag, row.data, ...row.rootLevelFieldValues] + ) } - } else { - const projectedColumnsSql = activeRootLevelFieldColumns.length > 0 - ? `, ${activeRootLevelFieldColumns.map((column) => quoteIdentifier(column.columnName)).join(", ")}` - : "" - const projectedValuesSql = activeRootLevelFieldColumns - .map((_, index) => `$${index + 5}`) - .join(", ") - yield* exec( - `INSERT INTO "${tableName}" (id, _namespace, _etag, data${projectedColumnsSql}) VALUES ($1, $2, $3, $4${ - projectedValuesSql.length > 0 ? `, ${projectedValuesSql}` : "" - })`, - [row.id, ns, row._etag, row.data, ...row.rootLevelFieldValues] - ) - } - return row.item - }) + return row.item + }) - const bulkSetInternal = (items: NonEmptyReadonlyArray, ns: string) => - sql - .withTransaction(Effect.forEach(items, (e) => setInternal(e, ns))) - .pipe( - Effect.orDie, - Effect.map((_) => _ as unknown as NonEmptyReadonlyArray) - ) + const bulkSetInternal = (items: NonEmptyReadonlyArray, ns: string) => + sql + .withTransaction(Effect.forEach(items, (e) => setInternal(e, ns))) + .pipe( + Effect.orDie, + Effect.map((_) => _ as unknown as NonEmptyReadonlyArray) + ) - const ctx = yield* Effect.context() - const seedCache = new Map>() - const makeSeedEffect = Effect.fnUntraced(function*(ns: string) { - yield* ensureTable - if (!seed) return - const existing = yield* exec( - `SELECT id FROM "_migrations" WHERE id = $1 AND version = $2`, - [`${tableName}::${ns}`, tableName] - ) - if ((existing as any[]).length > 0) return - yield* InfraLogger.logInfo(`Seeding data for ${name} (namespace: ${ns})`) - const items = yield* seed.pipe(Effect.provide(ctx), Effect.orDie) - const ne = toNonEmptyArray([...items]) - if (Option.isSome(ne)) yield* bulkSetInternal(ne.value, ns) - yield* exec( - `INSERT INTO "_migrations" (id, version) VALUES ($1, $2)`, - [`${tableName}::${ns}`, tableName] - ) - }) - const seedNamespace = (ns: string) => { - let cached = seedCache.get(ns) - if (!cached) { - cached = Effect.cached(Effect.uninterruptible(makeSeedEffect(ns))).pipe(Effect.runSync) - seedCache.set(ns, cached) + const ctx = yield* Effect.context() + const seedCache = new Map>() + const makeSeedEffect = Effect.fnUntraced(function*(ns: string) { + yield* ensureTable + if (!seed) return + const existing = yield* exec( + `SELECT id FROM "_migrations" WHERE id = $1 AND version = $2`, + [`${tableName}::${ns}`, tableName] + ) + if ((existing as any[]).length > 0) return + yield* InfraLogger.logInfo(`Seeding data for ${name} (namespace: ${ns})`) + const items = yield* seed.pipe(Effect.provide(ctx), Effect.orDie) + const ne = toNonEmptyArray([...items]) + if (Option.isSome(ne)) yield* bulkSetInternal(ne.value, ns) + yield* exec( + `INSERT INTO "_migrations" (id, version) VALUES ($1, $2)`, + [`${tableName}::${ns}`, tableName] + ) + }) + const seedNamespace = (ns: string) => { + let cached = seedCache.get(ns) + if (!cached) { + cached = Effect.cached(Effect.uninterruptible(makeSeedEffect(ns))).pipe(Effect.runSync) + seedCache.set(ns, cached) + } + return cached } - return cached - } - const s: Store = { - seedNamespace: (ns) => seedNamespace(ns), + const s: Store = { + seedNamespace: (ns) => seedNamespace(ns), - all: resolveNamespace.pipe( - Effect.flatMap((ns) => { - const sqlText = `SELECT id, _etag, data${selectColumnsSql} FROM "${tableName}" WHERE _namespace = $1` - return exec(sqlText, [ns]) - .pipe( - Effect.map((rows) => - (rows as any[]).map((r) => parseRow(r, idKey, defaultValues, activeRootLevelFieldColumns)) - ), - annotateDb({ - operation: "all", - system: "postgresql", - collection: tableName, - namespace: ns, - entity: name, - query: sqlText - }) - ) - }) - ), - - find: (id) => - resolveNamespace.pipe(Effect - .flatMap((ns) => { - const sqlText = `SELECT id, _etag, data${selectColumnsSql} FROM "${tableName}" WHERE id = $1 AND _namespace = $2` - return exec(sqlText, [id, ns]) + all: resolveNamespace.pipe( + Effect.flatMap((ns) => { + const sqlText = `SELECT id, _etag, data${selectColumnsSql} FROM "${tableName}" WHERE _namespace = $1` + return exec(sqlText, [ns]) .pipe( - Effect.map((rows) => { - const row = (rows as any[])[0] - return row - ? Option.some(parseRow(row, idKey, defaultValues, activeRootLevelFieldColumns)) - : Option.none() - }), + Effect.map((rows) => + (rows as any[]).map((r) => parseRow(r, idKey, defaultValues, activeRootLevelFieldColumns)) + ), annotateDb({ - operation: "find", + operation: "all", system: "postgresql", collection: tableName, namespace: ns, entity: name, - query: sqlText, - extra: { "app.entity.id": id } + query: sqlText }) ) - })), + }) + ), - filter: (f: FilterArgs) => { - const filter = f - .filter - type M = U extends undefined ? Encoded : Pick - return resolveNamespace.pipe(Effect.flatMap((ns) => - Effect - .sync(() => { - const q = buildWhereSQLQuery( - pgDialect, - idKey, - filter ? [{ t: "where-scope", result: filter, relation: "some" }] : [], - tableName, - defaultValues, - f.select as - | NonEmptyReadonlyArray< - string | { - key: string - subKeys: readonly string[] - } | { - key: string - computed: ComputedProjectionIrExpression - } - > - | undefined, - f.order, - f.skip, - f.limit, - undefined, - activeRootLevelFieldColumns - ) - const nsPlaceholder = pgDialect.placeholder(q.params.length + 1) - const hasWhere = q.sql.includes("WHERE") - const nsSql = hasWhere - ? q.sql.replace("WHERE", `WHERE _namespace = ${nsPlaceholder} AND`) - : q.sql.replace( - `FROM "${tableName}"`, - `FROM "${tableName}" WHERE _namespace = ${nsPlaceholder}` - ) - return { sql: nsSql, params: [...q.params, ns] } - }) - .pipe( - Effect.tap((q) => logQuery(q)), - Effect.tap((q) => Effect.annotateCurrentSpan({ "db.query.text": q.sql })), - Effect.flatMap((q) => - exec(q.sql, q.params).pipe( + find: (id) => + resolveNamespace.pipe(Effect + .flatMap((ns) => { + const sqlText = + `SELECT id, _etag, data${selectColumnsSql} FROM "${tableName}" WHERE id = $1 AND _namespace = $2` + return exec(sqlText, [id, ns]) + .pipe( Effect.map((rows) => { - if (f.select) { - return (rows as any[]).map((r) => { - const selected = parseSelectRow(r, idKey, {}, activeRootLevelFieldColumns) - return { - ...Struct.pick( - defaultValues as any, - f.select!.filter((_) => typeof _ === "string") as never[] - ), - ...selected - } as M - }) - } - return (rows as any[]).map((r) => - parseRow(r, idKey, defaultValues, activeRootLevelFieldColumns) as any as M - ) + const row = (rows as any[])[0] + return row + ? Option.some(parseRow(row, idKey, defaultValues, activeRootLevelFieldColumns)) + : Option.none() + }), + annotateDb({ + operation: "find", + system: "postgresql", + collection: tableName, + namespace: ns, + entity: name, + query: sqlText, + extra: { "app.entity.id": id } }) ) - ), + })), + + filter: (f: FilterArgs) => { + const filter = f + .filter + type M = U extends undefined ? Encoded : Pick + return resolveNamespace.pipe(Effect.flatMap((ns) => + Effect + .sync(() => { + const q = buildWhereSQLQuery( + pgDialect, + idKey, + filter ? [{ t: "where-scope", result: filter, relation: "some" }] : [], + tableName, + defaultValues, + f.select as + | NonEmptyReadonlyArray< + string | { + key: string + subKeys: readonly string[] + } | { + key: string + computed: ComputedProjectionIrExpression + } + > + | undefined, + f.order, + f.skip, + f.limit, + undefined, + activeRootLevelFieldColumns + ) + const nsPlaceholder = pgDialect.placeholder(q.params.length + 1) + const hasWhere = q.sql.includes("WHERE") + const nsSql = hasWhere + ? q.sql.replace("WHERE", `WHERE _namespace = ${nsPlaceholder} AND`) + : q.sql.replace( + `FROM "${tableName}"`, + `FROM "${tableName}" WHERE _namespace = ${nsPlaceholder}` + ) + return { sql: nsSql, params: [...q.params, ns] } + }) + .pipe( + Effect.tap((q) => logQuery(q)), + Effect.tap((q) => Effect.annotateCurrentSpan({ "db.query.text": q.sql })), + Effect.flatMap((q) => + exec(q.sql, q.params).pipe( + Effect.map((rows) => { + if (f.select) { + return (rows as any[]).map((r) => { + const selected = parseSelectRow(r, idKey, {}, activeRootLevelFieldColumns) + return { + ...Struct.pick( + defaultValues as any, + f.select!.filter((_) => typeof _ === "string") as never[] + ), + ...selected + } as M + }) + } + return (rows as any[]).map((r) => + parseRow(r, idKey, defaultValues, activeRootLevelFieldColumns) as any as M + ) + }) + ) + ), + annotateDb({ + operation: "filter", + system: "postgresql", + collection: tableName, + namespace: ns, + entity: name + }) + ) + )) + }, + + set: (e) => + resolveNamespace.pipe(Effect.flatMap((ns) => + setInternal(e, ns).pipe( annotateDb({ - operation: "filter", + operation: "set", + system: "postgresql", + collection: tableName, + namespace: ns, + entity: name, + extra: { "app.entity.id": e[idKey] } + }) + ) + )), + + batchSet: (items) => + resolveNamespace.pipe(Effect.flatMap((ns) => + bulkSetInternal(items, ns).pipe( + annotateDb({ + operation: "batchSet", system: "postgresql", collection: tableName, namespace: ns, entity: name }) ) - )) - }, + )), - set: (e) => - resolveNamespace.pipe(Effect.flatMap((ns) => - setInternal(e, ns).pipe( - annotateDb({ - operation: "set", - system: "postgresql", - collection: tableName, - namespace: ns, - entity: name, - extra: { "app.entity.id": e[idKey] } - }) - ) - )), + bulkSet: (items) => + resolveNamespace.pipe(Effect.flatMap((ns) => + bulkSetInternal(items, ns).pipe( + annotateDb({ + operation: "bulkSet", + system: "postgresql", + collection: tableName, + namespace: ns, + entity: name + }) + ) + )), - batchSet: (items) => - resolveNamespace.pipe(Effect.flatMap((ns) => - bulkSetInternal(items, ns).pipe( - annotateDb({ - operation: "batchSet", - system: "postgresql", - collection: tableName, - namespace: ns, - entity: name - }) - ) - )), + batchRemove: (ids) => { + const placeholders = ids.map((_, i) => `$${i + 1}`).join(", ") + const nsPlaceholder = `$${ids.length + 1}` + return resolveNamespace.pipe(Effect.flatMap((ns) => { + const sqlText = + `DELETE FROM "${tableName}" WHERE id IN (${placeholders}) AND _namespace = ${nsPlaceholder}` + return exec(sqlText, [...ids, ns]) + .pipe( + Effect.asVoid, + annotateDb({ + operation: "batchRemove", + system: "postgresql", + collection: tableName, + namespace: ns, + entity: name, + query: sqlText + }) + ) + })) + }, - bulkSet: (items) => - resolveNamespace.pipe(Effect.flatMap((ns) => - bulkSetInternal(items, ns).pipe( + queryRaw: (query) => + s.all.pipe( + Effect.map(query.memory), annotateDb({ - operation: "bulkSet", + operation: "queryRaw", system: "postgresql", collection: tableName, - namespace: ns, entity: name }) ) - )), - - batchRemove: (ids) => { - const placeholders = ids.map((_, i) => `$${i + 1}`).join(", ") - const nsPlaceholder = `$${ids.length + 1}` - return resolveNamespace.pipe(Effect.flatMap((ns) => { - const sqlText = `DELETE FROM "${tableName}" WHERE id IN (${placeholders}) AND _namespace = ${nsPlaceholder}` - return exec(sqlText, [...ids, ns]) - .pipe( - Effect.asVoid, - annotateDb({ - operation: "batchRemove", - system: "postgresql", - collection: tableName, - namespace: ns, - entity: name, - query: sqlText - }) - ) - })) - }, - - queryRaw: (query) => - s.all.pipe( - Effect.map(query.memory), - annotateDb({ - operation: "queryRaw", - system: "postgresql", - collection: tableName, - entity: name - }) - ) - } + } - // Eagerly seed primary namespace on initialization - yield* seedNamespace("primary") + // Eagerly seed primary namespace on initialization + yield* seedNamespace("primary") - return s - }) + return s + }) + } } -}) +) export function PgStoreLayer(cfg: StorageConfig) { return StoreMaker diff --git a/packages/infra/src/Store/SQL/query.ts b/packages/infra/src/Store/SQL/query.ts index 718e481f2..ccebf95a9 100644 --- a/packages/infra/src/Store/SQL/query.ts +++ b/packages/infra/src/Store/SQL/query.ts @@ -5,8 +5,8 @@ import { assertUnreachable } from "effect-app/utils" import { InfraLogger } from "../../logger.js" import type { FilterR, FilterResult } from "../../Model/filter/filterApi.js" import type { AggregateIrExpression, ComputedProjectionIrExpression, ComputedProjectionMathIrExpression } from "../../Model/query.js" -import type { RootLevelFieldColumn, RootLevelFieldColumnKind } from "../rootLevelFields.js" import { isRelationCheck } from "../codeFilter.js" +import type { RootLevelFieldColumn, RootLevelFieldColumnKind } from "../rootLevelFields.js" export interface SQLDialect { readonly jsonExtract: (path: string) => string diff --git a/packages/infra/src/Store/rootLevelFields.ts b/packages/infra/src/Store/rootLevelFields.ts index 47f390e3c..8834a77d9 100644 --- a/packages/infra/src/Store/rootLevelFields.ts +++ b/packages/infra/src/Store/rootLevelFields.ts @@ -37,7 +37,8 @@ const getScalarKind = (ast: SchemaAST.AST): RootLevelFieldColumnKind | undefined return undefined } case "Union": { - const scalarKinds = encoded.types + const scalarKinds = encoded + .types .flatMap((type) => { const kind = getScalarKind(type) return kind === undefined ? [] : [kind] diff --git a/packages/infra/test/sql-root-level-columns.test.ts b/packages/infra/test/sql-root-level-columns.test.ts deleted file mode 100644 index 9f913d239..000000000 --- a/packages/infra/test/sql-root-level-columns.test.ts +++ /dev/null @@ -1,133 +0,0 @@ -import { SqliteClient } from "@effect/sql-sqlite-node" -import * as Effect from "effect-app/Effect" -import * as Layer from "effect-app/Layer" -import * as S from "effect-app/Schema" -import { SqlClient } from "effect/unstable/sql" -import * as Redacted from "effect/Redacted" -import { describe, expect, it } from "vitest" -import { setupRequestContextFromCurrent } from "../src/api/setupRequest.js" -import { where } from "../src/Model/query.js" -import { makeRepo } from "../src/Model/Repository.js" -import { RepositoryRegistryLive } from "../src/Model/Repository/Registry.js" -import { SQLiteStoreLayer } from "../src/Store/SQL.js" - -class ProjectedItem extends S.Class("ProjectedItem")({ - id: S.String, - name: S.String, - age: S.Number, - active: S.Boolean, - createdAt: S.Date, - meta: S.Struct({ city: S.String }) -}) {} - -const tableName = "test_projectedItems" - -const makeStoreLayer = (rootLevelFieldsWhenAvailable?: boolean) => - Layer - .merge( - SQLiteStoreLayer({ - url: Redacted.make("sqlite://"), - prefix: "test_", - dbName: "test", - rootLevelFieldsWhenAvailable - }), - RepositoryRegistryLive - ) - .pipe(Layer.provide(SqliteClient.layer({ filename: ":memory:" }))) - -describe("SQLite root-level projected columns", () => { - it("creates and writes projected root-level columns when enabled per repository", async () => { - const createdAt = new Date("2024-01-02T03:04:05.000Z") - const result = await Effect - .gen(function*() { - const repo = yield* makeRepo("projectedItem", ProjectedItem, { - config: { rootLevelFieldsWhenAvailable: true } - }) - - yield* repo.saveAndPublish([{ - id: "1", - name: "Alice", - age: 30, - active: true, - createdAt, - meta: { city: "Oslo" } - }]) - - const sql = yield* SqlClient.SqlClient - const columns = yield* sql.unsafe(`PRAGMA table_info(${JSON.stringify(tableName)})`) as Array<{ name: string }> - const row = (yield* sql.unsafe( - `SELECT id, "__root_name", "__root_age", "__root_active", "__root_createdAt", data FROM "${tableName}"` - ) as any[])[0] - - return { columns, row } - }) - .pipe(setupRequestContextFromCurrent("sql root-level columns")) - .pipe(Effect.provide(makeStoreLayer())) - .pipe(Effect.runPromise) - - const columnNames = result.columns.map((column) => column.name) - expect(columnNames).toEqual(expect.arrayContaining([ - "__root_name", - "__root_age", - "__root_active", - "__root_createdAt" - ])) - expect(columnNames).not.toContain("__root_meta") - expect(result.row.__root_name).toBe("Alice") - expect(result.row.__root_age).toBe(30) - expect(result.row.__root_active).toBe(1) - expect(result.row.__root_createdAt).toBe(createdAt.toJSON()) - expect(JSON.parse(result.row.data)).toMatchObject({ - name: "Alice", - age: 30, - active: true, - meta: { city: "Oslo" } - }) - }) - - it("allows repository config to disable an adapter-level default", async () => { - const columnNames = await Effect - .gen(function*() { - yield* makeRepo("projectedItem", ProjectedItem, { - config: { rootLevelFieldsWhenAvailable: false } - }) - const sql = yield* SqlClient.SqlClient - const columns = yield* sql.unsafe(`PRAGMA table_info(${JSON.stringify(tableName)})`) as Array<{ name: string }> - return columns.map((column) => column.name) - }) - .pipe(setupRequestContextFromCurrent("sql root-level columns override")) - .pipe(Effect.provide(makeStoreLayer(true))) - .pipe(Effect.runPromise) - - expect(columnNames).not.toContain("__root_name") - expect(columnNames).not.toContain("__root_age") - }) - - it("queries legacy rows through JSON fallback when projected columns are null", async () => { - const rows = await Effect - .gen(function*() { - const repo = yield* makeRepo("projectedItem", ProjectedItem, { - config: { rootLevelFieldsWhenAvailable: true } - }) - const sql = yield* SqlClient.SqlClient - yield* sql.unsafe( - `INSERT INTO "${tableName}" (id, _namespace, _etag, data) VALUES (?, ?, ?, ?)`, - ["legacy", "primary", "etag1", JSON.stringify({ - name: "Legacy", - age: 42, - active: true, - createdAt: "2024-01-02T03:04:05.000Z", - meta: { city: "Oslo" } - })] - ) - return yield* repo.query(where("age", "gt", 40)) - }) - .pipe(setupRequestContextFromCurrent("sql root-level columns legacy")) - .pipe(Effect.provide(makeStoreLayer())) - .pipe(Effect.runPromise) - - expect(rows).toHaveLength(1) - expect(rows[0]!.id).toBe("legacy") - expect(rows[0]!.age).toBe(42) - }) -}) diff --git a/packages/infra/test/sql-store.test.ts b/packages/infra/test/sql-store.test.ts index d9b79c3e8..774ed1d37 100644 --- a/packages/infra/test/sql-store.test.ts +++ b/packages/infra/test/sql-store.test.ts @@ -1,8 +1,9 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ import type Sqlite from "better-sqlite3" import BetterSqlite from "better-sqlite3" +import * as S from "effect-app/Schema" import { describe, expect, it } from "vitest" -import type { RootLevelFieldColumn } from "../src/Store/rootLevelFields.js" +import { makeRootLevelFieldColumns, type RootLevelFieldColumn } from "../src/Store/rootLevelFields.js" import { parseRow } from "../src/Store/SQL.js" import { buildWhereSQLQuery, pgDialect, sqliteDialect } from "../src/Store/SQL/query.js" import { makeETag } from "../src/Store/utils.js" @@ -16,6 +17,29 @@ const projectedRootLevelFields: readonly RootLevelFieldColumn[] = [ { key: "flag", columnName: "__root_flag", kind: "boolean" } ] +describe("root-level projected field metadata", () => { + it("derives supported root-level scalar fields from encoded schema", () => { + const schema = S.Struct({ + id: S.String, + name: S.String, + age: S.Number, + active: S.Boolean, + createdAt: S.Date, + status: S.NullOr(S.String), + meta: S.Struct({ city: S.String }), + tags: S.Array(S.String) + }) + + expect(makeRootLevelFieldColumns(schema, "id")).toEqual([ + { key: "name", columnName: "__root_name", kind: "string" }, + { key: "age", columnName: "__root_age", kind: "number" }, + { key: "active", columnName: "__root_active", kind: "boolean" }, + { key: "createdAt", columnName: "__root_createdAt", kind: "string" }, + { key: "status", columnName: "__root_status", kind: "string" } + ]) + }) +}) + // --- Query builder unit tests --- describe("SQL query builder (SQLite dialect)", () => { From abcf7d1ead544252e3822af0b77ea5ea06916229 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 22 May 2026 18:59:38 +0000 Subject: [PATCH 03/11] refactor: tidy sql projection helpers Agent-Logs-Url: https://github.com/effect-app/libs/sessions/f324ecdc-fbb7-4471-9947-241bf85170ff Co-authored-by: patroza <42661+patroza@users.noreply.github.com> --- packages/infra/src/Store/SQL.ts | 12 ++++++------ packages/infra/src/Store/SQL/query.ts | 4 +++- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/packages/infra/src/Store/SQL.ts b/packages/infra/src/Store/SQL.ts index 7644857bc..469fa19b6 100644 --- a/packages/infra/src/Store/SQL.ts +++ b/packages/infra/src/Store/SQL.ts @@ -249,9 +249,9 @@ function makeSQLStoreInt(system: DbSystem, dialect: SQLDialect, jsonColumnType: ) }) const seedNamespace = (ns: string) => { - const cached = seedCache.get(ns) - if (cached) { - return cached + const existingEffect = seedCache.get(ns) + if (existingEffect) { + return existingEffect } const next = Effect.cached(Effect.uninterruptible(makeSeedEffect(ns))).pipe(Effect.runSync) seedCache.set(ns, next) @@ -636,9 +636,9 @@ function makeSQLiteStorePerNs( ) }) const seedNamespace = (ns: string) => { - const cached = seedCache.get(ns) - if (cached) { - return cached + const existingEffect = seedCache.get(ns) + if (existingEffect) { + return existingEffect } const next = Effect.cached(Effect.uninterruptible(makeSeedEffect(ns))).pipe(Effect.runSync) seedCache.set(ns, next) diff --git a/packages/infra/src/Store/SQL/query.ts b/packages/infra/src/Store/SQL/query.ts index ccebf95a9..dd92f2136 100644 --- a/packages/infra/src/Store/SQL/query.ts +++ b/packages/infra/src/Store/SQL/query.ts @@ -257,6 +257,8 @@ export function buildWhereSQLQuery( return dialect.placeholder(paramIndex++) } + const isRootLevelPath = (path: string, relation?: string) => !relation && !path.includes(".") + const fieldExpr = (path: string, relation?: string): string => { if (path === idKey || path === "id") return "id" if (relation && path.includes(".-1.")) { @@ -271,7 +273,7 @@ export function buildWhereSQLQuery( const arrPath = dottedToJsonPath(path.slice(0, -".length".length)) return dialect.arrayLength(arrPath) } - if (!relation && !path.includes(".")) { + if (isRootLevelPath(path, relation)) { const projected = projectedColumns.get(path) if (projected) { const expr = projectedColumnFieldExpr(dialect, projected) From d5b5d5f48ae1826b7128b0ce9f353e1886633ec9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 22 May 2026 19:00:21 +0000 Subject: [PATCH 04/11] refactor: rename cached seed effect Agent-Logs-Url: https://github.com/effect-app/libs/sessions/f324ecdc-fbb7-4471-9947-241bf85170ff Co-authored-by: patroza <42661+patroza@users.noreply.github.com> --- packages/infra/src/Store/SQL.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/infra/src/Store/SQL.ts b/packages/infra/src/Store/SQL.ts index 469fa19b6..a1c31e97c 100644 --- a/packages/infra/src/Store/SQL.ts +++ b/packages/infra/src/Store/SQL.ts @@ -253,9 +253,9 @@ function makeSQLStoreInt(system: DbSystem, dialect: SQLDialect, jsonColumnType: if (existingEffect) { return existingEffect } - const next = Effect.cached(Effect.uninterruptible(makeSeedEffect(ns))).pipe(Effect.runSync) - seedCache.set(ns, next) - return next + const seedEffect = Effect.cached(Effect.uninterruptible(makeSeedEffect(ns))).pipe(Effect.runSync) + seedCache.set(ns, seedEffect) + return seedEffect } const s: Store = { seedNamespace: (ns) => seedNamespace(ns), @@ -640,9 +640,9 @@ function makeSQLiteStorePerNs( if (existingEffect) { return existingEffect } - const next = Effect.cached(Effect.uninterruptible(makeSeedEffect(ns))).pipe(Effect.runSync) - seedCache.set(ns, next) - return next + const seedEffect = Effect.cached(Effect.uninterruptible(makeSeedEffect(ns))).pipe(Effect.runSync) + seedCache.set(ns, seedEffect) + return seedEffect } const s: Store = { From 24782ef139c8c556cc0a5828a178f32f8770e1f3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 22 May 2026 19:35:39 +0000 Subject: [PATCH 05/11] docs: outline relational follow-up plan Agent-Logs-Url: https://github.com/effect-app/libs/sessions/ffdbe97c-c62f-4ae1-8a79-57e8f539221a Co-authored-by: patroza <42661+patroza@users.noreply.github.com> --- packages/infra/src/Model/Repository/makeRepo.ts | 8 ++++++++ packages/infra/src/Store/service.ts | 13 +++++++++++++ 2 files changed, 21 insertions(+) diff --git a/packages/infra/src/Model/Repository/makeRepo.ts b/packages/infra/src/Model/Repository/makeRepo.ts index 3853844df..0fd308486 100644 --- a/packages/infra/src/Model/Repository/makeRepo.ts +++ b/packages/infra/src/Model/Repository/makeRepo.ts @@ -40,6 +40,14 @@ export interface RepositoryOptions< * use the config.defaultValues instead for simple default values */ jitM?: (pm: Encoded) => Encoded + /** + * Repository storage options. + * + * `rootLevelFieldsWhenAvailable` is the Phase 1 hybrid mode: keep document-style + * JSON storage for nested values, while letting SQL adapters project eligible + * root-level scalar fields into real columns. The planned next step for relational + * fields is a separate explicit mode with relation metadata and join planning. + */ config?: Omit, "partitionValue"> & { partitionValue?: (e?: Encoded) => string } diff --git a/packages/infra/src/Store/service.ts b/packages/infra/src/Store/service.ts index bdec69c22..eb90cbd61 100644 --- a/packages/infra/src/Store/service.ts +++ b/packages/infra/src/Store/service.ts @@ -15,6 +15,14 @@ import type { RootLevelFieldColumn } from "./rootLevelFields.js" export interface StoreConfig { partitionValue: (e?: E) => string + /** + * Phase 1: keep the document store model, but project eligible root-level scalar + * fields into real SQL columns when the adapter supports it. + * + * Nested objects / arrays and the current relation-like JSON-array semantics stay + * in `data`. The next step for actual relational fields is a separate mode with + * explicit PK/FK metadata, ownership/cardinality, migrations, and join planning. + */ rootLevelFieldsWhenAvailable?: boolean rootLevelFieldColumns?: readonly RootLevelFieldColumn[] /** @@ -233,5 +241,10 @@ export interface StorageConfig { url: Redacted.Redacted prefix: string dbName: string + /** + * Adapter default for Phase 1 root-level projected columns. Repositories can still + * opt in / out per table. Phase 2 relational support is expected to use a separate + * mode rather than extending this flag. + */ rootLevelFieldsWhenAvailable?: boolean } From c58eb8d44a877706cd05f7cfdb3fdf224ac1f329 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 22 May 2026 19:45:28 +0000 Subject: [PATCH 06/11] feat: project root fields without duplicating data Agent-Logs-Url: https://github.com/effect-app/libs/sessions/2c26dc3a-b352-4999-b5d5-98349e52640c Co-authored-by: patroza <42661+patroza@users.noreply.github.com> --- packages/infra/src/Store/SQL.ts | 18 +- packages/infra/src/Store/SQL/Pg.ts | 10 +- packages/infra/src/Store/SQL/query.ts | 255 ++++++++++++++++++-- packages/infra/src/Store/rootLevelFields.ts | 32 ++- packages/infra/test/sql-store.test.ts | 173 ++++++++++++- 5 files changed, 445 insertions(+), 43 deletions(-) diff --git a/packages/infra/src/Store/SQL.ts b/packages/infra/src/Store/SQL.ts index a1c31e97c..447d69633 100644 --- a/packages/infra/src/Store/SQL.ts +++ b/packages/infra/src/Store/SQL.ts @@ -15,7 +15,7 @@ import type { FieldValues } from "../Model/filter/types.js" import type { ComputedProjectionIrExpression } from "../Model/query.js" import { annotateDb, type DbSystem } from "../otel.js" import { storeId } from "./Memory.js" -import type { RootLevelFieldColumn } from "./rootLevelFields.js" +import { omitRootLevelFieldColumnsFromData, type RootLevelFieldColumn } from "./rootLevelFields.js" import { type FilterArgs, type PersistenceModelType, type StorageConfig, type Store, type StoreConfig, StoreMaker } from "./service.js" import { buildWhereSQLQuery, logQuery, normalizeProjectedColumnValue, projectedColumnBackfillExpr, projectedColumnSqlType, quoteIdentifier, type SQLDialect, sqliteDialect } from "./SQL/query.js" import { makeETag } from "./utils.js" @@ -76,14 +76,12 @@ const parseSelectRow = ( return result } -const serializeProjectedValue = ( - system: DbSystem, - column: RootLevelFieldColumn, - value: unknown -) => - column.kind === "boolean" && system === "sqlite" && value !== null && value !== undefined +const serializeProjectedValue = (system: DbSystem, column: RootLevelFieldColumn, value: unknown) => + column.kind === "json" + ? value === undefined ? null : JSON.stringify(value) + : column.kind === "boolean" && system === "sqlite" && value !== null && value !== undefined ? value === true ? 1 : 0 - : value + : value ?? null function makeSQLStoreInt(system: DbSystem, dialect: SQLDialect, jsonColumnType: string) { return Effect.fnUntraced( @@ -164,7 +162,7 @@ function makeSQLStoreInt(system: DbSystem, dialect: SQLDialect, jsonColumnType: const newE = makeETag(e) const id = newE[idKey] as string const { _etag, [idKey]: _id, ...rest } = newE as any - const data = JSON.stringify(rest) + const data = JSON.stringify(omitRootLevelFieldColumnsFromData(rest, activeRootLevelFieldColumns)) const rootLevelFieldValues = activeRootLevelFieldColumns.map((column) => serializeProjectedValue(system, column, rest[column.key]) ) @@ -503,7 +501,7 @@ function makeSQLiteStorePerNs( const newE = makeETag(e) const id = newE[idKey] as string const { _etag, [idKey]: _id, ...rest } = newE as any - const data = JSON.stringify(rest) + const data = JSON.stringify(omitRootLevelFieldColumnsFromData(rest, activeRootLevelFieldColumns)) const rootLevelFieldValues = activeRootLevelFieldColumns.map((column) => serializeProjectedValue("sqlite", column, rest[column.key]) ) diff --git a/packages/infra/src/Store/SQL/Pg.ts b/packages/infra/src/Store/SQL/Pg.ts index 8de59d092..c689213cc 100644 --- a/packages/infra/src/Store/SQL/Pg.ts +++ b/packages/infra/src/Store/SQL/Pg.ts @@ -12,7 +12,7 @@ import type { FieldValues } from "../../Model/filter/types.js" import type { ComputedProjectionIrExpression } from "../../Model/query.js" import { annotateDb } from "../../otel.js" import { storeId } from "../Memory.js" -import type { RootLevelFieldColumn } from "../rootLevelFields.js" +import { omitRootLevelFieldColumnsFromData, type RootLevelFieldColumn } from "../rootLevelFields.js" import { type FilterArgs, type PersistenceModelType, type StorageConfig, type Store, type StoreConfig, StoreMaker } from "../service.js" import { makeETag } from "../utils.js" import { buildWhereSQLQuery, logQuery, normalizeProjectedColumnValue, pgDialect, projectedColumnBackfillExpr, projectedColumnSqlType, quoteIdentifier } from "./query.js" @@ -131,8 +131,12 @@ const makePgStore = Effect.fnUntraced( const newE = makeETag(e) const id = newE[idKey] as string const { _etag, [idKey]: _id, ...rest } = newE as any - const data = JSON.stringify(rest) - const rootLevelFieldValues = activeRootLevelFieldColumns.map((column) => rest[column.key]) + const data = JSON.stringify(omitRootLevelFieldColumnsFromData(rest, activeRootLevelFieldColumns)) + const rootLevelFieldValues = activeRootLevelFieldColumns.map((column) => + column.kind === "json" + ? rest[column.key] === undefined ? null : JSON.stringify(rest[column.key]) + : rest[column.key] ?? null + ) return { id, _etag: newE._etag!, data, item: newE, rootLevelFieldValues } } diff --git a/packages/infra/src/Store/SQL/query.ts b/packages/infra/src/Store/SQL/query.ts index dd92f2136..2b3b26647 100644 --- a/packages/infra/src/Store/SQL/query.ts +++ b/packages/infra/src/Store/SQL/query.ts @@ -158,6 +158,8 @@ const projectedColumnJsonFallbackExpr = ( return dialect.jsonColumnType === "JSON" ? expr : `(${expr})::double precision` case "boolean": return dialect.jsonColumnType === "JSON" ? expr : `(${expr})::boolean` + case "json": + return extractJsonFromSourceExpr(dialect, "data", column.key, true) default: return assertUnreachable(column.kind) } @@ -187,6 +189,8 @@ export const projectedColumnSqlType = ( return dialect.jsonColumnType === "JSON" ? "REAL" : "DOUBLE PRECISION" case "boolean": return dialect.jsonColumnType === "JSON" ? "INTEGER" : "BOOLEAN" + case "json": + return dialect.jsonColumnType default: return assertUnreachable(kind) } @@ -201,6 +205,16 @@ export const normalizeProjectedColumnValue = ( column: RootLevelFieldColumn, value: unknown ) => { + if (column.kind === "json") { + if (typeof value === "string") { + try { + return JSON.parse(value) + } catch { + return value + } + } + return value + } if (column.kind === "boolean") { if (typeof value === "number") { return value !== 0 @@ -219,6 +233,137 @@ const dottedToJsonPath = (path: string) => .filter((p) => p !== "-1") .join(".") +const sqliteJsonPathArg = (path: string) => `'${(path.length > 0 ? `$.${path}` : "$").replaceAll("'", "''")}'` + +const extractJsonFromSourceExpr = ( + dialect: SQLDialect, + sourceExpr: string, + path: string, + asJson: boolean +) => { + if (dialect.jsonColumnType === "JSON") { + const pathArg = sqliteJsonPathArg(path) + if (!asJson) { + return `json_extract(${sourceExpr}, ${pathArg})` + } + return `CASE WHEN json_type(${sourceExpr}, ${pathArg}) IS NULL THEN NULL WHEN json_type(${sourceExpr}, ${pathArg}) = 'true' THEN 'true' WHEN json_type(${sourceExpr}, ${pathArg}) = 'false' THEN 'false' ELSE json_quote(json_extract(${sourceExpr}, ${pathArg})) END` + } + + const parts: ReadonlyArray = path.length > 0 ? path.split(".") : [] + if (parts.length === 0) { + return asJson ? `(${sourceExpr})` : `(${sourceExpr})#>> '{}'` + } + if (!asJson) { + const last = parts.at(-1)! + const parents = parts.slice(0, -1) + return `(${sourceExpr})${parents.map((part) => `->'${part}'`).join("")}->>'${last}'` + } + return `(${sourceExpr})${parts.map((part) => `->'${part}'`).join("")}` +} + +const arrayLengthFromSourceExpr = ( + dialect: SQLDialect, + sourceExpr: string, + path: string +) => + dialect.jsonColumnType === "JSON" + ? `json_array_length(${sourceExpr}, ${sqliteJsonPathArg(path)})` + : `jsonb_array_length(${extractJsonFromSourceExpr(dialect, sourceExpr, path, true)})` + +const jsonEachFromSourceExpr = ( + dialect: SQLDialect, + sourceExpr: string, + path: string, + alias: string +) => + dialect.jsonColumnType === "JSON" + ? `json_each(${sourceExpr}, ${sqliteJsonPathArg(path)}) AS ${alias}` + : `jsonb_array_elements(${extractJsonFromSourceExpr(dialect, sourceExpr, path, true)}) AS ${alias}` + +const jsonArrayContainsFromSourceExpr = ( + dialect: SQLDialect, + sourceExpr: string, + path: string, + valPlaceholder: string +) => + dialect.jsonColumnType === "JSON" + ? `EXISTS(SELECT 1 FROM json_each(${sourceExpr}, ${sqliteJsonPathArg(path)}) WHERE value = ${valPlaceholder})` + : `${extractJsonFromSourceExpr(dialect, sourceExpr, path, true)} @> ${valPlaceholder}::jsonb` + +const jsonArrayNotContainsFromSourceExpr = ( + dialect: SQLDialect, + sourceExpr: string, + path: string, + valPlaceholder: string +) => + dialect.jsonColumnType === "JSON" + ? `NOT EXISTS(SELECT 1 FROM json_each(${sourceExpr}, ${sqliteJsonPathArg(path)}) WHERE value = ${valPlaceholder})` + : `NOT (${extractJsonFromSourceExpr(dialect, sourceExpr, path, true)} @> ${valPlaceholder}::jsonb)` + +const jsonArrayContainsAnyFromSourceExpr = ( + dialect: SQLDialect, + sourceExpr: string, + path: string, + valPlaceholders: readonly string[] +) => + dialect.jsonColumnType === "JSON" + ? `EXISTS(SELECT 1 FROM json_each(${sourceExpr}, ${sqliteJsonPathArg(path)}) WHERE value IN (${ + valPlaceholders.join(", ") + }))` + : `(${ + valPlaceholders.map((v) => `${extractJsonFromSourceExpr(dialect, sourceExpr, path, true)} @> ${v}::jsonb`).join( + " OR " + ) + })` + +const jsonArrayNotContainsAnyFromSourceExpr = ( + dialect: SQLDialect, + sourceExpr: string, + path: string, + valPlaceholders: readonly string[] +) => + dialect.jsonColumnType === "JSON" + ? `NOT EXISTS(SELECT 1 FROM json_each(${sourceExpr}, ${sqliteJsonPathArg(path)}) WHERE value IN (${ + valPlaceholders.join(", ") + }))` + : `NOT (${ + valPlaceholders.map((v) => `${extractJsonFromSourceExpr(dialect, sourceExpr, path, true)} @> ${v}::jsonb`).join( + " OR " + ) + })` + +const jsonArrayContainsAllFromSourceExpr = ( + dialect: SQLDialect, + sourceExpr: string, + path: string, + valPlaceholders: readonly string[] +) => + dialect.jsonColumnType === "JSON" + ? valPlaceholders + .map((v) => `EXISTS(SELECT 1 FROM json_each(${sourceExpr}, ${sqliteJsonPathArg(path)}) WHERE value = ${v})`) + .join(" AND ") + : valPlaceholders.map((v) => `${extractJsonFromSourceExpr(dialect, sourceExpr, path, true)} @> ${v}::jsonb`).join( + " AND " + ) + +const jsonArrayNotContainsAllFromSourceExpr = ( + dialect: SQLDialect, + sourceExpr: string, + path: string, + valPlaceholders: readonly string[] +) => + dialect.jsonColumnType === "JSON" + ? `NOT (${ + valPlaceholders + .map((v) => `EXISTS(SELECT 1 FROM json_each(${sourceExpr}, ${sqliteJsonPathArg(path)}) WHERE value = ${v})`) + .join(" AND ") + })` + : `NOT (${ + valPlaceholders.map((v) => `${extractJsonFromSourceExpr(dialect, sourceExpr, path, true)} @> ${v}::jsonb`).join( + " AND " + ) + })` + const sqlStringLiteral = (value: string) => `'${value.replaceAll("'", "''")}'` export function buildWhereSQLQuery( @@ -251,12 +396,31 @@ export function buildWhereSQLQuery( const params: unknown[] = [] let paramIndex = 1 const projectedColumns = new Map(rootLevelFieldColumns.map((column) => [column.key, column] as const)) + const projectedJsonColumnForPath = (path: string, relation?: string) => { + if (relation) { + return + } + const [topKey, ...subPathParts] = path.split(".") + const projected = projectedColumns.get(topKey) + if (!projected || projected.kind !== "json") { + return + } + return { + sourceExpr: projectedColumnFieldExpr(dialect, projected), + subPath: subPathParts.join(".") + } + } const addParam = (value: unknown): string => { params.push(dialect.serializeScalar(value)) return dialect.placeholder(paramIndex++) } + const addJsonParam = (value: unknown): string => { + params.push(dialect.jsonColumnType === "JSON" ? JSON.stringify(value) : JSON.stringify(value)) + return dialect.placeholder(paramIndex++) + } + const isRootLevelPath = (path: string, relation?: string) => !relation && !path.includes(".") const fieldExpr = (path: string, relation?: string): string => { @@ -271,6 +435,10 @@ export function buildWhereSQLQuery( } if (path.endsWith(".length")) { const arrPath = dottedToJsonPath(path.slice(0, -".length".length)) + const projectedJson = projectedJsonColumnForPath(arrPath, relation) + if (projectedJson) { + return arrayLengthFromSourceExpr(dialect, projectedJson.sourceExpr, projectedJson.subPath) + } return dialect.arrayLength(arrPath) } if (isRootLevelPath(path, relation)) { @@ -278,11 +446,17 @@ export function buildWhereSQLQuery( if (projected) { const expr = projectedColumnFieldExpr(dialect, projected) if (path in defaultValues) { - return `COALESCE(${expr}, ${addParam(defaultValues[path])})` + return `COALESCE(${expr}, ${ + projected.kind === "json" ? addJsonParam(defaultValues[path]) : addParam(defaultValues[path]) + })` } return expr } } + const projectedJson = projectedJsonColumnForPath(path, relation) + if (projectedJson) { + return extractJsonFromSourceExpr(dialect, projectedJson.sourceExpr, projectedJson.subPath, false) + } const jsonPath = dottedToJsonPath(path) const expr = dialect.jsonExtract(jsonPath) const topKey = path.split(".")[0] @@ -295,6 +469,14 @@ export function buildWhereSQLQuery( const statement = (x: FilterR, relation?: string): string => { const resolvedPath = x.path === idKey ? "id" : x.path const k = fieldExpr(resolvedPath, relation) + const projectedRoot = isRootLevelPath(resolvedPath, relation) + ? projectedColumns.get(resolvedPath) + : undefined + const isProjectedJsonRoot = projectedRoot?.kind === "json" + const jsonComparisonParam = (value: unknown) => { + const placeholder = addJsonParam(value) + return dialect.jsonColumnType === "JSON" ? placeholder : `${placeholder}::jsonb` + } switch (x.op) { case "in": { @@ -303,7 +485,7 @@ export function buildWhereSQLQuery( const nonNullVals = vals.filter((v) => v != null) const parts: string[] = [] if (nonNullVals.length > 0) { - const placeholders = nonNullVals.map((v) => addParam(v)) + const placeholders = nonNullVals.map((v) => isProjectedJsonRoot ? jsonComparisonParam(v) : addParam(v)) parts.push(`${k} IN (${placeholders.join(", ")})`) } if (hasNull) parts.push(`${k} IS NULL`) @@ -315,7 +497,7 @@ export function buildWhereSQLQuery( const nonNullVals = vals.filter((v) => v != null) const parts: string[] = [] if (nonNullVals.length > 0) { - const placeholders = nonNullVals.map((v) => addParam(v)) + const placeholders = nonNullVals.map((v) => isProjectedJsonRoot ? jsonComparisonParam(v) : addParam(v)) parts.push(`${k} NOT IN (${placeholders.join(", ")})`) } if (hasNull) parts.push(`${k} IS NOT NULL`) @@ -325,38 +507,66 @@ export function buildWhereSQLQuery( case "includes": { const arrPath = dottedToJsonPath(resolvedPath) const v = addParam(x.value) - return dialect.jsonArrayContains(arrPath, v) + const projectedJson = projectedJsonColumnForPath(arrPath, relation) + return projectedJson + ? jsonArrayContainsFromSourceExpr(dialect, projectedJson.sourceExpr, projectedJson.subPath, v) + : dialect.jsonArrayContains(arrPath, v) } case "notIncludes": { const arrPath = dottedToJsonPath(resolvedPath) const v = addParam(x.value) - return dialect.jsonArrayNotContains(arrPath, v) + const projectedJson = projectedJsonColumnForPath(arrPath, relation) + return projectedJson + ? jsonArrayNotContainsFromSourceExpr(dialect, projectedJson.sourceExpr, projectedJson.subPath, v) + : dialect.jsonArrayNotContains(arrPath, v) } case "includes-any": { const arrPath = dottedToJsonPath(resolvedPath) const vals = x.value as unknown as readonly unknown[] const placeholders = vals.map((v) => addParam(dialect.serializeJsonValue(v))) - return dialect.jsonArrayContainsAny(arrPath, placeholders) + const projectedJson = projectedJsonColumnForPath(arrPath, relation) + return projectedJson + ? jsonArrayContainsAnyFromSourceExpr(dialect, projectedJson.sourceExpr, projectedJson.subPath, placeholders) + : dialect.jsonArrayContainsAny(arrPath, placeholders) } case "notIncludes-any": { const arrPath = dottedToJsonPath(resolvedPath) const vals = x.value as unknown as readonly unknown[] const placeholders = vals.map((v) => addParam(dialect.serializeJsonValue(v))) - return dialect.jsonArrayNotContainsAny(arrPath, placeholders) + const projectedJson = projectedJsonColumnForPath(arrPath, relation) + return projectedJson + ? jsonArrayNotContainsAnyFromSourceExpr( + dialect, + projectedJson.sourceExpr, + projectedJson.subPath, + placeholders + ) + : dialect.jsonArrayNotContainsAny(arrPath, placeholders) } case "includes-all": { const arrPath = dottedToJsonPath(resolvedPath) const vals = x.value as unknown as readonly unknown[] const placeholders = vals.map((v) => addParam(dialect.serializeJsonValue(v))) - return dialect.jsonArrayContainsAll(arrPath, placeholders) + const projectedJson = projectedJsonColumnForPath(arrPath, relation) + return projectedJson + ? jsonArrayContainsAllFromSourceExpr(dialect, projectedJson.sourceExpr, projectedJson.subPath, placeholders) + : dialect.jsonArrayContainsAll(arrPath, placeholders) } case "notIncludes-all": { const arrPath = dottedToJsonPath(resolvedPath) const vals = x.value as unknown as readonly unknown[] const placeholders = vals.map((v) => addParam(dialect.serializeJsonValue(v))) - return dialect.jsonArrayNotContainsAll(arrPath, placeholders) + const projectedJson = projectedJsonColumnForPath(arrPath, relation) + return projectedJson + ? jsonArrayNotContainsAllFromSourceExpr( + dialect, + projectedJson.sourceExpr, + projectedJson.subPath, + placeholders + ) + : dialect.jsonArrayNotContainsAll(arrPath, placeholders) } case "contains": { @@ -402,13 +612,13 @@ export function buildWhereSQLQuery( } case "neq": { if (x.value === null) return `${k} IS NOT NULL` - const v = addParam(x.value) + const v = isProjectedJsonRoot ? jsonComparisonParam(x.value) : addParam(x.value) return `${k} <> ${v}` } case undefined: case "eq": { if (x.value === null) return `${k} IS NULL` - const v = addParam(x.value) + const v = isProjectedJsonRoot ? jsonComparisonParam(x.value) : addParam(x.value) return `${k} = ${v}` } default: @@ -420,7 +630,10 @@ export function buildWhereSQLQuery( // Optimize tautological/contradictory conditions if (every && inner === "1=1") return "1=1" if (!every && inner === "1=0") return "1=0" - const from = dialect.jsonEachFrom(rel, `_${rel}`) + const projectedJson = projectedJsonColumnForPath(rel) + const from = projectedJson + ? jsonEachFromSourceExpr(dialect, projectedJson.sourceExpr, projectedJson.subPath, `_${rel}`) + : dialect.jsonEachFrom(rel, `_${rel}`) // ∀x.P(x) ≡ ¬∃x.¬P(x), i.e. NOT EXISTS(... WHERE NOT P) return every ? `NOT EXISTS(SELECT 1 FROM ${from} WHERE NOT (${inner}))` @@ -484,7 +697,10 @@ export function buildWhereSQLQuery( const computedSelectExpr = (key: string, computed: ComputedProjectionIrExpression): string => { const relationPath = dottedToJsonPath(computed.path) const relationAlias = `_${computed.path}` - const relationFrom = dialect.jsonEachFrom(relationPath, relationAlias) + const projectedJson = projectedJsonColumnForPath(relationPath) + const relationFrom = projectedJson + ? jsonEachFromSourceExpr(dialect, projectedJson.sourceExpr, projectedJson.subPath, relationAlias) + : dialect.jsonEachFrom(relationPath, relationAlias) const toNumber = (expr: string) => dialect.jsonColumnType === "JSON" ? `CAST(${expr} AS REAL)` : `(${expr})::numeric` const compileExpr = (expression: ComputedProjectionMathIrExpression): string => { @@ -563,7 +779,11 @@ export function buildWhereSQLQuery( } case "relation-length": { const arrPath = dottedToJsonPath(computed.path) - return `${dialect.arrayLength(arrPath)} AS "${key}"` + const projectedJson = projectedJsonColumnForPath(arrPath) + const expr = projectedJson + ? arrayLengthFromSourceExpr(dialect, projectedJson.sourceExpr, projectedJson.subPath) + : dialect.arrayLength(arrPath) + return `${expr} AS "${key}"` } case "relation-collect-fields": { const branches = computed.fields.map((field) => { @@ -612,7 +832,12 @@ export function buildWhereSQLQuery( : false const getSelectExpr = (): string => { - if (!select) return "id, _etag, data" + if (!select) { + const projectedSelect = rootLevelFieldColumns.length > 0 + ? `, ${rootLevelFieldColumns.map((column) => quoteIdentifier(column.columnName)).join(", ")}` + : "" + return `id, _etag, data${projectedSelect}` + } const fields = select.map((s) => { if (typeof s === "string") { if (s === idKey || s === "id") return `id` diff --git a/packages/infra/src/Store/rootLevelFields.ts b/packages/infra/src/Store/rootLevelFields.ts index 8834a77d9..7d1868dc4 100644 --- a/packages/infra/src/Store/rootLevelFields.ts +++ b/packages/infra/src/Store/rootLevelFields.ts @@ -1,7 +1,7 @@ import type * as S from "effect-app/Schema" import * as SchemaAST from "effect/SchemaAST" -export type RootLevelFieldColumnKind = "string" | "number" | "boolean" +export type RootLevelFieldColumnKind = "string" | "number" | "boolean" | "json" export interface RootLevelFieldColumn { readonly key: string @@ -16,7 +16,7 @@ const walkTransformation = (ast: SchemaAST.AST): SchemaAST.AST => { return ast } -const getScalarKind = (ast: SchemaAST.AST): RootLevelFieldColumnKind | undefined => { +const getColumnKind = (ast: SchemaAST.AST): RootLevelFieldColumnKind => { const encoded = walkTransformation(SchemaAST.toEncoded(ast)) switch (encoded._tag) { case "String": @@ -34,20 +34,20 @@ const getScalarKind = (ast: SchemaAST.AST): RootLevelFieldColumnKind | undefined case "boolean": return "boolean" default: - return undefined + return "json" } case "Union": { const scalarKinds = encoded .types .flatMap((type) => { - const kind = getScalarKind(type) - return kind === undefined ? [] : [kind] + const kind = getColumnKind(type) + return kind === "json" ? [] : [kind] }) const distinctKinds = [...new Set(scalarKinds)] - return distinctKinds.length === 1 ? distinctKinds[0] : undefined + return distinctKinds.length === 1 ? distinctKinds[0]! : "json" } default: - return undefined + return "json" } } @@ -67,9 +67,19 @@ export const makeRootLevelFieldColumns = ( if (key === String(idKey) || key === "id" || key === "_etag") { return [] } - const kind = getScalarKind(property.type) - return kind === undefined - ? [] - : [{ key, kind, columnName: makeColumnName(key) }] + return [{ key, kind: getColumnKind(property.type), columnName: makeColumnName(key) }] }) } + +export const omitRootLevelFieldColumnsFromData = >( + data: A, + rootLevelFieldColumns: readonly RootLevelFieldColumn[] +) => { + if (rootLevelFieldColumns.length === 0) { + return data + } + const rootLevelFieldKeys = new Set(rootLevelFieldColumns.map((column) => column.key)) + return Object.fromEntries( + Object.entries(data).filter(([key]) => !rootLevelFieldKeys.has(key)) + ) +} diff --git a/packages/infra/test/sql-store.test.ts b/packages/infra/test/sql-store.test.ts index 774ed1d37..468d95ed2 100644 --- a/packages/infra/test/sql-store.test.ts +++ b/packages/infra/test/sql-store.test.ts @@ -17,8 +17,13 @@ const projectedRootLevelFields: readonly RootLevelFieldColumn[] = [ { key: "flag", columnName: "__root_flag", kind: "boolean" } ] +const projectedJsonRootLevelFields: readonly RootLevelFieldColumn[] = [ + { key: "meta", columnName: "__root_meta", kind: "json" }, + { key: "items", columnName: "__root_items", kind: "json" } +] + describe("root-level projected field metadata", () => { - it("derives supported root-level scalar fields from encoded schema", () => { + it("derives root-level projected columns for scalar and complex encoded fields", () => { const schema = S.Struct({ id: S.String, name: S.String, @@ -35,7 +40,9 @@ describe("root-level projected field metadata", () => { { key: "age", columnName: "__root_age", kind: "number" }, { key: "active", columnName: "__root_active", kind: "boolean" }, { key: "createdAt", columnName: "__root_createdAt", kind: "string" }, - { key: "status", columnName: "__root_status", kind: "string" } + { key: "status", columnName: "__root_status", kind: "string" }, + { key: "meta", columnName: "__root_meta", kind: "json" }, + { key: "tags", columnName: "__root_tags", kind: "json" } ]) }) }) @@ -96,6 +103,26 @@ describe("SQL query builder (SQLite dialect)", () => { expect(result.sql).toContain(`COALESCE("__root_age", json_extract(data, '$.age')) > ?`) }) + it("uses projected JSON root columns for nested filters", () => { + const result = buildWhereSQLQuery( + sqliteDialect, + "id", + [{ t: "where", path: "meta.city", op: "eq", value: "NYC" }], + "users", + {}, + undefined, + undefined, + undefined, + undefined, + undefined, + projectedJsonRootLevelFields + ) + expect(result.sql).toContain( + `json_extract(COALESCE("__root_meta", CASE WHEN json_type(data, '$.meta') IS NULL THEN NULL` + ) + expect(result.sql).toContain(`'$.city'`) + }) + it("where or", () => { const result = buildWhereSQLQuery( sqliteDialect, @@ -298,6 +325,33 @@ describe("SQL query builder (SQLite dialect)", () => { expect(result.params).toContain("%picked%") }) + it("computed relation count uses projected JSON root arrays when available", () => { + const result = buildWhereSQLQuery( + sqliteDialect, + "id", + [], + "users", + {}, + [{ + key: "pickedCount", + computed: { + _tag: "relation-count", + path: "items", + filter: [{ t: "where", path: "items.-1.description", op: "contains", value: "picked" }] + } + }], + undefined, + undefined, + undefined, + undefined, + projectedJsonRootLevelFields + ) + expect(result.sql).toContain( + `json_each(COALESCE("__root_items", CASE WHEN json_type(data, '$.items') IS NULL THEN NULL` + ) + expect(result.sql).toContain(`AS _items`) + }) + it("computed relation any projection (sqlite bool encoding)", () => { const result = buildWhereSQLQuery( sqliteDialect, @@ -776,6 +830,23 @@ describe("SQL query builder (PostgreSQL dialect)", () => { ) expect(result.sql).toContain(`COALESCE("__root_flag", (data->>'flag')::boolean) = $1`) }) + + it("uses projected JSON root columns for pg nested filters", () => { + const result = buildWhereSQLQuery( + pgDialect, + "id", + [{ t: "where", path: "meta.city", op: "eq", value: "NYC" }], + "users", + {}, + undefined, + undefined, + undefined, + undefined, + undefined, + projectedJsonRootLevelFields + ) + expect(result.sql).toContain(`(COALESCE("__root_meta", (data)->'meta'))->>'city' = $1`) + }) }) // --- Integration tests with in-memory SQLite (direct, no Effect SQL client) --- @@ -908,6 +979,62 @@ describe("SQL Store (SQLite integration)", () => { expect((rows[0] as any).id).toBe("1") })) + it("projected root JSON columns can replace data storage for nested fields", () => + withDb((db) => { + db.exec( + `CREATE TABLE IF NOT EXISTS "test_projected_json" (id TEXT PRIMARY KEY, _etag TEXT, data JSON NOT NULL, "__root_meta" JSON, "__root_items" JSON)` + ) + db + .prepare( + `INSERT INTO "test_projected_json" (id, _etag, data, "__root_meta", "__root_items") VALUES (?, ?, ?, ?, ?)` + ) + .run( + "1", + "etag1", + JSON.stringify({ untouched: true }), + JSON.stringify({ city: "NYC", zip: "10001" }), + JSON.stringify([{ description: "picked" }, { description: "open" }]) + ) + + const nested = buildWhereSQLQuery( + sqliteDialect, + "id", + [{ t: "where", path: "meta.city", op: "eq", value: "NYC" }], + "test_projected_json", + {}, + undefined, + undefined, + undefined, + undefined, + undefined, + projectedJsonRootLevelFields + ) + expect(query(db, nested.sql, nested.params)).toHaveLength(1) + + const relation = buildWhereSQLQuery( + sqliteDialect, + "id", + [], + "test_projected_json", + {}, + [{ + key: "pickedCount", + computed: { + _tag: "relation-count", + path: "items", + filter: [{ t: "where", path: "items.-1.description", op: "contains", value: "picked" }] + } + }], + undefined, + undefined, + undefined, + undefined, + projectedJsonRootLevelFields + ) + const [row] = query(db, relation.sql, relation.params) + expect((row as any).pickedCount).toBe(1) + })) + it("queries work when data does not contain id", () => withDb((db) => { db.exec( @@ -1667,11 +1794,16 @@ describe("sqliteDialect.jsonExtractJson boolean round-trip", () => { describe("toRow strips _etag and id from data", () => { // Replicate the toRow logic from SQL.ts to test in isolation - const toRow = (e: any, idKey: IdKey) => { + const toRow = ( + e: any, + idKey: IdKey, + rootLevelFieldColumns: readonly RootLevelFieldColumn[] = [] + ) => { const newE = makeETag(e) const id = newE[idKey] as string const { _etag, [idKey]: _id, ...rest } = newE as any - const data = JSON.stringify(rest) + const projectedKeys = new Set(rootLevelFieldColumns.map((column) => column.key)) + const data = JSON.stringify(Object.fromEntries(Object.entries(rest).filter(([key]) => !projectedKeys.has(key)))) return { id, _etag: newE._etag!, data, item: newE } } @@ -1719,6 +1851,21 @@ describe("toRow strips _etag and id from data", () => { expect(parsed).not.toHaveProperty("id") expect(parsed).not.toHaveProperty("_etag") }) + + it("omits projected root fields from data", () => { + const row = toRow( + { id: "1", _etag: undefined, name: "Alice", meta: { city: "NYC" }, untouched: true }, + "id", + [ + { key: "name", columnName: "__root_name", kind: "string" }, + { key: "meta", columnName: "__root_meta", kind: "json" } + ] + ) + const parsed = JSON.parse(row.data) as any + expect(parsed).not.toHaveProperty("name") + expect(parsed).not.toHaveProperty("meta") + expect(parsed.untouched).toBe(true) + }) }) describe("parseRow reconstructs full object from row", () => { @@ -1822,4 +1969,22 @@ describe("parseRow reconstructs full object from row", () => { expect(result.name).toBe("Alice") expect(result.flag).toBe(true) }) + + it("reconstructs projected JSON root-level columns when available", () => { + const result: any = parseRow( + { + id: "1", + _etag: "e1", + data: JSON.stringify({ untouched: true }), + __root_meta: JSON.stringify({ city: "NYC" }), + __root_items: JSON.stringify([{ description: "picked" }]) + }, + "id", + {}, + projectedJsonRootLevelFields + ) + expect(result.meta).toEqual({ city: "NYC" }) + expect(result.items).toEqual([{ description: "picked" }]) + expect(result.untouched).toBe(true) + }) }) From ddd83a7a24d66b4dd2e9986a8c1edc51fef65244 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 22 May 2026 19:46:33 +0000 Subject: [PATCH 07/11] refactor: simplify projected json params Agent-Logs-Url: https://github.com/effect-app/libs/sessions/2c26dc3a-b352-4999-b5d5-98349e52640c Co-authored-by: patroza <42661+patroza@users.noreply.github.com> --- packages/infra/src/Store/SQL/query.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/infra/src/Store/SQL/query.ts b/packages/infra/src/Store/SQL/query.ts index 2b3b26647..e8466b8ee 100644 --- a/packages/infra/src/Store/SQL/query.ts +++ b/packages/infra/src/Store/SQL/query.ts @@ -417,7 +417,7 @@ export function buildWhereSQLQuery( } const addJsonParam = (value: unknown): string => { - params.push(dialect.jsonColumnType === "JSON" ? JSON.stringify(value) : JSON.stringify(value)) + params.push(JSON.stringify(value)) return dialect.placeholder(paramIndex++) } From 93db22d079b47767dab6512db128cdf7822a4e92 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 22 May 2026 19:48:26 +0000 Subject: [PATCH 08/11] refactor: tidy projected sql helpers Agent-Logs-Url: https://github.com/effect-app/libs/sessions/2c26dc3a-b352-4999-b5d5-98349e52640c Co-authored-by: patroza <42661+patroza@users.noreply.github.com> --- packages/infra/src/Store/SQL.ts | 15 +++++++++------ packages/infra/src/Store/SQL/Pg.ts | 11 ++++++----- packages/infra/src/Store/SQL/query.ts | 2 +- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/packages/infra/src/Store/SQL.ts b/packages/infra/src/Store/SQL.ts index 447d69633..ec79a6cf9 100644 --- a/packages/infra/src/Store/SQL.ts +++ b/packages/infra/src/Store/SQL.ts @@ -76,12 +76,15 @@ const parseSelectRow = ( return result } -const serializeProjectedValue = (system: DbSystem, column: RootLevelFieldColumn, value: unknown) => - column.kind === "json" - ? value === undefined ? null : JSON.stringify(value) - : column.kind === "boolean" && system === "sqlite" && value !== null && value !== undefined - ? value === true ? 1 : 0 - : value ?? null +const serializeProjectedValue = (system: DbSystem, column: RootLevelFieldColumn, value: unknown) => { + if (column.kind === "json") { + return value === undefined ? null : JSON.stringify(value) + } + if (column.kind === "boolean" && system === "sqlite" && value !== null && value !== undefined) { + return value === true ? 1 : 0 + } + return value ?? null +} function makeSQLStoreInt(system: DbSystem, dialect: SQLDialect, jsonColumnType: string) { return Effect.fnUntraced( diff --git a/packages/infra/src/Store/SQL/Pg.ts b/packages/infra/src/Store/SQL/Pg.ts index c689213cc..de343742b 100644 --- a/packages/infra/src/Store/SQL/Pg.ts +++ b/packages/infra/src/Store/SQL/Pg.ts @@ -132,11 +132,12 @@ const makePgStore = Effect.fnUntraced( const id = newE[idKey] as string const { _etag, [idKey]: _id, ...rest } = newE as any const data = JSON.stringify(omitRootLevelFieldColumnsFromData(rest, activeRootLevelFieldColumns)) - const rootLevelFieldValues = activeRootLevelFieldColumns.map((column) => - column.kind === "json" - ? rest[column.key] === undefined ? null : JSON.stringify(rest[column.key]) - : rest[column.key] ?? null - ) + const rootLevelFieldValues = activeRootLevelFieldColumns.map((column) => { + if (column.kind === "json") { + return rest[column.key] === undefined ? null : JSON.stringify(rest[column.key]) + } + return rest[column.key] ?? null + }) return { id, _etag: newE._etag!, data, item: newE, rootLevelFieldValues } } diff --git a/packages/infra/src/Store/SQL/query.ts b/packages/infra/src/Store/SQL/query.ts index e8466b8ee..5319d77d1 100644 --- a/packages/infra/src/Store/SQL/query.ts +++ b/packages/infra/src/Store/SQL/query.ts @@ -254,7 +254,7 @@ const extractJsonFromSourceExpr = ( return asJson ? `(${sourceExpr})` : `(${sourceExpr})#>> '{}'` } if (!asJson) { - const last = parts.at(-1)! + const last = parts[parts.length - 1]! const parents = parts.slice(0, -1) return `(${sourceExpr})${parents.map((part) => `->'${part}'`).join("")}->>'${last}'` } From 34fb23efe61e420f75c43e5d47fd04ba079a98a4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 23 May 2026 05:17:33 +0000 Subject: [PATCH 09/11] fix: make root column mode exclusive Agent-Logs-Url: https://github.com/effect-app/libs/sessions/18e97927-7b93-4bc4-aefc-a0461aa0331e Co-authored-by: patroza <42661+patroza@users.noreply.github.com> --- .../infra/src/Model/Repository/makeRepo.ts | 8 +- packages/infra/src/Store/SQL.ts | 79 ++++++------------- packages/infra/src/Store/SQL/Pg.ts | 39 ++++----- packages/infra/src/Store/SQL/query.ts | 32 +------- packages/infra/src/Store/service.ts | 16 ++-- packages/infra/test/sql-store.test.ts | 55 +++++++++---- 6 files changed, 96 insertions(+), 133 deletions(-) diff --git a/packages/infra/src/Model/Repository/makeRepo.ts b/packages/infra/src/Model/Repository/makeRepo.ts index 0fd308486..8f964afe1 100644 --- a/packages/infra/src/Model/Repository/makeRepo.ts +++ b/packages/infra/src/Model/Repository/makeRepo.ts @@ -43,10 +43,10 @@ export interface RepositoryOptions< /** * Repository storage options. * - * `rootLevelFieldsWhenAvailable` is the Phase 1 hybrid mode: keep document-style - * JSON storage for nested values, while letting SQL adapters project eligible - * root-level scalar fields into real columns. The planned next step for relational - * fields is a separate explicit mode with relation metadata and join planning. + * `rootLevelFieldsWhenAvailable` switches SQL repositories from reading/writing + * derived root fields via `data` to reading/writing those fields via dedicated + * root columns instead. Existing table/data migrations are expected to be handled + * offline rather than as runtime fallback behavior. */ config?: Omit, "partitionValue"> & { partitionValue?: (e?: Encoded) => string diff --git a/packages/infra/src/Store/SQL.ts b/packages/infra/src/Store/SQL.ts index ec79a6cf9..213b5e05b 100644 --- a/packages/infra/src/Store/SQL.ts +++ b/packages/infra/src/Store/SQL.ts @@ -17,7 +17,7 @@ import { annotateDb, type DbSystem } from "../otel.js" import { storeId } from "./Memory.js" import { omitRootLevelFieldColumnsFromData, type RootLevelFieldColumn } from "./rootLevelFields.js" import { type FilterArgs, type PersistenceModelType, type StorageConfig, type Store, type StoreConfig, StoreMaker } from "./service.js" -import { buildWhereSQLQuery, logQuery, normalizeProjectedColumnValue, projectedColumnBackfillExpr, projectedColumnSqlType, quoteIdentifier, type SQLDialect, sqliteDialect } from "./SQL/query.js" +import { buildWhereSQLQuery, logQuery, normalizeProjectedColumnValue, projectedColumnSqlType, quoteIdentifier, type SQLDialect, sqliteDialect } from "./SQL/query.js" import { makeETag } from "./utils.js" export type WithNsTransactionFn = (effect: Effect.Effect) => Effect.Effect @@ -33,7 +33,10 @@ export const parseRow = ( defaultValues: Partial, rootLevelFieldColumns: readonly RootLevelFieldColumn[] = [] ): PersistenceModelType => { - const data = (typeof row.data === "string" ? JSON.parse(row.data) : row.data) as object + const data = omitRootLevelFieldColumnsFromData( + (typeof row.data === "string" ? JSON.parse(row.data) : row.data) as Record, + rootLevelFieldColumns + ) const projectedFields = rootLevelFieldColumns.reduce>((acc, column) => { const value = row[column.columnName] if (value !== null && value !== undefined) { @@ -109,6 +112,15 @@ function makeSQLStoreInt(system: DbSystem, dialect: SQLDialect, jsonColumnType: const selectColumnsSql = activeRootLevelFieldColumns.length > 0 ? `, ${activeRootLevelFieldColumns.map((column) => quoteIdentifier(column.columnName)).join(", ")}` : "" + const projectedColumnsDefinitionSql = activeRootLevelFieldColumns.length > 0 + ? `, ${ + activeRootLevelFieldColumns + .map((column) => + `${quoteIdentifier(column.columnName)} ${projectedColumnSqlType(dialect, column.kind)}` + ) + .join(", ") + }` + : "" const resolveNamespace = !config?.allowNamespace ? Effect.succeed("primary") @@ -122,33 +134,9 @@ function makeSQLStoreInt(system: DbSystem, dialect: SQLDialect, jsonColumnType: const exec = (query: string, params?: readonly unknown[]) => sql.unsafe(query, params as any).pipe(Effect.orDie) - const ensureRootLevelFieldColumns = Effect.gen(function*() { - if (activeRootLevelFieldColumns.length === 0) { - return - } - const existing = (yield* exec(`PRAGMA table_info(${JSON.stringify(tableName)})`)) as Array< - { name: string } - > - const existingColumns = new Set(existing.map((column) => column.name)) - for (const column of activeRootLevelFieldColumns) { - if (!existingColumns.has(column.columnName)) { - yield* exec( - `ALTER TABLE "${tableName}" ADD COLUMN ${quoteIdentifier(column.columnName)} ${ - projectedColumnSqlType(dialect, column.kind) - }` - ) - } - yield* exec( - `UPDATE "${tableName}" SET ${quoteIdentifier(column.columnName)} = ${ - projectedColumnBackfillExpr(dialect, column) - } WHERE ${quoteIdentifier(column.columnName)} IS NULL` - ) - } - }) - const ensureTable = sql .unsafe( - `CREATE TABLE IF NOT EXISTS "${tableName}" (id TEXT NOT NULL, _namespace TEXT NOT NULL DEFAULT 'primary', _etag TEXT, data ${jsonColumnType} NOT NULL, PRIMARY KEY (id, _namespace))` + `CREATE TABLE IF NOT EXISTS "${tableName}" (id TEXT NOT NULL, _namespace TEXT NOT NULL DEFAULT 'primary', _etag TEXT, data ${jsonColumnType} NOT NULL${projectedColumnsDefinitionSql}, PRIMARY KEY (id, _namespace))` ) .pipe( Effect.andThen( @@ -156,7 +144,6 @@ function makeSQLStoreInt(system: DbSystem, dialect: SQLDialect, jsonColumnType: `CREATE TABLE IF NOT EXISTS "_migrations" (id TEXT NOT NULL, version TEXT NOT NULL, PRIMARY KEY (id, version))` ) ), - Effect.andThen(ensureRootLevelFieldColumns), Effect.orDie, Effect.asVoid ) @@ -490,6 +477,15 @@ function makeSQLiteStorePerNs( const selectColumnsSql = activeRootLevelFieldColumns.length > 0 ? `, ${activeRootLevelFieldColumns.map((column) => quoteIdentifier(column.columnName)).join(", ")}` : "" + const projectedColumnsDefinitionSql = activeRootLevelFieldColumns.length > 0 + ? `, ${ + activeRootLevelFieldColumns + .map((column) => + `${quoteIdentifier(column.columnName)} ${projectedColumnSqlType(sqliteDialect, column.kind)}` + ) + .join(", ") + }` + : "" const resolveNamespace = !config?.allowNamespace ? Effect.succeed("primary") @@ -518,7 +514,7 @@ function makeSQLiteStorePerNs( withNsSql(ns, (sql) => sql .unsafe( - `CREATE TABLE IF NOT EXISTS "${tableName}" (id TEXT NOT NULL PRIMARY KEY, _etag TEXT, data JSON NOT NULL)` + `CREATE TABLE IF NOT EXISTS "${tableName}" (id TEXT NOT NULL PRIMARY KEY, _etag TEXT, data JSON NOT NULL${projectedColumnsDefinitionSql})` ) .pipe( Effect.andThen( @@ -526,31 +522,6 @@ function makeSQLiteStorePerNs( `CREATE TABLE IF NOT EXISTS "_migrations" (id TEXT NOT NULL, version TEXT NOT NULL, PRIMARY KEY (id, version))` ) ), - Effect.andThen(Effect.gen(function*() { - if (activeRootLevelFieldColumns.length === 0) { - return - } - const existing = (yield* exec(ns, `PRAGMA table_info(${JSON.stringify(tableName)})`)) as Array< - { name: string } - > - const existingColumns = new Set(existing.map((column) => column.name)) - for (const column of activeRootLevelFieldColumns) { - if (!existingColumns.has(column.columnName)) { - yield* exec( - ns, - `ALTER TABLE "${tableName}" ADD COLUMN ${quoteIdentifier(column.columnName)} ${ - projectedColumnSqlType(sqliteDialect, column.kind) - }` - ) - } - yield* exec( - ns, - `UPDATE "${tableName}" SET ${quoteIdentifier(column.columnName)} = ${ - projectedColumnBackfillExpr(sqliteDialect, column) - } WHERE ${quoteIdentifier(column.columnName)} IS NULL` - ) - } - })), Effect.orDie, Effect.asVoid )) diff --git a/packages/infra/src/Store/SQL/Pg.ts b/packages/infra/src/Store/SQL/Pg.ts index de343742b..cff72d3ab 100644 --- a/packages/infra/src/Store/SQL/Pg.ts +++ b/packages/infra/src/Store/SQL/Pg.ts @@ -15,7 +15,7 @@ import { storeId } from "../Memory.js" import { omitRootLevelFieldColumnsFromData, type RootLevelFieldColumn } from "../rootLevelFields.js" import { type FilterArgs, type PersistenceModelType, type StorageConfig, type Store, type StoreConfig, StoreMaker } from "../service.js" import { makeETag } from "../utils.js" -import { buildWhereSQLQuery, logQuery, normalizeProjectedColumnValue, pgDialect, projectedColumnBackfillExpr, projectedColumnSqlType, quoteIdentifier } from "./query.js" +import { buildWhereSQLQuery, logQuery, normalizeProjectedColumnValue, pgDialect, projectedColumnSqlType, quoteIdentifier } from "./query.js" const parseRow = ( row: { id: string; _etag: string | null; data: unknown } & Record, @@ -23,7 +23,10 @@ const parseRow = ( defaultValues: Partial, rootLevelFieldColumns: readonly RootLevelFieldColumn[] = [] ): PersistenceModelType => { - const data = (typeof row.data === "string" ? JSON.parse(row.data) : row.data) as object + const data = omitRootLevelFieldColumnsFromData( + (typeof row.data === "string" ? JSON.parse(row.data) : row.data) as Record, + rootLevelFieldColumns + ) const projectedFields = rootLevelFieldColumns.reduce>((acc, column) => { const value = row[column.columnName] if (value !== null && value !== undefined) { @@ -80,6 +83,15 @@ const makePgStore = Effect.fnUntraced( const selectColumnsSql = activeRootLevelFieldColumns.length > 0 ? `, ${activeRootLevelFieldColumns.map((column) => quoteIdentifier(column.columnName)).join(", ")}` : "" + const projectedColumnsDefinitionSql = activeRootLevelFieldColumns.length > 0 + ? `, ${ + activeRootLevelFieldColumns + .map((column) => + `${quoteIdentifier(column.columnName)} ${projectedColumnSqlType(pgDialect, column.kind)}` + ) + .join(", ") + }` + : "" const resolveNamespace = !config?.allowNamespace ? Effect.succeed("primary") @@ -94,7 +106,7 @@ const makePgStore = Effect.fnUntraced( const ensureTable = sql .unsafe( - `CREATE TABLE IF NOT EXISTS "${tableName}" (id TEXT NOT NULL, _namespace TEXT NOT NULL DEFAULT 'primary', _etag TEXT, data JSONB NOT NULL, PRIMARY KEY (id, _namespace))` + `CREATE TABLE IF NOT EXISTS "${tableName}" (id TEXT NOT NULL, _namespace TEXT NOT NULL DEFAULT 'primary', _etag TEXT, data JSONB NOT NULL${projectedColumnsDefinitionSql}, PRIMARY KEY (id, _namespace))` ) .pipe( Effect.andThen( @@ -102,27 +114,6 @@ const makePgStore = Effect.fnUntraced( `CREATE TABLE IF NOT EXISTS "_migrations" (id TEXT NOT NULL, version TEXT NOT NULL, PRIMARY KEY (id, version))` ) ), - Effect.andThen( - Effect.forEach( - activeRootLevelFieldColumns, - (column) => - exec( - `ALTER TABLE "${tableName}" ADD COLUMN IF NOT EXISTS ${quoteIdentifier(column.columnName)} ${ - projectedColumnSqlType(pgDialect, column.kind) - }` - ) - .pipe( - Effect.andThen( - exec( - `UPDATE "${tableName}" SET ${quoteIdentifier(column.columnName)} = ${ - projectedColumnBackfillExpr(pgDialect, column) - } WHERE ${quoteIdentifier(column.columnName)} IS NULL` - ) - ) - ), - { discard: true } - ) - ), Effect.orDie, Effect.asVoid ) diff --git a/packages/infra/src/Store/SQL/query.ts b/packages/infra/src/Store/SQL/query.ts index 5319d77d1..32a8c6649 100644 --- a/packages/infra/src/Store/SQL/query.ts +++ b/packages/infra/src/Store/SQL/query.ts @@ -146,29 +146,10 @@ export function logQuery(q: { sql: string; params: unknown[] }) { export const quoteIdentifier = (value: string) => `"${value.replaceAll("\"", "\"\"")}"` -const projectedColumnJsonFallbackExpr = ( - dialect: SQLDialect, - column: RootLevelFieldColumn -) => { - const expr = dialect.jsonExtract(column.key) - switch (column.kind) { - case "string": - return expr - case "number": - return dialect.jsonColumnType === "JSON" ? expr : `(${expr})::double precision` - case "boolean": - return dialect.jsonColumnType === "JSON" ? expr : `(${expr})::boolean` - case "json": - return extractJsonFromSourceExpr(dialect, "data", column.key, true) - default: - return assertUnreachable(column.kind) - } -} - export const projectedColumnFieldExpr = ( - dialect: SQLDialect, + _dialect: SQLDialect, column: RootLevelFieldColumn -) => `COALESCE(${quoteIdentifier(column.columnName)}, ${projectedColumnJsonFallbackExpr(dialect, column)})` +) => quoteIdentifier(column.columnName) export const projectedColumnSelectExpr = ( dialect: SQLDialect, @@ -196,11 +177,6 @@ export const projectedColumnSqlType = ( } } -export const projectedColumnBackfillExpr = ( - dialect: SQLDialect, - column: RootLevelFieldColumn -) => projectedColumnJsonFallbackExpr(dialect, column) - export const normalizeProjectedColumnValue = ( column: RootLevelFieldColumn, value: unknown @@ -406,7 +382,7 @@ export function buildWhereSQLQuery( return } return { - sourceExpr: projectedColumnFieldExpr(dialect, projected), + sourceExpr: quoteIdentifier(projected.columnName), subPath: subPathParts.join(".") } } @@ -444,7 +420,7 @@ export function buildWhereSQLQuery( if (isRootLevelPath(path, relation)) { const projected = projectedColumns.get(path) if (projected) { - const expr = projectedColumnFieldExpr(dialect, projected) + const expr = quoteIdentifier(projected.columnName) if (path in defaultValues) { return `COALESCE(${expr}, ${ projected.kind === "json" ? addJsonParam(defaultValues[path]) : addParam(defaultValues[path]) diff --git a/packages/infra/src/Store/service.ts b/packages/infra/src/Store/service.ts index eb90cbd61..f57e29565 100644 --- a/packages/infra/src/Store/service.ts +++ b/packages/infra/src/Store/service.ts @@ -16,12 +16,13 @@ import type { RootLevelFieldColumn } from "./rootLevelFields.js" export interface StoreConfig { partitionValue: (e?: E) => string /** - * Phase 1: keep the document store model, but project eligible root-level scalar - * fields into real SQL columns when the adapter supports it. + * For SQL adapters, switch storage/querying for derived root-level fields from the + * document `data` column to dedicated root columns. * - * Nested objects / arrays and the current relation-like JSON-array semantics stay - * in `data`. The next step for actual relational fields is a separate mode with - * explicit PK/FK metadata, ownership/cardinality, migrations, and join planning. + * When disabled, reads/writes/queries use `data`. + * When enabled, derived root fields are read/written/queried via dedicated columns + * instead. Schema/data migrations for existing tables are expected to happen + * offline rather than on the fly. */ rootLevelFieldsWhenAvailable?: boolean rootLevelFieldColumns?: readonly RootLevelFieldColumn[] @@ -242,9 +243,8 @@ export interface StorageConfig { prefix: string dbName: string /** - * Adapter default for Phase 1 root-level projected columns. Repositories can still - * opt in / out per table. Phase 2 relational support is expected to use a separate - * mode rather than extending this flag. + * Adapter default for switching derived root-level fields from the `data` column + * to dedicated SQL columns. Repositories can still opt in / out per table. */ rootLevelFieldsWhenAvailable?: boolean } diff --git a/packages/infra/test/sql-store.test.ts b/packages/infra/test/sql-store.test.ts index 468d95ed2..72810fe33 100644 --- a/packages/infra/test/sql-store.test.ts +++ b/packages/infra/test/sql-store.test.ts @@ -86,7 +86,7 @@ describe("SQL query builder (SQLite dialect)", () => { expect(result.params).toContain(18) }) - it("uses projected root columns with JSON fallback for root-level filters", () => { + it("uses projected root columns directly for root-level filters", () => { const result = buildWhereSQLQuery( sqliteDialect, "id", @@ -100,7 +100,7 @@ describe("SQL query builder (SQLite dialect)", () => { undefined, projectedRootLevelFields ) - expect(result.sql).toContain(`COALESCE("__root_age", json_extract(data, '$.age')) > ?`) + expect(result.sql).toContain(`"__root_age" > ?`) }) it("uses projected JSON root columns for nested filters", () => { @@ -117,9 +117,7 @@ describe("SQL query builder (SQLite dialect)", () => { undefined, projectedJsonRootLevelFields ) - expect(result.sql).toContain( - `json_extract(COALESCE("__root_meta", CASE WHEN json_type(data, '$.meta') IS NULL THEN NULL` - ) + expect(result.sql).toContain(`json_extract("__root_meta", '$.city')`) expect(result.sql).toContain(`'$.city'`) }) @@ -325,7 +323,7 @@ describe("SQL query builder (SQLite dialect)", () => { expect(result.params).toContain("%picked%") }) - it("computed relation count uses projected JSON root arrays when available", () => { + it("computed relation count uses projected JSON root arrays directly", () => { const result = buildWhereSQLQuery( sqliteDialect, "id", @@ -346,9 +344,7 @@ describe("SQL query builder (SQLite dialect)", () => { undefined, projectedJsonRootLevelFields ) - expect(result.sql).toContain( - `json_each(COALESCE("__root_items", CASE WHEN json_type(data, '$.items') IS NULL THEN NULL` - ) + expect(result.sql).toContain(`json_each("__root_items", '$')`) expect(result.sql).toContain(`AS _items`) }) @@ -814,7 +810,7 @@ describe("SQL query builder (PostgreSQL dialect)", () => { expect(result.sql).toContain(`COALESCE(jsonb_agg(DISTINCT _items->>'articleId'), '[]'::jsonb)`) }) - it("uses projected root columns with JSON fallback for pg filters", () => { + it("uses projected root columns directly for pg filters", () => { const result = buildWhereSQLQuery( pgDialect, "id", @@ -828,7 +824,7 @@ describe("SQL query builder (PostgreSQL dialect)", () => { undefined, projectedRootLevelFields ) - expect(result.sql).toContain(`COALESCE("__root_flag", (data->>'flag')::boolean) = $1`) + expect(result.sql).toContain(`"__root_flag" = $1`) }) it("uses projected JSON root columns for pg nested filters", () => { @@ -845,7 +841,7 @@ describe("SQL query builder (PostgreSQL dialect)", () => { undefined, projectedJsonRootLevelFields ) - expect(result.sql).toContain(`(COALESCE("__root_meta", (data)->'meta'))->>'city' = $1`) + expect(result.sql).toContain(`("__root_meta")->>'city' = $1`) }) }) @@ -947,7 +943,7 @@ describe("SQL Store (SQLite integration)", () => { expect(query(db, q3.sql, q3.params).length).toBe(2) })) - it("projected root columns still fall back to JSON data for legacy rows", () => + it("projected root columns do not fall back to legacy JSON data", () => withDb((db) => { db.exec( `CREATE TABLE IF NOT EXISTS "test_projected" (id TEXT PRIMARY KEY, _etag TEXT, data JSON NOT NULL, "__root_name" TEXT, "__root_age" REAL, "__root_flag" INTEGER)` @@ -975,8 +971,7 @@ describe("SQL Store (SQLite integration)", () => { projectedRootLevelFields ) const rows = query(db, q.sql, q.params) - expect(rows).toHaveLength(1) - expect((rows[0] as any).id).toBe("1") + expect(rows).toHaveLength(0) })) it("projected root JSON columns can replace data storage for nested fields", () => @@ -1970,6 +1965,21 @@ describe("parseRow reconstructs full object from row", () => { expect(result.flag).toBe(true) }) + it("does not fall back to data for projected root-level columns", () => { + const result: any = parseRow( + { + id: "1", + _etag: "e1", + data: JSON.stringify({ name: "legacy", flag: true }) + }, + "id", + {}, + projectedRootLevelFields + ) + expect(result.name).toBeUndefined() + expect(result.flag).toBeUndefined() + }) + it("reconstructs projected JSON root-level columns when available", () => { const result: any = parseRow( { @@ -1987,4 +1997,19 @@ describe("parseRow reconstructs full object from row", () => { expect(result.items).toEqual([{ description: "picked" }]) expect(result.untouched).toBe(true) }) + + it("does not fall back to data for projected JSON root-level columns", () => { + const result: any = parseRow( + { + id: "1", + _etag: "e1", + data: JSON.stringify({ meta: { city: "legacy" }, items: [{ description: "legacy" }] }) + }, + "id", + {}, + projectedJsonRootLevelFields + ) + expect(result.meta).toBeUndefined() + expect(result.items).toBeUndefined() + }) }) From 0b3b0b90fa498d5203c827edb4cd786040e86e75 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 23 May 2026 05:18:53 +0000 Subject: [PATCH 10/11] refactor: simplify projected column helpers Agent-Logs-Url: https://github.com/effect-app/libs/sessions/18e97927-7b93-4bc4-aefc-a0461aa0331e Co-authored-by: patroza <42661+patroza@users.noreply.github.com> --- packages/infra/src/Store/SQL/query.ts | 11 ++++------- packages/infra/test/sql-store.test.ts | 1 - 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/packages/infra/src/Store/SQL/query.ts b/packages/infra/src/Store/SQL/query.ts index 32a8c6649..27ac41dac 100644 --- a/packages/infra/src/Store/SQL/query.ts +++ b/packages/infra/src/Store/SQL/query.ts @@ -146,18 +146,15 @@ export function logQuery(q: { sql: string; params: unknown[] }) { export const quoteIdentifier = (value: string) => `"${value.replaceAll("\"", "\"\"")}"` -export const projectedColumnFieldExpr = ( - _dialect: SQLDialect, - column: RootLevelFieldColumn -) => quoteIdentifier(column.columnName) +export const projectedColumnFieldExpr = (column: RootLevelFieldColumn) => quoteIdentifier(column.columnName) export const projectedColumnSelectExpr = ( dialect: SQLDialect, column: RootLevelFieldColumn ) => column.kind === "boolean" && dialect.jsonColumnType === "JSON" - ? `CASE ${projectedColumnFieldExpr(dialect, column)} WHEN 1 THEN 'true' WHEN 0 THEN 'false' ELSE 'null' END` - : projectedColumnFieldExpr(dialect, column) + ? `CASE ${projectedColumnFieldExpr(column)} WHEN 1 THEN 'true' WHEN 0 THEN 'false' ELSE 'null' END` + : projectedColumnFieldExpr(column) export const projectedColumnSqlType = ( dialect: SQLDialect, @@ -420,7 +417,7 @@ export function buildWhereSQLQuery( if (isRootLevelPath(path, relation)) { const projected = projectedColumns.get(path) if (projected) { - const expr = quoteIdentifier(projected.columnName) + const expr = projectedColumnFieldExpr(projected) if (path in defaultValues) { return `COALESCE(${expr}, ${ projected.kind === "json" ? addJsonParam(defaultValues[path]) : addParam(defaultValues[path]) diff --git a/packages/infra/test/sql-store.test.ts b/packages/infra/test/sql-store.test.ts index 72810fe33..663f377aa 100644 --- a/packages/infra/test/sql-store.test.ts +++ b/packages/infra/test/sql-store.test.ts @@ -118,7 +118,6 @@ describe("SQL query builder (SQLite dialect)", () => { projectedJsonRootLevelFields ) expect(result.sql).toContain(`json_extract("__root_meta", '$.city')`) - expect(result.sql).toContain(`'$.city'`) }) it("where or", () => { From bcaf8e24e079a71084b8b6c2394abbebc7e58c23 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 28 May 2026 07:42:05 +0000 Subject: [PATCH 11/11] style: format merge resolution --- packages/effect-app/src/Model/Repository/internal/internal.ts | 2 +- packages/effect-app/src/rootLevelFields.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/effect-app/src/Model/Repository/internal/internal.ts b/packages/effect-app/src/Model/Repository/internal/internal.ts index 4a2d0b9d8..3ced56f33 100644 --- a/packages/effect-app/src/Model/Repository/internal/internal.ts +++ b/packages/effect-app/src/Model/Repository/internal/internal.ts @@ -19,11 +19,11 @@ import * as Context from "../../../Context.js" import * as Effect from "../../../Effect.js" import { flatMapOption } from "../../../Effect.js" import * as Option from "../../../Option.js" +import { makeRootLevelFieldColumns } from "../../../rootLevelFields.js" import * as S from "../../../Schema.js" import { type Codec, NonNegativeInt } from "../../../Schema.js" import { setupRequestContextFromCurrent } from "../../../setupRequest.ts" import { type FilterArgs, getContextMap, type PersistenceModelType, type StoreConfig, storeId, StoreMaker } from "../../../Store.js" -import { makeRootLevelFieldColumns } from "../../../rootLevelFields.js" import type { FieldValues } from "../../filter/types.js" import * as Q from "../../query.js" import type { ChangeFeed, ChangeFeedEvent, Repository } from "../service.js" diff --git a/packages/effect-app/src/rootLevelFields.ts b/packages/effect-app/src/rootLevelFields.ts index 84335b3df..88ef3adec 100644 --- a/packages/effect-app/src/rootLevelFields.ts +++ b/packages/effect-app/src/rootLevelFields.ts @@ -1,6 +1,6 @@ +import * as SchemaAST from "effect/SchemaAST" import type * as S from "./Schema.js" import type { RootLevelFieldColumn, RootLevelFieldColumnKind } from "./Store.js" -import * as SchemaAST from "effect/SchemaAST" const walkTransformation = (ast: SchemaAST.AST): SchemaAST.AST => { if (ast._tag === "Declaration" && ast.typeParameters.length > 0) {