diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 5d01665..be018d8 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -45,6 +45,8 @@ jobs: run: npm run format:check - name: Run unit tests run: npm test + env: + SCHEMA_CODEGEN_NATIVE_TESTS: skip - name: Semantic Release if: ${{ github.event_name == 'push' }} uses: cycjimmy/semantic-release-action@b12c8f6015dc215fe37bc154d4ad456dd3833c90 # v6.0.0 diff --git a/.github/workflows/verify-pr.yaml b/.github/workflows/verify-pr.yaml index 1a3e377..2d8473b 100644 --- a/.github/workflows/verify-pr.yaml +++ b/.github/workflows/verify-pr.yaml @@ -58,3 +58,25 @@ jobs: run: npm ci - name: Run unit tests run: npm test + env: + SCHEMA_CODEGEN_NATIVE_TESTS: skip + + native: + name: Generated Swift and Kotlin + runs-on: macos-latest + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: '.nvmrc' + cache: npm + - name: Install the Kotlin compiler + run: command -v kotlinc || brew install kotlin + - name: Install dependencies + run: npm ci + - name: Compile and run the generated packages + run: npx vitest run utils/client/native.test.js + env: + SCHEMA_CODEGEN_NATIVE_TESTS: require diff --git a/README.md b/README.md index beaa404..c69072f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # @HarperFast/Schema-Codegen -Schema Codegen will generate TypeScript types for your GraphQL schemas, making it easier to work with your data in TypeScript and JavaScript applications. +Schema Codegen will generate TypeScript types for your GraphQL schemas, making it easier to work with your data in TypeScript and JavaScript applications. It can also generate Swift and Kotlin client models, plus a schema IR that describes your tables and sync profiles for client SDKs — see [Client models](#client-models-swift-and-kotlin). ## Installation @@ -40,6 +40,9 @@ When you `harper dev`, it will generate types based on the schema that's actuall | `includeDatabases` | (all) | List of database names to generate types for. When set, only matching databases are emitted. Supports `*` as a wildcard, e.g. `metrics-*`. | | `excludeDatabases` | (none) | List of database names to skip. Useful on a shared instance to keep other applications' databases out of your generated types. Supports `*` as a wildcard. | +Output paths are relative to your application directory. Client model options are described +[below](#client-models-swift-and-kotlin). + `includeDatabases`/`excludeDatabases` scope generation to your application's own databases. On a shared local instance, codegen otherwise sees every database on the instance — including ones belonging to other projects — so scoping keeps @@ -111,6 +114,140 @@ declare module 'harper' { } ``` +## Client models (Swift and Kotlin) + +Schema Codegen can also generate the typed models a mobile client works with: a Swift package and a +Kotlin module, built from a schema IR (a versioned JSON description of your tables, nested types and +sync profiles). Turn it on with any of these options: + +| Option | Default | Description | +| --------------- | --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `schemaIR` | — | Path to write the schema IR (`.json`). Set it whenever you generate `swift` or `kotlin`: it is the baseline that keeps generated type names stable and lets a regeneration warn about breaking changes. | +| `swift` | — | Directory of the Swift package to generate. | +| `swiftModule` | `HarperModels` | The Swift module (and target) name. | +| `kotlin` | — | Directory of the Kotlin module to generate. | +| `kotlinPackage` | `harper.models` | The Kotlin package. | +| `syncProfiles` | — | Path to a `sync.yaml` describing sync profiles; edits regenerate without a restart. | + +```yaml +'@harperfast/schema-codegen': + package: '@harperfast/schema-codegen' + schemaIR: 'client/harper-schema.json' + syncProfiles: 'sync.yaml' + swift: 'ios/HarperModels' + kotlin: 'android/harper-models' + kotlinPackage: 'com.example.harper' +``` + +Each table becomes a Swift `struct` and a Kotlin `data class` with one typed property per attribute, +`_version` (the record's version), and — unless the table is `@sealed` — `_extra`, which keeps +attributes the schema does not declare so they survive a round trip. Every model also has a nested +`New` (insert body) and `Patch` (patch body) type, and a schema descriptor for the device's SQLite +replica. For the `Order` table of a schema like this: + +```graphql +type Order @table @export { + id: ID @primaryKey + customerId: ID @indexed + status: String @indexed + amount: Float + points: Float @computed(from: "Math.round((amount || 0) * 10)") +} +``` + +```swift +import HarperModels + +var order = try Order(row: row, version: version) // row: HarperRow, e.g. [String: HarperValue] +order.status = "closed" +let body = try order.encodeUpsertRow() // PUT body: key, writable attributes, _extra +let patch = Order.Patch(status: "closed", clear: [.amount]).encodeRow() +``` + +```kotlin +val order = Order.decode(row, version) +val patch = Order.Patch(status = "closed", clear = setOf(Order.Patch.Field.amount)).encodeRow() +``` + +How Harper types map, and why, is recorded in [docs/design/client-codegen.md](docs/design/client-codegen.md): + +| Harper | Swift | Kotlin | +| -------------- | ------------------------- | ------------------------- | +| `ID`, `String` | `String` | `String` | +| `Int` | `Int` | `Int` | +| `Long` | `Int64` | `Long` | +| `Float` | `Double` | `Double` | +| `BigInt` | `HarperBigInt` | `java.math.BigInteger` | +| `Boolean` | `Bool` | `Boolean` | +| `Date` | `Date` | `java.time.Instant` | +| `Bytes` | `Data` | `ByteArray` | +| `Blob` | `HarperValue?`, read-only | `HarperValue?`, read-only | +| `Any` | `HarperValue` | `HarperValue` | +| `[T]` | `[T]` | `List` | +| nested type | generated `struct` | generated `data class` | + +Attributes are optional unless they are the primary key, server-managed (`@createdTime`, +`@updatedTime`), or declared non-null (`String!`). Computed attributes are read-only and optional; +relationships are not stored on the model. A table with a `Blob` attribute cannot be replaced in full +(`encodeUpsertRow` throws), because a client has no way to send a blob back yet; use a patch. A +`Blob` is always optional on the model, since sync does not deliver blob content; declaring one +required (`Blob!`) makes client inserts into its table fail, and generation warns about it. + +The generated Swift package needs Swift 5.9 (iOS 13, macOS 10.15); the Kotlin module targets the JVM +and needs Android API 26 (or core library desugaring) for `java.time`. `Package.swift` and +`build.gradle.kts` are created once and then left to you; the sources are regenerated. + +### Sync profiles + +A `sync.yaml` declares which tables replicate to devices, scoped to the rows each device may see: + +```yaml +profiles: + storefront: + retention: 14d # how long a device may stay offline and still catch up + tables: [Product] + my-orders: + direction: pull # pull (default), push or bidirectional + retention: 30d + scope: + customerId: $token.sub # a token claim; $user. reads the user record + tables: [Order] + regional: + tables: + Store: + database: retail + scope: { region: [us-east, us-west] } # literal values; a list means "any of" +``` + +Profiles fail closed: a profile with any error — an unknown table or attribute, a mistyped key, an +invalid value — is left out entirely and reported, so a typo can never widen a scope to every row. +Scoping by an attribute without `@indexed` works but logs a warning. + +### Schema hashes and compatibility + +The IR records a SHA-256 hash of each table's wire and storage contract, of the whole schema, and of +the tables in each sync profile (the value a device sends when it connects). When a regeneration +would break clients built from the previous IR — for example, a changed attribute type, a removed +table, a new required attribute, or a new computed attribute or relationship on a table that is not +`@sealed` (older models keep it in `_extra` and send it back, which the server rejects) — the +component logs which changes break reads, writes or the device's storage. To enforce this in CI, +compare two IR files with the CLI; an IR file whose hashes do not match its contents is refused. + +### CLI + +The `schema-codegen` command generates client packages from an IR file, without a running Harper: + +```bash +npx @harperfast/schema-codegen generate-client --ir client/harper-schema.json --swift ios/HarperModels --kotlin android/harper-models --kotlin-package com.example.harper +``` + +```bash +npx @harperfast/schema-codegen diff previous-schema.json client/harper-schema.json --fail-on read,write +``` + +`diff` exits with 1 when the change breaks any of the listed axes (`read`, `write`, `storage`; all +three by default), and `--json` prints the full report. + ## Development To use this in an application, first link it: diff --git a/bin/schema-codegen.js b/bin/schema-codegen.js new file mode 100755 index 0000000..5a7f7a1 --- /dev/null +++ b/bin/schema-codegen.js @@ -0,0 +1,8 @@ +#!/usr/bin/env node +import { runCli } from '../utils/client/cli.js'; + +process.exitCode = runCli(process.argv.slice(2), { + stdout: (text) => process.stdout.write(text), + stderr: (text) => process.stderr.write(text), + cwd: process.cwd(), +}); diff --git a/docs/design/client-codegen.md b/docs/design/client-codegen.md new file mode 100644 index 0000000..13eb16c --- /dev/null +++ b/docs/design/client-codegen.md @@ -0,0 +1,506 @@ +# Client codegen: schema IR, Swift/Kotlin emitters, sync profiles, schema hash + +Status: design note for HarperFast/harper-client-sdk-spike#8 (M8 in the device-sync design, +`docs/DESIGN.md` rev 3 in the spike repo). It is also the decision record the issue asks for: +type-mapping and model-shape decisions are made here once and every emitter follows them. Issue numbers in this note (`#10`, `#18`, …) are issues in +HarperFast/harper-client-sdk-spike, not in this repository. + +## Goal + +`@harperfast/schema-codegen` already turns the live schema into TypeScript/JSDoc in dev mode. +The client SDK needs, from the same live schema: + +1. Swift and Kotlin models shaped like the spike's hand-written `ios/HarperSpike/Harper.swift`: + typed columns, an `_extra` overflow bag on non-`@sealed` tables, and a version column, with + core's read/insert/upsert/patch/query projection rules (`resources/defineTable.ts`) ported. +2. Sync-profile awareness in an emitted IR (tables per profile, scope attributes, retention) — + the same IR the Studio profile editor (#16) and the sync gateway (#10) will consume. +3. Type-mapping decisions recorded once: `Long`/`BigInt` width, `Any`/`Bytes`/`Blob`, + undefined-means-nullable. +4. A schema hash for the `hello`-frame negotiation, and the additive-delta story. + +Acceptance: the codegen component (dev mode) or CLI generates a Swift package and a Kotlin +module; the spike's demo app replaces its hand-written models with the generated package and +builds. + +**Invariant this change enforces:** the IR is a deterministic function of the live schema plus +`sync.yaml` (and, for type names only, the previous IR), every client artifact is a deterministic +function of the IR plus emitter options, and the mapping/projection/nullability rules exist in +exactly one module. Equal contract hashes mean an identical **wire and storage contract** — what a +device stores and sends — not identical generated source: type names, the generator version and +emitter options can change public symbols without changing the hash. + +## Approaches considered + +| Axis | Candidate | Fact that decides | +| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Different layer** | (a) Harper core: a `harper generate-client` subcommand and a core IR endpoint beside `defineTable`. (b) The sync gateway (pro, #10), which will own `sync.yaml`. (c) SwiftPM/Gradle build plugins that shell out to a Node generator. | (a) Core already exposes everything needed — live `Table.attributes` plus the GraphQL type registry a component reaches as `scope.resources.allTypes` (traced below) — so no core change is required, and the design doc's M8 assigns the work to this package ("extends it rather than starting over"); a core subcommand would also tie emitter iterations to core releases. (b) The gateway does not exist yet, but codegen is a phase-0 mechanism (design doc M8: thin and cached strategies need typed models before any sync machinery). Both are served by building the IR here and **exporting** it (`buildSchemaIR`) so the gateway and Studio import one implementation. (c) Needs the same IR and emitters plus two adapters and Node in mobile builds; it removes no deliverable, and the CLI already allows build-time invocation. | +| **Deeper cause** | Make core's schema carry a canonical client contract — e.g. build the IR from `Table.properties` (the JSON-Schema fragments from `resources/jsonSchemaTypes.ts`). | `attributeToFragment` emits `nullable` only when `true` (GraphQL never sets `true`, so undefined-means-nullable and `[T!]` element requiredness are unrecoverable) and omits computed expressions, relationship targets and `sealed`; nested types appear as a bare type name (`address: {type: 'Address'}`, verified live). Extending it is a core API change for one consumer. | +| **Do less** | (a) Use the `describe_all`/`describe_table` operation output as the IR. (b) Emit only the IR and let the iOS/Android SDKs generate models in Swift/Kotlin. (c) Reuse the TS emitter's `mapType` and string-map its output. | (a) `dataLayer/schemaDescribe.ts` `pushAtt` reduces `elements` to a type name, nested `properties` to `{type, name}`, `computed` to `true`, and drops relationship targets — lossy for every non-scalar attribute. (b) The acceptance requires generated models in the demo app now, and generators written in Swift and Kotlin re-implement the naming/type rules twice more — the opposite of "recorded once". (c) `mapType` collapses `Int`/`Long`/`Float` to `number`, `Bytes`/`Blob`/`Any` to `any` and `Date` to `string`. The smallest sufficient change is the chosen one: extend `regenerateAll` right after its existing filtered `collectTables` call, building one IR from that same list when a client option is set. | +| **Chosen** | A language-neutral **IR v1** built in this package from live `Table.attributes`, the GraphQL type registry and normalized `sync.yaml` profiles; Swift and Kotlin emitters that read only the IR; contract hashing and a read/write/storage compatibility diff in the same module; component options and a small CLI as entry points; a public `index.js` for the gateway and Studio. | Beats the rejected options on the stated facts; keeps one implementation of every rule; needs no core or pro change. | + +The existing TS/JSDoc emitters are left byte-for-byte unchanged (users' generated files must not +churn); moving them onto the IR is a follow-up. + +## Traced facts this design relies on + +Core `origin/main` @ 33bea305 (2026-10-05), and a live `harper dev` instance (5.3.0-beta.2) with a +schema covering every attribute kind: + +- `resources/graphql.ts` `getProperty`: a plain field leaves `nullable` undefined, `!` sets + `nullable: false`; nothing sets `nullable: true`. List types become `{type: 'array', elements}` + with the element's own `nullable`. `@computed(from:)` keeps the source as + `computedFromExpression` and a `version`; `@createdTime`/`@updatedTime` set + `assignCreatedTime`/`assignUpdatedTime`; `@indexed` sets `indexed` to an object. `@fullText` + fields never enter `attributes`. Every type definition is registered in `resources.allTypes` + (graphql.ts:260), with nested references connected via non-enumerable `properties`/`definition`. +- **Live `Table.attributes` keep only the type name of a nested object** (`address: Address` → + `{type: 'Address'}`; `[PurchaseLine!]!` → `elements: {type: 'PurchaseLine', nullable: false}`). + Relationship attributes keep `relationship`, a non-enumerable `relationshipReference +{database, table}` (GraphQL), and `definition`. `scope.resources.allTypes` in a component holds + every GraphQL type with its connected structure, and `scope.directory` is the application + directory. `defineTable` relations carry `relationship` and a lazy `definition.tableClass` + instead of `relationshipReference`. +- `resources/defineTable.ts` projections: nullable → optional; server-managed and relation fields + read-only on the record and absent from every write; primary key required on read and upsert, + optional on insert, never patched; query = indexed fields, all optional. `compileTypeDef` leaves + PK and server-managed fields' `nullable` unmarked "because the PK machinery and server + assignment own their presence". +- `resources/Table.ts` `validate`: `Int` is int32; `Long` an integer with |v| ≤ 2^53; `Float` any + number; `ID` a string (or array of strings); `BigInt`, `Date` coerced from string/number; + `Bytes` a `Uint8Array` (strings coerced); `Blob` only a `Blob`, a string or a Buffer (anything + else is rejected); computed and relationship attributes may not be assigned; array elements are + validated against the element's `nullable`; a `sealed` table rejects unknown properties. + Nested-object sealing reads `attribute.sealed`, which no schema path sets on nested attributes + (only the table's primary attribute carries `sealed`, `resources/databases.ts`), so nested + objects are not sealed in practice. +- Every live table has `tableName`, `databaseName`, `primaryKey`, `attributes`, `sealed` statics. +- Node refuses to type-strip `.ts` files under `node_modules`, and this package is loaded from + `node_modules` by Harper's component loader (`componentLoader.ts`: `join(componentDirectory, +config.extensionModule)`); CI also tests Node 20. New modules therefore stay JavaScript with + JSDoc types, like the rest of the repository. + +## IR v1 + +JSON, deterministic (tables sorted by database then name, profiles and nested types by name, no +timestamps): + +```jsonc +{ + "irVersion": 1, + "generator": "@harperfast/schema-codegen@", // informational, not hashed + "schemaHash": "", + "tables": [ + { + "database": "data", + "name": "Order", + "typeName": "Order", + "primaryKey": "id", + "sealed": false, + "hash": "", + "profiles": ["my-orders"], + "attributes": [ + { + "name": "amount", + "type": { "kind": "scalar", "scalar": "Float" }, + // or { "kind": "array", "element": , "elementNullable": true } + // or { "kind": "object", "type": "OrderLine" } -> "types" entry + // or { "kind": "record", "database": "data", "table": "Customer" } (embedded table shape) + "nullable": true, + "primaryKey": false, + "indexed": false, + "serverManaged": null, // "createdTime" | "updatedTime" | null + "computed": null, // { "from": "" | null, "version": "" | null } + "readOnly": false, // serverManaged || computed + "description": "…", // optional, not hashed + }, + ], + "relations": [ + { + "name": "customer", + "cardinality": "one", + "from": "customerId", + "target": { "database": "data", "table": "Customer" }, + }, + ], + "projections": { + "record": [{ "name": "id", "optional": false }], + "insert": [], + "patch": [], + "query": [], + "upsert": [], // null when a full replacement is unsafe (Blob attributes) + "clearable": ["note"], // patch attributes that may be set to null + }, + "storage": { + "versionColumn": "_version", + "extraColumn": "_extra", + "columns": [{ "name": "id", "affinity": "TEXT" }], + }, + }, + ], + "types": [{ "name": "OrderLine", "typeName": "OrderLine", "attributes": [] }], + "profiles": [ + { + "name": "my-orders", + "direction": "pull", + "retention": "30d", + "retentionMs": 2592000000, + "hash": "", + "schemaHash": "", + "tables": [ + { + "database": "data", + "table": "Order", + "scope": { + "type": "match", + "conditions": [ + { + "attribute": "customerId", + "operator": "equals", + "source": "claim", + "claim": "sub", + }, + ], + }, + }, + ], + }, + ], + "diagnostics": [ + { "level": "error", "code": "SCOPE_ATTRIBUTE_UNKNOWN", "profile": "…", "message": "…" }, + ], +} +``` + +- **Inputs**: the filtered live tables (`collectTables` with the existing include/exclude + options — one collection feeds TS and client output), the GraphQL type registry + (`scope.resources.allTypes`) to resolve nested object types by name, the `sync.yaml` content, + and the previous IR (type-name stability only). +- **Nullability (read side, "undefined-means-nullable")**: an attribute is optional on read unless + it is the primary key, server-managed, or `nullable === false` (the existing `isNullable` + rule plus the server-managed exemption defineTable makes). Computed attributes are always + optional (an on-device materialization may not have run, issue #18). Array elements follow the + same rule: `[String]` has nullable elements, `[String!]` does not. Anything holding a `Blob` is + optional on read whatever its declaration, because sync does not deliver blob content: a + table attribute whose type contains a Blob at any depth (through the `record` projection, while + the attribute's `nullable` keeps what the server enforces, so `Blob` → `Blob!` still changes + the contract and breaks inserts, and a required Blob raises a `BLOB_REQUIRED` warning), and a + nested attribute or array element that is a Blob or an array of them. +- **Projections** are the defineTable rules over stored attributes, plus one client rule: + writable Blob-bearing attributes are excluded from every write projection and make `upsert` `null` + (a computed Blob is never written anyway), because the + transport has no blob contract yet and a full replacement would either delete the blob + (omitted) or be rejected by `validate` (a placeholder object). `record` = all stored attributes; + `insert` = writable attributes, PK optional; `upsert` = writable, PK required; `patch` = + writable non-PK, all optional; `clearable` = patch attributes whose read nullability allows + null; `query` = indexed attributes. Relations are not stored and appear only under `relations`. +- **`typeName`** is a language-neutral identifier: `_` prefix outside `data` (as the TS + emitter does), singularized table name, characters outside `[A-Za-z0-9_]` sanitized, leading + digit prefixed. Allocation covers tables and nested types in one namespace, avoids a reserved set + (Swift and Kotlin keywords, the standard types the generated code names, the generated runtime's + `Harper*` names, `New`, `Patch`, and the JVM classes Kotlin compiles the generated files' + top-level declarations into, such as `ModelsKt`), and resolves collisions by appending `Record`, then `_2`, + `_3`…. A table or nested type keeps the name it had in the previous IR while that name is still + valid and unclaimed, so adding a colliding table never renames an existing public type; the + diff reports `typeName` changes as source changes. +- **Storage** (the device replica's table shape): one column per stored attribute, plus a version + column (`REAL`) and, on non-sealed tables, an overflow column (`TEXT` JSON). The version/extra + column names are `_version`/`_extra` unless an attribute already uses the name, in which case an + extra leading underscore is added until unique. +- **Validation**: the CLI validates an imported IR (version, references, descriptors) before + emitting; unknown `irVersion` is an error, not a best effort. + +## Type mapping (the recorded decisions) + +| Harper | Swift | Kotlin | SQLite affinity | Row encoding (`HarperValue`) | +| --------------------- | ------------------------------------------ | ------------------------- | --------------- | ------------------------------------------- | +| `ID`, `String` | `String` | `String` | TEXT | string | +| `Int` (int32) | `Int` | `Int` | INTEGER | int | +| `Long` (\|v\| ≤ 2^53) | `Int64` | `Long` | INTEGER | int | +| `Float` | `Double` | `Double` | REAL | double (ints accepted) | +| `BigInt` (arbitrary) | `HarperBigInt` (decimal-string value type) | `java.math.BigInteger` | TEXT | string of digits (ints accepted) | +| `Boolean` | `Bool` | `Boolean` | INTEGER | bool (0/1 accepted) | +| `Date` | `Date` | `java.time.Instant` | REAL (epoch ms) | double epoch ms (ISO-8601 strings accepted) | +| `Bytes` | `Data` | `ByteArray` | BLOB | bytes (base64 strings accepted) | +| `Blob` | `HarperValue?`, read-only | `HarperValue?`, read-only | BLOB | whatever the transport delivered | +| `Any` | `HarperValue` | `HarperValue` | TEXT (JSON) | any | +| `[T]` | `[T]` / `[T?]` | `List` / `List` | TEXT (JSON) | array | +| nested type | generated `struct` | generated `data class` | TEXT (JSON) | object | +| relation | not a property | not a property | no column | — | + +Reasons for the non-obvious rows: + +- **Int → Swift `Int`, not `Int32`**: Swift `Int` is ≥32 bits on every Apple platform, so every + valid value fits; `Int32` would force conversions at every call site. Server validation stays + the guard against out-of-range writes. Kotlin `Int` is exactly int32. +- **Long → `Int64`/`Long`**: explicit 64-bit (watchOS `arm64_32` has a 32-bit Swift `Int`). + Harper bounds `Long` to ±2^53, so the JS core never loses precision in transit. +- **BigInt**: Swift has no arbitrary-precision integer, and `Decimal` silently rounds past 38 + digits; a decimal-string value type is lossless and typed (`int64Value` when it fits). On the + wire to devices BigInt travels as a decimal string (the gateway's job, #10); decoders also + accept integers. TEXT storage means local ordering on BigInt columns is lexicographic — local + query execution (RQL, #7) must compare numerically. +- **Date**: epoch milliseconds in `REAL` matches Harper's own timestamp scale and the version + column, sorts correctly, and is exact for millisecond values. `java.time.Instant` needs Android + API 26+ or core-library desugaring. +- **Blob**: content is fetch-on-demand and not inlined in sync (design doc open question 4) and the + transport placeholder is undecided, so the model exposes what the transport delivered as an + opaque, optional, read-only `HarperValue`, and the table has no upsert projection (see + Projections). A typed blob reference replaces this when blob sync is designed. +- **Any → `HarperValue`**: a closed JSON-like enum (`null/bool/int/double/string/bytes/array/ +object`) that is `Sendable`, `Hashable` and (Swift) `Codable`, so models stay value types. +- **Bytes in Kotlin**: `ByteArray` has identity equality, so generated data classes that hold + byte arrays, directly or in lists, override `equals`/`hashCode` with the runtime's + `HarperContent`, which compares them by content; `HarperValue.BytesValue` compares by content. + +## Client model shape + +Per table (Swift `struct` / Kotlin `data class`), named `typeName`: + +- one property per stored attribute, typed per the table above, optional per read nullability; +- `_version: Double?` — the record version (Harper's update timestamp); `nil` for records not + read from the replica (REST bodies carry it in headers, not the body); +- `_extra: [String: HarperValue]` / `Map` on non-sealed tables — attributes + the schema does not declare, preserved for round trips. Declared names (stored, computed, + relation) are filtered out of `_extra` when decoding **and when encoding**, so an injected + computed or relation key can never reach a write; +- `Order.New` (insert projection) and `Order.Patch` (patch projection) nested types, omitted when + the projection is empty. A patch distinguishes three states per attribute: unset (absent from + the body, unchanged), value, and cleared — `clear: Set` lists `clearable` + attributes to set to null; a set value wins over a clear; +- decode `init(row:version:)` (Swift, throwing) / `Order.decode(row, version)` (Kotlin) from a + `HarperRow` (`[String: HarperValue]`, keyed by attribute name — the logical record, extras at + the top level); `encodeRow()` (full record, for storage) and `encodeUpsertRow()` (PK + writable + attributes + `_extra`, for `PUT`; throws `HarperEncodingError.replacementUnsupported` when the + upsert projection is `null`). Absent optional values are omitted, never encoded as null; +- Swift: `let` for the PK and read-only attributes, `var` for writable ones (value semantics make + edit-then-`put` safe); `Hashable`, `Sendable`, and `Identifiable` (a computed `id` returns the + PK when it has another name and no attribute is called `id`). Kotlin: all `val` (`copy()` is the + edit idiom). A missing required value or a mismatched type throws a `HarperDecodingError` + naming type and attribute: the SDK decides whether to skip and report the row; the model never + guesses; +- property names: attribute names that are already identifiers are kept verbatim (claimed first); + others are sanitized like table names; keywords are escaped with backticks; names that collide + with the generated members (`_version`, `_extra`, `New`, `Patch`, `Field`, `clear`, + `encodeRow`, `encodeUpsertRow`, `harperValue`, `hashValue`, `Companion`), with a runtime type or + the owning type's own name (member bodies name those, and a same-named member would shadow + them), or with each other get a numeric suffix, rechecked against every claimed name. Other + generated types are never named inside member bodies (Kotlin reaches their converters through + file-level values), so adding a table never renames an existing property. The raw attribute + name is always what goes on the row; +- nested object types get the same treatment minus `_version`, `New` and `Patch`, and always keep + `_extra` (nested sealing is not enforced server-side). A nested type reachable from itself by + value (not through an array) cannot be a Swift struct; that edge is boxed in Swift as + `HarperBox` (an immutable final class) and stays a plain reference in Kotlin. + +Generated package layout — fixed file names, so a dropped table never leaves a stale file: + +- Swift: `/Package.swift` (written only if absent, then user-owned) and + `/Sources//{HarperRuntime,HarperSchema,Models}.swift`. +- Kotlin: `/build.gradle.kts` (written only if absent) and + `/src/main/kotlin//{HarperRuntime,HarperSchema,Models}.kt`. + +Every generated file starts with the same marker line carrying the IR's `schemaHash` and the +generator version, so a mixed generation is visible. Generated files elsewhere under the package +directory (a renamed module or package) are reported, never deleted, and a user-owned manifest +that does not name the current module is reported too. + +`HarperRuntime` holds the support types (`HarperValue`, `HarperRow`, `HarperRecord`, +`HarperBigInt`, `HarperBox`, `HarperTableSchema`, `HarperColumn`, `HarperSyncProfile`, +decoding/encoding errors and helpers). It lives in the generated package until the iOS/Android +SDKs (#12/#13) exist; the SDK then ships it and the emitter imports it instead. `HarperSchema` +holds `irVersion`, the schema hash, all table descriptors (columns with affinity, PK, indexed, +read-only, computed expression), and the valid sync profiles (name, direction, retention, tables, +profile `schemaHash`) with one named constant per profile (`HarperSyncProfile.storefront`). + +## Sync profiles (`sync.yaml`) + +No profile format exists in core or pro yet (searched both, 2026-10-05); the spike hard-codes +`server/app/profiles.js`, and the design doc sketches `sync.yaml`. This package parses that +documented format so the gateway (M1) inherits a tested normalizer: + +```yaml +profiles: + storefront: + direction: pull # pull (default) | push | bidirectional + retention: 14d # ms | s | m | h | d | w + tables: [Product] # list form: profile-level scope (default: all) + my-orders: + retention: 30d + scope: + customerId: $token.sub # claim reference; $user. for the user record + tables: [Order] + bench-seg10: + tables: + BenchItem: # map form: per-table database/scope override + scope: { seg: s10 } # literal equality; a list means membership +``` + +Scope values: a scalar is a literal equality; a string starting with `$token.`/`$user.` is a +claim/user reference (`$$` escapes a literal `$`); a list of scalars is literal membership; any +other `$` reference, empty list or object is invalid. Normalized to per-table `{type: 'all'}` or +`{type: 'match', conditions: [...]}`. + +**Profiles fail closed.** Any error — unknown table or database, unknown scope attribute, invalid +scope value or reference, empty table list, invalid direction or retention — invalidates the +whole profile: it is left out of `profiles` (and out of generated code), and the IR carries an +`error` diagnostic naming it. A scope that loses a condition can never degrade into `all`. +Warnings (a scope attribute without an index) keep the profile. Profiles bind tables by name in +`sync.yaml` because a `@sync` type directive does not survive into `Table` (the GraphQL parser +keeps only the directives it knows). + +`yaml` (already a Harper core dependency, zero transitive dependencies) parses it, with its alias +limit left on; it is loaded only when `syncProfiles` is configured. + +## Schema hash and the additive-delta story + +- **Contract**: per table — database, name, PK, sealed, the storage metadata column names, and per + attribute (sorted by name): name, type (object types and embedded tables by name), `nullable`, + PK, indexed, server-managed, computed `{from, version}`; the shapes of every object type and + embedded table reachable from the table (an embedded table with its sealing, attributes and + relations, since the same model decodes it), each listed once by name, so a nested change + reaches every table that carries it while shared and recursive types hash in linear time; + relations by name/cardinality/key/target. Excluded: descriptions, `typeName`, attribute order, + profile membership — none of them changes what a device stores or sends. +- `tables[].hash` = SHA-256 of the canonical JSON (sorted keys) of the table contract; + `schemaHash` = hash of `{irVersion, tables: [table contracts]}`; `profiles[].hash` = hash of the + profile definition (its "version"); `profiles[].schemaHash` = hash of the sorted + `{database, table, hash}` of the profile's tables — **the value the `hello` frame carries**. + `irVersion` is inside every hash and is bumped whenever a mapping or canonicalization change + alters what a device stores or sends, so such a change reads as unknown drift, never a false + match. +- **`diffSchemaIR(from, to)`** classifies each change by what it does to a client generated from + `from`: `read` (its decode can fail), `write` (the server can reject its writes, or a `PUT` + can drop data), `storage` (its replica layout cannot be extended in place — rebuild/resync), or + nothing (additive). Summary: `compatibility: {read, write, storage}`, each + `identical | additive | breaking`, plus `source` changes (`typeName` renames) and a `delta` an + old client can apply without regenerating: new tables' storage descriptors, added columns, + changed computed expressions (recompute pass, issue #18 finding). + +| Change (to a table unless noted) | read | write | storage | +| -------------------------------------------------------------------------------------------------------------------------- | ------------------------------- | ---------------------------------------------------------------------------- | ---------------------------------------- | +| table added | — | — | — (new table) | +| table removed; PK changed; attribute type changed | breaking | breaking | breaking | +| nullable attribute added, table not sealed; index changed | — | — | — (add column) | +| attribute added to a `@sealed` table | — | breaking (old model drops it on `PUT`) | — | +| required (`!`) attribute added | — | breaking (old inserts omit it) | — | +| attribute removed that the old model required on read | breaking | — | — | +| attribute removed from a `@sealed` table | — | breaking (old writes send it) | — | +| attribute became nullable where the old model required it | breaking | — | — | +| attribute became required on insert, or writable became read-only | — | breaking | — | +| element nullability: `[T!]` → `[T]` / `[T]` → `[T!]` | breaking / — | — / breaking | — | +| computed expression/version changed | — | — | — (recompute) | +| computed attribute or relationship added, table not sealed | — | breaking (old model echoes it from `_extra` on `PUT`; `validate` rejects it) | — (add column) | +| stored attribute turned into a relationship | — | breaking (old writes send it) | — | +| `sealed` changed | — | breaking | — (an orphaned `_extra` column is inert) | +| full replacement became unsupported (`upsert` became `null`: a writable Blob added, in nested types too) | — | breaking (an old `PUT` erases Blob content that sync does not deliver) | — | +| version/extra column relocated (attribute named `_version`/`_extra` added) | — | — | breaking | +| nested object type: same rules, never sealed; each type pair is compared once per table, at the first path that reaches it | | | | +| an embedded table's change (reported for each table embedding it, beside its own changes) | the embedded table's read break | the embedded table's write break | — | + +The gateway (#10) chooses by profile direction: `pull` checks `read`, `push` checks `write`, +`bidirectional` both; `storage` always forces a resync. With no consumer for the hello negotiation +yet, the diff has two consumers today: the dev-mode component warns when a regeneration is +breaking against the previous IR file, and `schema-codegen diff` exits non-zero on a breaking +change — the CI check the design doc's risk (4) asks for. + +## Entry points and publication + +Component options (dev mode, flat like the existing `schemaTypes`/`jsdoc`): + +```yaml +'@harperfast/schema-codegen': + package: '@harperfast/schema-codegen' + schemaIR: client/harper-schema.json + syncProfiles: sync.yaml + swift: ios/HarperModels + swiftModule: HarperModels # default HarperModels + kotlin: android/harper-models + kotlinPackage: com.example.harper # default harper.models +``` + +- Client generation runs only when `schemaIR`, `swift` or `kotlin` is set; otherwise nothing new + is imported, parsed or hashed. The client modules and `yaml` are loaded lazily. +- Paths resolve against `scope.directory` (the application directory). Today's outputs resolve + against the process working directory, which writes to the wrong place whenever Harper is not + started from the application directory (observed with `harper dev `); existing options get + the same fix. +- Event-driven regenerations (`updateTable`/`dropTable`/`dropDatabase`, and `sync.yaml` edits) + are coalesced into one trailing run and serialized, so a burst of schema events renders once and + an older run can never finish after a newer one. Each run reads the type registry afresh, and a + run still in flight when its scope closes stops before publishing. A scope closed before the initial five-second + delay installs nothing. +- Everything is rendered and validated in memory before any write. Every changed file is then + staged beside its target before any is renamed into place, so a failed write (a full disk) + publishes nothing, and a failed rename restores the files already replaced from the previous + contents; generated sources are renamed before the IR file, which is the baseline the next run + diffs against. Renames resolve symlinks (the link survives), and a target Windows holds + open is overwritten in place. Errors (YAML, IR validation, I/O) are logged at the component + boundary and leave the previous outputs in place; they never take Harper down. +- An IR read from a file is validated before use, including its hashes: each is recomputed from + the contents, so a hand-merged IR that kept an old hash is refused rather than diffed as + unchanged. The fields derived from the contract (`readOnly`, the projections, column + affinities) must match what the builder derives, and type names may not be reserved, so an + edited IR cannot emit a model that disagrees with its own hash. +- `swiftModule` must be a Swift identifier and `kotlinPackage` dot-separated identifiers (keyword + segments are backtick-escaped in source), so neither can traverse out of the output directory. + Every string emitted into source is escaped for the target language (Kotlin `$` included). + +CLI (`bin/schema-codegen.js`, `npx @harperfast/schema-codegen …`), working from an IR file so it +runs in CI without a Harper instance; errors exit non-zero with a one-line message: + +``` +schema-codegen generate-client --ir client/harper-schema.json --swift ios/HarperModels --kotlin android/harper-models +schema-codegen diff old.json new.json [--json] [--fail-on read,write,storage] +``` + +A remote `--url` mode is deliberately absent: the only remote schema surface today is +`describe_all`, which is lossy (do-less (a)); the gateway's IR endpoint (#16) is the right +source and will serve `buildSchemaIR` output. + +Package: `bin`, and `main` for `index.js`, which exports `buildSchemaIR`, `normalizeSyncProfiles`, +`diffSchemaIR`, `assertSchemaIR`, `emitSwiftPackage` and `emitKotlinModule` for the gateway and +Studio; `files` gains `index.js` and `bin/`. There is deliberately no `exports` map: it would block +the deep imports existing users may have, which is a breaking change for a feature release. +Toolchain floor for generated code: Swift 5.9 (iOS 13, +macOS 10.15), Kotlin 1.9 on the JVM (Android API 26 or desugaring for `java.time`/`Base64`). + +## Planning review resolutions (cross-model, codex, 2026-10-05) + +Framing verdict: `chosen-approach-sound`. Adopted: coalesced/lazy generation and reuse of the +filtered table list (P2); patch null states, Blob replacement safety, encode-time `_extra` +filtering (P1-2); the read/write/storage diff axes including sealed-table and metadata-relocation +rows, and "hash = wire/storage identity" (P1-3); full member/type/profile namespaces, verbatim +names claimed first, `id` synthesis, previous-IR type-name stability, empty projections skipped, +by-value cycles boxed (P1-4); render-then-atomic-publish, serialized runs, close-before-start, +boundary error handling, CLI exit codes (P1-5); fail-closed profiles, IR import validation, +module/package validation and per-language escaping (P1-6); compile + SQLite round-trip native +tests made mandatory on a macOS CI job, reordered-input byte equality, `defineTable` source in the +e2e (P1-7); packaging metadata and a pack-content test (P1-8). Overruled: "new source modules +must be TypeScript" — the package is loaded from `node_modules`, where Node does not type-strip, +and CI runs Node 20; a compile step for one package is out of scope, so the repository's +JavaScript + JSDoc convention stands. + +## Verification plan + +- Unit tests for the IR builder (nullability, projections incl. Blob and clearable, nested types + via the type registry, relations from GraphQL and `defineTable` metadata, naming collisions and + stability, storage columns), profile normalization and fail-closed diagnostics, hashing + (stability under input reordering, sensitivity to contract changes), every row of the diff + table, both emitters (escaping, keywords, empty projections), publication (atomic writes, + manifests written once, stale-file reports), coalescing and close-before-start, the CLI, and the + packed file list. +- Native tests (mandatory on the macOS CI job, skipped elsewhere): generated Swift compiled in + Swift 5 and 6 language modes and Kotlin with `kotlinc`, each driven by a harness that decodes + rows (JSON, Foundation/JVM maps), round-trips encode/decode, checks null states and extras, and + (Swift) writes and re-reads a SQLite replica built from the generated descriptors. +- End-to-end: the component in a hermetic `harper dev` instance writes the IR, Swift and Kotlin + for the spike's schema, a coverage schema and a `defineTable` table; the spike's iOS app is + switched to the generated package (models deleted, `SpikeStorage` driven by the generated + descriptors), built with `xcodebuild`, and run in the simulator against the spike gateway. + +## Not in scope (follow-ups) + +- Porting the TS/JSDoc emitters onto the IR. +- Typed query builders (#7/#12), relation navigation, blob references, invocable functions in the + IR (#18), the gateway's IR endpoint and hello negotiation (#10/#16). diff --git a/extensionModule.js b/extensionModule.js index 543e9ad..9bfaf61 100644 --- a/extensionModule.js +++ b/extensionModule.js @@ -1,10 +1,12 @@ /** @typedef {import('harperdb').Scope} Scope */ -import { setTimeout as delay } from 'node:timers/promises'; +import { createRegenerationScheduler } from './utils/createRegenerationScheduler.js'; import { setLogger } from './utils/logger.js'; -import { regenerateAll } from './utils/regenerateAll.js'; +import { hasClientOutputs, regenerateAll } from './utils/regenerateAll.js'; export const suppressHandleApplicationWarning = true; +const INITIAL_DELAY_MS = 5000; + /** * @param {Scope} scope */ @@ -16,53 +18,82 @@ export async function handleApplication(scope) { return; } - const watchConfig = scope.options.get(['watch']); + /** + * @param {string} key + * @returns {any} + */ + const option = (key) => scope.options.get([key]); + const watchConfig = option('watch'); const shouldWatch = watchConfig === true || watchConfig === undefined; - const globalTypes = /** @type {string} */ (scope.options.get(['globalTypes'])); - const schemaTypes = /** @type {string} */ (scope.options.get(['schemaTypes'])); - const jsdoc = /** @type {string | undefined} */ (scope.options.get(['jsdoc'])); - const moduleName = /** @type {string | undefined} */ (scope.options.get(['module'])); - const includeDatabases = /** @type {string[] | undefined} */ ( - scope.options.get(['includeDatabases']) - ); - const excludeDatabases = /** @type {string[] | undefined} */ ( - scope.options.get(['excludeDatabases']) - ); - const options = { module: moduleName, includeDatabases, excludeDatabases }; - - if (shouldWatch) { - scope.on('close', scopeClosed); - } - - // Do not await this. - delay(5000).then(() => { - // Initial generation - regenerateAll(globalTypes, schemaTypes, jsdoc, options); + const globalTypes = /** @type {string} */ (option('globalTypes')); + const schemaTypes = /** @type {string} */ (option('schemaTypes')); + const jsdoc = /** @type {string | undefined} */ (option('jsdoc')); + const client = { + schemaIR: /** @type {string | undefined} */ (option('schemaIR')), + syncProfiles: /** @type {string | undefined} */ (option('syncProfiles')), + swift: /** @type {string | undefined} */ (option('swift')), + swiftModule: /** @type {string | undefined} */ (option('swiftModule')), + kotlin: /** @type {string | undefined} */ (option('kotlin')), + kotlinPackage: /** @type {string | undefined} */ (option('kotlinPackage')), + }; + const options = { + module: /** @type {string | undefined} */ (option('module')), + includeDatabases: /** @type {string[] | undefined} */ (option('includeDatabases')), + excludeDatabases: /** @type {string[] | undefined} */ (option('excludeDatabases')), + baseDirectory: /** @type {any} */ (scope).directory, + }; + const closing = new AbortController(); + const scheduler = createRegenerationScheduler( + () => + regenerateAll(globalTypes, schemaTypes, jsdoc, { + ...options, + client: { + ...client, + types: /** @type {any} */ (scope).resources?.allTypes, + signal: closing.signal, + }, + }), + { + onError: (error) => + scope.logger.error?.( + `@harperfast/schema-codegen failed to regenerate: ${/** @type {Error} */ (error)?.stack ?? error}`, + ), + }, + ); + const schedule = () => scheduler.schedule(); + let started = false; + const initialTimer = setTimeout(() => { + started = true; + scheduler.schedule(); if (shouldWatch) { - // Watch for schema/database changes via events - scope.databaseEvents.on('updateTable', updateTable); - scope.databaseEvents.on('dropTable', dropTable); - scope.databaseEvents.on('dropDatabase', dropDatabase); + scope.databaseEvents.on('updateTable', schedule); + scope.databaseEvents.on('dropTable', schedule); + scope.databaseEvents.on('dropDatabase', schedule); } - }); - - function updateTable() { - regenerateAll(globalTypes, schemaTypes, jsdoc, options); - } + }, INITIAL_DELAY_MS); - function dropTable() { - regenerateAll(globalTypes, schemaTypes, jsdoc, options); + if (shouldWatch && client.syncProfiles && hasClientOutputs(client)) { + try { + /** @type {any} */ (scope).handleEntry({ files: client.syncProfiles }, () => { + if (started) scheduler.schedule(); + }); + } catch (error) { + scope.logger.warn?.( + `@harperfast/schema-codegen cannot watch ${client.syncProfiles}; edits apply on the next schema change: ${/** @type {Error} */ (error)?.message}`, + ); + } } - function dropDatabase() { - regenerateAll(globalTypes, schemaTypes, jsdoc, options); - } + scope.on('close', scopeClosed); function scopeClosed() { - scope.databaseEvents.off('updateTable', updateTable); - scope.databaseEvents.off('dropTable', dropTable); - scope.databaseEvents.off('dropDatabase', dropDatabase); + clearTimeout(initialTimer); + closing.abort(); + scheduler.close(); + scope.databaseEvents.off('updateTable', schedule); + scope.databaseEvents.off('dropTable', schedule); + scope.databaseEvents.off('dropDatabase', schedule); scope.off('close', scopeClosed); } } diff --git a/extensionModule.test.js b/extensionModule.test.js new file mode 100644 index 0000000..87a5e92 --- /dev/null +++ b/extensionModule.test.js @@ -0,0 +1,152 @@ +import { EventEmitter } from 'node:events'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const regenerateAll = vi.hoisted(() => vi.fn(async () => {})); +vi.mock('./utils/regenerateAll.js', async (importOriginal) => ({ + .../** @type {object} */ (await importOriginal()), + regenerateAll, +})); + +const { handleApplication } = await import('./extensionModule.js'); + +/** + * @param {Record} options + */ +function fakeScope(options) { + const scope = Object.assign(new EventEmitter(), { + logger: { trace: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, + options: { get: (/** @type {string[]} */ [key]) => options[key] }, + databaseEvents: new EventEmitter(), + directory: '/apps/demo', + resources: { allTypes: new Map() }, + handleEntry: vi.fn(), + }); + return scope; +} + +describe('handleApplication', () => { + beforeEach(() => { + vi.useFakeTimers(); + process.env.DEV_MODE = 'true'; + }); + afterEach(() => { + vi.useRealTimers(); + delete process.env.DEV_MODE; + vi.clearAllMocks(); + }); + + it('does nothing outside dev mode', async () => { + delete process.env.DEV_MODE; + const scope = fakeScope({ schemaTypes: 'types.ts' }); + await handleApplication(/** @type {any} */ (scope)); + await vi.advanceTimersByTimeAsync(10_000); + expect(regenerateAll).not.toHaveBeenCalled(); + }); + + it('generates after the initial delay, resolving outputs against the app directory', async () => { + const scope = fakeScope({ + schemaTypes: 'types.ts', + swift: 'ios/HarperModels', + swiftModule: 'Demo', + }); + await handleApplication(/** @type {any} */ (scope)); + await vi.advanceTimersByTimeAsync(4_999); + expect(regenerateAll).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(200); + expect(regenerateAll).toHaveBeenCalledTimes(1); + const [, schemaTypes, , options] = regenerateAll.mock.calls[0]; + expect(schemaTypes).toBe('types.ts'); + expect(options).toMatchObject({ + baseDirectory: '/apps/demo', + client: { swift: 'ios/HarperModels', swiftModule: 'Demo', types: scope.resources.allTypes }, + }); + }); + + it('reads the type registry on every run', async () => { + const scope = fakeScope({ schemaTypes: 'types.ts', schemaIR: 'ir.json' }); + await handleApplication(/** @type {any} */ (scope)); + await vi.advanceTimersByTimeAsync(5_200); + const replaced = new Map([['Address', { attributes: [] }]]); + scope.resources.allTypes = replaced; + scope.databaseEvents.emit('updateTable'); + await vi.advanceTimersByTimeAsync(200); + expect(regenerateAll.mock.calls.at(-1)?.[3]).toMatchObject({ client: { types: replaced } }); + }); + + it('aborts a run still in flight when the scope closes', async () => { + const scope = fakeScope({ schemaTypes: 'types.ts', schemaIR: 'ir.json' }); + await handleApplication(/** @type {any} */ (scope)); + await vi.advanceTimersByTimeAsync(5_200); + const { signal } = /** @type {any} */ (regenerateAll.mock.calls[0][3]).client; + expect(signal.aborted).toBe(false); + scope.emit('close'); + expect(signal.aborted).toBe(true); + }); + + it('coalesces a burst of schema events into one regeneration', async () => { + const scope = fakeScope({ schemaTypes: 'types.ts' }); + await handleApplication(/** @type {any} */ (scope)); + await vi.advanceTimersByTimeAsync(5_200); + regenerateAll.mockClear(); + for (let i = 0; i < 5; i++) scope.databaseEvents.emit('updateTable'); + scope.databaseEvents.emit('dropTable'); + await vi.advanceTimersByTimeAsync(200); + expect(regenerateAll).toHaveBeenCalledTimes(1); + }); + + it('installs nothing when the scope closes before the initial delay', async () => { + const scope = fakeScope({ schemaTypes: 'types.ts' }); + await handleApplication(/** @type {any} */ (scope)); + scope.emit('close'); + await vi.advanceTimersByTimeAsync(10_000); + expect(regenerateAll).not.toHaveBeenCalled(); + expect(scope.databaseEvents.listenerCount('updateTable')).toBe(0); + }); + + it('stops listening when the scope closes', async () => { + const scope = fakeScope({ schemaTypes: 'types.ts' }); + await handleApplication(/** @type {any} */ (scope)); + await vi.advanceTimersByTimeAsync(5_200); + scope.emit('close'); + regenerateAll.mockClear(); + scope.databaseEvents.emit('updateTable'); + await vi.advanceTimersByTimeAsync(1_000); + expect(regenerateAll).not.toHaveBeenCalled(); + expect(scope.databaseEvents.listenerCount('updateTable')).toBe(0); + }); + + it('logs a failed regeneration instead of throwing into Harper', async () => { + regenerateAll.mockRejectedValueOnce(new Error('disk full')); + const scope = fakeScope({ schemaTypes: 'types.ts' }); + await handleApplication(/** @type {any} */ (scope)); + await vi.advanceTimersByTimeAsync(5_200); + expect(scope.logger.error).toHaveBeenCalledWith(expect.stringContaining('disk full')); + }); + + it('watches sync.yaml only when a client output is configured', async () => { + const withClient = fakeScope({ syncProfiles: 'sync.yaml', kotlin: 'android' }); + await handleApplication(/** @type {any} */ (withClient)); + expect(withClient.handleEntry).toHaveBeenCalledWith( + { files: 'sync.yaml' }, + expect.any(Function), + ); + + const profilesOnly = fakeScope({ syncProfiles: 'sync.yaml' }); + await handleApplication(/** @type {any} */ (profilesOnly)); + expect(profilesOnly.handleEntry).not.toHaveBeenCalled(); + }); + + it('regenerates on sync.yaml changes once started', async () => { + const scope = fakeScope({ syncProfiles: 'sync.yaml', schemaIR: 'ir.json' }); + await handleApplication(/** @type {any} */ (scope)); + const onEntry = scope.handleEntry.mock.calls[0][1]; + onEntry({ eventType: 'add' }); + await vi.advanceTimersByTimeAsync(1_000); + expect(regenerateAll).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(4_200); + regenerateAll.mockClear(); + onEntry({ eventType: 'change' }); + await vi.advanceTimersByTimeAsync(200); + expect(regenerateAll).toHaveBeenCalledTimes(1); + }); +}); diff --git a/index.js b/index.js new file mode 100644 index 0000000..f414604 --- /dev/null +++ b/index.js @@ -0,0 +1,6 @@ +export { buildSchemaIR, IR_VERSION } from './utils/client/buildSchemaIR.js'; +export { diffSchemaIR } from './utils/client/diffSchemaIR.js'; +export { emitKotlinModule } from './utils/client/emitKotlin.js'; +export { emitSwiftPackage } from './utils/client/emitSwift.js'; +export { normalizeSyncProfiles } from './utils/client/syncProfiles.js'; +export { assertSchemaIR } from './utils/client/validateSchemaIR.js'; diff --git a/package-lock.json b/package-lock.json index f2aeeba..dba5106 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,9 @@ "name": "@harperfast/schema-codegen", "version": "2.0.0", "license": "Apache-2.0", + "dependencies": { + "yaml": "^2.9.1" + }, "devDependencies": { "@commitlint/cli": "^21.0.0", "@commitlint/config-conventional": "^21.0.0", @@ -8481,6 +8484,21 @@ "node": ">=10" } }, + "node_modules/yaml": { + "version": "2.9.1", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.1.tgz", + "integrity": "sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw==", + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + }, "node_modules/yargs": { "version": "18.0.0", "resolved": "https://registry.npmjs.org/yargs/-/yargs-18.0.0.tgz", diff --git a/package.json b/package.json index 8aae2a8..12f1ac2 100644 --- a/package.json +++ b/package.json @@ -16,13 +16,19 @@ "type": "git", "url": "git+https://github.com/HarperFast/schema-codegen.git" }, + "bin": { + "schema-codegen": "bin/schema-codegen.js" + }, "files": [ "extensionModule.js", "config.yaml", + "index.js", + "bin/", "utils/", "!utils/**/*.test.js" ], "type": "module", + "main": "./index.js", "scripts": { "commitlint": "commitlint --edit", "format:check": "oxfmt --check", @@ -34,6 +40,9 @@ "test:coverage": "vitest run --coverage", "test:watch": "vitest" }, + "dependencies": { + "yaml": "^2.9.1" + }, "devDependencies": { "@commitlint/cli": "^21.0.0", "@commitlint/config-conventional": "^21.0.0", diff --git a/package.test.js b/package.test.js new file mode 100644 index 0000000..4db31bd --- /dev/null +++ b/package.test.js @@ -0,0 +1,49 @@ +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import { describe, expect, it } from 'vitest'; + +describe('published package', () => { + it('ships the entry points, the client generator and its runtime sources, and no tests', () => { + const npm = process.platform === 'win32' ? 'npm.cmd' : 'npm'; + const [pack] = JSON.parse( + execFileSync(npm, ['pack', '--dry-run', '--json', '--ignore-scripts'], { + encoding: 'utf8', + shell: process.platform === 'win32', + }), + ); + const files = pack.files.map((/** @type {{ path: string }} */ file) => + file.path.replace(/\\/g, '/'), + ); + for (const required of [ + 'index.js', + 'extensionModule.js', + 'config.yaml', + 'bin/schema-codegen.js', + 'utils/client/buildSchemaIR.js', + 'utils/client/runtime/HarperRuntime.swift', + 'utils/client/runtime/HarperRuntime.kt', + ]) { + expect(files).toContain(required); + } + expect(files.filter((file) => file.endsWith('.test.js'))).toEqual([]); + expect(files.filter((file) => file.startsWith('test/') || file.startsWith('docs/'))).toEqual( + [], + ); + }); + + it('exposes the public API from the package entry point', async () => { + const { main } = JSON.parse( + fs.readFileSync(new URL('./package.json', import.meta.url), 'utf8'), + ); + const api = await import(new URL(main, import.meta.url).href); + expect(Object.keys(api).sort()).toEqual([ + 'IR_VERSION', + 'assertSchemaIR', + 'buildSchemaIR', + 'diffSchemaIR', + 'emitKotlinModule', + 'emitSwiftPackage', + 'normalizeSyncProfiles', + ]); + }); +}); diff --git a/test/fixtures/clientSchema.js b/test/fixtures/clientSchema.js new file mode 100644 index 0000000..1644949 --- /dev/null +++ b/test/fixtures/clientSchema.js @@ -0,0 +1,262 @@ +/** + * Live-table-shaped fixtures for client codegen tests: the device-sync spike's tables, a coverage + * database exercising every attribute kind, and adversarial names. Attribute objects mirror what + * Harper puts on `Table.attributes` (verified against a 5.3 instance): nested object types appear + * only by name and resolve through the GraphQL type registry passed as `types`. + */ + +/** @returns {any[]} */ +export function spikeTables() { + return [ + { + tableName: 'Product', + databaseName: 'data', + primaryKey: 'id', + attributes: [ + { name: 'id', type: 'ID', isPrimaryKey: true }, + { name: 'name', type: 'String' }, + { name: 'price', type: 'Float' }, + { name: 'category', type: 'String', indexed: {} }, + ], + }, + { + tableName: 'Order', + databaseName: 'data', + primaryKey: 'id', + attributes: [ + { name: 'id', type: 'ID', isPrimaryKey: true }, + { name: 'customerId', type: 'ID', indexed: {} }, + { name: 'status', type: 'String', indexed: {} }, + { name: 'amount', type: 'Float' }, + { + name: 'points', + type: 'Float', + computed: { from: () => 0 }, + computedFromExpression: "Math.round((amount || 0) * (status == 'open' ? 10 : 25))", + version: "Math.round((amount || 0) * (status == 'open' ? 10 : 25))", + }, + ], + }, + { + tableName: 'Report', + databaseName: 'data', + primaryKey: 'id', + attributes: [ + { name: 'id', type: 'ID', isPrimaryKey: true }, + { name: 'title', type: 'String' }, + { name: 'body', type: 'String' }, + ], + }, + ]; +} + +/** @returns {any[]} */ +export function coverageTables() { + return [ + { + tableName: 'Customer', + databaseName: 'coverage', + primaryKey: 'id', + attributes: [ + { name: 'id', type: 'ID', isPrimaryKey: true }, + { name: 'name', type: 'String', nullable: false }, + { name: 'email', type: 'String', indexed: {} }, + { name: 'createdAt', type: 'Date', assignCreatedTime: true }, + { name: 'updatedAt', type: 'Float', assignUpdatedTime: true }, + { + name: 'purchases', + type: 'array', + relationship: { to: 'customerId' }, + relationshipReference: { database: 'coverage', table: 'Purchase' }, + elements: { type: 'Purchase' }, + }, + { name: 'address', type: 'Address' }, + { name: 'tags', type: 'array', elements: { type: 'String', nullable: false } }, + { name: 'scores', type: 'array', elements: { type: 'Int' } }, + { name: 'big', type: 'BigInt' }, + { name: 'raw', type: 'Bytes' }, + { name: 'files', type: 'array', elements: { type: 'Bytes' } }, + { name: 'meta', type: 'Any' }, + { name: 'visits', type: 'Long' }, + { name: 'active', type: 'Boolean' }, + { name: 'tree', type: 'Node' }, + ], + }, + { + tableName: 'Purchase', + databaseName: 'coverage', + primaryKey: 'id', + sealed: true, + attributes: [ + { name: 'id', type: 'Long', isPrimaryKey: true }, + { name: 'customerId', type: 'ID', indexed: {} }, + { + name: 'customer', + type: 'Customer', + relationship: { from: 'customerId' }, + relationshipReference: { database: 'coverage', table: 'Customer' }, + }, + { name: 'total', type: 'Float', nullable: false }, + { + name: 'lines', + type: 'array', + nullable: false, + elements: { type: 'PurchaseLine', nullable: false }, + }, + ], + }, + { + tableName: 'Attachment', + databaseName: 'coverage', + primaryKey: 'id', + attributes: [ + { name: 'id', type: 'ID', isPrimaryKey: true }, + { name: 'label', type: 'String' }, + { name: 'file', type: 'Blob' }, + ], + }, + ]; +} + +/** @returns {any[]} */ +export function adversarialTables() { + return [ + { + tableName: 'Weird', + databaseName: 'data', + primaryKey: 'id', + attributes: [ + { name: 'id', type: 'ID', isPrimaryKey: true }, + { name: 'default', type: 'String' }, + { name: 'first-name', type: 'String' }, + { name: 'firstName', type: 'String' }, + { name: '_extra', type: 'String' }, + { name: '_version', type: 'Int' }, + { name: 'encodeRow', type: 'Int' }, + { name: 'Type', type: 'Int' }, + { name: 'self', type: 'String' }, + { name: 'row', type: 'String' }, + { name: 'version', type: 'Long' }, + { name: 'value', type: 'Any' }, + { name: 'in', type: 'Boolean' }, + { name: 'o"conn\\or $x', type: 'String' }, + { name: '9lives', type: 'Boolean', nullable: false }, + { name: 'HarperConverters', type: 'array', elements: { type: 'String' } }, + { name: 'Address', type: 'Address' }, + ], + }, + { + tableName: 'ModelsKt', + databaseName: 'data', + primaryKey: 'id', + attributes: [ + { name: 'id', type: 'ID', isPrimaryKey: true }, + { name: 'tags', type: 'array', elements: { type: 'String' } }, + ], + }, + { + tableName: 'Data', + databaseName: 'data', + primaryKey: 'key', + attributes: [ + { name: 'key', type: 'String', isPrimaryKey: true }, + { name: 'id', type: 'Int' }, + ], + }, + { + tableName: 'OnlyId', + databaseName: 'data', + primaryKey: 'id', + attributes: [{ name: 'id', type: 'ID', isPrimaryKey: true }], + }, + { + tableName: 'SkuItems', + databaseName: 'data', + primaryKey: 'sku', + attributes: [ + { name: 'sku', type: 'String', isPrimaryKey: true }, + { name: 'quantity', type: 'Int', nullable: false }, + ], + }, + ]; +} + +/** The GraphQL type registry (`resources.allTypes`) for the coverage database. */ +export function coverageTypes() { + return new Map([ + [ + 'Address', + { + attributes: [ + { name: 'street', type: 'String' }, + { name: 'city', type: 'String', nullable: false }, + { name: 'geo', type: 'Geo' }, + ], + }, + ], + [ + 'Geo', + { + attributes: [ + { name: 'lat', type: 'Float' }, + { name: 'lng', type: 'Float' }, + ], + }, + ], + [ + 'PurchaseLine', + { + attributes: [ + { name: 'sku', type: 'String', nullable: false }, + { name: 'qty', type: 'Int' }, + { name: 'note', type: 'String' }, + ], + }, + ], + [ + 'Node', + { + attributes: [ + { name: 'label', type: 'String' }, + { name: 'parent', type: 'Node' }, + { name: 'children', type: 'array', elements: { type: 'Node', nullable: false } }, + ], + }, + ], + ['Customer', { table: 'Customer', database: 'coverage', attributes: [] }], + ['Purchase', { table: 'Purchase', database: 'coverage', attributes: [] }], + ]); +} + +/** + * Forty nested types, each holding the next twice: every path through them is distinct, so + * anything that walks paths instead of types never finishes. + * @param {string} leafType + * @returns {Map} + */ +export function sharedChain(leafType) { + return new Map( + Array.from({ length: 40 }, (_, index) => [ + `T${index}`, + { + attributes: + index === 39 + ? [{ name: 'leaf', type: leafType }] + : [ + { name: 'left', type: `T${index + 1}` }, + { name: 'right', type: `T${index + 1}` }, + ], + }, + ]), + ); +} + +/** The spike's `profiles.js`, in `sync.yaml` form. */ +export function spikeProfiles() { + return { + profiles: { + storefront: { retention: '14d', tables: ['Product'] }, + 'my-orders': { retention: '30d', scope: { customerId: '$user.username' }, tables: ['Order'] }, + }, + }; +} diff --git a/test/fixtures/native/KotlinHarness.kt b/test/fixtures/native/KotlinHarness.kt new file mode 100644 index 0000000..274ec61 --- /dev/null +++ b/test/fixtures/native/KotlinHarness.kt @@ -0,0 +1,154 @@ +// Exercises generated Kotlin models against the semantics the IR promises. Compiled together with +// the generated sources by utils/client/native.test.js; prints "PASS " or the failures. +package harper.harness + +import java.math.BigInteger +import java.time.Instant +import kotlin.system.exitProcess + +private val failures = mutableListOf() +private var checks = 0 + +private fun check(condition: Boolean, message: String) { + checks++ + if (!condition) failures.add(message) +} + +private inline fun expectThrows(message: String, matching: (E) -> Boolean, body: () -> Unit) { + checks++ + try { + body() + failures.add("$message: did not throw") + } catch (error: Throwable) { + if (error !is E || !matching(error)) failures.add("$message: threw $error") + } +} + +@Suppress("UNCHECKED_CAST") +private fun row(vararg entries: Pair): HarperRow = + (HarperValue.of(mapOf(*entries)) as HarperValue.ObjectValue).value + +fun main() { + val order = Order.decode( + row("id" to "o1", "customerId" to "alice", "status" to "open", "amount" to 12.5, "points" to 125, "coupon" to "SPRING"), + 1791228562919.5, + ) + check(order.id == "o1" && order.customerId == "alice" && order.amount == 12.5 && order.points == 125.0, "order fields") + check(order._version == 1791228562919.5, "order version") + check(order._extra == mapOf("coupon" to HarperValue.StringValue("SPRING")), "undeclared attributes land in _extra") + check(Order.decode(order.encodeRow(), order._version) == order, "order round trip") + check("points" !in order.encodeUpsertRow(), "computed attributes are not upserted") + + expectThrows("a missing required attribute", { it.type == "coverage_Customer" && it.attribute == "name" }) { + coverage_Customer.decode(row("id" to "c1")) + } + expectThrows("a mismatched type", { it.attribute == "price" && it.expected == "double" && it.actual == "string" }) { + Product.decode(row("id" to "p1", "price" to "cheap")) + } + expectThrows("a date that is not a number", { it.attribute == "createdAt" && it.expected == "date" && it.actual == "out-of-range date" }) { + coverage_Customer.decode(row("id" to "c1", "name" to "Ada", "createdAt" to Double.NaN, "updatedAt" to 0.0)) + } + expectThrows("a Long at 2^63", { it.attribute == "visits" && it.expected == "int" && it.actual == "double" }) { + coverage_Customer.decode(row("id" to "c1", "name" to "Ada", "createdAt" to 0.0, "updatedAt" to 0.0, "visits" to 9.223372036854775808E18)) + } + + val customer = coverage_Customer.decode( + row( + "id" to "c1", + "name" to "Ada", + "createdAt" to "2026-10-05T12:34:56.789Z", + "updatedAt" to 1791203696000.0, + "address" to mapOf("street" to "1 Main", "city" to "Denver", "geo" to mapOf("lat" to 39.7, "lng" to -104.9), "unit" to "4B"), + "tags" to listOf("a", "b"), + "scores" to listOf(1, null, 3), + "big" to "123456789012345678901234567890", + "raw" to "AQID", + "files" to listOf("AQID", "BAU="), + "meta" to mapOf("nested" to listOf(true, 1, "x", null)), + "visits" to 9007199254740991L, + "active" to 1, + "tree" to mapOf("label" to "root", "parent" to mapOf("label" to "up"), "children" to listOf(mapOf("label" to "leaf"))), + "purchases" to listOf(mapOf("id" to 1)), + "loyalty" to "gold", + ), + ) + check(customer.createdAt == Instant.parse("2026-10-05T12:34:56.789Z"), "ISO instants") + val address = customer.address + check(address != null && address.city == "Denver" && address.geo?.lat == 39.7, "nested object types") + check(address?._extra == mapOf("unit" to HarperValue.StringValue("4B")), "nested types keep undeclared attributes") + check(customer.tags == listOf("a", "b") && customer.scores == listOf(1, null, 3), "lists with and without nullable elements") + check(customer.big == BigInteger("123456789012345678901234567890"), "BigInt beyond Long") + check(customer.raw?.contentEquals(byteArrayOf(1, 2, 3)) == true, "Bytes from base64") + check(customer.files?.map { it?.toList() } == listOf(listOf(1, 2, 3), listOf(4, 5)), "lists of Bytes") + check(customer.visits == 9007199254740991L && customer.active == true, "Long and 0/1 booleans") + val tree = customer.tree + check(tree != null && tree.parent?.label == "up" && tree.children?.first()?.label == "leaf", "recursive nested types") + check(customer._extra == mapOf("loyalty" to HarperValue.StringValue("gold")), "relations are never extras") + val again = coverage_Customer.decode(customer.encodeRow()) + check(again == customer && again.hashCode() == customer.hashCode(), "customer round trip compares ByteArray content, in lists too") + + val injected = customer.copy(_extra = customer._extra + mapOf("createdAt" to HarperValue.DoubleValue(0.0), "purchases" to HarperValue.ArrayValue(emptyList()))) + val upsert = injected.encodeUpsertRow() + check("createdAt" !in upsert && "updatedAt" !in upsert && "purchases" !in upsert, "upserts drop read-only, relation and injected keys") + check(upsert["loyalty"] == HarperValue.StringValue("gold"), "upserts keep undeclared attributes") + check(injected.encodeRow()["createdAt"] == HarperConverters.instant.encode(customer.createdAt), "typed values win over injected extras") + + val insert = coverage_Customer.New(name = "Grace", tags = listOf("x"), _extra = mapOf("referrer" to HarperValue.StringValue("ada"), "createdAt" to HarperValue.DoubleValue(1.0))) + check( + insert.encodeRow() == mapOf( + "name" to HarperValue.StringValue("Grace"), + "tags" to HarperValue.ArrayValue(listOf(HarperValue.StringValue("x"))), + "referrer" to HarperValue.StringValue("ada"), + ), + "insert body", + ) + + val patch = Order.Patch(status = "closed", clear = setOf(Order.Patch.Field.amount, Order.Patch.Field.status)) + check(patch.encodeRow() == mapOf("status" to HarperValue.StringValue("closed"), "amount" to HarperValue.Null), "patch: set, clear, and set wins over clear") + check(Order.Patch().encodeRow().isEmpty(), "an empty patch changes nothing") + + val attachment = coverage_Attachment.decode(row("id" to "f1", "label" to "deck", "file" to mapOf("contentType" to "application/pdf"))) + check(attachment.file == HarperValue.ObjectValue(mapOf("contentType" to HarperValue.StringValue("application/pdf"))), "Blob placeholders pass through untouched") + expectThrows("full replacement of a record holding a Blob", { it.attribute == "file" }) { + attachment.encodeUpsertRow() + } + + val purchase = coverage_Purchase.decode(row("id" to 7, "total" to 9.5, "lines" to listOf(mapOf("sku" to "s1", "qty" to 2)), "rogue" to true)) + check("rogue" !in purchase.encodeRow() && purchase.lines.first().sku == "s1", "sealed tables drop undeclared attributes") + + val weird = Weird.decode( + row( + "id" to "w1", + "default" to "d", + "first-name" to "kebab", + "firstName" to "camel", + "_extra" to "attribute", + "_version" to 4, + "encodeRow" to 5, + "Type" to 6, + "self" to "me", + "in" to true, + "o\"conn\\or \$x" to "quoted", + "9lives" to false, + "surprise" to 1, + ), + 8.0, + ) + check(weird.default == "d" && weird.firstName_2 == "kebab" && weird.firstName == "camel" && weird.`in` == true, "renamed and escaped properties") + check(weird._extra_2 == "attribute" && weird._version_2 == 4 && weird.encodeRow_2 == 5 && weird.Type == 6 && weird.self_2 == "me", "attributes named like generated members") + check(weird._extra == mapOf("surprise" to HarperValue.LongValue(1)) && weird._version == 8.0, "the generated members stay the generated members") + check(weird.o_conn_or__x == "quoted" && weird._9lives == false, "sanitized names") + check(Weird.decode(weird.encodeRow(), 8.0) == weird, "raw names round trip") + + check(HarperSchema.IR_VERSION == 1 && HarperSchema.SCHEMA_HASH.length == 64, "schema identity") + check(HarperSchema.table(Order::class) === Order, "table lookup by class") + check(HarperSchema.profiles.map { it.name } == listOf("my-orders", "storefront"), "valid profiles only") + check(HarperProfiles.myOrders.tables == listOf(Order.schema) && HarperProfiles.myOrders.retentionMs == 2_592_000_000L, "profile constants") + + if (failures.isEmpty()) { + println("PASS $checks") + } else { + println(failures.joinToString("\n")) + exitProcess(1) + } +} diff --git a/test/fixtures/native/SwiftHarness.swift b/test/fixtures/native/SwiftHarness.swift new file mode 100644 index 0000000..872212f --- /dev/null +++ b/test/fixtures/native/SwiftHarness.swift @@ -0,0 +1,256 @@ +// Exercises generated Swift models against the semantics the IR promises. Compiled together with +// the generated sources by utils/client/native.test.js; prints "PASS " or the failures. +import Foundation +import SQLite3 + +nonisolated(unsafe) var failures: [String] = [] +nonisolated(unsafe) var checks = 0 + +func check(_ condition: @autoclosure () -> Bool, _ message: String, line: Int = #line) { + checks += 1 + if !condition() { failures.append("line \(line): \(message)") } +} + +func expectThrows(_ message: String, line: Int = #line, _ body: () throws -> T, matching: (Error) -> Bool) { + checks += 1 + do { + _ = try body() + failures.append("line \(line): \(message): did not throw") + } catch { + if !matching(error) { failures.append("line \(line): \(message): threw \(error)") } + } +} + +func row(_ json: String) -> HarperRow { + let object = try! JSONSerialization.jsonObject(with: Data(json.utf8)) as! [String: Any] + return object.mapValues { HarperValue(any: $0) } +} + +func decodedRow(_ json: String) -> HarperRow { + guard case .object(let row) = try! JSONDecoder().decode(HarperValue.self, from: Data(json.utf8)) else { fatalError() } + return row +} + +// MARK: decoding the spike's models + +let orderJSON = #"{"id":"o1","customerId":"alice","status":"open","amount":12.5,"points":125,"coupon":"SPRING","_version":3}"# +let order = try! Order(row: row(orderJSON), version: 1791228562919.5) +check(order.id == "o1" && order.customerId == "alice" && order.amount == 12.5 && order.points == 125, "order fields") +check(order._version == 1791228562919.5, "order version") +check(order._extra == ["coupon": .string("SPRING"), "_version": .int(3)], "undeclared attributes land in _extra: \(order._extra)") +check(try! Order(row: decodedRow(orderJSON), version: 1791228562919.5) == order, "JSONDecoder and JSONSerialization rows decode alike") +check(try! Order(row: order.encodeRow(), version: order._version) == order, "order round trip") +check((try! order.encodeUpsertRow())["points"] == nil, "computed attributes are not upserted") +check(Order.tableName == "Order" && Order.schema.columns.first { $0.name == "points" }?.computedFrom != nil, "schema carries the computed expression") + +expectThrows("a missing required attribute", { try coverage_Customer(row: ["id": .string("c1")], version: nil) }) { + ($0 as? HarperDecodingError) == .missingValue(type: "coverage_Customer", attribute: "name") +} +expectThrows("a mismatched type", { try Product(row: ["id": .string("p1"), "price": .string("cheap")], version: nil) }) { + ($0 as? HarperDecodingError) == .typeMismatch(type: "Product", attribute: "price", expected: "double", actual: "string") +} +expectThrows("a date that is not a number", { try coverage_Customer(row: ["id": .string("c1"), "name": .string("Ada"), "createdAt": .double(.nan), "updatedAt": .double(0)], version: nil) }) { + ($0 as? HarperDecodingError) == .typeMismatch(type: "coverage_Customer", attribute: "createdAt", expected: "date", actual: "out-of-range date") +} +expectThrows("a Long at 2^63", { try coverage_Customer(row: ["id": .string("c1"), "name": .string("Ada"), "createdAt": .double(0), "updatedAt": .double(0), "visits": .double(9.223372036854775808e18)], version: nil) }) { + ($0 as? HarperDecodingError) == .typeMismatch(type: "coverage_Customer", attribute: "visits", expected: "int", actual: "double") +} + +// MARK: every type mapping + +let customerJSON = #""" +{"id":"c1","name":"Ada","email":"ada@example.com","createdAt":"2026-10-05T12:34:56.789Z","updatedAt":1791203696000, + "address":{"street":"1 Main","city":"Denver","geo":{"lat":39.7,"lng":-104.9},"unit":"4B"}, + "tags":["a","b"],"scores":[1,null,3],"big":"123456789012345678901234567890","raw":"AQID","files":["AQID","BAU="], + "meta":{"nested":[true,1,"x",null]},"visits":9007199254740991,"active":1, + "tree":{"label":"root","parent":{"label":"up"},"children":[{"label":"leaf"}]}, + "purchases":[{"id":1}],"loyalty":"gold"} +"""# +let customer = try! coverage_Customer(row: row(customerJSON), version: nil) +check(abs(customer.createdAt.timeIntervalSince1970 - 1791203696.789) < 0.0005, "ISO dates with fractional seconds") +check(customer.updatedAt == 1791203696000, "server-managed Float") +check(customer.address?.city == "Denver" && customer.address?.geo?.lat == 39.7, "nested object types") +check(customer.address?._extra == ["unit": .string("4B")], "nested types keep undeclared attributes") +check(customer.tags == ["a", "b"] && customer.scores == [1, nil, 3], "arrays with and without nullable elements") +check(customer.big?.decimalString == "123456789012345678901234567890" && customer.big?.int64Value == nil, "BigInt beyond Int64") +check(customer.raw == Data([1, 2, 3]), "Bytes from base64") +check(customer.files == [Data([1, 2, 3]), Data([4, 5])], "lists of Bytes") +check(customer.visits == 9007199254740991, "Long at the 2^53 bound") +check(customer.active == true, "Boolean from SQLite's 0/1") +check(customer.tree?.parent?.value.label == "up" && customer.tree?.children?.first?.label == "leaf", "boxed recursion") +check(customer._extra == ["loyalty": .string("gold")], "relations are never extras: \(customer._extra)") +check(try! coverage_Customer(row: customer.encodeRow(), version: nil) == customer, "customer round trip") +check(HarperBigInt("-0010")?.decimalString == "-10" && HarperBigInt("-0")?.decimalString == "0" && HarperBigInt("1e3") == nil, "BigInt canonical form") +check(HarperBigInt("-10")! < HarperBigInt("-9")! && HarperBigInt("99")! < HarperBigInt("100")! && !(HarperBigInt("5")! < HarperBigInt("5")!), "BigInt ordering") + +// MARK: writes + +var injected = customer +injected._extra["createdAt"] = .double(0) +injected._extra["purchases"] = .array([]) +let upsert = try! injected.encodeUpsertRow() +check(upsert["createdAt"] == nil && upsert["updatedAt"] == nil && upsert["purchases"] == nil, "upserts drop read-only, relation and injected keys") +check(upsert["loyalty"] == .string("gold"), "upserts keep undeclared attributes") +check(injected.encodeRow()["createdAt"] == customer.createdAt.harperValue, "typed values win over injected extras") + +let insert = coverage_Customer.New(name: "Grace", tags: ["x"], _extra: ["referrer": .string("ada"), "createdAt": .double(1)]) +check(insert.encodeRow() == ["name": .string("Grace"), "tags": .array([.string("x")]), "referrer": .string("ada")], "insert body: \(insert.encodeRow())") + +let patch = Order.Patch(status: "closed", clear: [.amount, .status]) +check(patch.encodeRow() == ["status": .string("closed"), "amount": .null], "patch: set, clear, and set wins over clear") +check(Order.Patch().encodeRow().isEmpty, "an empty patch changes nothing") + +let attachment = try! coverage_Attachment(row: ["id": .string("f1"), "label": .string("deck"), "file": .object(["contentType": .string("application/pdf")])], version: nil) +check(attachment.file == .object(["contentType": .string("application/pdf")]), "Blob placeholders pass through untouched") +expectThrows("full replacement of a record holding a Blob", { try attachment.encodeUpsertRow() }) { + ($0 as? HarperEncodingError) == .replacementUnsupported(type: "coverage_Attachment", attribute: "file") +} +check(!coverage_Attachment.schema.replaceable && Order.schema.replaceable, "replaceable flag") + +let purchase = try! coverage_Purchase(row: ["id": .int(7), "total": .double(9.5), "lines": .array([.object(["sku": .string("s1"), "qty": .int(2)])]), "rogue": .bool(true)], version: nil) +check(purchase.encodeRow()["rogue"] == nil && purchase.lines.first?.sku == "s1", "sealed tables drop undeclared attributes") + +// MARK: raw attribute names + +let weirdRow: HarperRow = [ + "id": .string("w1"), "default": .string("d"), "first-name": .string("kebab"), "firstName": .string("camel"), + "_extra": .string("attribute"), "_version": .int(4), "encodeRow": .int(5), "Type": .int(6), "self": .string("me"), + "in": .bool(true), "o\"conn\\or $x": .string("quoted"), "9lives": .bool(false), "surprise": .int(1), +] +let weird = try! Weird(row: weirdRow, version: 8) +check(weird.`default` == "d" && weird.firstName_2 == "kebab" && weird.firstName == "camel", "renamed and escaped properties") +check(weird._extra_2 == "attribute" && weird._version_2 == 4 && weird.encodeRow_2 == 5 && weird.`Type` == 6 && weird.self_2 == "me", "attributes named like generated members") +check(weird._extra == ["surprise": .int(1)] && weird._version == 8, "the generated members stay the generated members") +check(weird.o_conn_or__x == "quoted" && weird._9lives == false, "sanitized names") +check(try! Weird(row: weird.encodeRow(), version: 8) == weird, "raw names round trip") +check(Weird.schema.versionColumn == "__version" && Weird.schema.extraColumn == "__extra", "metadata columns move off attribute names") +check(SkuItem(sku: "s", quantity: 1).id == "s", "Identifiable from a differently named key") + +// MARK: schema and profiles + +check(HarperSchema.irVersion == 1 && HarperSchema.hash.count == 64, "schema identity") +check(HarperSchema.recordTypes.map { $0.tableName }.contains("Order"), "record types") +check(HarperSchema.profiles.map(\.name) == ["my-orders", "storefront"], "valid profiles only: \(HarperSchema.profiles.map(\.name))") +check(HarperSyncProfile.myOrders.tables == [Order.schema] && HarperSyncProfile.myOrders.retentionMs == 2_592_000_000, "profile constants") + +// MARK: a reopened SQLite replica built from the descriptors + +func sqliteValue(_ statement: OpaquePointer?, _ index: Int32, _ column: HarperColumn) -> HarperValue { + switch sqlite3_column_type(statement, index) { + case SQLITE_NULL: + return .null + case SQLITE_INTEGER: + return .int(sqlite3_column_int64(statement, index)) + case SQLITE_FLOAT: + return .double(sqlite3_column_double(statement, index)) + case SQLITE_BLOB: + let count = Int(sqlite3_column_bytes(statement, index)) + return .bytes(count == 0 ? Data() : Data(bytes: sqlite3_column_blob(statement, index), count: count)) + default: + let text = String(cString: sqlite3_column_text(statement, index)) + switch column.type { + case .array, .object, .any: return try! JSONDecoder().decode(HarperValue.self, from: Data(text.utf8)) + default: return .string(text) + } + } +} + +func bind(_ statement: OpaquePointer?, _ index: Int32, _ value: HarperValue?, _ column: HarperColumn?) { + let transient = unsafeBitCast(-1, to: sqlite3_destructor_type.self) + guard let value, value != .null else { + sqlite3_bind_null(statement, index) + return + } + if let column, [.array, .object, .any].contains(column.type) { + sqlite3_bind_text(statement, index, String(data: try! JSONEncoder().encode(value), encoding: .utf8), -1, transient) + return + } + switch value { + case .bool(let flag): sqlite3_bind_int64(statement, index, flag ? 1 : 0) + case .int(let integer): sqlite3_bind_int64(statement, index, integer) + case .double(let double): sqlite3_bind_double(statement, index, double) + case .string(let text): sqlite3_bind_text(statement, index, text, -1, transient) + case .bytes(let data): _ = data.withUnsafeBytes { sqlite3_bind_blob(statement, index, $0.baseAddress, Int32(data.count), transient) } + default: sqlite3_bind_text(statement, index, String(data: try! JSONEncoder().encode(value), encoding: .utf8), -1, transient) + } +} + +func quoted(_ name: String) -> String { "\"\(name.replacingOccurrences(of: "\"", with: "\"\""))\"" } + +func store(_ records: [T], at path: String) { + var db: OpaquePointer? + sqlite3_open(path, &db) + let schema = T.schema + var definitions = schema.columns.map { "\(quoted($0.name)) \($0.affinity.rawValue)\($0.primaryKey ? " PRIMARY KEY" : "")" } + definitions.append("\(quoted(schema.versionColumn)) REAL") + if let extra = schema.extraColumn { definitions.append("\(quoted(extra)) TEXT") } + sqlite3_exec(db, "CREATE TABLE \(quoted(schema.name)) (\(definitions.joined(separator: ", ")))", nil, nil, nil) + let names = schema.columns.map(\.name) + [schema.versionColumn] + (schema.extraColumn.map { [$0] } ?? []) + let sql = "INSERT INTO \(quoted(schema.name)) (\(names.map(quoted).joined(separator: ", "))) VALUES (\(names.map { _ in "?" }.joined(separator: ", ")))" + for record in records { + var statement: OpaquePointer? + sqlite3_prepare_v2(db, sql, -1, &statement, nil) + let encoded = record.encodeRow() + var index: Int32 = 1 + for column in schema.columns { + bind(statement, index, encoded[column.name], column) + index += 1 + } + bind(statement, index, record._version.map { .double($0) }, nil) + index += 1 + if schema.extraColumn != nil { + let declared = Set(schema.columns.map(\.name)) + bind(statement, index, .object(encoded.filter { !declared.contains($0.key) }), HarperColumn(name: "", type: .object, affinity: .text, nullable: true)) + } + sqlite3_step(statement) + sqlite3_finalize(statement) + } + sqlite3_close(db) +} + +func load(_ type: T.Type, at path: String) -> [T] { + var db: OpaquePointer? + sqlite3_open(path, &db) + defer { sqlite3_close(db) } + let schema = T.schema + var statement: OpaquePointer? + sqlite3_prepare_v2(db, "SELECT * FROM \(quoted(schema.name))", -1, &statement, nil) + defer { sqlite3_finalize(statement) } + var records: [T] = [] + let columns = Dictionary(uniqueKeysWithValues: schema.columns.map { ($0.name, $0) }) + while sqlite3_step(statement) == SQLITE_ROW { + var row: HarperRow = [:] + var version: Double? + for index in 0..} */ ({ + ID: 'TEXT', + String: 'TEXT', + Int: 'INTEGER', + Long: 'INTEGER', + Boolean: 'INTEGER', + Float: 'REAL', + Date: 'REAL', + BigInt: 'TEXT', + Bytes: 'BLOB', + Blob: 'BLOB', + Any: 'TEXT', +}); + +/** + * The fields of a live Harper table (or a plain description of one) the IR reads. + * @typedef {Object} SourceTable + * @property {string} tableName + * @property {string} [databaseName] + * @property {string} [primaryKey] + * @property {boolean} [sealed] + * @property {any[]} [attributes] + */ + +/** + * A GraphQL type definition as registered in `resources.allTypes`. + * @typedef {{ attributes?: any[], table?: string | null, database?: string | null, description?: string }} SourceTypeDef + */ + +/** + * @typedef {Object} BuildContext + * @property {Map | Record | undefined} types + * @property {Set} tableKeys + * @property {Map} pendingTypes + * @property {string[]} typeQueue + * @property {IRDiagnostic[]} diagnostics + */ + +/** + * @param {string} database + * @param {string} table + * @returns {string} + */ +function tableKey(database, table) { + return `${database}\u0000${table}`; +} + +/** + * @param {BuildContext['types']} types + * @param {string} name + * @returns {SourceTypeDef | undefined} + */ +function lookupType(types, name) { + if (!types) return undefined; + if (types instanceof Map) return types.get(name); + return Object.prototype.hasOwnProperty.call(types, name) ? types[name] : undefined; +} + +/** + * Builds the schema IR from live tables, the GraphQL type registry and `sync.yaml`. + * @param {Object} input + * @param {Iterable} input.tables the tables to describe (already database-filtered) + * @param {Map | Record} [input.types] resolves nested object types by name (`scope.resources.allTypes`) + * @param {unknown} [input.syncProfiles] the parsed `sync.yaml` + * @param {SchemaIR} [input.previous] the previous IR; tables and types keep their `typeName` + * @param {string} [input.generator] recorded in the IR, not hashed + * @returns {SchemaIR} + */ +export function buildSchemaIR({ tables: sourceTables, types, syncProfiles, previous, generator }) { + const sorted = [...sourceTables].sort( + (a, b) => + compareText(a.databaseName ?? 'data', b.databaseName ?? 'data') || + compareText(a.tableName, b.tableName), + ); + /** @type {BuildContext} */ + const context = { + types, + tableKeys: new Set( + sorted.map((table) => tableKey(table.databaseName ?? 'data', table.tableName)), + ), + pendingTypes: new Map(), + typeQueue: [], + diagnostics: [], + }; + + const tables = sorted.map((source) => describeTable(source, context)); + + /** @type {Map} */ + const objectTypes = new Map(); + while (context.typeQueue.length > 0) { + const name = /** @type {string} */ (context.typeQueue.shift()); + const pending = /** @type {{ attributes: any[], description?: string }} */ ( + context.pendingTypes.get(name) + ); + /** @type {IRNestedAttribute[]} */ + const attributes = []; + for (const source of pending.attributes) { + if (!source?.name || source.relationship) continue; + const type = toType(source, context, { table: name, attribute: source.name }); + attributes.push({ + name: source.name, + type, + nullable: source.nullable !== false || holdsBlob(type), + ...(source.description ? { description: String(source.description) } : {}), + }); + } + objectTypes.set(name, { + name, + typeName: '', + ...(pending.description ? { description: String(pending.description) } : {}), + attributes, + }); + } + const typeList = [...objectTypes.values()].sort((a, b) => compareText(a.name, b.name)); + const tablesByKey = new Map(tables.map((table) => [tableKey(table.database, table.name), table])); + + for (const table of tables) { + table.projections = projectionsOf(table, objectTypes, tablesByKey, context.diagnostics); + } + allocateTypeNames(tables, typeList, previous); + + const { tableHashes, schemaHash } = hashContracts(tables, typeList); + for (const table of tables) { + table.hash = /** @type {string} */ (tableHashes.get(tableKey(table.database, table.name))); + } + + /** @type {IRProfile[]} */ + const profiles = normalizeSyncProfiles(syncProfiles, tables, context.diagnostics).map( + (profile) => ({ ...profile, ...hashProfile(profile, tableHashes) }), + ); + for (const profile of profiles) { + for (const entry of profile.tables) { + tablesByKey.get(tableKey(entry.database, entry.table))?.profiles.push(profile.name); + } + } + + return { + irVersion: IR_VERSION, + ...(generator ? { generator } : {}), + schemaHash, + tables: tables.map((table) => ({ + database: table.database, + name: table.name, + typeName: table.typeName, + primaryKey: table.primaryKey, + sealed: table.sealed, + hash: table.hash, + profiles: table.profiles, + attributes: table.attributes, + relations: table.relations, + projections: table.projections, + storage: table.storage, + })), + types: typeList, + profiles, + diagnostics: context.diagnostics, + }; +} + +/** + * @param {SourceTable} source + * @param {BuildContext} context + * @returns {IRTable} + */ +function describeTable(source, context) { + const database = source.databaseName ?? 'data'; + const sourceAttributes = (source.attributes ?? []).filter((attribute) => attribute?.name); + const flaggedPrimaryKey = sourceAttributes.find((attribute) => attribute.isPrimaryKey)?.name; + const primaryKey = + flaggedPrimaryKey ?? + (sourceAttributes.some((attribute) => attribute.name === source.primaryKey) + ? source.primaryKey + : undefined) ?? + null; + const derived = derivedAttributeNames(sourceAttributes); + + /** @type {IRAttribute[]} */ + const attributes = []; + /** @type {IRRelation[]} */ + const relations = []; + for (const attribute of sourceAttributes) { + const where = { table: source.tableName, attribute: attribute.name }; + if (attribute.relationship) { + relations.push(describeRelation(attribute, context, where)); + continue; + } + const isPrimaryKey = attribute.name === primaryKey; + /** @type {IRAttribute['serverManaged']} */ + const serverManaged = + attribute.assignCreatedTime || attribute.name === '__createdtime__' + ? 'createdTime' + : attribute.assignUpdatedTime || attribute.name === '__updatedtime__' + ? 'updatedTime' + : derived.has(attribute.name) + ? 'derived' + : null; + const computed = attribute.computed + ? { + from: + typeof attribute.computedFromExpression === 'string' + ? attribute.computedFromExpression + : null, + version: attribute.version == null ? null : String(attribute.version), + } + : null; + const type = toType(attribute, context, where); + const nullable = + computed || serverManaged === 'derived' + ? true + : isPrimaryKey || serverManaged + ? false + : isNullable({ ...attribute, isPrimaryKey: false }); + attributes.push({ + name: attribute.name, + type, + nullable, + primaryKey: isPrimaryKey, + indexed: Boolean(attribute.indexed), + serverManaged, + computed, + readOnly: Boolean(serverManaged || computed), + ...(attribute.description ? { description: String(attribute.description) } : {}), + }); + } + + const sealed = source.sealed === true; + const attributeNames = new Set(attributes.map((attribute) => attribute.name)); + /** @type {IRStorage} */ + const storage = { + versionColumn: metadataColumn('_version', attributeNames), + extraColumn: sealed ? null : metadataColumn('_extra', attributeNames), + columns: attributes.map((attribute) => ({ + name: attribute.name, + affinity: affinityOf(attribute.type), + })), + }; + return { + database, + name: source.tableName, + typeName: '', + primaryKey, + sealed, + hash: '', + profiles: [], + attributes, + relations, + projections: /** @type {IRProjections} */ ({}), + storage, + }; +} + +/** + * `@embed` targets and `@decide` targets (with their confidence and decision fields) are written + * by the server from a source field, never by the client. + * @param {any[]} attributes + * @returns {Set} + */ +function derivedAttributeNames(attributes) { + const names = new Set(); + for (const attribute of attributes) { + if (attribute.embed) names.add(attribute.name); + if (attribute.decide) { + names.add(attribute.name); + if (attribute.decide.confidence) names.add(attribute.decide.confidence); + if (attribute.decide.decision) names.add(attribute.decide.decision); + } + } + return names; +} + +/** + * @param {string} base + * @param {Set} attributeNames + * @returns {string} + */ +function metadataColumn(base, attributeNames) { + let name = base; + while (attributeNames.has(name)) name = `_${name}`; + return name; +} + +/** + * A Blob, or an array of them. Sync does not deliver blob content, so such a value is optional on + * read wherever it appears. + * @param {IRType} type + * @returns {boolean} + */ +function holdsBlob(type) { + if (type.kind === 'array') return holdsBlob(type.element); + return type.kind === 'scalar' && type.scalar === 'Blob'; +} + +/** + * Whether the server rejects an insert that omits the attribute. + * @param {IRAttribute} attribute + * @returns {boolean} + */ +export function requiredOnInsert(attribute) { + return !attribute.readOnly && !attribute.primaryKey && !attribute.nullable; +} + +/** + * The replica column affinity for an attribute type: composite values are stored as JSON text. + * @param {IRType} type + * @returns {IRAffinity} + */ +export function affinityOf(type) { + return type.kind === 'scalar' ? AFFINITY[type.scalar] : 'TEXT'; +} + +/** + * @param {any} source an attribute or array-element descriptor + * @param {BuildContext} context + * @param {{ table: string, attribute: string }} where + * @returns {IRType} + */ +function toType(source, context, where) { + const type = source?.type; + if (type === 'array' || type === 'Array') { + const elements = source.elements; + const element = elements ? toType(elements, context, where) : ANY_TYPE; + return { + kind: 'array', + element, + elementNullable: elements?.nullable !== false || holdsBlob(element), + }; + } + if (type === undefined || type === null || type === '' || type === 'object' || type === 'Object') + return ANY_TYPE; + if (SCALAR_TYPES.has(type)) return { kind: 'scalar', scalar: type }; + + const typeDef = lookupType(context.types, type); + if (typeDef?.table) { + const database = typeDef.database ?? 'data'; + if (context.tableKeys.has(tableKey(database, typeDef.table))) { + return { kind: 'record', database, table: typeDef.table }; + } + context.diagnostics.push({ + level: 'warning', + code: 'TABLE_NOT_GENERATED', + ...where, + message: `${where.table}.${where.attribute} embeds table ${database}.${typeDef.table}, which is not generated (its database is excluded), so it is generated as a nested type`, + }); + } + const attributes = typeDef?.attributes ?? source.properties; + if (Array.isArray(attributes)) { + if (!context.pendingTypes.has(type)) { + context.pendingTypes.set(type, { attributes, description: typeDef?.description }); + context.typeQueue.push(type); + } + return { kind: 'object', type }; + } + context.diagnostics.push({ + level: 'warning', + code: 'TYPE_UNRESOLVED', + ...where, + message: `type "${type}" of ${where.table}.${where.attribute} is not a known schema type; it is generated as Any`, + }); + return ANY_TYPE; +} + +/** + * @param {any} attribute + * @param {BuildContext} context + * @param {{ table: string, attribute: string }} where + * @returns {IRRelation} + */ +function describeRelation(attribute, context, where) { + const many = attribute.type === 'array' || attribute.type === 'Array'; + /** @type {IRRelation} */ + const relation = { + name: attribute.name, + cardinality: many ? 'many' : 'one', + ...(attribute.relationship.from ? { from: String(attribute.relationship.from) } : {}), + ...(attribute.relationship.to ? { to: String(attribute.relationship.to) } : {}), + target: relationTarget(attribute), + }; + if (relation.target.database === null) { + context.diagnostics.push({ + level: 'warning', + code: 'RELATION_TARGET_UNRESOLVED', + ...where, + message: `relationship ${where.table}.${where.attribute} does not resolve to a table`, + }); + } + return relation; +} + +/** + * GraphQL relationships carry `relationshipReference`; `defineTable` relations carry a lazy + * `definition.tableClass`. + * @param {any} attribute + * @returns {IRRelation['target']} + */ +function relationTarget(attribute) { + const reference = attribute.relationshipReference ?? attribute.elements?.relationshipReference; + if (reference?.table) + return { database: reference.database ?? 'data', table: String(reference.table) }; + const definition = attribute.definition ?? attribute.elements?.definition; + /** @type {any} */ + let tableClass; + try { + tableClass = definition?.tableClass; + } catch { + tableClass = undefined; + } + if (tableClass?.tableName) + return { database: tableClass.databaseName ?? 'data', table: tableClass.tableName }; + if (definition?.table) + return { database: definition.database ?? 'data', table: String(definition.table) }; + const typeName = + attribute.type === 'array' || attribute.type === 'Array' + ? attribute.elements?.type + : attribute.type; + return { database: null, table: String(typeName ?? '') }; +} + +/** + * @param {IRType} type + * @param {Map} objectTypes + * @param {Map} tablesByKey + * @param {Set} [visiting] + * @returns {boolean} + */ +function containsBlob(type, objectTypes, tablesByKey, visiting = new Set()) { + switch (type.kind) { + case 'scalar': + return type.scalar === 'Blob'; + case 'array': + return containsBlob(type.element, objectTypes, tablesByKey, visiting); + case 'object': { + if (visiting.has(`o:${type.type}`)) return false; + visiting.add(`o:${type.type}`); + const attributes = objectTypes.get(type.type)?.attributes ?? []; + return attributes.some((attribute) => + containsBlob(attribute.type, objectTypes, tablesByKey, visiting), + ); + } + case 'record': { + const key = tableKey(type.database, type.table); + if (visiting.has(`r:${key}`)) return false; + visiting.add(`r:${key}`); + const attributes = tablesByKey.get(key)?.attributes ?? []; + return attributes.some((attribute) => + containsBlob(attribute.type, objectTypes, tablesByKey, visiting), + ); + } + } +} + +/** + * The defineTable projections over stored attributes, with Blob-bearing attributes kept out of + * every write: the transport has no blob contract yet, so a write could only delete or corrupt one. + * For the same reason a Blob-bearing attribute is optional on read whatever its schema nullability. + * @param {IRTable} table + * @param {Map} objectTypes + * @param {Map} tablesByKey + * @param {IRDiagnostic[]} diagnostics + * @returns {IRProjections} + */ +export function projectionsOf(table, objectTypes, tablesByKey, diagnostics) { + const blobBearing = new Set( + table.attributes + .filter((attribute) => containsBlob(attribute.type, objectTypes, tablesByKey)) + .map((attribute) => attribute.name), + ); + const unwritable = table.attributes + .filter((attribute) => !attribute.readOnly && blobBearing.has(attribute.name)) + .map((attribute) => attribute.name); + for (const attribute of table.attributes) { + if (blobBearing.has(attribute.name) && requiredOnInsert(attribute)) { + diagnostics.push({ + level: 'warning', + code: 'BLOB_REQUIRED', + table: table.name, + attribute: attribute.name, + message: `${table.name}.${attribute.name} is required and holds a Blob, which clients cannot send, so client inserts into ${table.name} are rejected`, + }); + } + } + const writable = table.attributes.filter( + (attribute) => !attribute.readOnly && !blobBearing.has(attribute.name), + ); + const patchable = writable.filter((attribute) => !attribute.primaryKey); + return { + record: table.attributes.map((attribute) => ({ + name: attribute.name, + optional: attribute.nullable || (!attribute.primaryKey && blobBearing.has(attribute.name)), + })), + insert: writable.map((attribute) => ({ + name: attribute.name, + optional: attribute.primaryKey || attribute.nullable, + })), + upsert: + unwritable.length > 0 + ? null + : writable.map((attribute) => ({ + name: attribute.name, + optional: !attribute.primaryKey && attribute.nullable, + })), + patch: patchable.map((attribute) => ({ name: attribute.name, optional: true })), + clearable: patchable + .filter((attribute) => attribute.nullable) + .map((attribute) => attribute.name), + unwritable, + query: table.attributes + .filter((attribute) => attribute.indexed) + .map((attribute) => ({ name: attribute.name, optional: true })), + }; +} + +/** + * @param {IRTable[]} tables sorted + * @param {IRObjectType[]} types sorted + * @param {SchemaIR | undefined} previous + */ +function allocateTypeNames(tables, types, previous) { + const allocator = createNameAllocator(RESERVED_TYPE_NAMES); + /** @type {Map} */ + const previousNames = new Map(); + for (const table of previous?.tables ?? []) { + previousNames.set(`t:${tableKey(table.database, table.name)}`, table.typeName); + } + for (const type of previous?.types ?? []) { + previousNames.set(`o:${type.name}`, type.typeName); + } + /** @type {{ key: string, base: string, assign: (name: string) => void }[]} */ + const entries = [ + ...tables.map((table) => ({ + key: `t:${tableKey(table.database, table.name)}`, + base: `${table.database === 'data' ? '' : `${toCodeIdentifier(table.database)}_`}${toCodeIdentifier(singularize(table.name))}`, + assign: (/** @type {string} */ name) => { + table.typeName = name; + }, + })), + ...types.map((type) => ({ + key: `o:${type.name}`, + base: toCodeIdentifier(type.name), + assign: (/** @type {string} */ name) => { + type.typeName = name; + }, + })), + ]; + const assigned = new Set(); + for (const entry of entries) { + const name = previousNames.get(entry.key); + if (name && /^[A-Za-z_][A-Za-z0-9_]*$/.test(name) && allocator.claimExact(name)) { + entry.assign(name); + assigned.add(entry.key); + } + } + for (const entry of entries) { + if (assigned.has(entry.key)) continue; + entry.assign( + allocator.claim(RESERVED_TYPE_NAMES.has(entry.base) ? `${entry.base}Record` : entry.base), + ); + } +} diff --git a/utils/client/buildSchemaIR.test.js b/utils/client/buildSchemaIR.test.js new file mode 100644 index 0000000..5e82d77 --- /dev/null +++ b/utils/client/buildSchemaIR.test.js @@ -0,0 +1,733 @@ +import { describe, expect, it } from 'vitest'; +import { + adversarialTables, + coverageTables, + coverageTypes, + sharedChain, + spikeProfiles, + spikeTables, +} from '../../test/fixtures/clientSchema.js'; +import { buildSchemaIR, IR_VERSION } from './buildSchemaIR.js'; +import { serializeSchemaIR } from './clientOutputs.js'; +import { assertSchemaIR } from './validateSchemaIR.js'; + +/** + * @param {any[]} tables + * @param {Partial[0]>} [rest] + */ +function build(tables, rest = {}) { + const ir = buildSchemaIR({ tables, types: coverageTypes(), ...rest }); + assertSchemaIR(ir); + return ir; +} + +/** + * @param {import('./irTypes.js').SchemaIR} ir + * @param {string} name + */ +function table(ir, name) { + const found = ir.tables.find((candidate) => candidate.name === name); + if (!found) throw new Error(`no table ${name}`); + return found; +} + +/** + * @param {import('./irTypes.js').IRTable} irTable + * @param {string} name + */ +function attribute(irTable, name) { + const found = irTable.attributes.find((candidate) => candidate.name === name); + if (!found) throw new Error(`no attribute ${name}`); + return found; +} + +describe('buildSchemaIR', () => { + const ir = build([...spikeTables(), ...coverageTables(), ...adversarialTables()], { + syncProfiles: spikeProfiles(), + }); + + it('sorts tables by database then name and records the IR version', () => { + expect(ir.irVersion).toBe(IR_VERSION); + expect(ir.tables.map((entry) => `${entry.database}.${entry.name}`)).toEqual([ + 'coverage.Attachment', + 'coverage.Customer', + 'coverage.Purchase', + 'data.Data', + 'data.ModelsKt', + 'data.OnlyId', + 'data.Order', + 'data.Product', + 'data.Report', + 'data.SkuItems', + 'data.Weird', + ]); + }); + + describe('nullability (undefined means nullable)', () => { + const customer = table(ir, 'Customer'); + + it('makes plain attributes optional and `!` attributes required', () => { + expect(attribute(customer, 'email').nullable).toBe(true); + expect(attribute(customer, 'name').nullable).toBe(false); + }); + + it('requires the primary key and server-managed timestamps on read', () => { + expect(attribute(customer, 'id').nullable).toBe(false); + expect(attribute(customer, 'createdAt')).toMatchObject({ + nullable: false, + serverManaged: 'createdTime', + readOnly: true, + }); + expect(attribute(customer, 'updatedAt')).toMatchObject({ + nullable: false, + serverManaged: 'updatedTime', + readOnly: true, + }); + }); + + it('keeps computed attributes optional and read-only, with their expression', () => { + expect(attribute(table(ir, 'Order'), 'points')).toMatchObject({ + nullable: true, + readOnly: true, + computed: { + from: "Math.round((amount || 0) * (status == 'open' ? 10 : 25))", + version: "Math.round((amount || 0) * (status == 'open' ? 10 : 25))", + }, + }); + }); + + it('applies the same rule to array elements', () => { + expect(attribute(customer, 'tags').type).toEqual({ + kind: 'array', + element: { kind: 'scalar', scalar: 'String' }, + elementNullable: false, + }); + expect(attribute(customer, 'scores').type).toEqual({ + kind: 'array', + element: { kind: 'scalar', scalar: 'Int' }, + elementNullable: true, + }); + }); + + it('reads a required Blob as optional but keeps it required, and warns that inserts fail', () => { + const withRequiredBlob = build([ + { + tableName: 'Files', + primaryKey: 'id', + attributes: [ + { name: 'id', type: 'ID', isPrimaryKey: true }, + { name: 'file', type: 'Blob', nullable: false }, + ], + }, + ]); + const files = table(withRequiredBlob, 'Files'); + expect(attribute(files, 'file').nullable).toBe(false); + expect(files.projections.record).toContainEqual({ name: 'file', optional: true }); + expect(withRequiredBlob.diagnostics).toContainEqual( + expect.objectContaining({ code: 'BLOB_REQUIRED', table: 'Files', attribute: 'file' }), + ); + }); + + it('treats @embed and @decide outputs and legacy timestamps as server-written', () => { + const derived = build([ + { + tableName: 'Docs', + primaryKey: 'id', + attributes: [ + { name: 'id', type: 'ID', isPrimaryKey: true }, + { name: 'body', type: 'String' }, + { + name: 'vector', + type: 'array', + elements: { type: 'Float' }, + embed: { source: 'body', model: 'm' }, + }, + { + name: 'label', + type: 'String', + decide: { source: 'body', confidence: 'labelConfidence', decision: 'labelDecision' }, + }, + { name: 'labelConfidence', type: 'Float' }, + { name: 'labelDecision', type: 'String' }, + { name: '__createdtime__', type: 'Float' }, + ], + }, + ]); + const docs = table(derived, 'Docs'); + for (const name of ['vector', 'label', 'labelConfidence', 'labelDecision']) { + expect(attribute(docs, name)).toMatchObject({ + serverManaged: 'derived', + readOnly: true, + nullable: true, + }); + } + expect(attribute(docs, '__createdtime__')).toMatchObject({ + serverManaged: 'createdTime', + nullable: false, + }); + expect(docs.projections.insert.map((field) => field.name)).toEqual(['id', 'body']); + }); + }); + + describe('projections (the defineTable rules)', () => { + const customer = table(ir, 'Customer'); + + it('reads every stored attribute and never includes relations', () => { + expect(customer.projections.record.map((field) => field.name)).not.toContain('purchases'); + expect(customer.projections.record.find((field) => field.name === 'name')).toEqual({ + name: 'name', + optional: false, + }); + }); + + it('inserts writable attributes with the primary key optional', () => { + expect(customer.projections.insert).toContainEqual({ name: 'id', optional: true }); + expect(customer.projections.insert).toContainEqual({ name: 'name', optional: false }); + expect(customer.projections.insert.map((field) => field.name)).not.toContain('createdAt'); + }); + + it('upserts writable attributes with the primary key required', () => { + expect(customer.projections.upsert).toContainEqual({ name: 'id', optional: false }); + expect(customer.projections.upsert?.map((field) => field.name)).not.toContain('updatedAt'); + }); + + it('patches writable non-key attributes, all optional, and lists the clearable ones', () => { + expect(customer.projections.patch.map((field) => field.name)).toEqual([ + 'name', + 'email', + 'address', + 'tags', + 'scores', + 'big', + 'raw', + 'files', + 'meta', + 'visits', + 'active', + 'tree', + ]); + expect(customer.projections.patch.every((field) => field.optional)).toBe(true); + expect(customer.projections.clearable).not.toContain('name'); + expect(customer.projections.clearable).toContain('email'); + }); + + it('queries indexed attributes only', () => { + expect(customer.projections.query).toEqual([{ name: 'email', optional: true }]); + }); + + it('keeps Blob attributes out of every write and makes full replacement unavailable', () => { + const attachment = table(ir, 'Attachment'); + expect(attachment.projections.unwritable).toEqual(['file']); + expect(attachment.projections.upsert).toBeNull(); + expect(attachment.projections.insert.map((field) => field.name)).toEqual(['id', 'label']); + expect(attachment.projections.patch.map((field) => field.name)).toEqual(['label']); + }); + + it('reads every value holding a Blob as optional, at any depth', () => { + const types = new Map([ + ['Scan', { attributes: [{ name: 'image', type: 'Blob', nullable: false }] }], + ]); + const media = build( + [ + { + tableName: 'Media', + primaryKey: 'id', + attributes: [ + { name: 'id', type: 'ID', isPrimaryKey: true }, + { + name: 'pages', + type: 'array', + nullable: false, + elements: { type: 'Blob', nullable: false }, + }, + { name: 'scan', type: 'Scan', nullable: false }, + ], + }, + ], + { types }, + ); + const projected = table(media, 'Media').projections.record; + expect(projected).toContainEqual({ name: 'pages', optional: true }); + expect(projected).toContainEqual({ name: 'scan', optional: true }); + expect(attribute(table(media, 'Media'), 'pages').type).toMatchObject({ + elementNullable: true, + }); + expect(media.types.find((type) => type.name === 'Scan')?.attributes[0].nullable).toBe(true); + }); + + it('still allows full replacement when the only Blob is computed', () => { + const rendered = build([ + { + tableName: 'Thumbnails', + primaryKey: 'id', + attributes: [ + { name: 'id', type: 'ID', isPrimaryKey: true }, + { name: 'preview', type: 'Blob', computed: { from: () => null } }, + ], + }, + ]); + const thumbnails = table(rendered, 'Thumbnails'); + expect(thumbnails.projections.unwritable).toEqual([]); + expect(thumbnails.projections.upsert).toEqual([{ name: 'id', optional: false }]); + }); + }); + + describe('nested types and relations', () => { + it('resolves nested object types through the type registry, transitively', () => { + expect(attribute(table(ir, 'Customer'), 'address').type).toEqual({ + kind: 'object', + type: 'Address', + }); + expect(ir.types.map((type) => type.name)).toEqual(['Address', 'Geo', 'Node', 'PurchaseLine']); + const address = /** @type {import('./irTypes.js').IRObjectType} */ ( + ir.types.find((type) => type.name === 'Address') + ); + expect(address.attributes).toEqual([ + { name: 'street', type: { kind: 'scalar', scalar: 'String' }, nullable: true }, + { name: 'city', type: { kind: 'scalar', scalar: 'String' }, nullable: false }, + { name: 'geo', type: { kind: 'object', type: 'Geo' }, nullable: true }, + ]); + }); + + it('records relationships from GraphQL references without storing them', () => { + expect(table(ir, 'Purchase').relations).toEqual([ + { + name: 'customer', + cardinality: 'one', + from: 'customerId', + target: { database: 'coverage', table: 'Customer' }, + }, + ]); + expect(table(ir, 'Customer').relations).toEqual([ + { + name: 'purchases', + cardinality: 'many', + to: 'customerId', + target: { database: 'coverage', table: 'Purchase' }, + }, + ]); + expect(table(ir, 'Purchase').storage.columns.map((column) => column.name)).not.toContain( + 'customer', + ); + }); + + it('resolves defineTable relation targets through the lazy table class', () => { + const defined = build([ + ...spikeTables(), + { + tableName: 'Note', + primaryKey: 'id', + attributes: [ + { name: 'id', type: 'ID', isPrimaryKey: true }, + { name: 'productId', type: 'ID', indexed: true, nullable: false }, + { + name: 'product', + relationship: { from: 'productId' }, + get type() { + return 'Product'; + }, + definition: { + get tableClass() { + return { tableName: 'Product', databaseName: 'data' }; + }, + }, + }, + ], + }, + ]); + expect(table(defined, 'Note').relations).toEqual([ + { + name: 'product', + cardinality: 'one', + from: 'productId', + target: { database: 'data', table: 'Product' }, + }, + ]); + expect(defined.diagnostics).toEqual([]); + }); + + it('embeds a table-typed attribute as a record reference when that table is generated', () => { + const embedded = build([ + ...coverageTables(), + { + tableName: 'Snapshot', + primaryKey: 'id', + attributes: [ + { name: 'id', type: 'ID', isPrimaryKey: true }, + { name: 'buyer', type: 'Customer' }, + ], + }, + ]); + expect(attribute(table(embedded, 'Snapshot'), 'buyer').type).toEqual({ + kind: 'record', + database: 'coverage', + table: 'Customer', + }); + }); + + it('falls back to Any with a warning for an unknown type', () => { + const unknown = build([ + { + tableName: 'Things', + primaryKey: 'id', + attributes: [ + { name: 'id', type: 'ID', isPrimaryKey: true }, + { name: 'shape', type: 'Mystery' }, + ], + }, + ]); + expect(attribute(table(unknown, 'Things'), 'shape').type).toEqual({ + kind: 'scalar', + scalar: 'Any', + }); + expect(unknown.diagnostics).toEqual([ + expect.objectContaining({ level: 'warning', code: 'TYPE_UNRESOLVED', attribute: 'shape' }), + ]); + }); + + it('warns when an embedded table is not generated and falls back to a nested type', () => { + const types = new Map([ + [ + 'Account', + { table: 'Account', database: 'billing', attributes: [{ name: 'id', type: 'ID' }] }, + ], + ]); + const partial = build( + [ + { + tableName: 'Invoice', + primaryKey: 'id', + attributes: [ + { name: 'id', type: 'ID', isPrimaryKey: true }, + { name: 'account', type: 'Account' }, + ], + }, + ], + { types }, + ); + expect(attribute(table(partial, 'Invoice'), 'account').type).toEqual({ + kind: 'object', + type: 'Account', + }); + expect(partial.diagnostics).toContainEqual( + expect.objectContaining({ code: 'TABLE_NOT_GENERATED', attribute: 'account' }), + ); + }); + + it('uses inline nested properties when there is no type registry', () => { + const inline = buildSchemaIR({ + tables: [ + { + tableName: 'Things', + primaryKey: 'id', + attributes: [ + { name: 'id', type: 'ID', isPrimaryKey: true }, + { + name: 'point', + type: 'Point', + properties: [{ name: 'x', type: 'Float', nullable: false }], + }, + ], + }, + ], + }); + expect(inline.types).toEqual([ + { + name: 'Point', + typeName: 'Point', + attributes: [{ name: 'x', type: { kind: 'scalar', scalar: 'Float' }, nullable: false }], + }, + ]); + }); + }); + + describe('type names', () => { + it('singularizes, prefixes non-data databases, and avoids reserved names', () => { + expect(table(ir, 'Customer').typeName).toBe('coverage_Customer'); + expect(table(ir, 'SkuItems').typeName).toBe('SkuItem'); + expect(table(ir, 'Data').typeName).toBe('DataRecord'); + }); + + it('dedupes tables and nested types that share a name', () => { + const clashing = build([ + { + tableName: 'Users', + primaryKey: 'id', + attributes: [{ name: 'id', type: 'ID', isPrimaryKey: true }], + }, + { + tableName: 'User', + primaryKey: 'id', + attributes: [ + { name: 'id', type: 'ID', isPrimaryKey: true }, + { name: 'geo', type: 'Geo' }, + ], + }, + { + tableName: 'Geos', + primaryKey: 'id', + attributes: [{ name: 'id', type: 'ID', isPrimaryKey: true }], + }, + ]); + expect(clashing.tables.map((entry) => entry.typeName)).toEqual(['Geo', 'User', 'User_2']); + expect(clashing.types.map((type) => type.typeName)).toEqual(['Geo_2']); + }); + + it('keeps names from the previous IR so an added table never renames an existing type', () => { + const before = build([ + { + tableName: 'Users', + primaryKey: 'id', + attributes: [{ name: 'id', type: 'ID', isPrimaryKey: true }], + }, + ]); + expect(before.tables[0].typeName).toBe('User'); + const after = build( + [ + { + tableName: 'Users', + primaryKey: 'id', + attributes: [{ name: 'id', type: 'ID', isPrimaryKey: true }], + }, + { + tableName: 'User', + primaryKey: 'id', + attributes: [{ name: 'id', type: 'ID', isPrimaryKey: true }], + }, + ], + { previous: before }, + ); + expect(table(after, 'Users').typeName).toBe('User'); + expect(table(after, 'User').typeName).toBe('User_2'); + }); + }); + + describe('storage', () => { + it('maps affinities and adds version and overflow columns', () => { + const customer = table(ir, 'Customer'); + expect(customer.storage.versionColumn).toBe('_version'); + expect(customer.storage.extraColumn).toBe('_extra'); + expect( + Object.fromEntries( + customer.storage.columns.map((column) => [column.name, column.affinity]), + ), + ).toEqual({ + id: 'TEXT', + name: 'TEXT', + email: 'TEXT', + createdAt: 'REAL', + updatedAt: 'REAL', + address: 'TEXT', + tags: 'TEXT', + scores: 'TEXT', + big: 'TEXT', + raw: 'BLOB', + files: 'TEXT', + meta: 'TEXT', + visits: 'INTEGER', + active: 'INTEGER', + tree: 'TEXT', + }); + }); + + it('has no overflow column on sealed tables', () => { + expect(table(ir, 'Purchase').storage.extraColumn).toBeNull(); + }); + + it('moves metadata columns off attribute names', () => { + const weird = table(ir, 'Weird'); + expect(weird.storage.versionColumn).toBe('__version'); + expect(weird.storage.extraColumn).toBe('__extra'); + }); + }); + + describe('primary keys', () => { + it('falls back to the table primaryKey when no attribute is flagged', () => { + const unflagged = build([ + { + tableName: 'Legacy', + primaryKey: 'code', + attributes: [ + { name: 'code', type: 'String' }, + { name: 'label', type: 'String' }, + ], + }, + ]); + expect(table(unflagged, 'Legacy').primaryKey).toBe('code'); + expect(attribute(table(unflagged, 'Legacy'), 'code')).toMatchObject({ + primaryKey: true, + nullable: false, + }); + }); + + it('allows a table without a primary key', () => { + const keyless = build([ + { tableName: 'Log', attributes: [{ name: 'message', type: 'String' }] }, + ]); + expect(table(keyless, 'Log').primaryKey).toBeNull(); + }); + }); + + describe('profiles', () => { + it('lists valid profiles and records membership on tables', () => { + expect(ir.profiles.map((profile) => profile.name)).toEqual(['my-orders', 'storefront']); + expect(table(ir, 'Order').profiles).toEqual(['my-orders']); + expect(table(ir, 'Report').profiles).toEqual([]); + }); + + it('hashes the profile definition and the tables it covers separately', () => { + const changedScope = build([...spikeTables()], { + syncProfiles: { + profiles: { + ...spikeProfiles().profiles, + 'my-orders': { + retention: '30d', + scope: { customerId: '$token.sub' }, + tables: ['Order'], + }, + }, + }, + }); + const before = /** @type {import('./irTypes.js').IRProfile} */ ( + ir.profiles.find((profile) => profile.name === 'my-orders') + ); + const after = /** @type {import('./irTypes.js').IRProfile} */ ( + changedScope.profiles.find((profile) => profile.name === 'my-orders') + ); + expect(after.hash).not.toBe(before.hash); + expect(after.schemaHash).toBe(before.schemaHash); + }); + }); +}); + +describe('contract hashes', () => { + const base = () => [...spikeTables(), ...coverageTables()]; + + it('are byte-for-byte deterministic regardless of input order', () => { + const forward = buildSchemaIR({ + tables: base(), + types: coverageTypes(), + syncProfiles: spikeProfiles(), + }); + const reversed = buildSchemaIR({ + tables: base().reverse(), + types: new Map([...coverageTypes()].reverse()), + syncProfiles: { + profiles: Object.fromEntries(Object.entries(spikeProfiles().profiles).reverse()), + }, + }); + expect(serializeSchemaIR(reversed)).toBe(serializeSchemaIR(forward)); + }); + + it('ignore attribute order, descriptions and type names', () => { + const original = buildSchemaIR({ tables: spikeTables() }); + const shuffled = spikeTables(); + shuffled[1].attributes.reverse(); + shuffled[1].attributes[0].description = 'documentation only'; + const reordered = buildSchemaIR({ tables: shuffled }); + expect(table(reordered, 'Order').hash).toBe(table(original, 'Order').hash); + expect(reordered.schemaHash).toBe(original.schemaHash); + }); + + /** @type {[string, (tables: any[]) => void][]} */ + const contractChanges = [ + ['a type change', (tables) => (tables[0].attributes[2].type = 'String')], + ['a nullability change', (tables) => (tables[0].attributes[1].nullable = false)], + ['an index change', (tables) => (tables[0].attributes[1].indexed = {})], + ['sealing', (tables) => (tables[0].sealed = true)], + ['a new attribute', (tables) => tables[0].attributes.push({ name: 'sku', type: 'String' })], + ]; + it.each(contractChanges)('change with %s', (_, mutate) => { + const original = buildSchemaIR({ tables: spikeTables() }); + const tables = spikeTables(); + mutate(tables); + const changed = buildSchemaIR({ tables }); + expect(table(changed, 'Product').hash).not.toBe(table(original, 'Product').hash); + expect(table(changed, 'Order').hash).toBe(table(original, 'Order').hash); + expect(changed.schemaHash).not.toBe(original.schemaHash); + }); + + it('change when a computed expression changes', () => { + const tables = spikeTables(); + tables[1].attributes[4].computedFromExpression = 'amount * 2'; + expect(table(buildSchemaIR({ tables }), 'Order').hash).not.toBe( + table(buildSchemaIR({ tables: spikeTables() }), 'Order').hash, + ); + }); + + it('change when an embedded table changes, and survive a table that embeds itself', () => { + const snapshot = () => ({ + tableName: 'Snapshot', + primaryKey: 'id', + attributes: [ + { name: 'id', type: 'ID', isPrimaryKey: true }, + { name: 'buyer', type: 'Customer' }, + ], + }); + const original = buildSchemaIR({ + tables: [...coverageTables(), snapshot()], + types: coverageTypes(), + }); + const changedTables = coverageTables(); + changedTables[0].attributes.push({ name: 'nickname', type: 'String' }); + changedTables[0].attributes.push({ name: 'referrer', type: 'Customer' }); + const changed = buildSchemaIR({ + tables: [...changedTables, snapshot()], + types: coverageTypes(), + }); + expect(table(changed, 'Snapshot').hash).not.toBe(table(original, 'Snapshot').hash); + expect(table(changed, 'Purchase').hash).toBe(table(original, 'Purchase').hash); + }); + + it('change when an embedded table is sealed', () => { + const snapshot = () => ({ + tableName: 'Snapshot', + primaryKey: 'id', + attributes: [ + { name: 'id', type: 'ID', isPrimaryKey: true }, + { name: 'buyer', type: 'Customer' }, + ], + }); + const original = buildSchemaIR({ + tables: [...coverageTables(), snapshot()], + types: coverageTypes(), + }); + const sealed = coverageTables(); + sealed[0].sealed = true; + const changed = buildSchemaIR({ tables: [...sealed, snapshot()], types: coverageTypes() }); + expect(table(changed, 'Snapshot').hash).not.toBe(table(original, 'Snapshot').hash); + }); + + it('change when a Blob becomes required', () => { + const tables = coverageTables(); + tables[2].attributes[2].nullable = false; + const original = buildSchemaIR({ tables: coverageTables(), types: coverageTypes() }); + const changed = buildSchemaIR({ tables, types: coverageTypes() }); + expect(table(changed, 'Attachment').hash).not.toBe(table(original, 'Attachment').hash); + }); + + it('cover widely shared nested types in linear time', () => { + const tables = [ + { + tableName: 'Deep', + primaryKey: 'id', + attributes: [ + { name: 'id', type: 'ID', isPrimaryKey: true }, + { name: 'root', type: 'T0' }, + ], + }, + ]; + const original = buildSchemaIR({ tables, types: sharedChain('String') }); + const changed = buildSchemaIR({ tables, types: sharedChain('Int') }); + expect(table(changed, 'Deep').hash).not.toBe(table(original, 'Deep').hash); + }); + + it('change when a nested type changes', () => { + const types = coverageTypes(); + /** @type {any} */ (types.get('Geo')).attributes.push({ name: 'alt', type: 'Float' }); + const original = buildSchemaIR({ tables: coverageTables(), types: coverageTypes() }); + const changed = buildSchemaIR({ tables: coverageTables(), types }); + expect(table(changed, 'Customer').hash).not.toBe(table(original, 'Customer').hash); + expect(table(changed, 'Purchase').hash).toBe(table(original, 'Purchase').hash); + }); +}); diff --git a/utils/client/canonicalHash.js b/utils/client/canonicalHash.js new file mode 100644 index 0000000..3967e0b --- /dev/null +++ b/utils/client/canonicalHash.js @@ -0,0 +1,29 @@ +import { createHash } from 'node:crypto'; + +/** + * Object keys are sorted at every depth; `undefined` members are dropped, as `JSON.stringify` does. + * @param {unknown} value + * @returns {string} + */ +export function canonicalJSON(value) { + if (value === null || typeof value !== 'object') { + return JSON.stringify(value) ?? 'null'; + } + if (Array.isArray(value)) { + return `[${value.map((item) => canonicalJSON(item === undefined ? null : item)).join(',')}]`; + } + const record = /** @type {Record} */ (value); + const members = Object.keys(record) + .filter((key) => record[key] !== undefined) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonicalJSON(record[key])}`); + return `{${members.join(',')}}`; +} + +/** + * @param {unknown} value + * @returns {string} lowercase hex SHA-256 of the value's canonical JSON + */ +export function hashCanonical(value) { + return createHash('sha256').update(canonicalJSON(value)).digest('hex'); +} diff --git a/utils/client/canonicalHash.test.js b/utils/client/canonicalHash.test.js new file mode 100644 index 0000000..0c39436 --- /dev/null +++ b/utils/client/canonicalHash.test.js @@ -0,0 +1,23 @@ +import { describe, expect, it } from 'vitest'; +import { canonicalJSON, hashCanonical } from './canonicalHash.js'; + +describe('canonicalJSON', () => { + it('sorts object keys at every depth and keeps array order', () => { + expect(canonicalJSON({ b: 1, a: { d: [3, 1], c: null } })).toBe( + '{"a":{"c":null,"d":[3,1]},"b":1}', + ); + }); + + it('drops undefined members like JSON.stringify', () => { + expect(canonicalJSON({ a: undefined, b: 2 })).toBe('{"b":2}'); + expect(canonicalJSON([undefined])).toBe('[null]'); + }); +}); + +describe('hashCanonical', () => { + it('is independent of key order and sensitive to values', () => { + expect(hashCanonical({ a: 1, b: 2 })).toBe(hashCanonical({ b: 2, a: 1 })); + expect(hashCanonical({ a: 1 })).not.toBe(hashCanonical({ a: 2 })); + expect(hashCanonical({ a: 1 })).toMatch(/^[0-9a-f]{64}$/); + }); +}); diff --git a/utils/client/cli.js b/utils/client/cli.js new file mode 100644 index 0000000..4ddb083 --- /dev/null +++ b/utils/client/cli.js @@ -0,0 +1,169 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { parseArgs } from 'node:util'; +import { + findStaleGeneratedFiles, + publishClientOutputs, + renderClientOutputs, +} from './clientOutputs.js'; +import { diffSchemaIR } from './diffSchemaIR.js'; +import { GENERATOR } from './generator.js'; +import { assertSchemaIR } from './validateSchemaIR.js'; + +const USAGE = `Usage: + schema-codegen generate-client --ir [--swift ] [--swift-module ] + [--kotlin ] [--kotlin-package ] + schema-codegen diff [--json] [--fail-on read,write,storage] + +generate-client Writes a Swift package and/or Kotlin module from a schema IR file. +diff Compares two schema IR files; exits 1 when the change breaks an axis in --fail-on. +`; + +const AXES = ['read', 'write', 'storage']; + +/** + * @typedef {Object} CliIO + * @property {(text: string) => void} stdout + * @property {(text: string) => void} stderr + * @property {string} cwd + */ + +/** + * Runs the CLI; returns the exit code (0 success, 1 breaking diff, 2 usage or input error). + * @param {string[]} argv the arguments after the executable and script + * @param {CliIO} io + * @returns {number} + */ +export function runCli(argv, io) { + const [command, ...rest] = argv; + try { + switch (command) { + case 'generate-client': + return generateClient(rest, io); + case 'diff': + return diff(rest, io); + case '--version': + case '-v': + io.stdout(`${GENERATOR}\n`); + return 0; + case undefined: + case '--help': + case '-h': + io.stdout(USAGE); + return command === undefined ? 2 : 0; + default: + io.stderr(`schema-codegen: unknown command "${command}"\n${USAGE}`); + return 2; + } + } catch (error) { + io.stderr(`schema-codegen: ${/** @type {Error} */ (error).message}\n`); + return 2; + } +} + +/** + * @param {string[]} args + * @param {CliIO} io + * @returns {number} + */ +function generateClient(args, io) { + const { values } = parseArgs({ + args, + options: { + ir: { type: 'string' }, + swift: { type: 'string' }, + 'swift-module': { type: 'string' }, + kotlin: { type: 'string' }, + 'kotlin-package': { type: 'string' }, + }, + strict: true, + }); + if (!values.ir) throw new Error('generate-client needs --ir '); + if (!values.swift && !values.kotlin) + throw new Error('generate-client needs --swift and/or --kotlin '); + const ir = readIR(path.resolve(io.cwd, values.ir)); + const options = { + baseDirectory: io.cwd, + swift: values.swift, + swiftModule: values['swift-module'], + kotlin: values.kotlin, + kotlinPackage: values['kotlin-package'], + }; + const outputs = renderClientOutputs(ir, options); + const { written, createdScaffolds } = publishClientOutputs(outputs); + for (const file of [...createdScaffolds, ...written]) + io.stdout(`wrote ${path.relative(io.cwd, file)}\n`); + if (written.length + createdScaffolds.length === 0) io.stdout('generated files are up to date\n'); + for (const directory of [options.swift, options.kotlin]) { + if (!directory) continue; + for (const stale of findStaleGeneratedFiles(path.resolve(io.cwd, directory), outputs)) { + io.stderr( + `warning: ${path.relative(io.cwd, stale)} was generated for a different module or package and is no longer updated\n`, + ); + } + } + return 0; +} + +/** + * @param {string[]} args + * @param {CliIO} io + * @returns {number} + */ +function diff(args, io) { + const { values, positionals } = parseArgs({ + args, + options: { json: { type: 'boolean' }, 'fail-on': { type: 'string' } }, + allowPositionals: true, + strict: true, + }); + if (positionals.length !== 2) throw new Error('diff needs two schema IR files'); + const failOn = (values['fail-on'] ?? AXES.join(',')) + .split(',') + .map((axis) => axis.trim()) + .filter(Boolean); + const unknown = failOn.filter((axis) => !AXES.includes(axis)); + if (unknown.length > 0) + throw new Error(`--fail-on accepts ${AXES.join(', ')}; got ${unknown.join(', ')}`); + + const from = readIR(path.resolve(io.cwd, positionals[0])); + const to = readIR(path.resolve(io.cwd, positionals[1])); + const result = diffSchemaIR(from, to); + if (values.json) { + io.stdout(`${JSON.stringify(result, null, '\t')}\n`); + } else { + const { read, write, storage } = result.compatibility; + io.stdout(`read: ${read}, write: ${write}, storage: ${storage}\n`); + for (const change of result.changes) { + io.stdout( + ` ${change.breaks.length > 0 ? `breaks ${change.breaks.join('+')}` : 'additive'}: ${change.message}\n`, + ); + } + for (const rename of result.source) + io.stdout(` type renamed: ${rename.from} -> ${rename.to}\n`); + for (const profile of result.profiles) + io.stdout(` profile ${profile.kind}: ${profile.name}\n`); + } + const breaking = failOn.some( + (axis) => + result.compatibility[/** @type {'read' | 'write' | 'storage'} */ (axis)] === 'breaking', + ); + return breaking ? 1 : 0; +} + +/** + * @param {string} filePath + * @returns {import('./irTypes.js').SchemaIR} + */ +function readIR(filePath) { + let ir; + try { + ir = JSON.parse(fs.readFileSync(filePath, 'utf8')); + } catch (error) { + throw new Error(`cannot read schema IR ${filePath}: ${/** @type {Error} */ (error).message}`, { + cause: error, + }); + } + assertSchemaIR(ir); + return ir; +} diff --git a/utils/client/cli.test.js b/utils/client/cli.test.js new file mode 100644 index 0000000..94d3cab --- /dev/null +++ b/utils/client/cli.test.js @@ -0,0 +1,136 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { spikeProfiles, spikeTables } from '../../test/fixtures/clientSchema.js'; +import { buildSchemaIR } from './buildSchemaIR.js'; +import { serializeSchemaIR } from './clientOutputs.js'; +import { runCli } from './cli.js'; + +describe('schema-codegen CLI', () => { + /** @type {string} */ + let directory; + /** @type {string[]} */ + let stdout; + /** @type {string[]} */ + let stderr; + beforeEach(() => { + directory = fs.mkdtempSync(path.join(os.tmpdir(), 'codegen-cli-')); + stdout = []; + stderr = []; + }); + afterEach(() => { + fs.rmSync(directory, { recursive: true, force: true }); + }); + + const run = (/** @type {string[]} */ argv) => + runCli(argv, { + stdout: (text) => stdout.push(text), + stderr: (text) => stderr.push(text), + cwd: directory, + }); + /** + * @param {string} name + * @param {any[]} [tables] + */ + const writeIR = (name, tables = spikeTables()) => { + fs.writeFileSync( + path.join(directory, name), + serializeSchemaIR(buildSchemaIR({ tables, syncProfiles: spikeProfiles() })), + ); + return name; + }; + + it('generates Swift and Kotlin from an IR file, then reports nothing to do', () => { + writeIR('ir.json'); + expect( + run([ + 'generate-client', + '--ir', + 'ir.json', + '--swift', + 'ios', + '--kotlin', + 'android', + '--kotlin-package', + 'com.example', + ]), + ).toBe(0); + expect( + fs.existsSync(path.join(directory, 'ios', 'Sources', 'HarperModels', 'Models.swift')), + ).toBe(true); + expect( + fs.existsSync( + path.join(directory, 'android', 'src', 'main', 'kotlin', 'com', 'example', 'Models.kt'), + ), + ).toBe(true); + expect(stdout.join('')).toContain(`wrote ${path.join('ios', 'Package.swift')}`); + stdout = []; + expect( + run([ + 'generate-client', + '--ir', + 'ir.json', + '--swift', + 'ios', + '--kotlin', + 'android', + '--kotlin-package', + 'com.example', + ]), + ).toBe(0); + expect(stdout.join('')).toBe('generated files are up to date\n'); + }); + + it('exits 2 with a message for bad input', () => { + fs.writeFileSync(path.join(directory, 'bad.json'), '{"irVersion": 7}'); + expect(run(['generate-client', '--ir', 'bad.json', '--swift', 'ios'])).toBe(2); + expect(stderr.join('')).toContain('irVersion'); + expect(run(['generate-client', '--ir', 'missing.json', '--swift', 'ios'])).toBe(2); + expect(run(['generate-client', '--ir', 'bad.json'])).toBe(2); + expect(run(['generate-client', '--swfit', 'ios'])).toBe(2); + expect(run(['explode'])).toBe(2); + expect(run([])).toBe(2); + }); + + it('diffs two IRs and exits 1 only when an axis in --fail-on breaks', () => { + const before = writeIR('before.json'); + const added = spikeTables(); + added[0].attributes.push({ name: 'sku', type: 'String', nullable: false }); + const after = writeIR('after.json', added); + + expect(run(['diff', before, before])).toBe(0); + expect(stdout.join('')).toContain('read: identical, write: identical, storage: identical'); + stdout = []; + expect(run(['diff', before, after])).toBe(1); + expect(stdout.join('')).toContain('breaks write: data.Product.sku added'); + expect(run(['diff', before, after, '--fail-on', 'read,storage'])).toBe(0); + expect(run(['diff', before, after, '--fail-on', 'sideways'])).toBe(2); + }); + + it('prints the diff as JSON', () => { + const before = writeIR('before.json'); + stdout = []; + expect(run(['diff', before, before, '--json'])).toBe(0); + expect(JSON.parse(stdout.join('')).compatibility).toEqual({ + read: 'identical', + write: 'identical', + storage: 'identical', + }); + }); + + it('runs as an executable and sets the exit code', () => { + const bin = fileURLToPath(new URL('../../bin/schema-codegen.js', import.meta.url)); + expect(execFileSync(process.execPath, [bin, '--version'], { encoding: 'utf8' })).toMatch( + /^@harperfast\/schema-codegen@/, + ); + const result = spawnSync(process.execPath, [bin, 'diff', 'a.json'], { + cwd: directory, + encoding: 'utf8', + }); + expect(result.status).toBe(2); + expect(result.stderr).toContain('diff needs two schema IR files'); + }); +}); diff --git a/utils/client/clientOutputs.js b/utils/client/clientOutputs.js new file mode 100644 index 0000000..975ddea --- /dev/null +++ b/utils/client/clientOutputs.js @@ -0,0 +1,220 @@ +/** @import { SchemaIR } from './irTypes.js' */ +/** @import { StagedFile } from '../writeIfChanged.js' */ +import fs from 'node:fs'; +import path from 'node:path'; +import { commitStaged, discardStaged, readIfExists, stageFile } from '../writeIfChanged.js'; +import { GENERATED_MARKER } from './emitSupport.js'; +import { emitKotlinModule } from './emitKotlin.js'; +import { emitSwiftPackage } from './emitSwift.js'; + +/** Package-root folders holding copies of generated sources that are not stale (Gradle's output). */ +const ROOT_BUILD_DIRECTORIES = new Set(['build', 'node_modules']); + +/** + * Where client outputs go. Relative paths resolve against `baseDirectory`. + * @typedef {Object} ClientOutputOptions + * @property {string} [baseDirectory] + * @property {string} [schemaIR] the IR JSON file + * @property {string} [swift] the Swift package directory + * @property {string} [swiftModule] + * @property {string} [kotlin] the Kotlin module directory + * @property {string} [kotlinPackage] + */ + +/** + * @typedef {Object} ClientOutput + * @property {string} path absolute + * @property {string} content + * @property {boolean} scaffold written only when absent + * @property {'swift' | 'kotlin' | 'ir'} target + */ + +/** + * @param {SchemaIR} ir + * @returns {string} + */ +export function serializeSchemaIR(ir) { + return `${JSON.stringify(ir, null, '\t')}\n`; +} + +/** + * Renders every requested output in memory, the IR file last: it is the baseline the next run + * diffs against, so it must never be published ahead of the sources generated from it. + * @param {SchemaIR} ir + * @param {ClientOutputOptions} options + * @returns {ClientOutput[]} + */ +export function renderClientOutputs(ir, options) { + const base = options.baseDirectory ?? ''; + /** @type {ClientOutput[]} */ + const outputs = []; + if (options.swift) { + const directory = path.resolve(base, options.swift); + for (const file of emitSwiftPackage(ir, { module: options.swiftModule })) { + outputs.push({ + path: path.join(directory, ...file.path.split('/')), + content: file.content, + scaffold: file.scaffold, + target: 'swift', + }); + } + } + if (options.kotlin) { + const directory = path.resolve(base, options.kotlin); + for (const file of emitKotlinModule(ir, { packageName: options.kotlinPackage })) { + outputs.push({ + path: path.join(directory, ...file.path.split('/')), + content: file.content, + scaffold: file.scaffold, + target: 'kotlin', + }); + } + } + if (options.schemaIR) { + outputs.push({ + path: path.resolve(base, options.schemaIR), + content: serializeSchemaIR(ir), + scaffold: false, + target: 'ir', + }); + } + return outputs; +} + +/** + * Publishes changed outputs, and scaffolds that do not exist yet, as one generation: every file is + * staged beside its target before any is moved into place, in order, and a failure while moving + * them restores the files already replaced, so a failed write leaves the previous generation. + * @param {ClientOutput[]} outputs + * @returns {{ written: string[], createdScaffolds: string[] }} + */ +export function publishClientOutputs(outputs) { + /** @type {{ output: ClientOutput, staged: StagedFile, previous: string | undefined }[]} */ + const pending = []; + try { + for (const output of outputs) { + const previous = readIfExists(output.path); + if (output.scaffold ? previous !== undefined : previous === output.content) continue; + fs.mkdirSync(path.dirname(output.path), { recursive: true }); + pending.push({ output, staged: stageFile(output.path, output.content), previous }); + } + } catch (error) { + for (const { staged } of pending) discardStaged(staged); + throw error; + } + /** @type {string[]} */ + const written = []; + /** @type {string[]} */ + const createdScaffolds = []; + pending.forEach(({ output, staged }, index) => { + try { + commitStaged(staged); + } catch (error) { + for (const rest of pending.slice(index + 1)) discardStaged(rest.staged); + throw restorePrevious(pending.slice(0, index + 1), error); + } + (output.scaffold ? createdScaffolds : written).push(output.path); + }); + return { written, createdScaffolds }; +} + +/** + * @param {{ staged: StagedFile, previous: string | undefined }[]} committed + * @param {unknown} cause + * @returns {unknown} the error to throw: the cause, or an aggregate when a restore failed too + */ +function restorePrevious(committed, cause) { + const failures = []; + for (const { staged, previous } of committed.reverse()) { + try { + if (previous === undefined) fs.rmSync(staged.target, { force: true }); + else commitStaged(stageFile(staged.target, previous)); + } catch (failure) { + failures.push(failure); + } + } + return failures.length === 0 + ? cause + : new AggregateError( + [cause, ...failures], + 'publishing client outputs failed and the previous generation could not be fully restored; regenerate to repair it', + ); +} + +/** + * Generated sources a renamed Swift module or Kotlin package left behind: reported, never deleted. + * @param {string} directory a package directory + * @param {ClientOutput[]} outputs + * @returns {string[]} + */ +export function findStaleGeneratedFiles(directory, outputs) { + const current = new Set(outputs.map((output) => output.path)); + /** @type {string[]} */ + const stale = []; + /** + * @param {string} folder + * @param {number} depth + */ + const walk = (folder, depth) => { + if (depth > 16) return; + /** @type {fs.Dirent[]} */ + let entries; + try { + entries = fs.readdirSync(folder, { withFileTypes: true }); + } catch { + return; + } + for (const entry of entries) { + const entryPath = path.join(folder, entry.name); + if (entry.isDirectory()) { + const skipped = + entry.name.startsWith('.') || (depth === 0 && ROOT_BUILD_DIRECTORIES.has(entry.name)); + if (!skipped) walk(entryPath, depth + 1); + } else if ( + (entry.isFile() || (entry.isSymbolicLink() && isFile(entryPath))) && + /\.(swift|kt)$/.test(entry.name) && + !current.has(entryPath) && + isGenerated(entryPath) + ) { + stale.push(entryPath); + } + } + }; + walk(directory, 0); + return stale.sort(); +} + +/** + * @param {string} filePath + * @returns {boolean} + */ +function isFile(filePath) { + try { + return fs.statSync(filePath).isFile(); + } catch { + return false; + } +} + +/** + * @param {string} filePath + * @returns {boolean} false for a file that cannot be read + */ +function isGenerated(filePath) { + /** @type {number} */ + let handle; + try { + handle = fs.openSync(filePath, 'r'); + } catch { + return false; + } + try { + const buffer = Buffer.alloc(GENERATED_MARKER.length); + const bytesRead = fs.readSync(handle, buffer, 0, buffer.length, 0); + return buffer.toString('utf8', 0, bytesRead) === GENERATED_MARKER; + } catch { + return false; + } finally { + fs.closeSync(handle); + } +} diff --git a/utils/client/clientOutputs.test.js b/utils/client/clientOutputs.test.js new file mode 100644 index 0000000..f3a1a4b --- /dev/null +++ b/utils/client/clientOutputs.test.js @@ -0,0 +1,201 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { spikeProfiles, spikeTables } from '../../test/fixtures/clientSchema.js'; +import { buildSchemaIR } from './buildSchemaIR.js'; +import { + findStaleGeneratedFiles, + publishClientOutputs, + renderClientOutputs, + serializeSchemaIR, +} from './clientOutputs.js'; + +const ir = buildSchemaIR({ + tables: spikeTables(), + syncProfiles: spikeProfiles(), + generator: 'test', +}); + +describe('client outputs', () => { + /** @type {string} */ + let directory; + beforeEach(() => { + directory = fs.mkdtempSync(path.join(os.tmpdir(), 'codegen-outputs-')); + }); + afterEach(() => { + vi.restoreAllMocks(); + fs.rmSync(directory, { recursive: true, force: true }); + }); + + const options = () => ({ + baseDirectory: directory, + schemaIR: 'client/schema.json', + swift: 'ios/Models', + kotlin: 'android/models', + kotlinPackage: 'com.example.models', + }); + + it('renders the requested outputs with the IR file last, writing nothing', () => { + const outputs = renderClientOutputs(ir, options()); + expect(outputs.at(-1)).toEqual({ + path: path.join(directory, 'client', 'schema.json'), + content: serializeSchemaIR(ir), + scaffold: false, + target: 'ir', + }); + expect(outputs.filter((output) => output.target === 'swift')).toHaveLength(4); + expect(outputs.filter((output) => output.target === 'kotlin')).toHaveLength(4); + expect(fs.readdirSync(directory)).toEqual([]); + }); + + it('renders nothing for an invalid option, before any file exists', () => { + expect(() => renderClientOutputs(ir, { ...options(), kotlinPackage: 'a/b' })).toThrow(); + expect(fs.readdirSync(directory)).toEqual([]); + }); + + it('writes generated files, creates scaffolds once, and reports only real changes', () => { + const outputs = renderClientOutputs(ir, options()); + const first = publishClientOutputs(outputs); + expect(first.createdScaffolds.map((file) => path.basename(file)).sort()).toEqual([ + 'Package.swift', + 'build.gradle.kts', + ]); + expect(first.written).toHaveLength(outputs.length - 2); + + const manifest = path.join(directory, 'ios', 'Models', 'Package.swift'); + fs.writeFileSync(manifest, '// edited by the app developer\n'); + const second = publishClientOutputs(outputs); + expect(second).toEqual({ written: [], createdScaffolds: [] }); + expect(fs.readFileSync(manifest, 'utf8')).toBe('// edited by the app developer\n'); + }); + + it('publishes nothing when any file of the generation cannot be written', () => { + const outputs = renderClientOutputs(ir, options()); + publishClientOutputs(outputs); + const changed = renderClientOutputs( + buildSchemaIR({ tables: spikeTables().slice(0, 1), generator: 'test' }), + options(), + ); + const before = outputs.map((output) => fs.readFileSync(output.path, 'utf8')); + const writeFileSync = fs.writeFileSync; + let writes = 0; + vi.spyOn(fs, 'writeFileSync').mockImplementation((...args) => { + if (++writes === 3) { + throw Object.assign(new Error('no space left on device'), { code: 'ENOSPC' }); + } + return writeFileSync(...args); + }); + expect(() => publishClientOutputs(changed)).toThrow(/no space left/); + expect(outputs.map((output) => fs.readFileSync(output.path, 'utf8'))).toEqual(before); + const all = /** @type {string[]} */ (fs.readdirSync(directory, { recursive: true })); + expect(all.filter((file) => file.endsWith('.tmp'))).toEqual([]); + }); + + it('restores the files already moved into place when a later move fails', () => { + const outputs = renderClientOutputs(ir, options()); + publishClientOutputs(outputs); + const before = outputs.map((output) => fs.readFileSync(output.path, 'utf8')); + const changed = renderClientOutputs( + buildSchemaIR({ tables: spikeTables().slice(0, 1), generator: 'test' }), + options(), + ); + const renameSync = fs.renameSync; + let renames = 0; + vi.spyOn(fs, 'renameSync').mockImplementation((...args) => { + if (++renames === 3) { + throw Object.assign(new Error('cross-device link not permitted'), { code: 'EXDEV' }); + } + return renameSync(...args); + }); + expect(() => publishClientOutputs(changed)).toThrow(/cross-device/); + vi.restoreAllMocks(); + expect(outputs.map((output) => fs.readFileSync(output.path, 'utf8'))).toEqual(before); + const all = /** @type {string[]} */ (fs.readdirSync(directory, { recursive: true })); + expect(all.filter((file) => file.endsWith('.tmp'))).toEqual([]); + }); + + it('restores the file whose in-place fallback failed part-way', () => { + const outputs = renderClientOutputs(ir, options()); + publishClientOutputs(outputs); + const before = outputs.map((output) => fs.readFileSync(output.path, 'utf8')); + const changed = renderClientOutputs( + buildSchemaIR({ tables: spikeTables().slice(0, 1), generator: 'test' }), + options(), + ); + const renameSync = fs.renameSync; + const writeFileSync = fs.writeFileSync; + let renames = 0; + vi.spyOn(fs, 'renameSync').mockImplementation((...args) => { + if (++renames === 2) { + throw Object.assign(new Error('operation not permitted'), { code: 'EPERM' }); + } + return renameSync(...args); + }); + vi.spyOn(fs, 'writeFileSync').mockImplementation((file, content, ...rest) => { + if (renames === 2 && !String(file).endsWith('.tmp')) { + writeFileSync(file, String(content).slice(0, 10)); + vi.mocked(fs.writeFileSync).mockRestore(); + throw Object.assign(new Error('no space left on device'), { code: 'ENOSPC' }); + } + return writeFileSync(file, content, ...rest); + }); + expect(() => publishClientOutputs(changed)).toThrow(/no space left/); + vi.restoreAllMocks(); + expect(outputs.map((output) => fs.readFileSync(output.path, 'utf8'))).toEqual(before); + }); + + it('skips build folders and unreadable files when looking for stale sources', () => { + const outputs = renderClientOutputs(ir, options()); + publishClientOutputs(outputs); + const packageDirectory = path.join(directory, 'ios', 'Models'); + const runtime = fs.readFileSync( + path.join(packageDirectory, 'Sources', 'HarperModels', 'HarperRuntime.swift'), + ); + for (const folder of ['.build/debug', 'build/out']) { + fs.mkdirSync(path.join(packageDirectory, folder), { recursive: true }); + fs.writeFileSync(path.join(packageDirectory, folder, 'Copied.swift'), runtime); + } + expect(findStaleGeneratedFiles(packageDirectory, outputs)).toEqual([]); + const nestedBuild = path.join(packageDirectory, 'Sources', 'build'); + fs.mkdirSync(nestedBuild); + fs.writeFileSync(path.join(nestedBuild, 'Renamed.swift'), runtime); + expect(findStaleGeneratedFiles(packageDirectory, outputs)).toEqual([ + path.join(nestedBuild, 'Renamed.swift'), + ]); + fs.rmSync(nestedBuild, { recursive: true }); + fs.writeFileSync(path.join(packageDirectory, 'Sources', 'Locked.swift'), runtime); + vi.spyOn(fs, 'openSync').mockImplementation(() => { + throw Object.assign(new Error('permission denied'), { code: 'EACCES' }); + }); + expect(findStaleGeneratedFiles(packageDirectory, outputs)).toEqual([]); + }); + + it('leaves no temporary files behind', () => { + publishClientOutputs(renderClientOutputs(ir, options())); + const all = /** @type {string[]} */ (fs.readdirSync(directory, { recursive: true })); + expect(all.filter((file) => file.endsWith('.tmp'))).toEqual([]); + }); + + it('finds generated sources a renamed module or package left behind, ignoring other files', () => { + publishClientOutputs(renderClientOutputs(ir, { ...options(), swiftModule: 'OldModels' })); + const handWritten = path.join( + directory, + 'ios', + 'Models', + 'Sources', + 'OldModels', + 'Extensions.swift', + ); + fs.writeFileSync(handWritten, 'import Foundation\n'); + const outputs = renderClientOutputs(ir, options()); + publishClientOutputs(outputs); + const stale = findStaleGeneratedFiles(path.join(directory, 'ios', 'Models'), outputs); + expect(stale.map((file) => path.relative(directory, file).split(path.sep).join('/'))).toEqual([ + 'ios/Models/Sources/OldModels/HarperRuntime.swift', + 'ios/Models/Sources/OldModels/HarperSchema.swift', + 'ios/Models/Sources/OldModels/Models.swift', + ]); + expect(findStaleGeneratedFiles(path.join(directory, 'android', 'models'), outputs)).toEqual([]); + }); +}); diff --git a/utils/client/contractHash.js b/utils/client/contractHash.js new file mode 100644 index 0000000..4066fb6 --- /dev/null +++ b/utils/client/contractHash.js @@ -0,0 +1,178 @@ +/** @import { IRObjectType, IRProfile, IRTable, IRType } from './irTypes.js' */ +import { hashCanonical } from './canonicalHash.js'; +import { IR_VERSION } from './irConstants.js'; +import { compareText } from './syncProfiles.js'; + +/** + * @param {string} database + * @param {string} table + * @returns {string} + */ +function tableKey(database, table) { + return `${database}\u0000${table}`; +} + +/** + * Hashes each table's wire/storage contract, and all of them together. A table's contract lists + * every object type and embedded table shape reachable from it once, by name, so a change to a + * nested shape changes the hash of every table that carries it while shared and recursive types + * cost linear time. + * @param {IRTable[]} tables + * @param {IRObjectType[]} types + * @returns {{ tableHashes: Map, schemaHash: string }} table hashes keyed by `database\0name` + */ +export function hashContracts(tables, types) { + const objectTypes = new Map(types.map((type) => [type.name, type])); + const tablesByKey = new Map(tables.map((table) => [tableKey(table.database, table.name), table])); + const sorted = [...tables].sort( + (a, b) => compareText(a.database, b.database) || compareText(a.name, b.name), + ); + const contracts = sorted.map((table) => tableContract(table, objectTypes, tablesByKey)); + return { + tableHashes: new Map( + sorted.map((table, index) => [ + tableKey(table.database, table.name), + hashCanonical({ irVersion: IR_VERSION, table: contracts[index] }), + ]), + ), + schemaHash: hashCanonical({ irVersion: IR_VERSION, tables: contracts }), + }; +} + +/** + * @param {Omit} profile + * @param {Map} tableHashes from `hashContracts` + * @returns {{ hash: string, schemaHash: string }} the definition's hash and the hello-frame hash of its tables + */ +export function hashProfile(profile, tableHashes) { + const tables = [...profile.tables].sort( + (a, b) => compareText(a.database, b.database) || compareText(a.table, b.table), + ); + return { + hash: hashCanonical({ + irVersion: IR_VERSION, + profile: { + name: profile.name, + direction: profile.direction, + retentionMs: profile.retentionMs, + tables, + }, + }), + schemaHash: hashCanonical({ + irVersion: IR_VERSION, + tables: tables.map((entry) => ({ + database: entry.database, + table: entry.table, + hash: tableHashes.get(tableKey(entry.database, entry.table)), + })), + }), + }; +} + +/** + * @param {IRTable} table + * @param {Map} objectTypes + * @param {Map} tablesByKey + * @returns {object} + */ +function tableContract(table, objectTypes, tablesByKey) { + const own = `r:${tableKey(table.database, table.name)}`; + /** @type {Record} */ + const shapes = {}; + /** @type {{ key: string, describe: () => object }[]} */ + const pending = []; + /** + * @param {string} key + * @param {() => object} describe + */ + const reach = (key, describe) => { + if (key === own || key in shapes) return; + shapes[key] = {}; + pending.push({ key, describe }); + }; + /** + * @param {IRType} type + * @returns {object} + */ + const reference = (type) => { + switch (type.kind) { + case 'scalar': + return type; + case 'array': + return { + kind: 'array', + element: reference(type.element), + elementNullable: type.elementNullable, + }; + case 'object': + reach(`o:${type.type}`, () => ({ + attributes: sortedByName(objectTypes.get(type.type)?.attributes ?? []).map( + (attribute) => ({ + name: attribute.name, + type: reference(attribute.type), + nullable: attribute.nullable, + }), + ), + })); + return { kind: 'object', type: type.type }; + case 'record': { + const key = tableKey(type.database, type.table); + const embedded = tablesByKey.get(key); + if (embedded) reach(`r:${key}`, () => recordShape(embedded, reference)); + return { kind: 'record', database: type.database, table: type.table }; + } + } + }; + + const contract = { + ...recordShape(table, reference), + database: table.database, + name: table.name, + versionColumn: table.storage.versionColumn, + extraColumn: table.storage.extraColumn, + shapes, + }; + for (let next = pending.pop(); next; next = pending.pop()) { + shapes[next.key] = next.describe(); + } + return contract; +} + +/** + * What a table contributes wherever it appears, on its own or embedded in another table: an + * embedded record decodes and encodes with the same model. + * @param {IRTable} table + * @param {(type: IRType) => object} reference + * @returns {object} + */ +function recordShape(table, reference) { + return { + primaryKey: table.primaryKey, + sealed: table.sealed, + attributes: sortedByName(table.attributes).map((attribute) => ({ + name: attribute.name, + type: reference(attribute.type), + nullable: attribute.nullable, + primaryKey: attribute.primaryKey, + indexed: attribute.indexed, + serverManaged: attribute.serverManaged, + computed: attribute.computed, + })), + relations: sortedByName(table.relations).map((relation) => ({ + name: relation.name, + cardinality: relation.cardinality, + from: relation.from ?? null, + to: relation.to ?? null, + target: relation.target, + })), + }; +} + +/** + * @template {{ name: string }} T + * @param {T[]} entries + * @returns {T[]} + */ +function sortedByName(entries) { + return [...entries].sort((a, b) => compareText(a.name, b.name)); +} diff --git a/utils/client/diffSchemaIR.js b/utils/client/diffSchemaIR.js new file mode 100644 index 0000000..9642dec --- /dev/null +++ b/utils/client/diffSchemaIR.js @@ -0,0 +1,623 @@ +/** @import { IRAffinity, IRObjectType, IRStorage, IRTable, IRType, SchemaIR } from './irTypes.js' */ +import { affinityOf, requiredOnInsert } from './buildSchemaIR.js'; +import { canonicalJSON } from './canonicalHash.js'; +import { compareText } from './syncProfiles.js'; +import { assertSchemaIR } from './validateSchemaIR.js'; + +/** @typedef {'read' | 'write' | 'storage'} BreakAxis */ + +/** + * @typedef {Object} SchemaChange + * @property {string} kind + * @property {string} database + * @property {string} table + * @property {string} [attribute] + * @property {string} [path] the nested attribute path, for changes inside object types + * @property {BreakAxis[]} breaks what the change breaks for a client generated from the old IR + * @property {string} message + */ + +/** + * @typedef {Object} SchemaDelta + * @property {{ database: string, table: string, storage: IRStorage }[]} addedTables + * @property {{ database: string, table: string, addColumns: { name: string, affinity: IRAffinity }[], extraColumn: string | null, recompute: { attribute: string, from: string | null, version: string | null }[] }[]} tables + */ + +/** + * @typedef {Object} SchemaDiff + * @property {string} fromHash + * @property {string} toHash + * @property {{ read: Compatibility, write: Compatibility, storage: Compatibility }} compatibility + * @property {SchemaChange[]} changes + * @property {{ kind: 'typeRenamed', name: string, from: string, to: string }[]} source changed public type names + * @property {{ kind: 'added' | 'removed' | 'changed', name: string }[]} profiles + * @property {SchemaDelta} delta what a client generated from the old IR applies to keep syncing + */ + +/** @typedef {'identical' | 'additive' | 'breaking'} Compatibility */ + +/** + * @typedef {Object} TypeFinding + * @property {string} path + * @property {string} kind + * @property {boolean} read + * @property {boolean} write + * @property {string} message + */ + +/** + * Classifies how moving from one IR to another affects clients generated from the first: whether + * their decoding (`read`), their writes (`write`) or their replica layout (`storage`) breaks, and + * what an additive change asks them to apply. + * @param {SchemaIR} from + * @param {SchemaIR} to + * @returns {SchemaDiff} + */ +export function diffSchemaIR(from, to) { + assertSchemaIR(from); + assertSchemaIR(to); + /** @type {SchemaChange[]} */ + const changes = []; + /** @type {SchemaDiff['source']} */ + const source = []; + /** @type {SchemaDelta} */ + const delta = { addedTables: [], tables: [] }; + const fromTypes = new Map(from.types.map((type) => [type.name, type])); + const toTypes = new Map(to.types.map((type) => [type.name, type])); + const fromTables = new Map(from.tables.map((table) => [keyOf(table), table])); + const toTables = new Map(to.tables.map((table) => [keyOf(table), table])); + const keys = [...new Set([...fromTables.keys(), ...toTables.keys()])].sort(compareText); + /** @type {{ table: IRTable, itemized: boolean }[]} */ + const changedTables = []; + + for (const key of keys) { + const before = fromTables.get(key); + const after = toTables.get(key); + if (!before && after) { + changes.push({ + kind: 'tableAdded', + database: after.database, + table: after.name, + breaks: [], + message: `table ${after.database}.${after.name} added`, + }); + delta.addedTables.push({ + database: after.database, + table: after.name, + storage: after.storage, + }); + continue; + } + if (before && !after) { + changes.push({ + kind: 'tableRemoved', + database: before.database, + table: before.name, + breaks: ['read', 'write', 'storage'], + message: `table ${before.database}.${before.name} removed`, + }); + continue; + } + if (!before || !after) continue; + if (before.typeName !== after.typeName) { + source.push({ + kind: 'typeRenamed', + name: `${after.database}.${after.name}`, + from: before.typeName, + to: after.typeName, + }); + } + if (before.hash !== after.hash) { + const count = changes.length; + diffTable(before, after, fromTypes, toTypes, changes, delta); + changedTables.push({ table: after, itemized: changes.length > count }); + } + } + const ownChanges = [...changes]; + for (const { table, itemized } of changedTables) { + const embedded = embeddedTableKeys(table, toTypes, toTables); + const inherited = new Set( + ownChanges + .filter((change) => embedded.has(keyOf({ database: change.database, name: change.table }))) + .flatMap((change) => change.breaks), + ); + /** @type {BreakAxis[]} */ + const breaks = /** @type {BreakAxis[]} */ (['read', 'write']).filter((axis) => + inherited.has(axis), + ); + if (breaks.length > 0) { + changes.push({ + kind: 'embeddedTableChanged', + database: table.database, + table: table.name, + breaks, + message: `${table.database}.${table.name} embeds a table whose change breaks ${breaks.join(', ')}`, + }); + } else if (!itemized) { + changes.push({ + kind: 'contractChanged', + database: table.database, + table: table.name, + breaks: [], + message: `${table.database}.${table.name} changed through a nested type or an embedded table, whose own changes are listed`, + }); + } + } + for (const [name, type] of fromTypes) { + const next = toTypes.get(name); + if (next && next.typeName !== type.typeName) { + source.push({ kind: 'typeRenamed', name, from: type.typeName, to: next.typeName }); + } + } + + const identical = from.schemaHash === to.schemaHash; + /** + * @param {BreakAxis} axis + * @returns {Compatibility} + */ + const compatibilityOf = (axis) => + identical + ? 'identical' + : changes.some((change) => change.breaks.includes(axis)) + ? 'breaking' + : 'additive'; + + return { + fromHash: from.schemaHash, + toHash: to.schemaHash, + compatibility: { + read: compatibilityOf('read'), + write: compatibilityOf('write'), + storage: compatibilityOf('storage'), + }, + changes, + source, + profiles: diffProfiles(from, to), + delta, + }; +} + +/** + * @param {{ database: string, name: string }} table + * @returns {string} + */ +function keyOf(table) { + return `${table.database}\u0000${table.name}`; +} + +/** + * The tables a table embeds, directly or through nested types and other embedded tables; their + * models decode and encode inside its rows. + * @param {IRTable} table + * @param {Map} types + * @param {Map} tables + * @returns {Set} table keys + */ +function embeddedTableKeys(table, types, tables) { + const found = new Set(); + const visitedTypes = new Set(); + /** @param {IRType} type */ + const visit = (type) => { + if (type.kind === 'array') visit(type.element); + else if (type.kind === 'object' && !visitedTypes.has(type.type)) { + visitedTypes.add(type.type); + for (const attribute of types.get(type.type)?.attributes ?? []) visit(attribute.type); + } else if (type.kind === 'record') { + const key = keyOf({ database: type.database, name: type.table }); + if (found.has(key)) return; + found.add(key); + for (const attribute of tables.get(key)?.attributes ?? []) visit(attribute.type); + } + }; + for (const attribute of table.attributes) visit(attribute.type); + return found; +} + +/** + * @param {IRTable} table + * @returns {Map} attribute name → optional on read + */ +function readOptionality(table) { + return new Map(table.projections.record.map((field) => [field.name, field.optional])); +} + +/** + * @param {IRTable} before + * @param {IRTable} after + * @param {Map} fromTypes + * @param {Map} toTypes + * @param {SchemaChange[]} changes + * @param {SchemaDelta} delta + */ +function diffTable(before, after, fromTypes, toTypes, changes, delta) { + const table = { database: after.database, table: after.name }; + /** + * @param {string} kind + * @param {BreakAxis[]} breaks + * @param {string} message + * @param {{ attribute?: string, path?: string }} [where] + */ + const record = (kind, breaks, message, where = {}) => + changes.push({ kind, ...table, ...where, breaks, message }); + const name = `${after.database}.${after.name}`; + + if (before.primaryKey !== after.primaryKey) { + record( + 'primaryKeyChanged', + ['read', 'write', 'storage'], + `${name} primary key changed from ${before.primaryKey} to ${after.primaryKey}`, + ); + return; + } + const metadataMoved = + before.storage.versionColumn !== after.storage.versionColumn || + (before.storage.extraColumn !== null && + after.storage.extraColumn !== null && + before.storage.extraColumn !== after.storage.extraColumn); + if (metadataMoved) { + record( + 'metadataColumnMoved', + ['storage'], + `${name} replica metadata columns moved (an attribute now uses ${before.storage.versionColumn} or ${before.storage.extraColumn})`, + ); + } + if (before.sealed !== after.sealed) { + record( + 'sealedChanged', + ['write'], + after.sealed + ? `${name} became @sealed: clients that send undeclared attributes are rejected` + : `${name} is no longer @sealed: clients without an overflow bag drop undeclared attributes on PUT`, + ); + } + if ((before.projections.upsert === null) !== (after.projections.upsert === null)) { + record( + 'replacementChanged', + after.projections.upsert === null ? ['write'] : [], + after.projections.upsert === null + ? `${name} no longer supports full replacement: an older client's PUT erases Blob content that sync does not deliver` + : `${name} supports full replacement again`, + ); + } + + const beforeAttributes = new Map( + before.attributes.map((attribute) => [attribute.name, attribute]), + ); + const afterAttributes = new Map(after.attributes.map((attribute) => [attribute.name, attribute])); + const relationsBefore = new Map(before.relations.map((relation) => [relation.name, relation])); + const relationsAfter = new Map(after.relations.map((relation) => [relation.name, relation])); + const optionalBefore = readOptionality(before); + const optionalAfter = readOptionality(after); + /** @type {{ name: string, affinity: IRAffinity }[]} */ + const addColumns = []; + /** @type {{ attribute: string, from: string | null, version: string | null }[]} */ + const recompute = []; + const oldMetadataColumns = new Set([before.storage.versionColumn, before.storage.extraColumn]); + const seenTypes = new Set(); + const echoed = + 'older models keep it in their overflow and send it back on PUT, which the server rejects'; + + for (const [attributeName, previous] of beforeAttributes) { + if (afterAttributes.has(attributeName)) continue; + /** @type {BreakAxis[]} */ + const breaks = []; + if (!optionalBefore.get(attributeName)) breaks.push('read'); + if (!previous.readOnly && (after.sealed || relationsAfter.has(attributeName))) { + breaks.push('write'); + } + record('attributeRemoved', breaks, `${name}.${attributeName} removed`, { + attribute: attributeName, + }); + } + for (const [attributeName, current] of afterAttributes) { + const previous = beforeAttributes.get(attributeName); + const where = { attribute: attributeName }; + if (!previous) { + const echoedComputed = + current.computed !== null && !before.sealed && !relationsBefore.has(attributeName); + const dropped = !current.readOnly && (before.sealed || relationsBefore.has(attributeName)); + record( + 'attributeAdded', + requiredOnInsert(current) || dropped || echoedComputed ? ['write'] : [], + echoedComputed + ? `${name}.${attributeName} added as a computed attribute: ${echoed}` + : `${name}.${attributeName} added`, + where, + ); + if (!oldMetadataColumns.has(attributeName)) { + addColumns.push({ name: attributeName, affinity: affinityOf(current.type) }); + } + if (current.computed) { + recompute.push({ + attribute: attributeName, + from: current.computed.from, + version: current.computed.version, + }); + } + continue; + } + + const findings = compareTypes( + previous.type, + current.type, + fromTypes, + toTypes, + attributeName, + seenTypes, + ); + for (const finding of findings) { + /** @type {BreakAxis[]} */ + const breaks = []; + if (finding.read) breaks.push('read'); + if (finding.write) breaks.push('write'); + record(finding.kind, breaks, `${name}.${finding.message}`, { ...where, path: finding.path }); + } + if (affinityOf(previous.type) !== affinityOf(current.type)) { + record( + 'columnAffinityChanged', + ['storage'], + `${name}.${attributeName} is now stored as ${affinityOf(current.type)}`, + where, + ); + } + const readRelaxed = !optionalBefore.get(attributeName) && optionalAfter.get(attributeName); + if (readRelaxed || (!previous.nullable && current.nullable)) { + record( + 'attributeBecameNullable', + readRelaxed ? ['read'] : [], + `${name}.${attributeName} became nullable`, + where, + ); + } + if (!requiredOnInsert(previous) && requiredOnInsert(current)) { + record( + 'attributeBecameRequired', + ['write'], + `${name}.${attributeName} became required on insert`, + where, + ); + } + if (!previous.readOnly && current.readOnly) { + record( + 'attributeBecameReadOnly', + ['write'], + `${name}.${attributeName} became read-only`, + where, + ); + } + if (previous.readOnly && !current.readOnly) { + record('attributeBecameWritable', [], `${name}.${attributeName} became writable`, where); + } + const computedChanged = + current.computed && + (previous.computed?.from !== current.computed.from || + previous.computed?.version !== current.computed.version); + if (computedChanged && current.computed) { + record('computedChanged', [], `${name}.${attributeName} computed expression changed`, where); + recompute.push({ + attribute: attributeName, + from: current.computed.from, + version: current.computed.version, + }); + } + if ( + previous.serverManaged !== current.serverManaged && + previous.readOnly === current.readOnly + ) { + record( + 'serverManagedChanged', + [], + current.serverManaged + ? `${name}.${attributeName} is now server-managed (${current.serverManaged})` + : `${name}.${attributeName} is no longer server-managed`, + where, + ); + } + if (previous.indexed !== current.indexed) { + record( + 'indexChanged', + [], + `${name}.${attributeName} ${current.indexed ? 'is now' : 'is no longer'} indexed`, + where, + ); + } + } + + for (const [relationName, relation] of relationsAfter) { + const previous = relationsBefore.get(relationName); + const where = { attribute: relationName }; + if (!previous) { + const echoedRelation = !before.sealed && !beforeAttributes.has(relationName); + record( + 'relationAdded', + echoedRelation ? ['write'] : [], + echoedRelation + ? `${name}.${relationName} relationship added: ${echoed}` + : `${name}.${relationName} relationship added`, + where, + ); + } else if (canonicalJSON(previous) !== canonicalJSON(relation)) { + record('relationChanged', [], `${name}.${relationName} relationship changed`, where); + } + } + for (const relationName of relationsBefore.keys()) { + if (!relationsAfter.has(relationName)) { + record('relationRemoved', [], `${name}.${relationName} relationship removed`, { + attribute: relationName, + }); + } + } + + const extraColumn = before.storage.extraColumn === null ? after.storage.extraColumn : null; + if (addColumns.length > 0 || recompute.length > 0 || extraColumn !== null) { + delta.tables.push({ ...table, addColumns, extraColumn, recompute }); + } +} + +/** + * @param {IRType} before + * @param {IRType} after + * @param {Map} fromTypes + * @param {Map} toTypes + * @param {string} path + * @param {Set} seen object type pairs already compared for this table; each is reported + * once, at the first path that reaches it + * @returns {TypeFinding[]} + */ +function compareTypes(before, after, fromTypes, toTypes, path, seen) { + /** + * @param {string} kind + * @param {boolean} read + * @param {boolean} write + * @param {string} message + * @returns {TypeFinding} + */ + const finding = (kind, read, write, message) => ({ path, kind, read, write, message }); + if (before.kind !== after.kind) { + return [ + finding( + 'attributeTypeChanged', + true, + true, + `${path} type changed from ${before.kind} to ${after.kind}`, + ), + ]; + } + switch (before.kind) { + case 'scalar': { + const next = /** @type {typeof before} */ (after); + return before.scalar === next.scalar + ? [] + : [ + finding( + 'attributeTypeChanged', + true, + true, + `${path} type changed from ${before.scalar} to ${next.scalar}`, + ), + ]; + } + case 'record': { + const next = /** @type {typeof before} */ (after); + return before.database === next.database && before.table === next.table + ? [] + : [ + finding( + 'attributeTypeChanged', + true, + true, + `${path} now embeds ${next.database}.${next.table}`, + ), + ]; + } + case 'array': { + const next = /** @type {typeof before} */ (after); + const findings = compareTypes( + before.element, + next.element, + fromTypes, + toTypes, + `${path}[]`, + seen, + ); + if (!before.elementNullable && next.elementNullable) { + findings.push( + finding('elementBecameNullable', true, false, `${path} elements became nullable`), + ); + } + if (before.elementNullable && !next.elementNullable) { + findings.push( + finding('elementBecameRequired', false, true, `${path} elements became non-null`), + ); + } + return findings; + } + case 'object': { + const next = /** @type {typeof before} */ (after); + const pair = `${before.type}\u0000${next.type}`; + if (seen.has(pair)) return []; + seen.add(pair); + const beforeAttributes = new Map( + (fromTypes.get(before.type)?.attributes ?? []).map((a) => [a.name, a]), + ); + const afterAttributes = new Map( + (toTypes.get(next.type)?.attributes ?? []).map((a) => [a.name, a]), + ); + /** @type {TypeFinding[]} */ + const findings = []; + for (const [name, attribute] of beforeAttributes) { + if (!afterAttributes.has(name)) { + findings.push({ + ...finding( + 'nestedAttributeRemoved', + !attribute.nullable, + false, + `${path}.${name} removed`, + ), + path: `${path}.${name}`, + }); + } + } + for (const [name, attribute] of afterAttributes) { + const previous = beforeAttributes.get(name); + const nestedPath = `${path}.${name}`; + if (!previous) { + findings.push({ + ...finding( + 'nestedAttributeAdded', + false, + !attribute.nullable, + `${nestedPath} added${attribute.nullable ? '' : ' as required'}`, + ), + path: nestedPath, + }); + continue; + } + findings.push( + ...compareTypes(previous.type, attribute.type, fromTypes, toTypes, nestedPath, seen), + ); + if (!previous.nullable && attribute.nullable) { + findings.push({ + ...finding( + 'nestedAttributeBecameNullable', + true, + false, + `${nestedPath} became nullable`, + ), + path: nestedPath, + }); + } + if (previous.nullable && !attribute.nullable) { + findings.push({ + ...finding( + 'nestedAttributeBecameRequired', + false, + true, + `${nestedPath} became required`, + ), + path: nestedPath, + }); + } + } + return findings; + } + } +} + +/** + * @param {SchemaIR} from + * @param {SchemaIR} to + * @returns {SchemaDiff['profiles']} + */ +function diffProfiles(from, to) { + const before = new Map(from.profiles.map((profile) => [profile.name, profile.hash])); + const after = new Map(to.profiles.map((profile) => [profile.name, profile.hash])); + /** @type {SchemaDiff['profiles']} */ + const result = []; + for (const name of [...new Set([...before.keys(), ...after.keys()])].sort(compareText)) { + if (!before.has(name)) result.push({ kind: 'added', name }); + else if (!after.has(name)) result.push({ kind: 'removed', name }); + else if (before.get(name) !== after.get(name)) result.push({ kind: 'changed', name }); + } + return result; +} diff --git a/utils/client/diffSchemaIR.test.js b/utils/client/diffSchemaIR.test.js new file mode 100644 index 0000000..c9f6cf8 --- /dev/null +++ b/utils/client/diffSchemaIR.test.js @@ -0,0 +1,620 @@ +import { describe, expect, it } from 'vitest'; +import { + coverageTables, + coverageTypes, + sharedChain, + spikeProfiles, + spikeTables, +} from '../../test/fixtures/clientSchema.js'; +import { buildSchemaIR } from './buildSchemaIR.js'; +import { diffSchemaIR } from './diffSchemaIR.js'; + +/** + * @param {(tables: any[], types: Map) => void} [mutate] + * @param {unknown} [syncProfiles] + */ +function irOf(mutate, syncProfiles = spikeProfiles()) { + const tables = [...spikeTables(), ...coverageTables()]; + const types = coverageTypes(); + mutate?.(tables, types); + return buildSchemaIR({ tables, types, syncProfiles }); +} + +/** + * @param {any[]} tables + * @param {string} name + */ +function find(tables, name) { + return tables.find((table) => table.tableName === name); +} + +/** + * @param {(tables: any[], types: Map) => void} mutate + */ +function diffAfter(mutate) { + return diffSchemaIR(irOf(), irOf(mutate)); +} + +describe('diffSchemaIR', () => { + it('reports identical schemas', () => { + const result = diffSchemaIR(irOf(), irOf()); + expect(result.compatibility).toEqual({ + read: 'identical', + write: 'identical', + storage: 'identical', + }); + expect(result.changes).toEqual([]); + expect(result.delta).toEqual({ addedTables: [], tables: [] }); + }); + + it('treats an added table as additive and ships its storage', () => { + const result = diffAfter((tables) => + tables.push({ + tableName: 'Review', + primaryKey: 'id', + attributes: [{ name: 'id', type: 'ID', isPrimaryKey: true }], + }), + ); + expect(result.compatibility).toEqual({ + read: 'additive', + write: 'additive', + storage: 'additive', + }); + expect(result.delta.addedTables).toEqual([ + { + database: 'data', + table: 'Review', + storage: { + versionColumn: '_version', + extraColumn: '_extra', + columns: [{ name: 'id', affinity: 'TEXT' }], + }, + }, + ]); + }); + + it('breaks everything when a table is removed', () => { + const result = diffAfter((tables) => tables.splice(tables.indexOf(find(tables, 'Report')), 1)); + expect(result.compatibility).toEqual({ + read: 'breaking', + write: 'breaking', + storage: 'breaking', + }); + expect(result.changes).toEqual([ + expect.objectContaining({ + kind: 'tableRemoved', + table: 'Report', + breaks: ['read', 'write', 'storage'], + }), + ]); + }); + + it('breaks everything when the primary key changes', () => { + const result = diffAfter((tables) => { + const report = find(tables, 'Report'); + report.attributes[0].isPrimaryKey = false; + report.attributes[1].isPrimaryKey = true; + report.primaryKey = 'title'; + }); + expect(result.changes).toEqual([ + expect.objectContaining({ kind: 'primaryKeyChanged', breaks: ['read', 'write', 'storage'] }), + ]); + }); + + it('treats a nullable attribute added to an unsealed table as additive and ships the column', () => { + const result = diffAfter((tables) => + find(tables, 'Product').attributes.push({ name: 'sku', type: 'String', indexed: {} }), + ); + expect(result.compatibility).toEqual({ + read: 'additive', + write: 'additive', + storage: 'additive', + }); + expect(result.delta.tables).toEqual([ + { + database: 'data', + table: 'Product', + addColumns: [{ name: 'sku', affinity: 'TEXT' }], + extraColumn: null, + recompute: [], + }, + ]); + }); + + it('breaks writes when a required attribute is added', () => { + const result = diffAfter((tables) => + find(tables, 'Product').attributes.push({ name: 'sku', type: 'String', nullable: false }), + ); + expect(result.compatibility).toEqual({ + read: 'additive', + write: 'breaking', + storage: 'additive', + }); + }); + + it('breaks writes when any writable attribute is added to a sealed table', () => { + const result = diffAfter((tables) => + find(tables, 'Purchase').attributes.push({ name: 'note', type: 'String' }), + ); + expect(result.compatibility.write).toBe('breaking'); + expect(result.changes).toEqual([ + expect.objectContaining({ kind: 'attributeAdded', attribute: 'note', breaks: ['write'] }), + ]); + }); + + it('breaks reads when an attribute the old model required is removed', () => { + const result = diffAfter((tables) => { + const customer = find(tables, 'Customer'); + customer.attributes = customer.attributes.filter( + (/** @type {any} */ attribute) => attribute.name !== 'name', + ); + }); + expect(result.changes).toEqual([ + expect.objectContaining({ kind: 'attributeRemoved', attribute: 'name', breaks: ['read'] }), + ]); + }); + + it('treats removing an optional attribute from an unsealed table as additive', () => { + const result = diffAfter((tables) => { + const product = find(tables, 'Product'); + product.attributes = product.attributes.filter( + (/** @type {any} */ attribute) => attribute.name !== 'category', + ); + }); + expect(result.compatibility).toEqual({ + read: 'additive', + write: 'additive', + storage: 'additive', + }); + }); + + it('breaks writes when an attribute is removed from a sealed table', () => { + const result = diffAfter((tables) => { + const purchase = find(tables, 'Purchase'); + purchase.attributes = purchase.attributes.filter( + (/** @type {any} */ attribute) => attribute.name !== 'customerId', + ); + }); + expect(result.changes).toContainEqual( + expect.objectContaining({ + kind: 'attributeRemoved', + attribute: 'customerId', + breaks: ['write'], + }), + ); + }); + + it('breaks reads, writes and storage when an attribute type changes', () => { + const result = diffAfter((tables) => (find(tables, 'Product').attributes[2].type = 'String')); + expect(result.compatibility).toEqual({ + read: 'breaking', + write: 'breaking', + storage: 'breaking', + }); + expect(result.changes.map((change) => change.kind)).toEqual([ + 'attributeTypeChanged', + 'columnAffinityChanged', + ]); + }); + + it('does not break storage for a type change within the same affinity', () => { + const result = diffAfter((tables) => (find(tables, 'Product').attributes[1].type = 'ID')); + expect(result.compatibility).toEqual({ + read: 'breaking', + write: 'breaking', + storage: 'additive', + }); + }); + + it('breaks reads when an attribute the old model required becomes nullable', () => { + const result = diffAfter((tables) => delete find(tables, 'Customer').attributes[1].nullable); + expect(result.changes).toEqual([ + expect.objectContaining({ + kind: 'attributeBecameNullable', + attribute: 'name', + breaks: ['read'], + }), + ]); + }); + + it('breaks writes when an attribute becomes required on insert', () => { + const result = diffAfter((tables) => (find(tables, 'Product').attributes[1].nullable = false)); + expect(result.changes).toEqual([ + expect.objectContaining({ kind: 'attributeBecameRequired', breaks: ['write'] }), + ]); + expect(result.compatibility.read).toBe('additive'); + }); + + it('breaks writes when a writable attribute becomes server-managed', () => { + const result = diffAfter( + (tables) => (find(tables, 'Product').attributes[2].assignUpdatedTime = true), + ); + expect(result.changes).toContainEqual( + expect.objectContaining({ kind: 'attributeBecameReadOnly', breaks: ['write'] }), + ); + }); + + it('classifies array element nullability changes by direction', () => { + const loosened = diffAfter( + (tables) => + (find(tables, 'Customer').attributes.find( + (/** @type {any} */ a) => a.name === 'tags', + ).elements.nullable = undefined), + ); + expect(loosened.changes).toEqual([ + expect.objectContaining({ kind: 'elementBecameNullable', breaks: ['read'] }), + ]); + const tightened = diffAfter( + (tables) => + (find(tables, 'Customer').attributes.find( + (/** @type {any} */ a) => a.name === 'scores', + ).elements.nullable = false), + ); + expect(tightened.changes).toEqual([ + expect.objectContaining({ kind: 'elementBecameRequired', breaks: ['write'] }), + ]); + }); + + it('asks for a recompute when a computed expression changes', () => { + const result = diffAfter( + (tables) => (find(tables, 'Order').attributes[4].computedFromExpression = 'amount * 2'), + ); + expect(result.compatibility).toEqual({ + read: 'additive', + write: 'additive', + storage: 'additive', + }); + expect(result.delta.tables).toEqual([ + { + database: 'data', + table: 'Order', + addColumns: [], + extraColumn: null, + recompute: [ + { + attribute: 'points', + from: 'amount * 2', + version: "Math.round((amount || 0) * (status == 'open' ? 10 : 25))", + }, + ], + }, + ]); + }); + + it('treats an index change as additive', () => { + const result = diffAfter((tables) => (find(tables, 'Product').attributes[1].indexed = {})); + expect(result.compatibility).toEqual({ + read: 'additive', + write: 'additive', + storage: 'additive', + }); + expect(result.changes).toEqual([expect.objectContaining({ kind: 'indexChanged', breaks: [] })]); + }); + + it('breaks writes when sealing changes, and ships the overflow column when unsealing', () => { + const sealed = diffAfter((tables) => (find(tables, 'Product').sealed = true)); + expect(sealed.changes).toEqual([ + expect.objectContaining({ kind: 'sealedChanged', breaks: ['write'] }), + ]); + const unsealed = diffAfter((tables) => (find(tables, 'Purchase').sealed = false)); + expect(unsealed.changes).toEqual([ + expect.objectContaining({ kind: 'sealedChanged', breaks: ['write'] }), + ]); + expect(unsealed.delta.tables).toEqual([ + { + database: 'coverage', + table: 'Purchase', + addColumns: [], + extraColumn: '_extra', + recompute: [], + }, + ]); + }); + + it('breaks storage when an attribute takes over a metadata column name', () => { + const result = diffAfter((tables) => + find(tables, 'Product').attributes.push({ name: '_version', type: 'Int' }), + ); + expect(result.compatibility.storage).toBe('breaking'); + expect(result.changes).toContainEqual( + expect.objectContaining({ kind: 'metadataColumnMoved', breaks: ['storage'] }), + ); + expect(result.delta.tables).toEqual([]); + }); + + it('applies the attribute rules inside nested types, which are never sealed', () => { + const added = diffAfter((_, types) => + types.get('Address').attributes.push({ name: 'zip', type: 'String' }), + ); + expect(added.compatibility).toEqual({ + read: 'additive', + write: 'additive', + storage: 'additive', + }); + const required = diffAfter((_, types) => + types.get('Address').attributes.push({ name: 'zip', type: 'String', nullable: false }), + ); + expect(required.changes).toEqual([ + expect.objectContaining({ + kind: 'nestedAttributeAdded', + attribute: 'address', + path: 'address.zip', + breaks: ['write'], + }), + ]); + const retyped = diffAfter((_, types) => (types.get('Geo').attributes[0].type = 'String')); + expect(retyped.changes).toEqual([ + expect.objectContaining({ + kind: 'attributeTypeChanged', + path: 'address.geo.lat', + breaks: ['read', 'write'], + }), + ]); + expect(retyped.compatibility.storage).toBe('additive'); + }); + + it('reports a removed or changed relationship without breaking anything', () => { + const removed = diffAfter((tables) => { + const purchase = find(tables, 'Purchase'); + purchase.attributes = purchase.attributes.filter( + (/** @type {any} */ attribute) => attribute.name !== 'customer', + ); + }); + expect(removed.changes).toEqual([ + expect.objectContaining({ kind: 'relationRemoved', attribute: 'customer', breaks: [] }), + ]); + const changed = diffAfter((tables) => { + const purchases = find(tables, 'Customer').attributes.find( + (/** @type {any} */ attribute) => attribute.name === 'purchases', + ); + purchases.relationship = { to: 'buyerId' }; + }); + expect(changed.changes).toEqual([ + expect.objectContaining({ kind: 'relationChanged', attribute: 'purchases', breaks: [] }), + ]); + }); + + it('breaks writes when an unsealed table gains a relationship or computed attribute older models would echo', () => { + const relation = { + name: 'favorite', + type: 'Purchase', + relationship: { from: 'favoriteId' }, + relationshipReference: { database: 'coverage', table: 'Purchase' }, + }; + const computed = { name: 'rank', type: 'Int', computed: { from: () => 1 } }; + const unsealed = diffAfter((tables) => + find(tables, 'Customer').attributes.push(relation, computed), + ); + expect(unsealed.compatibility.write).toBe('breaking'); + expect(unsealed.changes).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + kind: 'relationAdded', + attribute: 'favorite', + breaks: ['write'], + message: expect.stringContaining('send it back on PUT'), + }), + expect.objectContaining({ kind: 'attributeAdded', attribute: 'rank', breaks: ['write'] }), + ]), + ); + const sealed = diffAfter((tables) => + find(tables, 'Purchase').attributes.push({ ...relation, name: 'gift' }, computed), + ); + expect(sealed.compatibility.write).toBe('additive'); + }); + + it('breaks writes when a stored attribute becomes a relationship', () => { + const result = diffAfter((tables) => { + const product = find(tables, 'Product'); + const category = product.attributes.find( + (/** @type {any} */ attribute) => attribute.name === 'category', + ); + Object.assign(category, { + type: 'Customer', + indexed: undefined, + relationship: { from: 'id' }, + relationshipReference: { database: 'coverage', table: 'Customer' }, + }); + }); + expect(result.changes).toEqual([ + expect.objectContaining({ + kind: 'attributeRemoved', + attribute: 'category', + breaks: ['write'], + }), + expect.objectContaining({ kind: 'relationAdded', attribute: 'category', breaks: [] }), + ]); + }); + + it('breaks inserts, not reads, when a Blob becomes required', () => { + /** @param {any[]} tables */ + const requireFile = (tables) => (find(tables, 'Attachment').attributes[2].nullable = false); + const required = diffAfter(requireFile); + expect(required.compatibility).toEqual({ + read: 'additive', + write: 'breaking', + storage: 'additive', + }); + expect(required.changes).toEqual([ + expect.objectContaining({ kind: 'attributeBecameRequired', attribute: 'file' }), + ]); + const relaxed = diffSchemaIR(irOf(requireFile), irOf()); + expect(relaxed.compatibility).toEqual({ + read: 'additive', + write: 'additive', + storage: 'additive', + }); + }); + + it('breaks writes when a table stops supporting full replacement', () => { + /** @param {any[]} tables */ + const addBlob = (tables) => + find(tables, 'Product').attributes.push({ name: 'manual', type: 'Blob' }); + const added = diffAfter(addBlob); + expect(added.compatibility.write).toBe('breaking'); + expect(added.changes).toEqual([ + expect.objectContaining({ kind: 'replacementChanged', table: 'Product', breaks: ['write'] }), + expect.objectContaining({ kind: 'attributeAdded', attribute: 'manual', breaks: [] }), + ]); + const nested = diffAfter((_, types) => + types.get('Address').attributes.push({ name: 'photo', type: 'Blob' }), + ); + expect(nested.changes).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + kind: 'replacementChanged', + table: 'Customer', + breaks: ['write'], + }), + ]), + ); + const removed = diffSchemaIR(irOf(addBlob), irOf()); + expect(removed.compatibility.write).toBe('additive'); + expect(removed.changes).toEqual( + expect.arrayContaining([ + expect.objectContaining({ kind: 'replacementChanged', table: 'Product', breaks: [] }), + ]), + ); + }); + + it('itemizes changes that alter a contract without breaking older clients', () => { + const timestamps = diffAfter((tables) => { + const createdAt = find(tables, 'Customer').attributes.find( + (/** @type {any} */ attribute) => attribute.name === 'createdAt', + ); + createdAt.assignCreatedTime = false; + createdAt.assignUpdatedTime = true; + }); + expect(timestamps.changes).toEqual([ + expect.objectContaining({ kind: 'serverManagedChanged', attribute: 'createdAt', breaks: [] }), + ]); + const nested = diffAfter((_, types) => + types.get('Geo').attributes.push({ name: 'alt', type: 'Float' }), + ); + expect(nested.changes).toEqual([ + expect.objectContaining({ + kind: 'nestedAttributeAdded', + path: 'address.geo.alt', + breaks: [], + }), + ]); + const snapshot = { + tableName: 'Snapshot', + databaseName: 'coverage', + primaryKey: 'id', + attributes: [ + { name: 'id', type: 'ID', isPrimaryKey: true }, + { name: 'buyer', type: 'Customer' }, + ], + }; + const embedded = diffSchemaIR( + irOf((tables) => tables.push({ ...snapshot })), + irOf((tables) => { + tables.push({ ...snapshot }); + find(tables, 'Customer').attributes.push({ name: 'nickname', type: 'String' }); + }), + ); + expect(embedded.changes.map((change) => [change.table, change.kind])).toEqual([ + ['Customer', 'attributeAdded'], + ['Snapshot', 'contractChanged'], + ]); + const required = diffSchemaIR( + irOf((tables) => tables.push({ ...snapshot })), + irOf((tables) => { + tables.push({ ...snapshot }); + find(tables, 'Customer').attributes.push({ name: 'tier', type: 'String', nullable: false }); + }), + ); + expect(required.changes.find((change) => change.table === 'Snapshot')).toMatchObject({ + kind: 'embeddedTableChanged', + breaks: ['write'], + }); + const both = diffSchemaIR( + irOf((tables) => tables.push({ ...snapshot, attributes: [...snapshot.attributes] })), + irOf((tables) => { + tables.push({ + ...snapshot, + attributes: [...snapshot.attributes, { name: 'takenAt', type: 'Date', indexed: {} }], + }); + find(tables, 'Customer').attributes.push({ name: 'tier', type: 'String', nullable: false }); + }), + ); + expect( + both.changes.filter((change) => change.table === 'Snapshot').map((change) => change.kind), + ).toEqual(['attributeAdded', 'embeddedTableChanged']); + }); + + it('keeps tables whose names contain dots apart', () => { + /** @param {string} database @param {string} name */ + const only = (database, name) => + buildSchemaIR({ + tables: [ + { + tableName: name, + databaseName: database, + primaryKey: 'id', + attributes: [{ name: 'id', type: 'ID', isPrimaryKey: true }], + }, + ], + }); + const result = diffSchemaIR(only('a.b', 'c'), only('a', 'b.c')); + expect(result.changes.map((change) => change.kind)).toEqual(['tableAdded', 'tableRemoved']); + }); + + it('compares widely shared nested types once per table', () => { + /** @param {string} leafType */ + const deep = (leafType) => + buildSchemaIR({ + tables: [ + { + tableName: 'Deep', + primaryKey: 'id', + attributes: [ + { name: 'id', type: 'ID', isPrimaryKey: true }, + { name: 'root', type: 'T0' }, + ], + }, + ], + types: sharedChain(leafType), + }); + const result = diffSchemaIR(deep('String'), deep('Int')); + expect(result.changes).toEqual([ + expect.objectContaining({ + kind: 'attributeTypeChanged', + path: `root${'.left'.repeat(39)}.leaf`, + breaks: ['read', 'write'], + }), + ]); + }); + + it('reports type renames as source changes', () => { + const before = irOf(); + const after = irOf(); + const customer = after.tables.find((table) => table.name === 'Customer'); + if (customer) customer.typeName = 'Buyer'; + expect(diffSchemaIR(before, after).source).toEqual([ + { kind: 'typeRenamed', name: 'coverage.Customer', from: 'coverage_Customer', to: 'Buyer' }, + ]); + }); + + it('reports profile additions, removals and changes', () => { + const result = diffSchemaIR( + irOf(), + irOf(undefined, { + profiles: { + storefront: { retention: '7d', tables: ['Product'] }, + reports: { tables: ['Report'] }, + }, + }), + ); + expect(result.profiles).toEqual([ + { kind: 'removed', name: 'my-orders' }, + { kind: 'added', name: 'reports' }, + { kind: 'changed', name: 'storefront' }, + ]); + }); + + it('rejects an IR it cannot read', () => { + expect(() => diffSchemaIR({ ...irOf(), irVersion: 99 }, irOf())).toThrow(/irVersion/); + }); +}); diff --git a/utils/client/emitKotlin.js b/utils/client/emitKotlin.js new file mode 100644 index 0000000..2f0ddbe --- /dev/null +++ b/utils/client/emitKotlin.js @@ -0,0 +1,708 @@ +/** @import { IRAttribute, IRNestedAttribute, IRObjectType, IRProfile, IRTable, IRType, SchemaIR } from './irTypes.js' */ +/** @import { EmittedFile } from './emitSwift.js' */ +import { commentLines, generatedHeader, kotlinString, readRuntimeSource } from './emitSupport.js'; +import { + allocateMemberNames, + createNameAllocator, + escapeKotlinEnumEntry, + escapeKotlinIdentifier, + reservedMemberNames, + reservedProfileNames, +} from './naming.js'; +import { assertSchemaIR } from './validateSchemaIR.js'; + +const KOTLIN_TYPES = /** @type {Record} */ ({ + ID: 'String', + String: 'String', + Int: 'Int', + Long: 'Long', + Float: 'Double', + BigInt: 'BigInteger', + Boolean: 'Boolean', + Date: 'Instant', + Bytes: 'ByteArray', + Blob: 'HarperValue', + Any: 'HarperValue', +}); +const SCALAR_CONVERTERS = /** @type {Record} */ ({ + ID: 'string', + String: 'string', + Int: 'int', + Long: 'long', + Float: 'double', + BigInt: 'bigInteger', + Boolean: 'boolean', + Date: 'instant', + Bytes: 'bytes', + Blob: 'value', + Any: 'value', +}); +const COLUMN_TYPES = /** @type {Record} */ ({ + ID: 'ID', + String: 'STRING', + Int: 'INT', + Long: 'LONG', + Float: 'FLOAT', + BigInt: 'BIG_INT', + Boolean: 'BOOLEAN', + Date: 'DATE', + Bytes: 'BYTES', + Blob: 'BLOB', + Any: 'ANY', +}); +const FIELD_ENTRY_RESERVED = [ + 'name', + 'ordinal', + 'entries', + 'values', + 'valueOf', + 'attribute', + 'Companion', +]; +const GENERATED_LOCALS = ['field', 'it', 'other', 'result', 'row', 'version']; +const PACKAGE_SEGMENT = /^[A-Za-z_][A-Za-z0-9_]*$/; +const FILE_SUPPRESSIONS = + '@file:Suppress("PropertyName", "ObjectPropertyName", "ConstructorParameterNaming", "EnumEntryName", "ClassName", "RedundantVisibilityModifier", "RemoveRedundantQualifierName", "unused")\n'; + +/** + * @typedef {Object} KotlinContext + * @property {Map} typesByName + * @property {Map} tablesByKey + * @property {Map} converters list converter expression → the file-level value holding it + * @property {ReturnType} converterNames + */ + +/** + * Renders a Kotlin/JVM module (build script, runtime, schema and models) from a schema IR. + * @param {SchemaIR} ir + * @param {{ packageName?: string }} [options] + * @returns {EmittedFile[]} + */ +export function emitKotlinModule(ir, { packageName = 'harper.models' } = {}) { + assertSchemaIR(ir); + const segments = packageName.split('.'); + if (!segments.every((segment) => PACKAGE_SEGMENT.test(segment))) { + throw new Error(`kotlinPackage "${packageName}" must be dot-separated identifiers`); + } + /** @type {KotlinContext} */ + const context = { + typesByName: new Map(ir.types.map((type) => [type.name, type])), + tablesByKey: new Map(ir.tables.map((table) => [`${table.database}\u0000${table.name}`, table])), + converters: new Map(), + converterNames: createNameAllocator(fileScopeNames(ir)), + }; + const packageLine = `package ${segments.map(escapeKotlinIdentifier).join('.')}\n`; + const header = generatedHeader(ir) + FILE_SUPPRESSIONS + '\n'; + const directory = `src/main/kotlin/${segments.join('/')}`; + const runtime = readRuntimeSource('HarperRuntime.kt').replace( + /^package harper\.models\n/, + packageLine, + ); + return [ + { path: 'build.gradle.kts', content: buildScript(), scaffold: true }, + { path: `${directory}/HarperRuntime.kt`, content: header + runtime, scaffold: false }, + { + path: `${directory}/HarperSchema.kt`, + content: header + packageLine + schemaSource(ir), + scaffold: false, + }, + { + path: `${directory}/Models.kt`, + content: header + packageLine + modelsSource(ir, context), + scaffold: false, + }, + ]; +} + +/** + * @returns {string} + */ +function buildScript() { + return `// Created once by @harperfast/schema-codegen; it is never regenerated, so edit it freely. +// Without a Kotlin plugin version on the root project's classpath, add one here. +plugins { + kotlin("jvm") +} +`; +} + +/** + * @param {IRType} type + * @param {KotlinContext} context + * @returns {string} + */ +function kotlinType(type, context) { + switch (type.kind) { + case 'scalar': + return KOTLIN_TYPES[type.scalar]; + case 'array': + return `List<${kotlinType(type.element, context)}${type.elementNullable ? '?' : ''}>`; + case 'object': + return /** @type {IRObjectType} */ (context.typesByName.get(type.type)).typeName; + case 'record': + return /** @type {IRTable} */ (context.tablesByKey.get(`${type.database}\u0000${type.table}`)) + .typeName; + } +} + +/** + * Every name a member, class or local of Models.kt can take, so a file-level converter value is + * never shadowed where it is used. + * @param {SchemaIR} ir + * @returns {Set} + */ +function fileScopeNames(ir) { + const names = new Set([...reservedProfileNames(ir), ...GENERATED_LOCALS]); + for (const owner of [...ir.tables, ...ir.types]) { + const reserved = reservedMemberNames(owner.typeName); + for (const name of reserved) names.add(name); + const members = allocateMemberNames( + owner.attributes.map((attribute) => attribute.name), + reserved, + ); + for (const member of members.values()) names.add(member); + } + return names; +} + +/** + * @param {IRType} type + * @param {KotlinContext} context + * @returns {string} + */ +function converterExpression(type, context) { + switch (type.kind) { + case 'scalar': + return `HarperConverters.${SCALAR_CONVERTERS[type.scalar]}`; + case 'array': { + const element = converterExpression(type.element, context); + return `HarperConverters.list(${type.elementNullable ? `HarperConverters.nullable(${element})` : element})`; + } + case 'object': + case 'record': + return kotlinType(type, context); + } +} + +/** + * @param {IRType} type + * @param {KotlinContext} context + * @returns {string} + */ +function converterStem(type, context) { + switch (type.kind) { + case 'scalar': + return SCALAR_CONVERTERS[type.scalar]; + case 'array': { + const element = converterStem(type.element, context); + return `${type.elementNullable ? `nullable${element[0].toUpperCase()}${element.slice(1)}` : element}List`; + } + case 'object': + case 'record': { + const name = kotlinType(type, context); + return `${name[0].toLowerCase()}${name.slice(1)}`; + } + } +} + +/** + * Converters other than the runtime's scalar ones are file-level values: a list converter is + * built once rather than on every decode and encode, and a generated type is never named inside a + * member body, where a member of the same name would shadow it. + * @param {IRType} type + * @param {KotlinContext} context + * @returns {string} + */ +function converterOf(type, context) { + const expression = converterExpression(type, context); + if (type.kind === 'scalar') return expression; + let name = context.converters.get(expression); + if (name === undefined) { + name = context.converterNames.claim(`${converterStem(type, context)}Converter`); + context.converters.set(expression, name); + } + return name; +} + +/** + * @param {IRType} type + * @returns {string} + */ +function columnType(type) { + if (type.kind === 'scalar') return COLUMN_TYPES[type.scalar]; + return type.kind === 'array' ? 'ARRAY' : 'OBJECT'; +} + +/** + * @param {string[]} lines + * @param {string} indent + * @returns {string} + */ +function kdoc(lines, indent) { + if (lines.length === 0) return ''; + if (lines.length === 1) return `${indent}/** ${lines[0]} */\n`; + return `${indent}/**\n${lines.map((line) => `${indent} *${line ? ` ${line}` : ''}\n`).join('')}${indent} */\n`; +} + +/** + * @typedef {Object} KotlinProperty + * @property {string} raw the attribute name + * @property {string} name the escaped Kotlin identifier + * @property {string} type the property type, nullability included + * @property {boolean} optional + * @property {string} converter + * @property {boolean} bytes holds byte arrays itself, directly or in lists + */ + +/** + * @param {IRAttribute | IRNestedAttribute} attribute + * @param {string} identifier + * @param {KotlinContext} context + * @param {boolean} optional + * @returns {KotlinProperty} + */ +function propertyOf(attribute, identifier, context, optional) { + const base = kotlinType(attribute.type, context); + return { + raw: attribute.name, + name: escapeKotlinIdentifier(identifier), + type: optional ? `${base}?` : base, + optional, + converter: converterOf(attribute.type, context), + bytes: holdsBytes(attribute.type), + }; +} + +/** + * Generated classes compare their own byte arrays, so only bytes outside them count. + * @param {IRType} type + * @returns {boolean} + */ +function holdsBytes(type) { + if (type.kind === 'scalar') return type.scalar === 'Bytes'; + return type.kind === 'array' && holdsBytes(type.element); +} + +/** + * @param {KotlinProperty[]} properties + * @param {string} indent + * @returns {string} + */ +function constructorParameters(properties, indent) { + return properties + .map( + (property) => + `${indent}public val ${property.name}: ${property.type}${property.optional ? ' = null' : ''},\n`, + ) + .join(''); +} + +/** + * @param {KotlinProperty[]} properties + * @param {string} receiver the class name used to qualify `this`, so builder members never shadow properties + * @param {string} indent + * @returns {string} + */ +function encodeStatements(properties, receiver, indent) { + return properties + .map((property) => + property.optional + ? `${indent}this@${receiver}.${property.name}?.let { put(${kotlinString(property.raw)}, ${property.converter}.encode(it)) }\n` + : `${indent}put(${kotlinString(property.raw)}, ${property.converter}.encode(this@${receiver}.${property.name}))\n`, + ) + .join(''); +} + +/** + * Data classes compare arrays by identity, so a class holding byte arrays gets content equality. + * @param {string} className + * @param {KotlinProperty[]} properties including the trailing system properties + * @param {string} indent + * @returns {string} + */ +function contentEquality(className, properties, indent) { + if (!properties.some((property) => property.bytes)) return ''; + const comparisons = properties.map((property) => + property.bytes + ? `HarperContent.same(this.${property.name}, other.${property.name})` + : `this.${property.name} == other.${property.name}`, + ); + const hashes = properties.map((property) => { + const ours = `this.${property.name}`; + if (property.bytes) return `HarperContent.hash(${ours})`; + return property.optional ? `(${ours}?.hashCode() ?: 0)` : `${ours}.hashCode()`; + }); + let source = `\n${indent}override fun equals(other: Any?): Boolean {\n`; + source += `${indent}\tif (this === other) return true\n`; + source += `${indent}\tif (other !is ${className}) return false\n`; + source += `${indent}\treturn ${comparisons.join(` &&\n${indent}\t\t`)}\n`; + source += `${indent}}\n\n`; + source += `${indent}override fun hashCode(): Int {\n`; + source += `${indent}\tvar result = 0\n`; + for (const hash of hashes) source += `${indent}\tresult = 31 * result + ${hash}\n`; + source += `${indent}\treturn result\n${indent}}\n`; + return source; +} + +/** + * @param {SchemaIR} ir + * @param {KotlinContext} context + * @returns {string} + */ +function modelsSource(ir, context) { + const body = [ + ...ir.tables.map((table) => tableModel(table, context)), + ...ir.types.map((type) => objectModel(type, context)), + ].join('\n'); + const imports = ['kotlin.reflect.KClass']; + if (/\bBigInteger\b/.test(body)) imports.push('java.math.BigInteger'); + if (/\bInstant\b/.test(body)) imports.push('java.time.Instant'); + const converters = [...context.converters] + .map(([expression, name]) => `private val ${name} = ${expression}\n`) + .join(''); + return `\n${imports + .sort() + .map((name) => `import ${name}\n`) + .join('')}\n${converters ? `${converters}\n` : ''}${body}`; +} + +/** + * @param {IRTable} table + * @param {KotlinContext} context + * @returns {string} + */ +function tableModel(table, context) { + const className = table.typeName; + const names = allocateMemberNames( + table.attributes.map((attribute) => attribute.name), + reservedMemberNames(className), + ); + const identifierOf = (/** @type {string} */ raw) => /** @type {string} */ (names.get(raw)); + const attributesByName = new Map( + table.attributes.map((attribute) => [attribute.name, attribute]), + ); + const record = table.projections.record.map((field) => + propertyOf( + /** @type {IRAttribute} */ (attributesByName.get(field.name)), + identifierOf(field.name), + context, + field.optional, + ), + ); + const recordByName = new Map(record.map((property) => [property.raw, property])); + const declared = [ + ...table.attributes.map((attribute) => attribute.name), + ...table.relations.map((relation) => relation.name), + ]; + const sealed = table.storage.extraColumn === null; + const typeLabel = kotlinString(className); + const extras = sealed + ? '' + : `putAll(this@${className}._extra.harperExtra(${className}.declaredAttributes))\n`; + /** @type {KotlinProperty[]} */ + const systemProperties = [ + { + raw: '_version', + name: '_version', + type: 'Double?', + optional: true, + converter: '', + bytes: false, + }, + ...(sealed + ? [] + : [ + { + raw: '_extra', + name: '_extra', + type: 'Map', + optional: false, + converter: '', + bytes: false, + }, + ]), + ]; + + let source = kdoc(commentLines(`Table \`${table.name}\` in database \`${table.database}\`.`), ''); + source += `public data class ${className}(\n`; + for (const property of record) { + const attribute = /** @type {IRAttribute} */ (attributesByName.get(property.raw)); + const lines = commentLines(attribute.description); + if (attribute.computed?.from) + lines.push(...commentLines(`Computed on the client from \`${attribute.computed.from}\`.`)); + source += kdoc(lines, '\t'); + source += `\tpublic val ${property.name}: ${property.type}${property.optional ? ' = null' : ''},\n`; + } + source += '\toverride val _version: Double? = null,\n'; + if (!sealed) source += '\tpublic val _extra: Map = emptyMap(),\n'; + source += ') : HarperRecord {\n'; + + source += '\toverride fun encodeRow(): HarperRow = buildMap {\n'; + if (extras) source += `\t\t${extras}`; + source += encodeStatements(record, className, '\t\t'); + source += '\t}\n\n'; + if (table.projections.upsert === null) { + source += `\toverride fun encodeUpsertRow(): HarperRow =\n\t\tthrow HarperEncodingError.ReplacementUnsupported(${typeLabel}, ${kotlinString(table.projections.unwritable[0])})\n`; + } else { + const upsert = table.projections.upsert.map( + (field) => /** @type {KotlinProperty} */ (recordByName.get(field.name)), + ); + source += '\toverride fun encodeUpsertRow(): HarperRow = buildMap {\n'; + if (extras) source += `\t\t${extras}`; + source += encodeStatements(upsert, className, '\t\t'); + source += '\t}\n'; + } + source += contentEquality(className, [...record, ...systemProperties], '\t'); + + if (table.projections.insert.length > 0) { + const insert = table.projections.insert.map((field) => + propertyOf( + /** @type {IRAttribute} */ (attributesByName.get(field.name)), + identifierOf(field.name), + context, + field.optional, + ), + ); + const newSystem = sealed + ? [] + : [ + { + raw: '_extra', + name: '_extra', + type: 'Map', + optional: false, + converter: '', + bytes: false, + }, + ]; + source += + '\n\t/** The insert (`POST`) body; without a primary key the server assigns one. */\n'; + source += '\tpublic data class New(\n'; + source += constructorParameters(insert, '\t\t'); + if (!sealed) source += '\t\tpublic val _extra: Map = emptyMap(),\n'; + source += '\t) : HarperWritable {\n'; + source += '\t\toverride fun encodeRow(): HarperRow = buildMap {\n'; + if (!sealed) + source += `\t\t\tputAll(this@New._extra.harperExtra(${className}.declaredAttributes))\n`; + source += encodeStatements(insert, 'New', '\t\t\t'); + source += '\t\t}\n'; + source += contentEquality('New', [...insert, ...newSystem], '\t\t'); + source += '\t}\n'; + } + + if (table.projections.patch.length > 0) { + const patch = table.projections.patch.map((field) => + propertyOf( + /** @type {IRAttribute} */ (attributesByName.get(field.name)), + identifierOf(field.name), + context, + true, + ), + ); + const clearable = table.projections.clearable; + const entryNames = allocateMemberNames( + clearable.map((raw) => identifierOf(raw)), + FIELD_ENTRY_RESERVED, + ); + source += + '\n\t/** The patch body: unset attributes are left unchanged, [clear] sets attributes to null, and a set value wins over a clear. */\n'; + source += '\tpublic data class Patch(\n'; + source += constructorParameters(patch, '\t\t'); + if (clearable.length > 0) source += '\t\tpublic val clear: Set = emptySet(),\n'; + source += '\t) : HarperWritable {\n'; + if (clearable.length > 0) { + source += '\t\tpublic enum class Field(public val attribute: String) {\n'; + for (const raw of clearable) { + const entry = escapeKotlinEnumEntry( + /** @type {string} */ (entryNames.get(identifierOf(raw))), + ); + source += `\t\t\t${entry}(${kotlinString(raw)}),\n`; + } + source += '\t\t}\n\n'; + } + source += '\t\toverride fun encodeRow(): HarperRow = buildMap {\n'; + if (clearable.length > 0) + source += '\t\t\tfor (field in this@Patch.clear) put(field.attribute, HarperValue.Null)\n'; + source += encodeStatements(patch, 'Patch', '\t\t\t'); + source += '\t\t}\n'; + source += contentEquality( + 'Patch', + [ + ...patch, + ...(clearable.length > 0 + ? [ + { + raw: 'clear', + name: 'clear', + type: 'Set', + optional: false, + converter: '', + bytes: false, + }, + ] + : []), + ], + '\t\t', + ); + source += '\t}\n'; + } + + source += `\n\tpublic companion object : HarperTable<${className}> {\n`; + source += `\t\toverride val schema: HarperTableSchema = ${tableSchemaLiteral(table)}\n`; + source += `\t\toverride val recordClass: KClass<${className}> = ${className}::class\n`; + source += `\t\tprivate val declaredAttributes: Set = setOf(${declared.map(kotlinString).join(', ')})\n\n`; + source += `\t\toverride fun decode(row: HarperRow, version: Double?): ${className} = ${className}(\n`; + for (const property of record) { + const call = property.optional ? 'harperOptional' : 'harperRequired'; + source += `\t\t\t${property.name} = row.${call}(${kotlinString(property.raw)}, ${typeLabel}, ${property.converter}),\n`; + } + source += '\t\t\t_version = version,\n'; + if (!sealed) source += '\t\t\t_extra = row.harperExtra(declaredAttributes),\n'; + source += '\t\t)\n\t}\n}\n'; + return source; +} + +/** + * @param {IRObjectType} type + * @param {KotlinContext} context + * @returns {string} + */ +function objectModel(type, context) { + const className = type.typeName; + const names = allocateMemberNames( + type.attributes.map((attribute) => attribute.name), + reservedMemberNames(className), + ); + const properties = type.attributes.map((attribute) => + propertyOf( + attribute, + /** @type {string} */ (names.get(attribute.name)), + context, + attribute.nullable, + ), + ); + const typeLabel = kotlinString(className); + let source = kdoc(commentLines(type.description ?? `Nested type \`${type.name}\`.`), ''); + source += `public data class ${className}(\n`; + type.attributes.forEach((attribute, index) => { + source += kdoc(commentLines(attribute.description), '\t'); + const property = properties[index]; + source += `\tpublic val ${property.name}: ${property.type}${property.optional ? ' = null' : ''},\n`; + }); + source += '\tpublic val _extra: Map = emptyMap(),\n'; + source += ') : HarperObject {\n'; + source += '\toverride fun encodeRow(): HarperRow = buildMap {\n'; + source += `\t\tputAll(this@${className}._extra.harperExtra(${className}.declaredAttributes))\n`; + source += encodeStatements(properties, className, '\t\t'); + source += '\t}\n'; + source += contentEquality( + className, + [ + ...properties, + { + raw: '_extra', + name: '_extra', + type: 'Map', + optional: false, + converter: '', + bytes: false, + }, + ], + '\t', + ); + source += `\n\tpublic companion object : HarperObjectType<${className}> {\n`; + source += `\t\tprivate val declaredAttributes: Set = setOf(${type.attributes.map((attribute) => kotlinString(attribute.name)).join(', ')})\n\n`; + source += `\t\toverride fun decode(row: HarperRow): ${className} = ${className}(\n`; + for (const property of properties) { + const call = property.optional ? 'harperOptional' : 'harperRequired'; + source += `\t\t\t${property.name} = row.${call}(${kotlinString(property.raw)}, ${typeLabel}, ${property.converter}),\n`; + } + source += '\t\t\t_extra = row.harperExtra(declaredAttributes),\n'; + source += '\t\t)\n\t}\n}\n'; + return source; +} + +/** + * @param {IRTable} table + * @returns {string} + */ +function tableSchemaLiteral(table) { + const attributesByName = new Map( + table.attributes.map((attribute) => [attribute.name, attribute]), + ); + const optional = new Map(table.projections.record.map((field) => [field.name, field.optional])); + const columns = table.storage.columns.map((column) => { + const attribute = /** @type {IRAttribute} */ (attributesByName.get(column.name)); + const parts = [ + kotlinString(column.name), + `HarperColumnType.${columnType(attribute.type)}`, + `HarperAffinity.${column.affinity}`, + `nullable = ${optional.get(column.name)}`, + ]; + if (attribute.primaryKey) parts.push('primaryKey = true'); + if (attribute.indexed) parts.push('indexed = true'); + if (attribute.readOnly) parts.push('readOnly = true'); + if (attribute.computed?.from) + parts.push(`computedFrom = ${kotlinString(attribute.computed.from)}`); + return `\t\t\t\tHarperColumn(${parts.join(', ')}),\n`; + }); + return `HarperTableSchema( + database = ${kotlinString(table.database)}, + name = ${kotlinString(table.name)}, + primaryKey = ${table.primaryKey === null ? 'null' : kotlinString(table.primaryKey)}, + sealed = ${table.sealed}, + hash = ${kotlinString(table.hash)}, + versionColumn = ${kotlinString(table.storage.versionColumn)}, + extraColumn = ${table.storage.extraColumn === null ? 'null' : kotlinString(table.storage.extraColumn)}, + replaceable = ${table.projections.upsert !== null}, + columns = ${columns.length > 0 ? `listOf(\n${columns.join('')}\t\t\t)` : 'emptyList()'}, + )`; +} + +/** + * @param {SchemaIR} ir + * @returns {string} + */ +function schemaSource(ir) { + const profileNames = allocateMemberNames( + ir.profiles.map((profile) => profile.name), + reservedProfileNames(ir), + ); + const constantOf = (/** @type {IRProfile} */ profile) => + escapeKotlinIdentifier(/** @type {string} */ (profileNames.get(profile.name))); + const tablesByKey = new Map( + ir.tables.map((table) => [`${table.database}\u0000${table.name}`, table]), + ); + let source = '\nimport kotlin.reflect.KClass\n\n'; + source += '/** The schema these models were generated from. */\n'; + source += 'public object HarperSchema {\n'; + source += `\tpublic const val IR_VERSION: Int = ${ir.irVersion}\n\n`; + source += '\t/** The wire and storage contract hash of every table below. */\n'; + source += `\tpublic const val SCHEMA_HASH: String = ${kotlinString(ir.schemaHash)}\n\n`; + source += `\tpublic val tables: List> = ${ir.tables.length > 0 ? `listOf(${ir.tables.map((table) => table.typeName).join(', ')})` : 'emptyList()'}\n\n`; + source += '\t/** The valid sync profiles from `sync.yaml`. */\n'; + source += `\tpublic val profiles: List = ${ir.profiles.length > 0 ? `listOf(${ir.profiles.map((profile) => `HarperProfiles.${constantOf(profile)}`).join(', ')})` : 'emptyList()'}\n\n`; + source += '\t@Suppress("UNCHECKED_CAST")\n'; + source += '\tpublic fun table(recordClass: KClass): HarperTable? =\n'; + source += '\t\ttables.firstOrNull { it.recordClass == recordClass } as HarperTable?\n'; + source += '}\n\n'; + source += '/** One constant per valid sync profile. */\n'; + source += 'public object HarperProfiles {\n'; + ir.profiles.forEach((profile, index) => { + const tables = profile.tables.map( + (entry) => + `${/** @type {IRTable} */ (tablesByKey.get(`${entry.database}\u0000${entry.table}`)).typeName}.schema`, + ); + if (index > 0) source += '\n'; + source += kdoc(commentLines(profile.description ?? `Sync profile \`${profile.name}\`.`), '\t'); + source += `\tpublic val ${constantOf(profile)}: HarperSyncProfile = HarperSyncProfile(\n`; + source += `\t\tname = ${kotlinString(profile.name)},\n`; + source += `\t\tdirection = HarperSyncProfile.Direction.${profile.direction.toUpperCase()},\n`; + source += `\t\tretentionMs = ${profile.retentionMs === null ? 'null' : `${profile.retentionMs}L`},\n`; + source += `\t\tschemaHash = ${kotlinString(profile.schemaHash)},\n`; + source += `\t\ttables = ${tables.length > 0 ? `listOf(${tables.join(', ')})` : 'emptyList()'},\n`; + source += '\t)\n'; + }); + source += '}\n'; + return source; +} diff --git a/utils/client/emitKotlin.test.js b/utils/client/emitKotlin.test.js new file mode 100644 index 0000000..7191afa --- /dev/null +++ b/utils/client/emitKotlin.test.js @@ -0,0 +1,173 @@ +import { describe, expect, it } from 'vitest'; +import { + adversarialTables, + coverageTables, + coverageTypes, + spikeProfiles, + spikeTables, +} from '../../test/fixtures/clientSchema.js'; +import { buildSchemaIR } from './buildSchemaIR.js'; +import { GENERATED_MARKER } from './emitSupport.js'; +import { emitKotlinModule } from './emitKotlin.js'; + +const ir = buildSchemaIR({ + tables: [...spikeTables(), ...coverageTables(), ...adversarialTables()], + types: coverageTypes(), + syncProfiles: spikeProfiles(), + generator: '@harperfast/schema-codegen@test', +}); +const files = emitKotlinModule(ir, { packageName: 'dev.harper.in.models' }); +const source = (/** @type {string} */ name) => { + const file = files.find((candidate) => candidate.path.endsWith(name)); + if (!file) throw new Error(`no ${name}`); + return file.content; +}; +const models = source('Models.kt'); + +/** + * @param {string} className + */ +function dataClass(className) { + const start = models.indexOf(`public data class ${className}(`); + if (start < 0) throw new Error(`no class ${className}`); + const next = models.indexOf('\n/** ', start + 1); + return models.slice(start, next < 0 ? undefined : next); +} + +describe('emitKotlinModule', () => { + it('lays out a Kotlin module under the package path with a user-owned build script', () => { + expect(files.map((file) => [file.path, file.scaffold])).toEqual([ + ['build.gradle.kts', true], + ['src/main/kotlin/dev/harper/in/models/HarperRuntime.kt', false], + ['src/main/kotlin/dev/harper/in/models/HarperSchema.kt', false], + ['src/main/kotlin/dev/harper/in/models/Models.kt', false], + ]); + }); + + it('escapes keyword package segments and refuses package names that are not identifiers', () => { + for (const file of files.filter((candidate) => !candidate.scaffold)) { + expect(file.content).toContain('\npackage dev.harper.`in`.models\n'); + expect(file.content.startsWith(GENERATED_MARKER)).toBe(true); + } + expect(() => emitKotlinModule(ir, { packageName: 'dev/../escape' })).toThrow( + /dot-separated identifiers/, + ); + expect(() => emitKotlinModule(ir, { packageName: 'dev..models' })).toThrow( + /dot-separated identifiers/, + ); + }); + + it('generates the spike model shape as a data class with a decoding companion', () => { + const product = dataClass('Product'); + expect(product).toContain('\tpublic val id: String,\n'); + expect(product).toContain('\tpublic val price: Double? = null,\n'); + expect(product).toContain('\toverride val _version: Double? = null,\n'); + expect(product).toContain('\tpublic val _extra: Map = emptyMap(),\n'); + expect(product).toContain('public companion object : HarperTable {'); + expect(product).toContain( + 'price = row.harperOptional("price", "Product", HarperConverters.double),', + ); + }); + + it('maps every Harper type to its Kotlin type and converter', () => { + const customer = dataClass('coverage_Customer'); + for (const declaration of [ + 'public val tags: List? = null', + 'public val scores: List? = null', + 'public val big: BigInteger? = null', + 'public val raw: ByteArray? = null', + 'public val meta: HarperValue? = null', + 'public val visits: Long? = null', + 'public val createdAt: Instant,', + 'scores = row.harperOptional("scores", "coverage_Customer", nullableIntListConverter),', + 'address = row.harperOptional("address", "coverage_Customer", addressConverter),', + ]) { + expect(customer).toContain(declaration); + } + expect(models).toContain('import java.math.BigInteger\n'); + expect(models).toContain('import java.time.Instant\n'); + }); + + it('keeps members from shadowing the runtime types their bodies name, without renaming them for unrelated tables', () => { + const weird = dataClass('Weird'); + expect(weird).toContain('public val HarperConverters_2: List? = null,'); + expect(weird).toContain('public val Address: Address? = null,'); + expect(weird).toContain('put("Address", addressConverter.encode(it))'); + expect(models).toContain('private val addressConverter = Address\n'); + expect(models).toContain('public data class ModelsKtRecord('); + }); + + it('builds each list converter once, as a file-level value no member can shadow', () => { + expect(models).toContain( + 'private val nullableIntListConverter = HarperConverters.list(HarperConverters.nullable(HarperConverters.int))\n', + ); + expect(models).toContain('private val nodeListConverter = HarperConverters.list(Node)\n'); + expect(models.match(/private val stringListConverter = /g)).toHaveLength(1); + expect(models).not.toMatch(/\(HarperConverters\.list\(/); + }); + + it('compares byte arrays by content, in lists too', () => { + const customer = dataClass('coverage_Customer'); + expect(customer).toContain('override fun equals(other: Any?): Boolean {'); + expect(customer).toContain('HarperContent.same(this.raw, other.raw)'); + expect(customer).toContain('HarperContent.same(this.files, other.files)'); + expect(customer).toContain('this.name == other.name'); + expect(customer).toContain('HarperContent.hash(this.files)'); + expect(dataClass('Product')).not.toContain('override fun equals'); + }); + + it('qualifies property reads inside map builders', () => { + expect(dataClass('Product')).toContain( + 'this@Product.price?.let { put("price", HarperConverters.double.encode(it)) }', + ); + }); + + it('refuses full replacement for tables with a Blob attribute', () => { + expect(dataClass('coverage_Attachment')).toContain( + 'throw HarperEncodingError.ReplacementUnsupported("coverage_Attachment", "file")', + ); + }); + + it('escapes keywords, soft keywords in enum entries, and $ in strings', () => { + const weird = dataClass('Weird'); + expect(weird).toContain('public val `in`: Boolean? = null,'); + expect(weird).toContain('public val value: HarperValue? = null,'); + expect(weird).toContain('\t\t\t`value`("value"),\n'); + expect(weird).toContain('"o\\"conn\\\\or \\$x"'); + }); + + it('names one constant per profile', () => { + const schema = source('HarperSchema.kt'); + expect(schema).toContain( + 'public val profiles: List = listOf(HarperProfiles.myOrders, HarperProfiles.storefront)', + ); + expect(schema).toContain('direction = HarperSyncProfile.Direction.PULL,'); + expect(schema).toContain('retentionMs = 2592000000L,'); + expect(schema).toContain(`public const val SCHEMA_HASH: String = "${ir.schemaHash}"`); + }); + + it('gives profiles whose names sanitize alike their own constants', () => { + const schema = + emitKotlinModule( + buildSchemaIR({ + tables: spikeTables(), + syncProfiles: { + profiles: { 'my-orders': { tables: ['Order'] }, myOrders: { tables: ['Order'] } }, + }, + }), + ).find((file) => file.path.endsWith('HarperSchema.kt'))?.content ?? ''; + expect(schema).toContain('listOf(HarperProfiles.myOrders_2, HarperProfiles.myOrders)'); + expect(schema).toContain( + 'public val myOrders_2: HarperSyncProfile = HarperSyncProfile(\n\t\tname = "my-orders",', + ); + expect(schema).toContain( + 'public val myOrders: HarperSyncProfile = HarperSyncProfile(\n\t\tname = "myOrders",', + ); + }); + + it('keeps the runtime package in step with the models', () => { + const runtime = source('HarperRuntime.kt'); + expect(runtime).not.toContain('package harper.models'); + expect(runtime).toContain('public sealed interface HarperValue'); + }); +}); diff --git a/utils/client/emitSupport.js b/utils/client/emitSupport.js new file mode 100644 index 0000000..5957073 --- /dev/null +++ b/utils/client/emitSupport.js @@ -0,0 +1,192 @@ +/** @import { IRObjectType, IRTable, IRType, SchemaIR } from './irTypes.js' */ +import fs from 'node:fs'; + +/** Every generated source file starts with this, so stale or mixed generations can be found. */ +export const GENERATED_MARKER = '// @generated by @harperfast/schema-codegen'; + +/** + * @param {SchemaIR} ir + * @returns {string} + */ +export function generatedHeader(ir) { + return `${GENERATED_MARKER} from ${ir.generator ?? 'a schema IR'} (schema ${ir.schemaHash}). Do not edit.\n`; +} + +/** + * @param {string} fileName a file in ./runtime + * @returns {string} with LF line endings, whatever the checkout used, so output is identical on every platform + */ +export function readRuntimeSource(fileName) { + return fs + .readFileSync(new URL(`./runtime/${fileName}`, import.meta.url), 'utf8') + .replace(/\r\n/g, '\n'); +} + +/** + * Splits a description into comment lines; `*\/` and `/*` are broken up so the text can never + * close or open a block comment. + * @param {string | undefined} text + * @returns {string[]} + */ +export function commentLines(text) { + if (!text) return []; + return String(text) + .replace(/\*\//g, '* /') + .replace(/\/\*/g, '/ *') + .split(/\r\n|\r|\n/) + .map((line) => line.trimEnd()); +} + +/** + * @param {number} codePoint + * @returns {boolean} + */ +function isControl(codePoint) { + return ( + codePoint < 0x20 || + (codePoint >= 0x7f && codePoint < 0xa0) || + codePoint === 0x2028 || + codePoint === 0x2029 + ); +} + +/** + * A Swift string literal for any text. Lone surrogates (impossible in a Swift string) become U+FFFD. + * @param {string} value + * @returns {string} + */ +export function swiftString(value) { + let literal = '"'; + for (const character of String(value)) { + const codePoint = /** @type {number} */ (character.codePointAt(0)); + if (character === '\\') literal += '\\\\'; + else if (character === '"') literal += '\\"'; + else if (character === '\n') literal += '\\n'; + else if (character === '\r') literal += '\\r'; + else if (character === '\t') literal += '\\t'; + else if (codePoint >= 0xd800 && codePoint <= 0xdfff) literal += '\\u{FFFD}'; + else if (isControl(codePoint)) literal += `\\u{${codePoint.toString(16).toUpperCase()}}`; + else literal += character; + } + return `${literal}"`; +} + +/** + * A Kotlin string literal for any text; `$` is escaped so nothing is interpolated. + * @param {string} value + * @returns {string} + */ +export function kotlinString(value) { + let literal = '"'; + for (const character of String(value)) { + const codePoint = /** @type {number} */ (character.codePointAt(0)); + if (character === '\\') literal += '\\\\'; + else if (character === '"') literal += '\\"'; + else if (character === '$') literal += '\\$'; + else if (character === '\n') literal += '\\n'; + else if (character === '\r') literal += '\\r'; + else if (character === '\t') literal += '\\t'; + else if (character === '\b') literal += '\\b'; + else if (isControl(codePoint) || (codePoint >= 0xd800 && codePoint <= 0xdfff)) { + literal += `\\u${codePoint.toString(16).toUpperCase().padStart(4, '0')}`; + } else literal += character; + } + return `${literal}"`; +} + +/** + * Finds the attribute edges that make a value type contain itself: a direct (not through an + * array) reference from a table or nested type to one in the same strongly connected component. + * Swift structs cannot hold such an edge by value. + * @param {SchemaIR} ir + * @returns {Set} keys `#`, where owner is a typeName + */ +export function recursiveValueEdges(ir) { + /** @type {Map} */ + const graph = new Map(); + const tablesByKey = new Map( + ir.tables.map((table) => [`${table.database}\u0000${table.name}`, table]), + ); + const typesByName = new Map(ir.types.map((type) => [type.name, type])); + /** + * @param {IRType} type + * @returns {string | null} + */ + const directTarget = (type) => { + if (type.kind === 'object') return typesByName.get(type.type)?.typeName ?? null; + if (type.kind === 'record') + return tablesByKey.get(`${type.database}\u0000${type.table}`)?.typeName ?? null; + return null; + }; + /** @param {IRTable | IRObjectType} owner */ + const addOwner = (owner) => { + const edges = []; + for (const attribute of owner.attributes) { + const target = directTarget(attribute.type); + if (target) edges.push({ attribute: attribute.name, target }); + } + graph.set(owner.typeName, edges); + }; + ir.tables.forEach(addOwner); + ir.types.forEach(addOwner); + + /** @type {Map} */ + const component = new Map(); + /** @type {Map} */ + const index = new Map(); + /** @type {Map} */ + const lowLink = new Map(); + /** @type {string[]} */ + const stack = []; + const onStack = new Set(); + let counter = 0; + let components = 0; + /** @param {string} node */ + const connect = (node) => { + index.set(node, counter); + lowLink.set(node, counter); + counter++; + stack.push(node); + onStack.add(node); + for (const { target } of graph.get(node) ?? []) { + if (!index.has(target)) { + connect(target); + lowLink.set( + node, + Math.min( + /** @type {number} */ (lowLink.get(node)), + /** @type {number} */ (lowLink.get(target)), + ), + ); + } else if (onStack.has(target)) { + lowLink.set( + node, + Math.min( + /** @type {number} */ (lowLink.get(node)), + /** @type {number} */ (index.get(target)), + ), + ); + } + } + if (lowLink.get(node) === index.get(node)) { + let member; + do { + member = /** @type {string} */ (stack.pop()); + onStack.delete(member); + component.set(member, components); + } while (member !== node); + components++; + } + }; + for (const node of graph.keys()) { + if (!index.has(node)) connect(node); + } + + const edges = new Set(); + for (const [owner, ownerEdges] of graph) { + for (const { attribute, target } of ownerEdges) { + if (component.get(owner) === component.get(target)) edges.add(`${owner}#${attribute}`); + } + } + return edges; +} diff --git a/utils/client/emitSupport.test.js b/utils/client/emitSupport.test.js new file mode 100644 index 0000000..c53b598 --- /dev/null +++ b/utils/client/emitSupport.test.js @@ -0,0 +1,83 @@ +import fs from 'node:fs'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { coverageTables, coverageTypes } from '../../test/fixtures/clientSchema.js'; +import { buildSchemaIR } from './buildSchemaIR.js'; +import { + commentLines, + kotlinString, + readRuntimeSource, + recursiveValueEdges, + swiftString, +} from './emitSupport.js'; + +describe('readRuntimeSource', () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('returns LF line endings from a CRLF checkout', () => { + vi.spyOn(fs, 'readFileSync').mockReturnValue('package harper.models\r\n\r\nobject X\r\n'); + expect(readRuntimeSource('HarperRuntime.kt')).toBe('package harper.models\n\nobject X\n'); + }); +}); + +describe('swiftString', () => { + it('escapes quotes, backslashes and control characters', () => { + expect(swiftString('o"conn\\or')).toBe('"o\\"conn\\\\or"'); + expect(swiftString('a\nb\tc\u0001')).toBe('"a\\nb\\tc\\u{1}"'); + expect(swiftString('\\(interpolation)')).toBe('"\\\\(interpolation)"'); + }); + + it('keeps other Unicode and replaces lone surrogates', () => { + expect(swiftString('naïve 🍕')).toBe('"naïve 🍕"'); + expect(swiftString('a\ud800b')).toBe('"a\\u{FFFD}b"'); + }); +}); + +describe('kotlinString', () => { + it('escapes $ so nothing is interpolated', () => { + expect(kotlinString('$price ${total}')).toBe('"\\$price \\${total}"'); + }); + + it('escapes quotes, backslashes, control characters and lone surrogates', () => { + expect(kotlinString('o"conn\\or\n')).toBe('"o\\"conn\\\\or\\n"'); + expect(kotlinString('\u0001\b')).toBe('"\\u0001\\b"'); + expect(kotlinString('a\udc00')).toBe('"a\\uDC00"'); + }); +}); + +describe('commentLines', () => { + it('breaks block-comment delimiters and splits lines', () => { + expect(commentLines('ends */ here\nopens /* there')).toEqual([ + 'ends * / here', + 'opens / * there', + ]); + expect(commentLines(undefined)).toEqual([]); + }); +}); + +describe('recursiveValueEdges', () => { + it('marks direct self and mutual references, but not references through arrays', () => { + const types = coverageTypes(); + types.set('Ping', { attributes: [{ name: 'pong', type: 'Pong' }] }); + types.set('Pong', { + attributes: [ + { name: 'ping', type: 'Ping' }, + { name: 'pings', type: 'array', elements: { type: 'Ping' } }, + ], + }); + const tables = [ + ...coverageTables(), + { + tableName: 'Rally', + primaryKey: 'id', + attributes: [ + { name: 'id', type: 'ID', isPrimaryKey: true }, + { name: 'start', type: 'Ping' }, + ], + }, + ]; + const ir = buildSchemaIR({ tables, types }); + expect([...recursiveValueEdges(ir)].sort()).toEqual(['Node#parent', 'Ping#pong', 'Pong#ping']); + }); +}); diff --git a/utils/client/emitSwift.js b/utils/client/emitSwift.js new file mode 100644 index 0000000..6664833 --- /dev/null +++ b/utils/client/emitSwift.js @@ -0,0 +1,524 @@ +/** @import { IRAttribute, IRNestedAttribute, IRObjectType, IRProfile, IRTable, IRType, SchemaIR } from './irTypes.js' */ +import { + commentLines, + generatedHeader, + readRuntimeSource, + recursiveValueEdges, + swiftString, +} from './emitSupport.js'; +import { + allocateMemberNames, + escapeSwiftIdentifier, + reservedMemberNames, + reservedProfileNames, +} from './naming.js'; +import { assertSchemaIR } from './validateSchemaIR.js'; + +const SWIFT_TYPES = /** @type {Record} */ ({ + ID: 'String', + String: 'String', + Int: 'Int', + Long: 'Int64', + Float: 'Double', + BigInt: 'HarperBigInt', + Boolean: 'Bool', + Date: 'Date', + Bytes: 'Data', + Blob: 'HarperValue', + Any: 'HarperValue', +}); +const COLUMN_TYPES = /** @type {Record} */ ({ + ID: 'id', + String: 'string', + Int: 'int', + Long: 'long', + Float: 'float', + BigInt: 'bigInt', + Boolean: 'boolean', + Date: 'date', + Bytes: 'bytes', + Blob: 'blob', + Any: 'any', +}); +const FIELD_CASE_RESERVED = ['rawValue', 'hashValue', 'init']; +const PROFILE_RESERVED = ['Direction', 'Type', 'Protocol', 'init', 'self', 'Self']; +const MODULE_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/; + +/** + * @typedef {Object} EmittedFile + * @property {string} path relative to the package directory, `/`-separated + * @property {string} content + * @property {boolean} scaffold written only when absent, then owned by the user + */ + +/** + * @typedef {Object} SwiftContext + * @property {Map} typesByName + * @property {Map} tablesByKey + * @property {Set} boxedEdges + */ + +/** + * Renders a Swift package (manifest, runtime, schema and models) from a schema IR. + * @param {SchemaIR} ir + * @param {{ module?: string }} [options] + * @returns {EmittedFile[]} + */ +export function emitSwiftPackage(ir, { module = 'HarperModels' } = {}) { + assertSchemaIR(ir); + if (!MODULE_NAME.test(module)) { + throw new Error(`swiftModule "${module}" must be a Swift identifier`); + } + const clash = [...ir.tables, ...ir.types].find((entry) => entry.typeName === module); + if (clash) { + throw new Error( + `swiftModule "${module}" is also a generated type name; choose a different module name`, + ); + } + /** @type {SwiftContext} */ + const context = { + typesByName: new Map(ir.types.map((type) => [type.name, type])), + tablesByKey: new Map(ir.tables.map((table) => [`${table.database}\u0000${table.name}`, table])), + boxedEdges: recursiveValueEdges(ir), + }; + const header = generatedHeader(ir); + const sources = `Sources/${module}`; + return [ + { path: 'Package.swift', content: packageManifest(module), scaffold: true }, + { + path: `${sources}/HarperRuntime.swift`, + content: header + readRuntimeSource('HarperRuntime.swift'), + scaffold: false, + }, + { path: `${sources}/HarperSchema.swift`, content: header + schemaSource(ir), scaffold: false }, + { + path: `${sources}/Models.swift`, + content: header + modelsSource(ir, context), + scaffold: false, + }, + ]; +} + +/** + * @param {string} module + * @returns {string} + */ +function packageManifest(module) { + return `// swift-tools-version:5.9 +import PackageDescription + +let package = Package( + name: ${swiftString(module)}, + platforms: [.iOS(.v13), .macOS(.v10_15), .tvOS(.v13), .watchOS(.v6)], + products: [.library(name: ${swiftString(module)}, targets: [${swiftString(module)}])], + targets: [.target(name: ${swiftString(module)})] +) +`; +} + +/** + * @param {IRType} type + * @param {SwiftContext} context + * @param {boolean} boxed + * @returns {string} + */ +function swiftType(type, context, boxed = false) { + switch (type.kind) { + case 'scalar': + return SWIFT_TYPES[type.scalar]; + case 'array': + return `[${swiftType(type.element, context)}${type.elementNullable ? '?' : ''}]`; + case 'object': { + const name = /** @type {IRObjectType} */ (context.typesByName.get(type.type)).typeName; + return boxed ? `HarperBox<${name}>` : name; + } + case 'record': { + const name = /** @type {IRTable} */ ( + context.tablesByKey.get(`${type.database}\u0000${type.table}`) + ).typeName; + return boxed ? `HarperBox<${name}>` : name; + } + } +} + +/** + * @param {IRType} type + * @returns {string} + */ +function columnType(type) { + if (type.kind === 'scalar') return COLUMN_TYPES[type.scalar]; + return type.kind === 'array' ? 'array' : 'object'; +} + +/** + * @param {string[]} lines + * @param {string} indent + * @returns {string} + */ +function docComment(lines, indent) { + return lines.map((line) => `${indent}///${line ? ` ${line}` : ''}\n`).join(''); +} + +/** + * @typedef {Object} SwiftProperty + * @property {string} raw the attribute name + * @property {string} name the escaped Swift identifier + * @property {string} type the property type, optional marker included + * @property {boolean} optional + */ + +/** + * @param {IRAttribute | IRNestedAttribute} attribute + * @param {string} identifier + * @param {string} ownerTypeName + * @param {SwiftContext} context + * @param {boolean} optional + * @returns {SwiftProperty} + */ +function propertyOf(attribute, identifier, ownerTypeName, context, optional) { + const boxed = context.boxedEdges.has(`${ownerTypeName}#${attribute.name}`); + const base = swiftType(attribute.type, context, boxed); + return { + raw: attribute.name, + name: escapeSwiftIdentifier(identifier), + type: optional ? `${base}?` : base, + optional, + }; +} + +/** + * @param {SwiftProperty[]} properties + * @param {string} indent + * @param {string[]} [trailing] extra parameters + * @returns {string} + */ +function initializer(properties, indent, trailing = []) { + const parameters = [ + ...properties.map( + (property) => `${property.name}: ${property.type}${property.optional ? ' = nil' : ''}`, + ), + ...trailing, + ]; + const assignments = properties.map( + (property) => `${indent}\tself.${property.name} = ${property.name}\n`, + ); + return `${indent}public init(${parameters.join(', ')}) {\n${assignments.join('')}`; +} + +/** + * @param {SwiftProperty[]} properties + * @param {string} indent + * @returns {string} + */ +function encodeAssignments(properties, indent) { + return properties + .map((property) => + property.optional + ? `${indent}if let value = self.${property.name} { row[${swiftString(property.raw)}] = value.harperValue }\n` + : `${indent}row[${swiftString(property.raw)}] = self.${property.name}.harperValue\n`, + ) + .join(''); +} + +/** + * @param {SchemaIR} ir + * @param {SwiftContext} context + * @returns {string} + */ +function modelsSource(ir, context) { + let source = 'import Foundation\n'; + for (const table of ir.tables) source += `\n${tableModel(table, context)}`; + for (const type of ir.types) source += `\n${objectModel(type, context)}`; + return source; +} + +/** + * @param {IRTable} table + * @param {SwiftContext} context + * @returns {string} + */ +function tableModel(table, context) { + const typeName = table.typeName; + const names = allocateMemberNames( + table.attributes.map((attribute) => attribute.name), + reservedMemberNames(typeName), + ); + const identifierOf = (/** @type {string} */ raw) => /** @type {string} */ (names.get(raw)); + const attributesByName = new Map( + table.attributes.map((attribute) => [attribute.name, attribute]), + ); + const patchable = new Set(table.projections.patch.map((field) => field.name)); + const record = table.projections.record.map((field) => + propertyOf( + /** @type {IRAttribute} */ (attributesByName.get(field.name)), + identifierOf(field.name), + typeName, + context, + field.optional, + ), + ); + const recordByName = new Map(record.map((property) => [property.raw, property])); + const declared = [ + ...table.attributes.map((attribute) => attribute.name), + ...table.relations.map((relation) => relation.name), + ]; + const sealed = table.storage.extraColumn === null; + const typeLabel = swiftString(typeName); + + const primaryKey = table.primaryKey === null ? undefined : recordByName.get(table.primaryKey); + const hasIdProperty = record.some((property) => property.name === 'id'); + const identifiable = primaryKey !== undefined && (primaryKey.name === 'id' || !hasIdProperty); + + let source = docComment( + commentLines(`Table \`${table.name}\` in database \`${table.database}\`.`), + '', + ); + source += `public struct ${typeName}: HarperRecord${identifiable ? ', Identifiable' : ''} {\n`; + source += `\tpublic static let schema = ${tableSchemaLiteral(table)}\n`; + source += `\tprivate static let declaredAttributes: Set = [${declared.map(swiftString).join(', ')}]\n\n`; + + for (const property of record) { + const attribute = /** @type {IRAttribute} */ (attributesByName.get(property.raw)); + const lines = commentLines(attribute.description); + if (attribute.computed?.from) + lines.push(`Computed on the client from \`${attribute.computed.from}\`.`); + source += docComment( + lines.flatMap((line) => commentLines(line)), + '\t', + ); + const keyword = patchable.has(attribute.name) ? 'var' : 'let'; + source += `\tpublic ${keyword} ${property.name}: ${property.type}\n`; + } + source += '\tpublic let _version: Double?\n'; + if (!sealed) source += '\tpublic var _extra: [String: HarperValue]\n'; + if (identifiable && primaryKey && primaryKey.name !== 'id') { + source += `\tpublic var id: ${primaryKey.type} { self.${primaryKey.name} }\n`; + } + + const trailing = [ + '_version: Double? = nil', + ...(sealed ? [] : ['_extra: [String: HarperValue] = [:]']), + ]; + source += `\n${initializer(record, '\t', trailing)}`; + source += '\t\tself._version = _version\n'; + if (!sealed) source += '\t\tself._extra = _extra\n'; + source += '\t}\n\n'; + + source += '\tpublic init(row: HarperRow, version: Double?) throws {\n'; + for (const property of record) { + const call = property.optional ? 'harperOptional' : 'harperRequired'; + source += `\t\tself.${property.name} = try row.${call}(${swiftString(property.raw)}, of: ${typeLabel})\n`; + } + source += '\t\tself._version = version\n'; + if (!sealed) source += '\t\tself._extra = row.harperExtra(excluding: Self.declaredAttributes)\n'; + source += '\t}\n\n'; + + const startRow = sealed + ? 'var row: HarperRow = [:]' + : 'var row = self._extra.harperExtra(excluding: Self.declaredAttributes)'; + source += '\tpublic func encodeRow() -> HarperRow {\n'; + source += `\t\t${startRow}\n`; + source += encodeAssignments(record, '\t\t'); + source += '\t\treturn row\n\t}\n\n'; + + source += '\tpublic func encodeUpsertRow() throws -> HarperRow {\n'; + if (table.projections.upsert === null) { + source += `\t\tthrow HarperEncodingError.replacementUnsupported(type: ${typeLabel}, attribute: ${swiftString(table.projections.unwritable[0])})\n`; + } else { + const upsert = table.projections.upsert.map( + (field) => /** @type {SwiftProperty} */ (recordByName.get(field.name)), + ); + source += `\t\t${startRow}\n`; + source += encodeAssignments(upsert, '\t\t'); + source += '\t\treturn row\n'; + } + source += '\t}\n'; + + if (table.projections.insert.length > 0) { + const insert = table.projections.insert.map((field) => + propertyOf( + /** @type {IRAttribute} */ (attributesByName.get(field.name)), + identifierOf(field.name), + typeName, + context, + field.optional, + ), + ); + source += '\n\t/// The insert (`POST`) body; without a primary key the server assigns one.\n'; + source += '\tpublic struct New: HarperWritable {\n'; + for (const property of insert) source += `\t\tpublic var ${property.name}: ${property.type}\n`; + if (!sealed) source += '\t\tpublic var _extra: [String: HarperValue]\n'; + source += `\n${initializer(insert, '\t\t', sealed ? [] : ['_extra: [String: HarperValue] = [:]'])}`; + if (!sealed) source += '\t\t\tself._extra = _extra\n'; + source += '\t\t}\n\n'; + source += '\t\tpublic func encodeRow() -> HarperRow {\n'; + source += sealed + ? '\t\t\tvar row: HarperRow = [:]\n' + : `\t\t\tvar row = self._extra.harperExtra(excluding: ${typeName}.declaredAttributes)\n`; + source += encodeAssignments(insert, '\t\t\t'); + source += '\t\t\treturn row\n\t\t}\n\t}\n'; + } + + if (table.projections.patch.length > 0) { + const patch = table.projections.patch.map((field) => + propertyOf( + /** @type {IRAttribute} */ (attributesByName.get(field.name)), + identifierOf(field.name), + typeName, + context, + true, + ), + ); + const clearable = table.projections.clearable; + const caseNames = allocateMemberNames( + clearable.map((raw) => identifierOf(raw)), + FIELD_CASE_RESERVED, + ); + source += + '\n\t/// The patch body: unset attributes are left unchanged, `clear` sets attributes to null, and a set value wins over a clear.\n'; + source += '\tpublic struct Patch: HarperWritable {\n'; + if (clearable.length > 0) { + source += '\t\tpublic enum Field: String, Sendable, Hashable {\n'; + for (const raw of clearable) { + source += `\t\t\tcase ${escapeSwiftIdentifier(/** @type {string} */ (caseNames.get(identifierOf(raw))))} = ${swiftString(raw)}\n`; + } + source += '\t\t}\n\n'; + } + for (const property of patch) source += `\t\tpublic var ${property.name}: ${property.type}\n`; + if (clearable.length > 0) source += '\t\tpublic var clear: Set\n'; + source += `\n${initializer(patch, '\t\t', clearable.length > 0 ? ['clear: Set = []'] : [])}`; + if (clearable.length > 0) source += '\t\t\tself.clear = clear\n'; + source += '\t\t}\n\n'; + source += '\t\tpublic func encodeRow() -> HarperRow {\n'; + source += '\t\t\tvar row: HarperRow = [:]\n'; + if (clearable.length > 0) + source += '\t\t\tfor field in self.clear { row[field.rawValue] = .null }\n'; + source += encodeAssignments(patch, '\t\t\t'); + source += '\t\t\treturn row\n\t\t}\n\t}\n'; + } + source += '}\n'; + return source; +} + +/** + * @param {IRObjectType} type + * @param {SwiftContext} context + * @returns {string} + */ +function objectModel(type, context) { + const names = allocateMemberNames( + type.attributes.map((attribute) => attribute.name), + reservedMemberNames(type.typeName), + ); + const properties = type.attributes.map((attribute) => + propertyOf( + attribute, + /** @type {string} */ (names.get(attribute.name)), + type.typeName, + context, + attribute.nullable, + ), + ); + const typeLabel = swiftString(type.typeName); + let source = docComment(commentLines(type.description ?? `Nested type \`${type.name}\`.`), ''); + source += `public struct ${type.typeName}: HarperObject {\n`; + source += `\tprivate static let declaredAttributes: Set = [${type.attributes.map((attribute) => swiftString(attribute.name)).join(', ')}]\n\n`; + type.attributes.forEach((attribute, index) => { + source += docComment(commentLines(attribute.description), '\t'); + source += `\tpublic var ${properties[index].name}: ${properties[index].type}\n`; + }); + source += '\tpublic var _extra: [String: HarperValue]\n'; + source += `\n${initializer(properties, '\t', ['_extra: [String: HarperValue] = [:]'])}`; + source += '\t\tself._extra = _extra\n\t}\n\n'; + source += '\tpublic init(row: HarperRow) throws {\n'; + for (const property of properties) { + const call = property.optional ? 'harperOptional' : 'harperRequired'; + source += `\t\tself.${property.name} = try row.${call}(${swiftString(property.raw)}, of: ${typeLabel})\n`; + } + source += '\t\tself._extra = row.harperExtra(excluding: Self.declaredAttributes)\n\t}\n\n'; + source += '\tpublic func encodeRow() -> HarperRow {\n'; + source += '\t\tvar row = self._extra.harperExtra(excluding: Self.declaredAttributes)\n'; + source += encodeAssignments(properties, '\t\t'); + source += '\t\treturn row\n\t}\n}\n'; + return source; +} + +/** + * @param {IRTable} table + * @returns {string} + */ +function tableSchemaLiteral(table) { + const attributesByName = new Map( + table.attributes.map((attribute) => [attribute.name, attribute]), + ); + const optional = new Map(table.projections.record.map((field) => [field.name, field.optional])); + const columns = table.storage.columns.map((column) => { + const attribute = /** @type {IRAttribute} */ (attributesByName.get(column.name)); + const parts = [ + `name: ${swiftString(column.name)}`, + `type: .${columnType(attribute.type)}`, + `affinity: .${column.affinity.toLowerCase()}`, + `nullable: ${optional.get(column.name)}`, + ]; + if (attribute.primaryKey) parts.push('primaryKey: true'); + if (attribute.indexed) parts.push('indexed: true'); + if (attribute.readOnly) parts.push('readOnly: true'); + if (attribute.computed?.from) + parts.push(`computedFrom: ${swiftString(attribute.computed.from)}`); + return `\t\t\tHarperColumn(${parts.join(', ')}),\n`; + }); + return `HarperTableSchema( + database: ${swiftString(table.database)}, + name: ${swiftString(table.name)}, + primaryKey: ${table.primaryKey === null ? 'nil' : swiftString(table.primaryKey)}, + sealed: ${table.sealed}, + hash: ${swiftString(table.hash)}, + versionColumn: ${swiftString(table.storage.versionColumn)}, + extraColumn: ${table.storage.extraColumn === null ? 'nil' : swiftString(table.storage.extraColumn)}, + replaceable: ${table.projections.upsert !== null}, + columns: [${columns.length > 0 ? `\n${columns.join('')}\t\t` : ''}] + )`; +} + +/** + * @param {SchemaIR} ir + * @returns {string} + */ +function schemaSource(ir) { + const profileNames = allocateMemberNames( + ir.profiles.map((profile) => profile.name), + [...PROFILE_RESERVED, ...reservedProfileNames(ir)], + ); + const constantOf = (/** @type {IRProfile} */ profile) => + escapeSwiftIdentifier(/** @type {string} */ (profileNames.get(profile.name))); + const tablesByKey = new Map( + ir.tables.map((table) => [`${table.database}\u0000${table.name}`, table]), + ); + let source = 'import Foundation\n\n'; + source += '/// The schema these models were generated from.\n'; + source += 'public enum HarperSchema {\n'; + source += `\tpublic static let irVersion = ${ir.irVersion}\n`; + source += `\t/// The wire and storage contract hash of every table below.\n`; + source += `\tpublic static let hash = ${swiftString(ir.schemaHash)}\n`; + source += `\tpublic static let tables: [HarperTableSchema] = [${ir.tables.map((table) => `${table.typeName}.schema`).join(', ')}]\n`; + source += `\tpublic static let recordTypes: [any HarperRecord.Type] = [${ir.tables.map((table) => `${table.typeName}.self`).join(', ')}]\n`; + source += `\t/// The valid sync profiles from \`sync.yaml\`.\n`; + source += `\tpublic static let profiles: [HarperSyncProfile] = [${ir.profiles.map((profile) => `.${constantOf(profile)}`).join(', ')}]\n`; + source += '}\n'; + if (ir.profiles.length > 0) { + source += '\nextension HarperSyncProfile {\n'; + for (const profile of ir.profiles) { + const tables = profile.tables.map( + (entry) => + `${/** @type {IRTable} */ (tablesByKey.get(`${entry.database}\u0000${entry.table}`)).typeName}.schema`, + ); + source += docComment( + commentLines(profile.description ?? `Sync profile \`${profile.name}\`.`), + '\t', + ); + source += `\tpublic static let ${constantOf(profile)} = HarperSyncProfile(name: ${swiftString(profile.name)}, direction: .${profile.direction}, retentionMs: ${profile.retentionMs ?? 'nil'}, schemaHash: ${swiftString(profile.schemaHash)}, tables: [${tables.join(', ')}])\n`; + } + source += '}\n'; + } + return source; +} diff --git a/utils/client/emitSwift.test.js b/utils/client/emitSwift.test.js new file mode 100644 index 0000000..1f80993 --- /dev/null +++ b/utils/client/emitSwift.test.js @@ -0,0 +1,186 @@ +import { describe, expect, it } from 'vitest'; +import { + adversarialTables, + coverageTables, + coverageTypes, + spikeProfiles, + spikeTables, +} from '../../test/fixtures/clientSchema.js'; +import { buildSchemaIR } from './buildSchemaIR.js'; +import { GENERATED_MARKER } from './emitSupport.js'; +import { emitSwiftPackage } from './emitSwift.js'; + +const ir = buildSchemaIR({ + tables: [...spikeTables(), ...coverageTables(), ...adversarialTables()], + types: coverageTypes(), + syncProfiles: { profiles: { ...spikeProfiles().profiles, init: { tables: ['OnlyId'] } } }, + generator: '@harperfast/schema-codegen@test', +}); +const files = emitSwiftPackage(ir); +const source = (/** @type {string} */ name) => { + const file = files.find((candidate) => candidate.path.endsWith(name)); + if (!file) throw new Error(`no ${name}`); + return file.content; +}; +const models = source('Models.swift'); + +/** + * @param {string} typeName + */ +function struct(typeName) { + const start = models.indexOf(`public struct ${typeName}: `); + if (start < 0) throw new Error(`no struct ${typeName}`); + const next = models.indexOf('\n/// ', start + 1); + return models.slice(start, next < 0 ? undefined : next); +} + +describe('emitSwiftPackage', () => { + it('lays out a Swift package with a user-owned manifest', () => { + expect(files.map((file) => [file.path, file.scaffold])).toEqual([ + ['Package.swift', true], + ['Sources/HarperModels/HarperRuntime.swift', false], + ['Sources/HarperModels/HarperSchema.swift', false], + ['Sources/HarperModels/Models.swift', false], + ]); + expect(source('Package.swift')).toContain('.target(name: "HarperModels")'); + }); + + it('marks every generated source with the generator and schema hash', () => { + for (const file of files.filter((candidate) => !candidate.scaffold)) { + expect( + file.content.startsWith( + `${GENERATED_MARKER} from @harperfast/schema-codegen@test (schema ${ir.schemaHash}). Do not edit.\n`, + ), + ).toBe(true); + } + }); + + it('honors the module name and refuses invalid or clashing ones', () => { + expect(emitSwiftPackage(ir, { module: 'Spike' }).map((file) => file.path)).toContain( + 'Sources/Spike/Models.swift', + ); + expect(() => emitSwiftPackage(ir, { module: '../Escape' })).toThrow( + /must be a Swift identifier/, + ); + expect(() => emitSwiftPackage(ir, { module: 'Order' })).toThrow(/also a generated type name/); + }); + + it('generates the spike model shape: typed columns, overflow bag and version', () => { + const product = struct('Product'); + expect(product).toContain('public struct Product: HarperRecord, Identifiable {'); + expect(product).toContain('\tpublic let id: String\n'); + expect(product).toContain('\tpublic var name: String?\n'); + expect(product).toContain('\tpublic var price: Double?\n'); + expect(product).toContain('\tpublic let _version: Double?\n'); + expect(product).toContain('\tpublic var _extra: [String: HarperValue]\n'); + expect(product).toContain('self.price = try row.harperOptional("price", of: "Product")'); + expect(product).toContain('self._extra = row.harperExtra(excluding: Self.declaredAttributes)'); + }); + + it('makes computed and server-managed attributes read-only and documents the expression', () => { + const order = struct('Order'); + expect(order).toContain( + "/// Computed on the client from `Math.round((amount || 0) * (status == 'open' ? 10 : 25))`.\n\tpublic let points: Double?", + ); + const customer = struct('coverage_Customer'); + expect(customer).toContain('\tpublic let createdAt: Date\n'); + expect(customer).not.toMatch(/if let value = self\.createdAt \{ row\["createdAt"\]/); + }); + + it('maps every Harper type to its Swift type', () => { + const customer = struct('coverage_Customer'); + for (const declaration of [ + 'public var tags: [String]?', + 'public var scores: [Int?]?', + 'public var big: HarperBigInt?', + 'public var raw: Data?', + 'public var meta: HarperValue?', + 'public var visits: Int64?', + 'public var active: Bool?', + 'public var address: Address?', + 'public let updatedAt: Double', + ]) { + expect(customer).toContain(declaration); + } + }); + + it('leaves the overflow bag off sealed tables', () => { + expect(struct('coverage_Purchase')).not.toContain('_extra'); + }); + + it('refuses full replacement for tables with a Blob attribute', () => { + const attachment = struct('coverage_Attachment'); + expect(attachment).toContain('public let file: HarperValue?'); + expect(attachment).toContain( + 'throw HarperEncodingError.replacementUnsupported(type: "coverage_Attachment", attribute: "file")', + ); + expect(attachment).toContain('replaceable: false'); + }); + + it('emits insert and patch types, with clear only for nullable attributes, and skips empty ones', () => { + const customer = struct('coverage_Customer'); + expect(customer).toContain('public struct New: HarperWritable {'); + expect(customer).toContain('public init(id: String? = nil, name: String, '); + expect(customer).toContain('public enum Field: String, Sendable, Hashable {'); + expect(customer).not.toContain('case name = "name"'); + expect(customer).toContain('for field in self.clear { row[field.rawValue] = .null }'); + const onlyId = struct('OnlyId'); + expect(onlyId).toContain('public struct New: HarperWritable {'); + expect(onlyId).not.toContain('struct Patch'); + }); + + it('keeps raw attribute names on the row while escaping and renaming properties', () => { + const weird = struct('Weird'); + expect(weird).toContain('public var `default`: String?'); + expect(weird).toContain('public var `Type`: Int?'); + expect(weird).toContain('public var firstName_2: String?'); + expect(weird).toContain('public var _extra_2: String?'); + expect(weird).toContain('public var _version_2: Int?'); + expect(weird).toContain('public var encodeRow_2: Int?'); + expect(weird).toContain('public var self_2: String?'); + expect(weird).toContain('public var o_conn_or__x: String?'); + expect(weird).toContain('public var _9lives: Bool'); + expect(weird).toContain('row["first-name"] = value.harperValue'); + expect(weird).toContain('row["o\\"conn\\\\or $x"] = value.harperValue'); + expect(weird).toContain('extraColumn: "__extra"'); + }); + + it('synthesizes Identifiable from a differently named key, and skips it when id is taken', () => { + expect(struct('SkuItem')).toContain('public var id: String { self.sku }'); + expect(struct('DataRecord')).toMatch(/public struct DataRecord: HarperRecord \{/); + }); + + it('boxes a nested type that contains itself by value', () => { + const node = struct('Node'); + expect(node).toContain('public var parent: HarperBox?'); + expect(node).toContain('public var children: [Node]?'); + }); + + it('lists tables and named profiles in HarperSchema', () => { + const schema = source('HarperSchema.swift'); + expect(schema).toContain(`public static let hash = "${ir.schemaHash}"`); + expect(schema).toContain( + 'public static let profiles: [HarperSyncProfile] = [.init_2, .myOrders, .storefront]', + ); + expect(schema).toMatch( + /public static let myOrders = HarperSyncProfile\(name: "my-orders", direction: \.pull, retentionMs: 2592000000, schemaHash: "[0-9a-f]{64}", tables: \[Order\.schema\]\)/, + ); + }); + + it('gives profiles whose names sanitize alike their own constants', () => { + const schema = + emitSwiftPackage( + buildSchemaIR({ + tables: spikeTables(), + syncProfiles: { + profiles: { 'my-orders': { tables: ['Order'] }, myOrders: { tables: ['Order'] } }, + }, + }), + ).find((file) => file.path.endsWith('HarperSchema.swift'))?.content ?? ''; + expect(schema).toContain( + 'public static let profiles: [HarperSyncProfile] = [.myOrders_2, .myOrders]', + ); + expect(schema).toContain('public static let myOrders_2 = HarperSyncProfile(name: "my-orders"'); + expect(schema).toContain('public static let myOrders = HarperSyncProfile(name: "myOrders"'); + }); +}); diff --git a/utils/client/generator.js b/utils/client/generator.js new file mode 100644 index 0000000..85bef2f --- /dev/null +++ b/utils/client/generator.js @@ -0,0 +1,8 @@ +import fs from 'node:fs'; + +const { name, version } = JSON.parse( + fs.readFileSync(new URL('../../package.json', import.meta.url), 'utf8'), +); + +/** This package's name and version, recorded in every IR and generated file header. */ +export const GENERATOR = `${name}@${version}`; diff --git a/utils/client/irConstants.js b/utils/client/irConstants.js new file mode 100644 index 0000000..901454e --- /dev/null +++ b/utils/client/irConstants.js @@ -0,0 +1,21 @@ +/** + * The schema IR format version. Bump it whenever a mapping or canonicalization change alters what + * a device stores or sends: it is part of every hash, so the change then reads as unknown drift + * instead of a false match. + */ +export const IR_VERSION = 1; + +/** The Harper attribute types the IR represents as scalars. */ +export const SCALAR_TYPES = new Set([ + 'ID', + 'String', + 'Int', + 'Float', + 'Long', + 'BigInt', + 'Boolean', + 'Date', + 'Bytes', + 'Blob', + 'Any', +]); diff --git a/utils/client/irTypes.js b/utils/client/irTypes.js new file mode 100644 index 0000000..4395f14 --- /dev/null +++ b/utils/client/irTypes.js @@ -0,0 +1,109 @@ +/** + * Shapes of the schema IR (`irVersion` 1). See docs/design/client-codegen.md. + * + * @typedef {'ID' | 'String' | 'Int' | 'Long' | 'Float' | 'BigInt' | 'Boolean' | 'Date' | 'Bytes' | 'Blob' | 'Any'} IRScalar + * + * @typedef {{ kind: 'scalar', scalar: IRScalar } + * | { kind: 'array', element: IRType, elementNullable: boolean } + * | { kind: 'object', type: string } + * | { kind: 'record', database: string, table: string }} IRType + * + * @typedef {Object} IRAttribute + * @property {string} name + * @property {IRType} type + * @property {boolean} nullable optional on read + * @property {boolean} primaryKey + * @property {boolean} indexed + * @property {'createdTime' | 'updatedTime' | 'derived' | null} serverManaged `derived`: an `@embed`/`@decide` output + * @property {{ from: string | null, version: string | null } | null} computed + * @property {boolean} readOnly server-managed or computed + * @property {string} [description] + * + * @typedef {Object} IRNestedAttribute + * @property {string} name + * @property {IRType} type + * @property {boolean} nullable + * @property {string} [description] + * + * @typedef {Object} IRRelation + * @property {string} name + * @property {'one' | 'many'} cardinality + * @property {string} [from] + * @property {string} [to] + * @property {{ database: string | null, table: string }} target + * + * @typedef {{ name: string, optional: boolean }} IRProjectionField + * + * @typedef {Object} IRProjections + * @property {IRProjectionField[]} record + * @property {IRProjectionField[]} insert + * @property {IRProjectionField[] | null} upsert null when a full replacement is unsafe + * @property {IRProjectionField[]} patch + * @property {string[]} clearable patch attributes that may be set to null + * @property {string[]} unwritable attributes kept out of every write because they have no client write contract (Blob) + * @property {IRProjectionField[]} query + * + * @typedef {'TEXT' | 'INTEGER' | 'REAL' | 'BLOB'} IRAffinity + * + * @typedef {Object} IRStorage + * @property {string} versionColumn + * @property {string | null} extraColumn null on sealed tables + * @property {{ name: string, affinity: IRAffinity }[]} columns + * + * @typedef {Object} IRTable + * @property {string} database + * @property {string} name + * @property {string} typeName + * @property {string | null} primaryKey + * @property {boolean} sealed + * @property {string} hash + * @property {string[]} profiles + * @property {string} [description] + * @property {IRAttribute[]} attributes stored attributes, in schema order + * @property {IRRelation[]} relations + * @property {IRProjections} projections + * @property {IRStorage} storage + * + * @typedef {Object} IRObjectType + * @property {string} name the schema type name + * @property {string} typeName + * @property {string} [description] + * @property {IRNestedAttribute[]} attributes + * + * @typedef {{ attribute: string, operator: 'equals', source: 'literal', value: string | number | boolean } + * | { attribute: string, operator: 'in', source: 'literal', value: (string | number | boolean)[] } + * | { attribute: string, operator: 'equals', source: 'claim', claim: string } + * | { attribute: string, operator: 'equals', source: 'user', field: string }} IRScopeCondition + * + * @typedef {{ type: 'all' } | { type: 'match', conditions: IRScopeCondition[] }} IRScope + * + * @typedef {{ database: string, table: string, scope: IRScope }} IRProfileTable + * + * @typedef {Object} IRProfile + * @property {string} name + * @property {string} [description] + * @property {'pull' | 'push' | 'bidirectional'} direction + * @property {string | null} retention + * @property {number | null} retentionMs + * @property {string} hash the profile definition's hash (its version) + * @property {string} schemaHash the hash a device sends in its hello frame for this profile + * @property {IRProfileTable[]} tables + * + * @typedef {Object} IRDiagnostic + * @property {'error' | 'warning'} level + * @property {string} code + * @property {string} message + * @property {string} [profile] + * @property {string} [table] + * @property {string} [attribute] + * + * @typedef {Object} SchemaIR + * @property {number} irVersion + * @property {string} [generator] + * @property {string} schemaHash + * @property {IRTable[]} tables + * @property {IRObjectType[]} types + * @property {IRProfile[]} profiles + * @property {IRDiagnostic[]} diagnostics + */ +export {}; diff --git a/utils/client/naming.js b/utils/client/naming.js new file mode 100644 index 0000000..0a66c09 --- /dev/null +++ b/utils/client/naming.js @@ -0,0 +1,402 @@ +/** + * Swift words that must be backtick-escaped to be used as a property name. Contextual keywords + * (`get`, `set`, `some`, `async`, …) are valid member names and are left out. + */ +const SWIFT_KEYWORDS = new Set([ + 'Any', + 'Protocol', + 'Self', + 'Type', + '_', + 'as', + 'associatedtype', + 'await', + 'break', + 'case', + 'catch', + 'class', + 'continue', + 'default', + 'defer', + 'deinit', + 'do', + 'else', + 'enum', + 'extension', + 'fallthrough', + 'false', + 'fileprivate', + 'for', + 'func', + 'guard', + 'if', + 'import', + 'in', + 'init', + 'inout', + 'internal', + 'is', + 'let', + 'nil', + 'operator', + 'precedencegroup', + 'private', + 'protocol', + 'public', + 'repeat', + 'rethrows', + 'return', + 'self', + 'static', + 'struct', + 'subscript', + 'super', + 'switch', + 'throw', + 'throws', + 'true', + 'try', + 'typealias', + 'var', + 'where', + 'while', +]); + +/** Kotlin hard keywords: valid identifiers only when backtick-escaped. */ +const KOTLIN_KEYWORDS = new Set([ + 'as', + 'break', + 'class', + 'continue', + 'do', + 'else', + 'false', + 'for', + 'fun', + 'if', + 'in', + 'interface', + 'is', + 'null', + 'object', + 'package', + 'return', + 'super', + 'this', + 'throw', + 'true', + 'try', + 'typealias', + 'typeof', + 'val', + 'var', + 'when', + 'while', +]); + +/** + * Kotlin soft and modifier keywords. They are valid property names, but at the start of an enum + * entry the parser can read them as modifiers (`value("value")` fails to parse), so enum entries + * escape them too. + */ +const KOTLIN_SOFT_KEYWORDS = new Set([ + 'abstract', + 'actual', + 'annotation', + 'by', + 'catch', + 'companion', + 'const', + 'constructor', + 'crossinline', + 'data', + 'delegate', + 'dynamic', + 'enum', + 'expect', + 'external', + 'field', + 'file', + 'final', + 'finally', + 'get', + 'import', + 'infix', + 'init', + 'inline', + 'inner', + 'internal', + 'lateinit', + 'noinline', + 'open', + 'operator', + 'out', + 'override', + 'param', + 'private', + 'property', + 'protected', + 'public', + 'receiver', + 'reified', + 'sealed', + 'set', + 'setparam', + 'suspend', + 'tailrec', + 'value', + 'vararg', + 'where', +]); + +/** Names the generated runtime and emitted code declare at the top level of the package. */ +const RUNTIME_TYPE_NAMES = [ + 'HarperAffinity', + 'HarperBigInt', + 'HarperBox', + 'HarperColumn', + 'HarperColumnType', + 'HarperContent', + 'HarperConverter', + 'HarperConverters', + 'HarperDecodingError', + 'HarperEncodingError', + 'HarperObject', + 'HarperObjectType', + 'HarperProfiles', + 'HarperRecord', + 'HarperRow', + 'HarperSchema', + 'HarperSyncProfile', + 'HarperTable', + 'HarperTableSchema', + 'HarperValue', + 'HarperValueConvertible', + 'HarperValueMismatch', + 'HarperWritable', +]; + +/** + * Type names a table or nested type may not take: keywords and contextual words of either + * language, the standard types the generated code names unqualified, the runtime's own types, + * and the names nested inside every generated record. + */ +export const RESERVED_TYPE_NAMES = new Set([ + ...SWIFT_KEYWORDS, + ...KOTLIN_KEYWORDS, + ...RUNTIME_TYPE_NAMES, + // The JVM classes Kotlin compiles the generated files' top-level declarations into. + 'HarperRuntimeKt', + 'HarperSchemaKt', + 'ModelsKt', + 'actor', + 'any', + 'async', + 'some', + 'Array', + 'Base64', + 'BigInteger', + 'Bool', + 'Boolean', + 'Byte', + 'ByteArray', + 'Character', + 'Codable', + 'Comparable', + 'Companion', + 'CustomStringConvertible', + 'Data', + 'Date', + 'DateTimeParseException', + 'Decoder', + 'DecodingError', + 'Dictionary', + 'Direction', + 'Double', + 'Encoder', + 'Enum', + 'Equatable', + 'Error', + 'Exception', + 'ExpressibleByIntegerLiteral', + 'Field', + 'Float', + 'Foundation', + 'Hashable', + 'ISO8601DateFormatter', + 'Identifiable', + 'IllegalArgumentException', + 'Instant', + 'Int', + 'Int64', + 'Iterable', + 'KClass', + 'List', + 'Long', + 'Map', + 'Math', + 'NSNull', + 'NSNumber', + 'Never', + 'New', + 'Nothing', + 'Number', + 'Optional', + 'Patch', + 'Sendable', + 'Set', + 'Short', + 'String', + 'Substring', + 'Suppress', + 'Swift', + 'Throwable', + 'Unit', + 'Void', +]); + +/** + * Member names every generated record, insert, patch or nested type may declare itself. An + * attribute whose identifier lands on one of these is given a suffixed property name instead. + */ +export const RESERVED_MEMBER_NAMES = [ + 'Companion', + 'Field', + 'New', + 'Patch', + '_extra', + '_version', + 'clear', + 'encodeRow', + 'encodeUpsertRow', + 'harperValue', + 'hashValue', + 'self', + 'this', +]; + +/** + * The names no member of the type `ownerTypeName` may take: the reserved members, plus the + * runtime types and the owner itself, which its member bodies refer to by name and a same-named + * member would shadow. Other generated types are never named there, so adding a table never + * renames an existing member. + * @param {string} ownerTypeName + * @returns {string[]} + */ +export function reservedMemberNames(ownerTypeName) { + return [...RESERVED_MEMBER_NAMES, ...RUNTIME_TYPE_NAMES, ownerTypeName]; +} + +/** + * The names no profile constant may take: profile initializers name every table they cover. + * @param {{ tables: { typeName: string }[], types: { typeName: string }[] }} ir + * @returns {string[]} + */ +export function reservedProfileNames(ir) { + return [ + ...RUNTIME_TYPE_NAMES, + ...ir.tables.map((table) => table.typeName), + ...ir.types.map((type) => type.typeName), + ]; +} + +/** + * Converts any schema name into an identifier valid in Swift, Kotlin and TypeScript: kebab-case + * becomes camelCase, every other character outside `[A-Za-z0-9_]` becomes `_`, and a leading + * digit (or an empty name) gains a `_` prefix. Unlike `toIdentifier`, `$` is not kept: neither + * Swift nor Kotlin allows it. + * @param {string} name + * @returns {string} + */ +export function toCodeIdentifier(name) { + let identifier = String(name) + .replace(/-([a-zA-Z0-9])/g, (_, character) => character.toUpperCase()) + .replace(/[^a-zA-Z0-9_]/g, '_'); + if (identifier === '' || /^[0-9]/.test(identifier)) { + identifier = `_${identifier}`; + } + return identifier; +} + +/** + * Hands out unique names: a requested name is returned as-is when free, otherwise suffixed + * `_2`, `_3`, … until the result is free as well. Comparison is case-sensitive. + * @param {Iterable} [reserved] + */ +export function createNameAllocator(reserved = []) { + const claimed = new Set(reserved); + return { + /** + * Claims `name` exactly; returns false when it is already taken. + * @param {string} name + * @returns {boolean} + */ + claimExact(name) { + if (claimed.has(name)) return false; + claimed.add(name); + return true; + }, + /** + * @param {string} name + * @returns {string} + */ + claim(name) { + let candidate = name; + for (let suffix = 2; claimed.has(candidate); suffix++) { + candidate = `${name}_${suffix}`; + } + claimed.add(candidate); + return candidate; + }, + }; +} + +/** + * Assigns a unique identifier to every raw name. Names that are already identifiers are claimed + * first, so `aB` keeps its name even when a later `a-b` also sanitizes to `aB`. + * @param {string[]} names raw names, in declaration order + * @param {Iterable} reserved + * @returns {Map} raw name → identifier + */ +export function allocateMemberNames(names, reserved) { + const allocator = createNameAllocator(reserved); + /** @type {Map} */ + const result = new Map(); + for (const name of names) { + if (toCodeIdentifier(name) === name && allocator.claimExact(name)) { + result.set(name, name); + } + } + for (const name of names) { + if (!result.has(name)) { + result.set(name, allocator.claim(toCodeIdentifier(name))); + } + } + return result; +} + +/** + * @param {string} identifier + * @returns {string} + */ +export function escapeSwiftIdentifier(identifier) { + return SWIFT_KEYWORDS.has(identifier) ? `\`${identifier}\`` : identifier; +} + +/** + * @param {string} identifier + * @returns {string} + */ +export function escapeKotlinIdentifier(identifier) { + return KOTLIN_KEYWORDS.has(identifier) ? `\`${identifier}\`` : identifier; +} + +/** + * @param {string} identifier + * @returns {string} + */ +export function escapeKotlinEnumEntry(identifier) { + return KOTLIN_KEYWORDS.has(identifier) || KOTLIN_SOFT_KEYWORDS.has(identifier) + ? `\`${identifier}\`` + : identifier; +} diff --git a/utils/client/naming.test.js b/utils/client/naming.test.js new file mode 100644 index 0000000..5a7f612 --- /dev/null +++ b/utils/client/naming.test.js @@ -0,0 +1,84 @@ +import { describe, expect, it } from 'vitest'; +import { + allocateMemberNames, + createNameAllocator, + escapeKotlinEnumEntry, + escapeKotlinIdentifier, + escapeSwiftIdentifier, + RESERVED_TYPE_NAMES, + toCodeIdentifier, +} from './naming.js'; + +describe('toCodeIdentifier', () => { + it('keeps identifiers and camelCases kebab-case', () => { + expect(toCodeIdentifier('customerId')).toBe('customerId'); + expect(toCodeIdentifier('first-name')).toBe('firstName'); + }); + + it('drops characters Swift and Kotlin cannot use, including $', () => { + expect(toCodeIdentifier('a.b c$d')).toBe('a_b_c_d'); + expect(toCodeIdentifier('o"conn\\or')).toBe('o_conn_or'); + }); + + it('prefixes leading digits and empty names', () => { + expect(toCodeIdentifier('9lives')).toBe('_9lives'); + expect(toCodeIdentifier('')).toBe('_'); + }); +}); + +describe('createNameAllocator', () => { + it('suffixes taken names and rechecks every suffix', () => { + const allocator = createNameAllocator(['User']); + expect(allocator.claimExact('User_2')).toBe(true); + expect(allocator.claim('User')).toBe('User_3'); + expect(allocator.claim('Order')).toBe('Order'); + expect(allocator.claim('Order')).toBe('Order_2'); + }); +}); + +describe('allocateMemberNames', () => { + it('lets names that are already identifiers keep them over later sanitized collisions', () => { + const names = allocateMemberNames(['first-name', 'firstName'], []); + expect(names.get('firstName')).toBe('firstName'); + expect(names.get('first-name')).toBe('firstName_2'); + }); + + it('moves attributes off the generated members', () => { + const names = allocateMemberNames( + ['_extra', '_version', 'encodeRow', 'New', 'id'], + ['_extra', '_version', 'encodeRow', 'New'], + ); + expect(names.get('_extra')).toBe('_extra_2'); + expect(names.get('_version')).toBe('_version_2'); + expect(names.get('encodeRow')).toBe('encodeRow_2'); + expect(names.get('New')).toBe('New_2'); + expect(names.get('id')).toBe('id'); + }); + + it('gives every raw name a distinct identifier', () => { + const raw = ['a-b', 'aB', 'a_b', 'a.b', 'a b']; + const names = allocateMemberNames(raw, []); + expect(new Set(names.values()).size).toBe(raw.length); + }); +}); + +describe('escaping', () => { + it('backticks Swift keywords, including the member names Swift reserves', () => { + expect(escapeSwiftIdentifier('default')).toBe('`default`'); + expect(escapeSwiftIdentifier('Type')).toBe('`Type`'); + expect(escapeSwiftIdentifier('value')).toBe('value'); + }); + + it('backticks Kotlin hard keywords, and soft keywords only in enum entries', () => { + expect(escapeKotlinIdentifier('in')).toBe('`in`'); + expect(escapeKotlinIdentifier('value')).toBe('value'); + expect(escapeKotlinEnumEntry('value')).toBe('`value`'); + expect(escapeKotlinEnumEntry('status')).toBe('status'); + }); + + it('reserves standard and runtime type names', () => { + for (const name of ['Data', 'String', 'Instant', 'HarperValue', 'New', 'Patch', 'class']) { + expect(RESERVED_TYPE_NAMES.has(name)).toBe(true); + } + }); +}); diff --git a/utils/client/native.test.js b/utils/client/native.test.js new file mode 100644 index 0000000..0275a45 --- /dev/null +++ b/utils/client/native.test.js @@ -0,0 +1,145 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterAll, describe, expect, it } from 'vitest'; +import { + adversarialTables, + coverageTables, + coverageTypes, + spikeProfiles, + spikeTables, +} from '../../test/fixtures/clientSchema.js'; +import { buildSchemaIR } from './buildSchemaIR.js'; +import { emitKotlinModule } from './emitKotlin.js'; +import { emitSwiftPackage } from './emitSwift.js'; + +/** + * `auto` (default) runs each language when its toolchain is found; `require` fails when one is + * missing, so a CI host meant to cover them cannot pass vacuously; `skip` runs nothing. + */ +const mode = process.env.SCHEMA_CODEGEN_NATIVE_TESTS ?? 'auto'; +const harnesses = new URL('../../test/fixtures/native/', import.meta.url); +const workspace = fs.mkdtempSync(path.join(os.tmpdir(), 'codegen-native-')); +afterAll(() => fs.rmSync(workspace, { recursive: true, force: true })); + +const ir = buildSchemaIR({ + tables: [...spikeTables(), ...coverageTables(), ...adversarialTables()], + types: coverageTypes(), + syncProfiles: spikeProfiles(), + generator: '@harperfast/schema-codegen@native-test', +}); + +/** + * @param {string} command + * @returns {string | null} + */ +function onPath(command) { + const result = spawnSync(process.platform === 'win32' ? 'where' : 'which', [command], { + encoding: 'utf8', + }); + return result.status === 0 ? result.stdout.split(/\r?\n/)[0].trim() : null; +} + +function swiftAvailable() { + if (mode === 'skip' || process.platform !== 'darwin') return false; + return spawnSync('xcrun', ['--find', 'swiftc'], { encoding: 'utf8' }).status === 0; +} + +function kotlinToolchain() { + if (mode === 'skip') return null; + const kotlinc = process.env.KOTLINC ?? onPath('kotlinc'); + const java = process.env.JAVA_HOME + ? path.join(process.env.JAVA_HOME, 'bin', 'java') + : onPath('java'); + return kotlinc && java && fs.existsSync(java) ? { kotlinc, java } : null; +} + +/** + * @param {import('./emitSwift.js').EmittedFile[]} files + * @param {string} directory + */ +function write(files, directory) { + for (const file of files) { + const target = path.join(directory, ...file.path.split('/')); + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.writeFileSync(target, file.content); + } +} + +const swift = swiftAvailable(); +const kotlin = kotlinToolchain(); + +describe('native toolchains', () => { + it.runIf(mode === 'require')('are present when required', () => { + expect(swift, 'swiftc (Xcode) is required').toBe(true); + expect(kotlin, 'kotlinc and java are required').toBeTruthy(); + }); +}); + +describe.runIf(swift)('generated Swift', () => { + it('compiles in Swift 5 and 6 language modes and keeps the model contract, through a reopened SQLite replica', () => { + const directory = path.join(workspace, 'swift'); + write(emitSwiftPackage(ir), directory); + const sources = path.join(directory, 'Sources', 'HarperModels'); + const main = path.join(sources, 'main.swift'); + fs.copyFileSync(new URL('SwiftHarness.swift', harnesses), main); + const inputs = ['HarperRuntime.swift', 'HarperSchema.swift', 'Models.swift', 'main.swift'].map( + (name) => path.join(sources, name), + ); + execFileSync('xcrun', ['swiftc', '-swift-version', '5', '-typecheck', ...inputs], { + encoding: 'utf8', + stdio: 'pipe', + }); + const binary = path.join(directory, 'harness'); + execFileSync( + 'xcrun', + [ + 'swiftc', + '-swift-version', + '6', + '-warnings-as-errors', + '-o', + binary, + ...inputs, + '-lsqlite3', + ], + { + encoding: 'utf8', + stdio: 'pipe', + }, + ); + const output = spawnSync(binary, { encoding: 'utf8' }); + expect(output.stdout + output.stderr).toMatch(/^PASS \d+\n$/); + expect(output.status).toBe(0); + }, 600_000); +}); + +describe.runIf(kotlin)('generated Kotlin', () => { + it('compiles without warnings and keeps the model contract', () => { + const toolchain = /** @type {{ kotlinc: string, java: string }} */ (kotlin); + const directory = path.join(workspace, 'kotlin'); + write(emitKotlinModule(ir, { packageName: 'harper.harness' }), directory); + const sources = path.join(directory, 'src', 'main', 'kotlin', 'harper', 'harness'); + fs.copyFileSync(new URL('KotlinHarness.kt', harnesses), path.join(sources, 'KotlinHarness.kt')); + const jar = path.join(directory, 'harness.jar'); + const compiled = spawnSync( + toolchain.kotlinc, + [ + '-Werror', + '-include-runtime', + '-d', + jar, + ...fs.readdirSync(sources).map((name) => path.join(sources, name)), + ], + { encoding: 'utf8', shell: process.platform === 'win32' }, + ); + expect(compiled.stderr, compiled.stderr).not.toMatch(/error:/); + expect(compiled.status).toBe(0); + const output = spawnSync(toolchain.java, ['-cp', jar, 'harper.harness.KotlinHarnessKt'], { + encoding: 'utf8', + }); + expect(output.stdout + output.stderr).toMatch(/^PASS \d+\r?\n$/); + expect(output.status).toBe(0); + }, 600_000); +}); diff --git a/utils/client/regenerateClient.js b/utils/client/regenerateClient.js new file mode 100644 index 0000000..4d80d00 --- /dev/null +++ b/utils/client/regenerateClient.js @@ -0,0 +1,127 @@ +/** @import { SchemaIR } from './irTypes.js' */ +/** @import { SourceTable, SourceTypeDef } from './buildSchemaIR.js' */ +/** @import { ClientOutputOptions } from './clientOutputs.js' */ +import fs from 'node:fs'; +import path from 'node:path'; +import { getLogger } from '../logger.js'; +import { buildSchemaIR } from './buildSchemaIR.js'; +import { + findStaleGeneratedFiles, + publishClientOutputs, + renderClientOutputs, +} from './clientOutputs.js'; +import { diffSchemaIR } from './diffSchemaIR.js'; +import { GENERATOR } from './generator.js'; +import { loadSyncProfilesFile } from './syncProfiles.js'; +import { assertSchemaIR } from './validateSchemaIR.js'; + +/** + * @typedef {ClientOutputOptions & { + * syncProfiles?: string, + * types?: Map | Record, + * signal?: AbortSignal, + * }} RegenerateClientOptions + */ + +/** + * @param {SourceTable[]} tables already database-filtered + * @param {RegenerateClientOptions} options `signal` aborts before anything is published; once + * publication starts it runs to completion without yielding + * @returns {Promise} undefined when aborted + */ +export async function regenerateClient(tables, options) { + const logger = getLogger(); + const base = options.baseDirectory ?? ''; + const irPath = options.schemaIR ? path.resolve(base, options.schemaIR) : undefined; + const previous = irPath ? readPublishedIR(irPath) : undefined; + + let syncProfiles; + if (options.syncProfiles) { + const profilesPath = path.resolve(base, options.syncProfiles); + syncProfiles = await loadSyncProfilesFile(profilesPath); + if (syncProfiles === undefined) { + logger?.warn?.( + `@harperfast/schema-codegen: sync profiles file ${profilesPath} does not exist; no profiles are generated`, + ); + } + } + if (options.signal?.aborted) return undefined; + + const ir = buildSchemaIR({ + tables, + types: options.types, + syncProfiles, + previous, + generator: GENERATOR, + }); + for (const diagnostic of ir.diagnostics) { + const message = `@harperfast/schema-codegen: ${diagnostic.message}`; + if (diagnostic.level === 'error') logger?.error?.(message); + else logger?.warn?.(message); + } + if (previous && previous.schemaHash !== ir.schemaHash) { + const diff = diffSchemaIR(previous, ir); + const breaking = /** @type {const} */ (['read', 'write', 'storage']).filter( + (axis) => diff.compatibility[axis] === 'breaking', + ); + if (breaking.length > 0) { + const changes = diff.changes + .filter((change) => change.breaks.length > 0) + .map((change) => `${change.message} (breaks ${change.breaks.join(', ')})`); + logger?.warn?.( + `@harperfast/schema-codegen: this schema change breaks ${breaking.join(', ')} compatibility for clients generated before it:\n ${changes.join('\n ')}`, + ); + } + for (const rename of diff.source) { + logger?.warn?.( + `@harperfast/schema-codegen: generated type ${rename.from} is now ${rename.to}`, + ); + } + } + + const outputs = renderClientOutputs(ir, options); + const swiftManifest = options.swift + ? path.resolve(base, options.swift, 'Package.swift') + : undefined; + const manifestExisted = swiftManifest !== undefined && fs.existsSync(swiftManifest); + publishClientOutputs(outputs); + + const module = options.swiftModule ?? 'HarperModels'; + if ( + swiftManifest && + manifestExisted && + !fs.readFileSync(swiftManifest, 'utf8').includes(`"${module}"`) + ) { + logger?.warn?.( + `@harperfast/schema-codegen: ${swiftManifest} does not declare the "${module}" target the models are generated into`, + ); + } + for (const directory of [options.swift, options.kotlin]) { + if (!directory) continue; + for (const stale of findStaleGeneratedFiles(path.resolve(base, directory), outputs)) { + logger?.warn?.( + `@harperfast/schema-codegen: ${stale} was generated for a different module or package and is no longer updated`, + ); + } + } + return ir; +} + +/** + * The previously published IR, when it is still readable by this generator. + * @param {string} irPath + * @returns {SchemaIR | undefined} + */ +function readPublishedIR(irPath) { + if (!fs.existsSync(irPath)) return undefined; + try { + const ir = JSON.parse(fs.readFileSync(irPath, 'utf8')); + assertSchemaIR(ir); + return ir; + } catch (error) { + getLogger()?.warn?.( + `@harperfast/schema-codegen: ignoring the existing ${irPath} (${/** @type {Error} */ (error).message}); type names may change and compatibility is not checked`, + ); + return undefined; + } +} diff --git a/utils/client/regenerateClient.test.js b/utils/client/regenerateClient.test.js new file mode 100644 index 0000000..c1aa28e --- /dev/null +++ b/utils/client/regenerateClient.test.js @@ -0,0 +1,153 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { spikeTables } from '../../test/fixtures/clientSchema.js'; +import { setLogger } from '../logger.js'; +import { regenerateClient } from './regenerateClient.js'; + +describe('regenerateClient', () => { + /** @type {string} */ + let directory; + const logger = { + error: vi.fn(), + warn: vi.fn(), + info: vi.fn(), + debug: vi.fn(), + trace: vi.fn(), + notify: vi.fn(), + fatal: vi.fn(), + }; + beforeEach(() => { + directory = fs.mkdtempSync(path.join(os.tmpdir(), 'codegen-regenerate-')); + setLogger(/** @type {any} */ (logger)); + }); + afterEach(() => { + vi.clearAllMocks(); + fs.rmSync(directory, { recursive: true, force: true }); + }); + + const options = (/** @type {object} */ overrides = {}) => ({ + baseDirectory: directory, + schemaIR: 'client/harper-schema.json', + syncProfiles: 'sync.yaml', + swift: 'ios/HarperModels', + kotlin: 'android/harper-models', + ...overrides, + }); + const warnings = () => logger.warn.mock.calls.map((call) => String(call[0])); + + it('publishes nothing once its signal is aborted', async () => { + fs.writeFileSync(path.join(directory, 'sync.yaml'), 'profiles: {}\n'); + const closing = new AbortController(); + const run = regenerateClient(spikeTables(), options({ signal: closing.signal })); + closing.abort(); + expect(await run).toBeUndefined(); + expect(fs.readdirSync(directory)).toEqual(['sync.yaml']); + }); + + it('writes the IR, Swift and Kotlin from the app directory, with profiles from sync.yaml', async () => { + fs.writeFileSync( + path.join(directory, 'sync.yaml'), + 'profiles:\n storefront:\n tables: [Product]\n', + ); + const ir = await regenerateClient(spikeTables(), options()); + expect(ir.profiles.map((profile) => profile.name)).toEqual(['storefront']); + const written = JSON.parse( + fs.readFileSync(path.join(directory, 'client', 'harper-schema.json'), 'utf8'), + ); + expect(written.schemaHash).toBe(ir.schemaHash); + expect(written.generator).toMatch(/^@harperfast\/schema-codegen@/); + expect( + fs.existsSync( + path.join(directory, 'ios', 'HarperModels', 'Sources', 'HarperModels', 'Models.swift'), + ), + ).toBe(true); + expect( + fs.existsSync( + path.join( + directory, + 'android', + 'harper-models', + 'src', + 'main', + 'kotlin', + 'harper', + 'models', + 'Models.kt', + ), + ), + ).toBe(true); + expect(logger.warn).not.toHaveBeenCalled(); + }); + + it('logs profile errors and warnings', async () => { + fs.writeFileSync( + path.join(directory, 'sync.yaml'), + 'profiles:\n broken:\n scope: { custmerId: $token.sub }\n tables: [Order]\n slow:\n scope: { amount: 5 }\n tables: [Order]\n', + ); + const ir = await regenerateClient(spikeTables(), options()); + expect(ir.profiles.map((profile) => profile.name)).toEqual(['slow']); + expect(logger.error).toHaveBeenCalledWith( + expect.stringContaining('"custmerId", which is not a stored attribute'), + ); + expect(warnings()).toContainEqual(expect.stringContaining('which is not @indexed')); + }); + + it('warns when sync.yaml is configured but missing', async () => { + await regenerateClient(spikeTables(), options()); + expect(warnings()).toContainEqual( + expect.stringContaining('does not exist; no profiles are generated'), + ); + }); + + it('warns when a regeneration breaks clients generated from the published IR', async () => { + await regenerateClient(spikeTables(), options({ syncProfiles: undefined })); + const changed = spikeTables(); + changed[0].attributes[2].type = 'String'; + await regenerateClient(changed, options({ syncProfiles: undefined })); + const warning = warnings().find((message) => + message.includes('breaks read, write, storage compatibility'), + ); + expect(warning).toContain( + 'data.Product.price type changed from Float to String (breaks read, write)', + ); + }); + + it('ignores a published IR it cannot read, and says so', async () => { + fs.mkdirSync(path.join(directory, 'client')); + fs.writeFileSync(path.join(directory, 'client', 'harper-schema.json'), '{"irVersion": 0}'); + await regenerateClient(spikeTables(), options({ syncProfiles: undefined })); + expect(warnings()).toContainEqual(expect.stringContaining('ignoring the existing')); + expect( + JSON.parse(fs.readFileSync(path.join(directory, 'client', 'harper-schema.json'), 'utf8')) + .irVersion, + ).toBe(1); + }); + + it('warns when the existing Package.swift does not declare the module, and about stale sources', async () => { + await regenerateClient(spikeTables(), options({ syncProfiles: undefined })); + await regenerateClient( + spikeTables(), + options({ syncProfiles: undefined, swiftModule: 'SpikeModels' }), + ); + expect(warnings()).toContainEqual( + expect.stringContaining('does not declare the "SpikeModels" target'), + ); + expect(warnings()).toContainEqual( + expect.stringContaining( + `${path.join('Sources', 'HarperModels', 'Models.swift')} was generated for a different module`, + ), + ); + }); + + it('writes nothing when an option is invalid', async () => { + await expect( + regenerateClient( + spikeTables(), + options({ syncProfiles: undefined, kotlinPackage: '../escape' }), + ), + ).rejects.toThrow(/dot-separated identifiers/); + expect(fs.readdirSync(directory)).toEqual([]); + }); +}); diff --git a/utils/client/runtime/HarperRuntime.kt b/utils/client/runtime/HarperRuntime.kt new file mode 100644 index 0000000..bc61984 --- /dev/null +++ b/utils/client/runtime/HarperRuntime.kt @@ -0,0 +1,370 @@ +package harper.models + +import java.math.BigInteger +import java.time.Instant +import java.time.format.DateTimeParseException +import java.util.Base64 +import kotlin.reflect.KClass + +/** A Harper attribute value: the closed set of shapes a record can hold. */ +public sealed interface HarperValue { + public data object Null : HarperValue + + public data class BoolValue(public val value: Boolean) : HarperValue + + public data class LongValue(public val value: Long) : HarperValue + + public data class DoubleValue(public val value: Double) : HarperValue + + public data class StringValue(public val value: String) : HarperValue + + /** Bytes compared by content; the array is copied in and out, so the value stays immutable. */ + public class BytesValue(value: ByteArray) : HarperValue { + private val bytes: ByteArray = value.copyOf() + public val value: ByteArray get() = bytes.copyOf() + + override fun equals(other: Any?): Boolean = other is BytesValue && bytes.contentEquals(other.bytes) + + override fun hashCode(): Int = bytes.contentHashCode() + + override fun toString(): String = "BytesValue(${Base64.getEncoder().encodeToString(bytes)})" + } + + public data class ArrayValue(public val value: List) : HarperValue + + public data class ObjectValue(public val value: Map) : HarperValue + + /** The kind name used in decoding errors. */ + public val kindName: String + get() = when (this) { + Null -> "null" + is BoolValue -> "bool" + is LongValue -> "int" + is DoubleValue -> "double" + is StringValue -> "string" + is BytesValue -> "bytes" + is ArrayValue -> "array" + is ObjectValue -> "object" + } + + public companion object { + /** + * Bridges a JVM value (maps, lists, boxed numbers, strings, byte arrays, instants) into a + * `HarperValue`. Instants become epoch milliseconds, the row encoding of `Date` attributes. + */ + public fun of(value: Any?): HarperValue = when (value) { + null -> Null + is HarperValue -> value + is Boolean -> BoolValue(value) + is Byte, is Short, is Int, is Long -> LongValue((value as Number).toLong()) + is Float, is Double -> DoubleValue((value as Number).toDouble()) + is BigInteger -> if (value.bitLength() < 64) LongValue(value.toLong()) else StringValue(value.toString()) + is String -> StringValue(value) + is ByteArray -> BytesValue(value) + is Instant -> HarperConverters.instant.encode(value) + is Map<*, *> -> ObjectValue(value.entries.associate { (key, item) -> key.toString() to of(item) }) + is Iterable<*> -> ArrayValue(value.map { of(it) }) + is Array<*> -> ArrayValue(value.map { of(it) }) + else -> StringValue(value.toString()) + } + } +} + +/** A record or nested object keyed by attribute name: the logical record, undeclared attributes included. */ +public typealias HarperRow = Map + +/** A conversion failure, before the type and attribute it happened in are known. */ +public class HarperValueMismatch(public val expected: String, public val actual: String) : + Exception("expected $expected but found $actual") + +/** A row that does not match the generated model. */ +public sealed class HarperDecodingError(message: String) : Exception(message) { + public class MissingValue(public val type: String, public val attribute: String) : + HarperDecodingError("$type.$attribute is required but the row has no value") + + public class TypeMismatch( + public val type: String, + public val attribute: String, + public val expected: String, + public val actual: String, + ) : HarperDecodingError("$type.$attribute expected $expected but the row has $actual") +} + +/** A write the generated model refuses to encode. */ +public sealed class HarperEncodingError(message: String) : Exception(message) { + /** A full replacement (`PUT`) would delete or corrupt this attribute; update the record with a patch instead. */ + public class ReplacementUnsupported(public val type: String, public val attribute: String) : + HarperEncodingError("$type cannot be replaced in full: $attribute has no client write contract yet; use a patch") +} + +/** Converts one Kotlin type to and from a `HarperValue`. */ +public interface HarperConverter { + public fun decode(value: HarperValue): T + + public fun encode(value: T): HarperValue +} + +public object HarperConverters { + public val string: HarperConverter = object : HarperConverter { + override fun decode(value: HarperValue): String = + (value as? HarperValue.StringValue)?.value ?: throw HarperValueMismatch("string", value.kindName) + + override fun encode(value: String): HarperValue = HarperValue.StringValue(value) + } + + public val int: HarperConverter = object : HarperConverter { + override fun decode(value: HarperValue): Int = when (value) { + is HarperValue.LongValue -> + if (value.value in Int.MIN_VALUE..Int.MAX_VALUE) value.value.toInt() else throw HarperValueMismatch("int", "out-of-range int") + is HarperValue.DoubleValue -> + value.value.toInt().takeIf { it.toDouble() == value.value } ?: throw HarperValueMismatch("int", "double") + else -> throw HarperValueMismatch("int", value.kindName) + } + + override fun encode(value: Int): HarperValue = HarperValue.LongValue(value.toLong()) + } + + public val long: HarperConverter = object : HarperConverter { + override fun decode(value: HarperValue): Long = when (value) { + is HarperValue.LongValue -> value.value + is HarperValue.DoubleValue -> exactLong(value.value) ?: throw HarperValueMismatch("int", "double") + else -> throw HarperValueMismatch("int", value.kindName) + } + + override fun encode(value: Long): HarperValue = HarperValue.LongValue(value) + } + + public val double: HarperConverter = object : HarperConverter { + override fun decode(value: HarperValue): Double = when (value) { + is HarperValue.DoubleValue -> value.value + is HarperValue.LongValue -> value.value.toDouble() + else -> throw HarperValueMismatch("double", value.kindName) + } + + override fun encode(value: Double): HarperValue = HarperValue.DoubleValue(value) + } + + public val boolean: HarperConverter = object : HarperConverter { + override fun decode(value: HarperValue): Boolean = when { + value is HarperValue.BoolValue -> value.value + value is HarperValue.LongValue && value.value == 0L -> false + value is HarperValue.LongValue && value.value == 1L -> true + else -> throw HarperValueMismatch("bool", value.kindName) + } + + override fun encode(value: Boolean): HarperValue = HarperValue.BoolValue(value) + } + + public val instant: HarperConverter = object : HarperConverter { + override fun decode(value: HarperValue): Instant = when (value) { + is HarperValue.DoubleValue -> instantOfMillis(value.value) + is HarperValue.LongValue -> + if (value.value in -MAX_DATE_MILLIS..MAX_DATE_MILLIS) Instant.ofEpochMilli(value.value) else throw HarperValueMismatch("date", "out-of-range date") + is HarperValue.StringValue -> + try { + Instant.parse(value.value) + } catch (error: DateTimeParseException) { + throw HarperValueMismatch("date", "string") + } + else -> throw HarperValueMismatch("date", value.kindName) + } + + override fun encode(value: Instant): HarperValue = + HarperValue.DoubleValue(value.epochSecond * 1000.0 + value.nano / 1_000_000.0) + } + + public val bytes: HarperConverter = object : HarperConverter { + override fun decode(value: HarperValue): ByteArray = when (value) { + is HarperValue.BytesValue -> value.value + is HarperValue.StringValue -> + try { + Base64.getDecoder().decode(value.value) + } catch (error: IllegalArgumentException) { + throw HarperValueMismatch("bytes", "string") + } + else -> throw HarperValueMismatch("bytes", value.kindName) + } + + override fun encode(value: ByteArray): HarperValue = HarperValue.BytesValue(value) + } + + public val bigInteger: HarperConverter = object : HarperConverter { + override fun decode(value: HarperValue): BigInteger = when (value) { + is HarperValue.LongValue -> BigInteger.valueOf(value.value) + is HarperValue.DoubleValue -> + exactLong(value.value)?.let { BigInteger.valueOf(it) } ?: throw HarperValueMismatch("bigint", "double") + is HarperValue.StringValue -> value.value.toBigIntegerOrNull() ?: throw HarperValueMismatch("bigint", "string") + else -> throw HarperValueMismatch("bigint", value.kindName) + } + + override fun encode(value: BigInteger): HarperValue = HarperValue.StringValue(value.toString()) + } + + public val value: HarperConverter = object : HarperConverter { + override fun decode(value: HarperValue): HarperValue = value + + override fun encode(value: HarperValue): HarperValue = value + } + + public fun nullable(converter: HarperConverter): HarperConverter = object : HarperConverter { + override fun decode(value: HarperValue): T? = if (value == HarperValue.Null) null else converter.decode(value) + + override fun encode(value: T?): HarperValue = if (value == null) HarperValue.Null else converter.encode(value) + } + + public fun list(element: HarperConverter): HarperConverter> = object : HarperConverter> { + override fun decode(value: HarperValue): List = + (value as? HarperValue.ArrayValue)?.value?.map { element.decode(it) } ?: throw HarperValueMismatch("array", value.kindName) + + override fun encode(value: List): HarperValue = HarperValue.ArrayValue(value.map { element.encode(it) }) + } + + /** `toLong()` saturates, and 2^63 saturates to a Long that reads back as the same Double. */ + private fun exactLong(value: Double): Long? = + if (value >= -9.223372036854775808E18 && value < 9.223372036854775808E18 && value == Math.rint(value)) value.toLong() else null + + /** JavaScript's Date range, which is all a Harper date can hold. */ + private const val MAX_DATE_MILLIS: Long = 8_640_000_000_000_000L + + private fun instantOfMillis(milliseconds: Double): Instant { + if (!(milliseconds >= -MAX_DATE_MILLIS.toDouble() && milliseconds <= MAX_DATE_MILLIS.toDouble())) { + throw HarperValueMismatch("date", "out-of-range date") + } + val seconds = Math.floorDiv(milliseconds.toLong(), 1000L) + val nanos = ((milliseconds - seconds * 1000.0) * 1_000_000.0).toLong() + return Instant.ofEpochSecond(seconds, nanos) + } +} + +/** Decodes a required attribute; a missing or null value throws [HarperDecodingError.MissingValue]. */ +public fun HarperRow.harperRequired(attribute: String, type: String, converter: HarperConverter): T { + val value = this[attribute] + if (value == null || value == HarperValue.Null) throw HarperDecodingError.MissingValue(type, attribute) + return try { + converter.decode(value) + } catch (mismatch: HarperValueMismatch) { + throw HarperDecodingError.TypeMismatch(type, attribute, mismatch.expected, mismatch.actual) + } +} + +/** Decodes an optional attribute; a missing or null value is `null`. */ +public fun HarperRow.harperOptional(attribute: String, type: String, converter: HarperConverter): T? { + val value = this[attribute] + if (value == null || value == HarperValue.Null) return null + return try { + converter.decode(value) + } catch (mismatch: HarperValueMismatch) { + throw HarperDecodingError.TypeMismatch(type, attribute, mismatch.expected, mismatch.actual) + } +} + +/** The entries whose keys the schema does not declare. */ +public fun HarperRow.harperExtra(declared: Set): Map = filterKeys { it !in declared } + +/** Equality for generated models: byte arrays compare by content, inside lists too, not by identity. */ +public object HarperContent { + public fun same(a: Any?, b: Any?): Boolean = when { + a is ByteArray && b is ByteArray -> a.contentEquals(b) + a is List<*> && b is List<*> -> a.size == b.size && a.indices.all { same(a[it], b[it]) } + else -> a == b + } + + public fun hash(value: Any?): Int = when (value) { + is ByteArray -> value.contentHashCode() + is List<*> -> value.fold(1) { result, element -> 31 * result + hash(element) } + else -> value?.hashCode() ?: 0 + } +} + +/** The logical type of a stored column. */ +public enum class HarperColumnType { ID, STRING, INT, LONG, FLOAT, BIG_INT, BOOLEAN, DATE, BYTES, BLOB, ANY, ARRAY, OBJECT } + +/** The SQLite storage affinity of a replica column. */ +public enum class HarperAffinity(public val sql: String) { TEXT("TEXT"), INTEGER("INTEGER"), REAL("REAL"), BLOB("BLOB") } + +/** A stored attribute as it appears in the device replica. */ +public data class HarperColumn( + public val name: String, + public val type: HarperColumnType, + public val affinity: HarperAffinity, + public val nullable: Boolean, + public val primaryKey: Boolean = false, + public val indexed: Boolean = false, + public val readOnly: Boolean = false, + /** The `@computed(from:)` expression the client evaluates when applying a row, if any. */ + public val computedFrom: String? = null, +) + +/** A table's replica shape: its columns plus the version and overflow columns. */ +public data class HarperTableSchema( + public val database: String, + public val name: String, + public val primaryKey: String?, + public val sealed: Boolean, + /** The table's wire and storage contract hash. */ + public val hash: String, + public val versionColumn: String, + /** The overflow column holding undeclared attributes as JSON; `null` on `@sealed` tables. */ + public val extraColumn: String?, + /** Whether the record can be replaced in full (`PUT`); false when an attribute has no client write contract. */ + public val replaceable: Boolean, + public val columns: List, +) + +/** A generated table model. */ +public interface HarperRecord { + /** The record version (Harper's last-updated timestamp), when known. */ + public val _version: Double? + + /** Every attribute, undeclared ones included, keyed by attribute name. */ + public fun encodeRow(): HarperRow + + /** The full-replacement (`PUT`) body; throws [HarperEncodingError.ReplacementUnsupported] when the table has none. */ + public fun encodeUpsertRow(): HarperRow +} + +/** A generated model's companion: its replica schema and decoder. */ +public interface HarperTable : HarperConverter { + public val schema: HarperTableSchema + public val recordClass: KClass + + /** Decodes a stored or fetched record; throws [HarperDecodingError] when the row does not match the schema. */ + public fun decode(row: HarperRow, version: Double? = null): T + + override fun decode(value: HarperValue): T = + decode((value as? HarperValue.ObjectValue)?.value ?: throw HarperValueMismatch("object", value.kindName), null) + + override fun encode(value: T): HarperValue = HarperValue.ObjectValue(value.encodeRow()) +} + +/** A generated nested object type. */ +public interface HarperObject { + public fun encodeRow(): HarperRow +} + +/** A generated nested object type's companion: its decoder. */ +public interface HarperObjectType : HarperConverter { + public fun decode(row: HarperRow): T + + override fun decode(value: HarperValue): T = + decode((value as? HarperValue.ObjectValue)?.value ?: throw HarperValueMismatch("object", value.kindName)) + + override fun encode(value: T): HarperValue = HarperValue.ObjectValue(value.encodeRow()) +} + +/** An insert (`POST`) or patch body. */ +public interface HarperWritable { + public fun encodeRow(): HarperRow +} + +/** A sync profile from `sync.yaml`: which tables replicate, in which direction, and how long a device may stay offline. */ +public data class HarperSyncProfile( + public val name: String, + public val direction: Direction, + public val retentionMs: Long?, + /** The schema hash a device sends in its `hello` frame for this profile. */ + public val schemaHash: String, + public val tables: List, +) { + public enum class Direction { PULL, PUSH, BIDIRECTIONAL } +} diff --git a/utils/client/runtime/HarperRuntime.swift b/utils/client/runtime/HarperRuntime.swift new file mode 100644 index 0000000..c30596d --- /dev/null +++ b/utils/client/runtime/HarperRuntime.swift @@ -0,0 +1,572 @@ +import Foundation + +/// A Harper attribute value: the closed set of shapes a record can hold. +public enum HarperValue: Sendable, Hashable { + case null + case bool(Bool) + case int(Int64) + case double(Double) + case string(String) + case bytes(Data) + case array([HarperValue]) + case object([String: HarperValue]) +} + +/// A record or nested object keyed by attribute name: the logical record, undeclared attributes included. +public typealias HarperRow = [String: HarperValue] + +extension HarperValue { + /// The kind name used in decoding errors. + public var kindName: String { + switch self { + case .null: return "null" + case .bool: return "bool" + case .int: return "int" + case .double: return "double" + case .string: return "string" + case .bytes: return "bytes" + case .array: return "array" + case .object: return "object" + } + } + + /// Bridges a Foundation value (`JSONSerialization` output, SQLite column values) into a `HarperValue`. + /// Dates become epoch milliseconds, the row encoding of `Date` attributes. + public init(any value: Any?) { + guard let value else { + self = .null + return + } + switch value { + case let harperValue as HarperValue: + self = harperValue + case is NSNull: + self = .null + case let string as String: + self = .string(string) + case let number as NSNumber: + if CFGetTypeID(number) == CFBooleanGetTypeID() { + self = .bool(number.boolValue) + } else if CFNumberIsFloatType(number) { + self = .double(number.doubleValue) + } else { + self = .int(number.int64Value) + } + case let data as Data: + self = .bytes(data) + case let date as Date: + self = .double(date.timeIntervalSince1970 * 1000) + case let array as [Any?]: + self = .array(array.map { HarperValue(any: $0) }) + case let dictionary as [String: Any?]: + self = .object(dictionary.mapValues { HarperValue(any: $0) }) + default: + self = .string(String(describing: value)) + } + } + + /// A `JSONSerialization`-compatible value; bytes become base64 strings and non-finite doubles become null. + public var jsonObject: Any { + switch self { + case .null: return NSNull() + case .bool(let value): return value + case .int(let value): return value + case .double(let value): return value.isFinite ? value : NSNull() + case .string(let value): return value + case .bytes(let value): return value.base64EncodedString() + case .array(let values): return values.map(\.jsonObject) + case .object(let values): return values.mapValues(\.jsonObject) + } + } +} + +extension HarperValue: Codable { + public init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + if container.decodeNil() { + self = .null + } else if let value = try? container.decode(Bool.self) { + self = .bool(value) + } else if let value = try? container.decode(Int64.self) { + self = .int(value) + } else if let value = try? container.decode(Double.self) { + self = .double(value) + } else if let value = try? container.decode(String.self) { + self = .string(value) + } else if let value = try? container.decode([HarperValue].self) { + self = .array(value) + } else { + self = .object(try container.decode([String: HarperValue].self)) + } + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.singleValueContainer() + switch self { + case .null: try container.encodeNil() + case .bool(let value): try container.encode(value) + case .int(let value): try container.encode(value) + case .double(let value): try value.isFinite ? container.encode(value) : container.encodeNil() + case .string(let value): try container.encode(value) + case .bytes(let value): try container.encode(value.base64EncodedString()) + case .array(let values): try container.encode(values) + case .object(let values): try container.encode(values) + } + } +} + +/// An arbitrary-precision integer carried as its canonical decimal string. +public struct HarperBigInt: Sendable, Hashable, Codable, Comparable, CustomStringConvertible, ExpressibleByIntegerLiteral { + /// Decimal digits with an optional leading `-`, no leading zeros, and no `-0`. + public let decimalString: String + + public init?(_ text: String) { + var digits = Substring(text) + var negative = false + if digits.first == "-" || digits.first == "+" { + negative = digits.first == "-" + digits = digits.dropFirst() + } + guard !digits.isEmpty, digits.allSatisfy({ $0 >= "0" && $0 <= "9" }) else { return nil } + let significant = digits.drop(while: { $0 == "0" }) + decimalString = significant.isEmpty ? "0" : (negative ? "-" : "") + significant + } + + public init(_ value: Int64) { + decimalString = String(value) + } + + public init(integerLiteral value: Int64) { + self.init(value) + } + + /// The value as an `Int64`, or `nil` when it does not fit. + public var int64Value: Int64? { Int64(decimalString) } + + public var description: String { decimalString } + + public init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + if let integer = try? container.decode(Int64.self) { + self.init(integer) + } else if let value = HarperBigInt(try container.decode(String.self)) { + self = value + } else { + throw DecodingError.dataCorruptedError(in: container, debugDescription: "expected a decimal integer") + } + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.singleValueContainer() + try container.encode(decimalString) + } + + public static func < (lhs: HarperBigInt, rhs: HarperBigInt) -> Bool { + let lhsNegative = lhs.decimalString.hasPrefix("-") + let rhsNegative = rhs.decimalString.hasPrefix("-") + if lhsNegative != rhsNegative { return lhsNegative } + let lhsDigits = lhsNegative ? lhs.decimalString.dropFirst() : Substring(lhs.decimalString) + let rhsDigits = rhsNegative ? rhs.decimalString.dropFirst() : Substring(rhs.decimalString) + if lhsDigits == rhsDigits { return false } + let magnitudeLess = lhsDigits.count != rhsDigits.count ? lhsDigits.count < rhsDigits.count : lhsDigits < rhsDigits + return lhsNegative ? !magnitudeLess : magnitudeLess + } +} + +/// A value that converts to and from a `HarperValue`. +public protocol HarperValueConvertible: Sendable { + init(harperValue: HarperValue) throws + var harperValue: HarperValue { get } +} + +/// A conversion failure, before the type and attribute it happened in are known. +public struct HarperValueMismatch: Error, Sendable { + public let expected: String + public let actual: String + + public init(expected: String, actual: String) { + self.expected = expected + self.actual = actual + } +} + +/// A row that does not match the generated model. +public enum HarperDecodingError: Error, Sendable, Hashable, CustomStringConvertible { + case missingValue(type: String, attribute: String) + case typeMismatch(type: String, attribute: String, expected: String, actual: String) + + public var description: String { + switch self { + case .missingValue(let type, let attribute): + return "\(type).\(attribute) is required but the row has no value" + case .typeMismatch(let type, let attribute, let expected, let actual): + return "\(type).\(attribute) expected \(expected) but the row has \(actual)" + } + } +} + +/// A write the generated model refuses to encode. +public enum HarperEncodingError: Error, Sendable, Hashable, CustomStringConvertible { + /// A full replacement (`PUT`) would delete or corrupt this attribute; update the record with a patch instead. + case replacementUnsupported(type: String, attribute: String) + + public var description: String { + switch self { + case .replacementUnsupported(let type, let attribute): + return "\(type) cannot be replaced in full: \(attribute) has no client write contract yet; use a patch" + } + } +} + +extension String: HarperValueConvertible { + public init(harperValue: HarperValue) throws { + guard case .string(let value) = harperValue else { + throw HarperValueMismatch(expected: "string", actual: harperValue.kindName) + } + self = value + } + + public var harperValue: HarperValue { .string(self) } +} + +extension Int: HarperValueConvertible { + public init(harperValue: HarperValue) throws { + switch harperValue { + case .int(let value): + guard let exact = Int(exactly: value) else { throw HarperValueMismatch(expected: "int", actual: "out-of-range int") } + self = exact + case .double(let value): + guard let exact = Int(exactly: value) else { throw HarperValueMismatch(expected: "int", actual: "double") } + self = exact + default: + throw HarperValueMismatch(expected: "int", actual: harperValue.kindName) + } + } + + public var harperValue: HarperValue { .int(Int64(self)) } +} + +extension Int64: HarperValueConvertible { + public init(harperValue: HarperValue) throws { + switch harperValue { + case .int(let value): + self = value + case .double(let value): + guard let exact = Int64(exactly: value) else { throw HarperValueMismatch(expected: "int", actual: "double") } + self = exact + default: + throw HarperValueMismatch(expected: "int", actual: harperValue.kindName) + } + } + + public var harperValue: HarperValue { .int(self) } +} + +extension Double: HarperValueConvertible { + public init(harperValue: HarperValue) throws { + switch harperValue { + case .double(let value): self = value + case .int(let value): self = Double(value) + default: throw HarperValueMismatch(expected: "double", actual: harperValue.kindName) + } + } + + public var harperValue: HarperValue { .double(self) } +} + +extension Bool: HarperValueConvertible { + public init(harperValue: HarperValue) throws { + switch harperValue { + case .bool(let value): self = value + case .int(0): self = false + case .int(1): self = true + default: throw HarperValueMismatch(expected: "bool", actual: harperValue.kindName) + } + } + + public var harperValue: HarperValue { .bool(self) } +} + +extension Date: HarperValueConvertible { + public init(harperValue: HarperValue) throws { + switch harperValue { + case .double(let milliseconds): + guard milliseconds >= -8.64e15, milliseconds <= 8.64e15 else { throw HarperValueMismatch(expected: "date", actual: "out-of-range date") } + self = Date(timeIntervalSince1970: milliseconds / 1000) + case .int(let milliseconds): + guard milliseconds >= -8_640_000_000_000_000, milliseconds <= 8_640_000_000_000_000 else { throw HarperValueMismatch(expected: "date", actual: "out-of-range date") } + self = Date(timeIntervalSince1970: Double(milliseconds) / 1000) + case .string(let text): + let formatter = ISO8601DateFormatter() + formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] + if let date = formatter.date(from: text) { + self = date + return + } + formatter.formatOptions = [.withInternetDateTime] + guard let date = formatter.date(from: text) else { throw HarperValueMismatch(expected: "date", actual: "string") } + self = date + default: + throw HarperValueMismatch(expected: "date", actual: harperValue.kindName) + } + } + + public var harperValue: HarperValue { .double(timeIntervalSince1970 * 1000) } +} + +extension Data: HarperValueConvertible { + public init(harperValue: HarperValue) throws { + switch harperValue { + case .bytes(let value): + self = value + case .string(let base64): + guard let value = Data(base64Encoded: base64) else { throw HarperValueMismatch(expected: "bytes", actual: "string") } + self = value + default: + throw HarperValueMismatch(expected: "bytes", actual: harperValue.kindName) + } + } + + public var harperValue: HarperValue { .bytes(self) } +} + +extension HarperBigInt: HarperValueConvertible { + public init(harperValue: HarperValue) throws { + switch harperValue { + case .int(let value): + self.init(value) + case .double(let value): + guard let exact = Int64(exactly: value) else { throw HarperValueMismatch(expected: "bigint", actual: "double") } + self.init(exact) + case .string(let text): + guard let value = HarperBigInt(text) else { throw HarperValueMismatch(expected: "bigint", actual: "string") } + self = value + default: + throw HarperValueMismatch(expected: "bigint", actual: harperValue.kindName) + } + } + + public var harperValue: HarperValue { .string(decimalString) } +} + +extension HarperValue: HarperValueConvertible { + public init(harperValue: HarperValue) throws { + self = harperValue + } + + public var harperValue: HarperValue { self } +} + +extension Optional: HarperValueConvertible where Wrapped: HarperValueConvertible { + public init(harperValue: HarperValue) throws { + if case .null = harperValue { + self = nil + } else { + self = try Wrapped(harperValue: harperValue) + } + } + + public var harperValue: HarperValue { self?.harperValue ?? .null } +} + +extension Array: HarperValueConvertible where Element: HarperValueConvertible { + public init(harperValue: HarperValue) throws { + guard case .array(let values) = harperValue else { + throw HarperValueMismatch(expected: "array", actual: harperValue.kindName) + } + self = try values.map { try Element(harperValue: $0) } + } + + public var harperValue: HarperValue { .array(map(\.harperValue)) } +} + +/// Holds a nested value by reference, so a generated struct can contain a type that contains it. +public final class HarperBox: HarperValueConvertible, Hashable, Sendable { + public let value: Wrapped + + public init(_ value: Wrapped) { + self.value = value + } + + public init(harperValue: HarperValue) throws { + value = try Wrapped(harperValue: harperValue) + } + + public var harperValue: HarperValue { value.harperValue } + + public static func == (lhs: HarperBox, rhs: HarperBox) -> Bool { + lhs.value == rhs.value + } + + public func hash(into hasher: inout Hasher) { + hasher.combine(value) + } +} + +extension Dictionary where Key == String, Value == HarperValue { + /// Decodes a required attribute; a missing or null value throws `HarperDecodingError.missingValue`. + public func harperRequired(_ attribute: String, of type: String) throws -> T { + guard let value = self[attribute], value != .null else { + throw HarperDecodingError.missingValue(type: type, attribute: attribute) + } + do { + return try T(harperValue: value) + } catch let mismatch as HarperValueMismatch { + throw HarperDecodingError.typeMismatch(type: type, attribute: attribute, expected: mismatch.expected, actual: mismatch.actual) + } + } + + /// Decodes an optional attribute; a missing or null value is `nil`. + public func harperOptional(_ attribute: String, of type: String) throws -> T? { + guard let value = self[attribute], value != .null else { return nil } + do { + return try T(harperValue: value) + } catch let mismatch as HarperValueMismatch { + throw HarperDecodingError.typeMismatch(type: type, attribute: attribute, expected: mismatch.expected, actual: mismatch.actual) + } + } + + /// The entries whose keys the schema does not declare. + public func harperExtra(excluding declared: Set) -> [String: HarperValue] { + filter { !declared.contains($0.key) } + } +} + +/// The logical type of a stored column. +public enum HarperColumnType: String, Sendable, Hashable { + case id, string, int, long, float, bigInt, boolean, date, bytes, blob, any, array, object +} + +/// The SQLite storage affinity of a replica column. +public enum HarperAffinity: String, Sendable, Hashable { + case text = "TEXT" + case integer = "INTEGER" + case real = "REAL" + case blob = "BLOB" +} + +/// A stored attribute as it appears in the device replica. +public struct HarperColumn: Sendable, Hashable { + public let name: String + public let type: HarperColumnType + public let affinity: HarperAffinity + public let nullable: Bool + public let primaryKey: Bool + public let indexed: Bool + public let readOnly: Bool + /// The `@computed(from:)` expression the client evaluates when applying a row, if any. + public let computedFrom: String? + + public init(name: String, type: HarperColumnType, affinity: HarperAffinity, nullable: Bool, primaryKey: Bool = false, indexed: Bool = false, readOnly: Bool = false, computedFrom: String? = nil) { + self.name = name + self.type = type + self.affinity = affinity + self.nullable = nullable + self.primaryKey = primaryKey + self.indexed = indexed + self.readOnly = readOnly + self.computedFrom = computedFrom + } +} + +/// A table's replica shape: its columns plus the version and overflow columns. +public struct HarperTableSchema: Sendable, Hashable { + public let database: String + public let name: String + public let primaryKey: String? + public let sealed: Bool + /// The table's wire and storage contract hash. + public let hash: String + public let versionColumn: String + /// The overflow column holding undeclared attributes as JSON; `nil` on `@sealed` tables. + public let extraColumn: String? + /// Whether the record can be replaced in full (`PUT`); false when an attribute has no client write contract. + public let replaceable: Bool + public let columns: [HarperColumn] + + public init(database: String, name: String, primaryKey: String?, sealed: Bool, hash: String, versionColumn: String, extraColumn: String?, replaceable: Bool, columns: [HarperColumn]) { + self.database = database + self.name = name + self.primaryKey = primaryKey + self.sealed = sealed + self.hash = hash + self.versionColumn = versionColumn + self.extraColumn = extraColumn + self.replaceable = replaceable + self.columns = columns + } +} + +/// A generated table model. +public protocol HarperRecord: HarperValueConvertible, Hashable { + static var schema: HarperTableSchema { get } + /// Decodes a stored or fetched record; throws `HarperDecodingError` when the row does not match the schema. + init(row: HarperRow, version: Double?) throws + /// The record version (Harper's last-updated timestamp), when known. + var _version: Double? { get } + /// Every attribute, undeclared ones included, keyed by attribute name. + func encodeRow() -> HarperRow + /// The full-replacement (`PUT`) body: the primary key, client-writable attributes and undeclared attributes. + func encodeUpsertRow() throws -> HarperRow +} + +extension HarperRecord { + public static var tableName: String { schema.name } + + public init(row: HarperRow) throws { + try self.init(row: row, version: nil) + } + + public init(harperValue: HarperValue) throws { + guard case .object(let row) = harperValue else { + throw HarperValueMismatch(expected: "object", actual: harperValue.kindName) + } + try self.init(row: row, version: nil) + } + + public var harperValue: HarperValue { .object(encodeRow()) } +} + +/// A generated nested object type. +public protocol HarperObject: HarperValueConvertible, Hashable { + init(row: HarperRow) throws + func encodeRow() -> HarperRow +} + +extension HarperObject { + public init(harperValue: HarperValue) throws { + guard case .object(let row) = harperValue else { + throw HarperValueMismatch(expected: "object", actual: harperValue.kindName) + } + try self.init(row: row) + } + + public var harperValue: HarperValue { .object(encodeRow()) } +} + +/// An insert (`POST`) or patch body. +public protocol HarperWritable: Sendable, Hashable { + func encodeRow() -> HarperRow +} + +/// A sync profile from `sync.yaml`: which tables replicate, in which direction, and how long a device may stay offline. +public struct HarperSyncProfile: Sendable, Hashable { + public enum Direction: String, Sendable, Hashable { + case pull, push, bidirectional + } + + public let name: String + public let direction: Direction + public let retentionMs: Int64? + /// The schema hash a device sends in its `hello` frame for this profile. + public let schemaHash: String + public let tables: [HarperTableSchema] + + public init(name: String, direction: Direction, retentionMs: Int64?, schemaHash: String, tables: [HarperTableSchema]) { + self.name = name + self.direction = direction + self.retentionMs = retentionMs + self.schemaHash = schemaHash + self.tables = tables + } +} diff --git a/utils/client/syncProfiles.js b/utils/client/syncProfiles.js new file mode 100644 index 0000000..7b37f7b --- /dev/null +++ b/utils/client/syncProfiles.js @@ -0,0 +1,379 @@ +/** @import { IRDiagnostic, IRProfile, IRProfileTable, IRScope, IRScopeCondition, IRTable } from './irTypes.js' */ +import fs from 'node:fs'; +import path from 'node:path'; + +const DIRECTIONS = new Set(['pull', 'push', 'bidirectional']); +const PROFILE_KEYS = new Set([ + 'description', + 'direction', + 'retention', + 'database', + 'scope', + 'tables', +]); +const TABLE_KEYS = new Set(['database', 'scope']); +const DURATION_UNITS = /** @type {Record} */ ({ + ms: 1, + s: 1000, + m: 60_000, + h: 3_600_000, + d: 86_400_000, + w: 604_800_000, +}); + +/** + * Parses a retention duration such as `14d`, `12h` or `1.5w` into milliseconds. + * @param {unknown} text + * @returns {number | null} null when the text is not a positive duration with a unit + */ +export function parseDuration(text) { + if (typeof text !== 'string') return null; + const match = /^\s*(\d+(?:\.\d+)?)\s*(ms|s|m|h|d|w)\s*$/.exec(text); + if (!match) return null; + const milliseconds = Math.round(Number(match[1]) * DURATION_UNITS[match[2]]); + return milliseconds > 0 && Number.isSafeInteger(milliseconds) ? milliseconds : null; +} + +/** + * Reads a `sync.yaml` (or JSON) file. The YAML parser is loaded only when this is called. + * @param {string} filePath + * @returns {Promise} the parsed document, or undefined when the file does not exist + */ +export async function loadSyncProfilesFile(filePath) { + if (!fs.existsSync(filePath)) return undefined; + const text = fs.readFileSync(filePath, 'utf8'); + if (path.extname(filePath).toLowerCase() === '.json') return JSON.parse(text); + const { parse } = await import('yaml'); + return parse(text); +} + +/** + * @param {unknown} value + * @returns {value is Record} + */ +function isPlainObject(value) { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +/** + * @param {unknown} value + * @returns {boolean} + */ +function isScalar(value) { + return ( + typeof value === 'string' || + typeof value === 'boolean' || + (typeof value === 'number' && Number.isFinite(value)) + ); +} + +/** + * Normalizes the `profiles` section of a `sync.yaml` document against the IR's tables. + * + * Profiles fail closed: any error invalidates the whole profile, which is then left out of the + * result and reported as an `error` diagnostic, so a mistyped scope can never widen into `all`. + * @param {unknown} document the parsed `sync.yaml` + * @param {IRTable[]} tables the IR tables profiles may reference + * @param {IRDiagnostic[]} diagnostics receives errors and warnings + * @returns {Omit[]} the valid profiles, sorted by name + */ +export function normalizeSyncProfiles(document, tables, diagnostics) { + if (document === undefined || document === null) return []; + if (!isPlainObject(document) || !isPlainObject(document.profiles)) { + diagnostics.push({ + level: 'error', + code: 'PROFILES_INVALID', + message: 'sync profiles must be a document with a `profiles` map', + }); + return []; + } + const tablesByKey = new Map( + tables.map((table) => [`${table.database}\u0000${table.name}`, table]), + ); + /** @type {Omit[]} */ + const profiles = []; + for (const name of Object.keys(document.profiles).sort()) { + /** + * @param {string} code + * @param {string} message + * @param {{ table?: string, attribute?: string }} [where] + */ + const report = (code, message, where = {}) => + diagnostics.push({ level: 'error', code, profile: name, ...where, message }); + const profile = normalizeProfile( + name, + document.profiles[name], + tablesByKey, + report, + diagnostics, + ); + if (profile) profiles.push(profile); + } + return profiles; +} + +/** + * @param {string} name + * @param {unknown} definition + * @param {Map} tablesByKey + * @param {(code: string, message: string, where?: { table?: string, attribute?: string }) => void} report + * @param {IRDiagnostic[]} diagnostics + * @returns {Omit | null} null when any part of the profile is invalid + */ +function normalizeProfile(name, definition, tablesByKey, report, diagnostics) { + let failed = false; + /** @type {typeof report} */ + const fail = (code, message, where) => { + failed = true; + report(code, message, where); + }; + if (!isPlainObject(definition)) { + fail('PROFILE_INVALID', `profile "${name}" must be a map`); + return null; + } + for (const key of Object.keys(definition)) { + if (!PROFILE_KEYS.has(key)) + fail('PROFILE_UNKNOWN_KEY', `profile "${name}" has an unknown key "${key}"`); + } + const direction = definition.direction ?? 'pull'; + if (typeof direction !== 'string' || !DIRECTIONS.has(direction)) { + fail( + 'PROFILE_DIRECTION_INVALID', + `profile "${name}" direction must be pull, push or bidirectional`, + ); + } + /** @type {string | null} */ + let retention = null; + /** @type {number | null} */ + let retentionMs = null; + if (definition.retention !== undefined && definition.retention !== null) { + retentionMs = parseDuration(definition.retention); + if (retentionMs === null) { + fail( + 'PROFILE_RETENTION_INVALID', + `profile "${name}" retention must be a duration such as 14d`, + ); + } else { + retention = String(definition.retention).trim(); + } + } + if (definition.description !== undefined && typeof definition.description !== 'string') { + fail('PROFILE_INVALID', `profile "${name}" description must be a string`); + } + const defaultDatabase = definition.database ?? 'data'; + if (typeof defaultDatabase !== 'string' || defaultDatabase === '') { + fail('PROFILE_DATABASE_INVALID', `profile "${name}" database must be a database name`); + } + + /** @type {{ table: string, database: unknown, scope: unknown }[]} */ + const entries = []; + const tablesDefinition = definition.tables; + if (Array.isArray(tablesDefinition)) { + for (const tableName of tablesDefinition) { + if (typeof tableName !== 'string' || tableName === '') { + fail('PROFILE_TABLES_INVALID', `profile "${name}" lists a table that is not a name`); + continue; + } + entries.push({ table: tableName, database: defaultDatabase, scope: definition.scope }); + } + } else if (isPlainObject(tablesDefinition)) { + for (const [tableName, tableDefinition] of Object.entries(tablesDefinition)) { + if (tableDefinition !== null && !isPlainObject(tableDefinition)) { + fail('PROFILE_TABLES_INVALID', `profile "${name}" table "${tableName}" must be a map`, { + table: tableName, + }); + continue; + } + const overrides = tableDefinition ?? {}; + for (const key of Object.keys(overrides)) { + if (!TABLE_KEYS.has(key)) { + fail( + 'PROFILE_UNKNOWN_KEY', + `profile "${name}" table "${tableName}" has an unknown key "${key}"`, + { + table: tableName, + }, + ); + } + } + entries.push({ + table: tableName, + database: overrides.database ?? defaultDatabase, + scope: 'scope' in overrides ? overrides.scope : definition.scope, + }); + } + } else { + fail('PROFILE_TABLES_INVALID', `profile "${name}" must list its tables`); + } + if (entries.length === 0) { + fail('PROFILE_TABLES_INVALID', `profile "${name}" must list at least one table`); + } + + /** @type {IRProfileTable[]} */ + const profileTables = []; + const seen = new Set(); + for (const entry of entries) { + if (typeof entry.database !== 'string' || entry.database === '') { + fail( + 'PROFILE_DATABASE_INVALID', + `profile "${name}" table "${entry.table}" database must be a name`, + { + table: entry.table, + }, + ); + continue; + } + const key = `${entry.database}\u0000${entry.table}`; + if (seen.has(key)) { + fail( + 'PROFILE_TABLE_DUPLICATE', + `profile "${name}" lists table "${entry.table}" more than once`, + { + table: entry.table, + }, + ); + continue; + } + seen.add(key); + const table = tablesByKey.get(key); + if (!table) { + fail( + 'PROFILE_TABLE_UNKNOWN', + `profile "${name}" references table "${entry.table}" in database "${entry.database}", which codegen does not see`, + { table: entry.table }, + ); + continue; + } + const scope = normalizeScope(name, table, entry.scope, fail, diagnostics); + if (scope) profileTables.push({ database: table.database, table: table.name, scope }); + } + profileTables.sort( + (a, b) => compareText(a.database, b.database) || compareText(a.table, b.table), + ); + if (failed) return null; + + return { + name, + ...(typeof definition.description === 'string' ? { description: definition.description } : {}), + direction: /** @type {IRProfile['direction']} */ (direction), + retention, + retentionMs, + tables: profileTables, + }; +} + +/** + * @param {string} profileName + * @param {IRTable} table + * @param {unknown} scope + * @param {(code: string, message: string, where?: { table?: string, attribute?: string }) => void} fail + * @param {IRDiagnostic[]} diagnostics + * @returns {IRScope | null} + */ +function normalizeScope(profileName, table, scope, fail, diagnostics) { + if (scope === undefined || scope === 'all') return { type: 'all' }; + const where = { table: table.name }; + if (!isPlainObject(scope) || Object.keys(scope).length === 0) { + fail( + 'SCOPE_INVALID', + `profile "${profileName}" scope for "${table.name}" must be "all" or a map of attribute to value`, + where, + ); + return null; + } + /** @type {IRScopeCondition[]} */ + const conditions = []; + for (const attributeName of Object.keys(scope).sort()) { + const attribute = table.attributes.find((candidate) => candidate.name === attributeName); + const at = { table: table.name, attribute: attributeName }; + if (!attribute) { + fail( + 'SCOPE_ATTRIBUTE_UNKNOWN', + `profile "${profileName}" scopes "${table.name}" by "${attributeName}", which is not a stored attribute`, + at, + ); + continue; + } + if (attribute.type.kind !== 'scalar') { + fail( + 'SCOPE_ATTRIBUTE_NOT_SCALAR', + `profile "${profileName}" scopes "${table.name}" by "${attributeName}", which is not a scalar attribute`, + at, + ); + continue; + } + if (!attribute.indexed && !attribute.primaryKey) { + diagnostics.push({ + level: 'warning', + code: 'SCOPE_ATTRIBUTE_NOT_INDEXED', + profile: profileName, + ...at, + message: `profile "${profileName}" scopes "${table.name}" by "${attributeName}", which is not @indexed; checkout scans the table`, + }); + } + const condition = normalizeCondition(attributeName, scope[attributeName]); + if (typeof condition === 'string') { + fail( + 'SCOPE_VALUE_INVALID', + `profile "${profileName}" scope "${table.name}.${attributeName}": ${condition}`, + at, + ); + continue; + } + conditions.push(condition); + } + return { type: 'match', conditions }; +} + +/** + * @param {string} attribute + * @param {unknown} value + * @returns {IRScopeCondition | string} the condition, or why the value is invalid + */ +function normalizeCondition(attribute, value) { + if (Array.isArray(value)) { + if (value.length === 0) return 'a membership list must not be empty'; + /** @type {(string | number | boolean)[]} */ + const values = []; + for (const item of value) { + const literal = literalOf(item); + if (typeof literal === 'object') return literal.error; + values.push(literal); + } + return { attribute, operator: 'in', source: 'literal', value: values }; + } + if (typeof value === 'string' && value.startsWith('$') && !value.startsWith('$$')) { + const reference = /^\$(token|user)\.(.+)$/.exec(value); + if (!reference) return `"${value}" is not a $token. or $user. reference`; + return reference[1] === 'token' + ? { attribute, operator: 'equals', source: 'claim', claim: reference[2] } + : { attribute, operator: 'equals', source: 'user', field: reference[2] }; + } + const literal = literalOf(value); + if (typeof literal === 'object') return literal.error; + return { attribute, operator: 'equals', source: 'literal', value: literal }; +} + +/** + * @param {unknown} value + * @returns {string | number | boolean | { error: string }} + */ +function literalOf(value) { + if (!isScalar(value)) return { error: 'values must be strings, numbers or booleans' }; + const scalar = /** @type {string | number | boolean} */ (value); + if (typeof scalar === 'string' && scalar.startsWith('$')) { + if (!scalar.startsWith('$$')) + return { error: 'references are not allowed inside a membership list' }; + return scalar.slice(1); + } + return scalar; +} + +/** + * @param {string} a + * @param {string} b + * @returns {number} + */ +export function compareText(a, b) { + return a < b ? -1 : a > b ? 1 : 0; +} diff --git a/utils/client/syncProfiles.test.js b/utils/client/syncProfiles.test.js new file mode 100644 index 0000000..4b15d60 --- /dev/null +++ b/utils/client/syncProfiles.test.js @@ -0,0 +1,298 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { coverageTables, coverageTypes, spikeTables } from '../../test/fixtures/clientSchema.js'; +import { buildSchemaIR } from './buildSchemaIR.js'; +import { loadSyncProfilesFile, normalizeSyncProfiles, parseDuration } from './syncProfiles.js'; + +const { tables } = buildSchemaIR({ + tables: [...spikeTables(), ...coverageTables()], + types: coverageTypes(), +}); + +/** + * @param {unknown} document + */ +function normalize(document) { + /** @type {import('./irTypes.js').IRDiagnostic[]} */ + const diagnostics = []; + const profiles = normalizeSyncProfiles(document, tables, diagnostics); + return { profiles, diagnostics }; +} + +describe('parseDuration', () => { + it('parses durations with units into milliseconds', () => { + expect(parseDuration('14d')).toBe(14 * 86_400_000); + expect(parseDuration(' 12h ')).toBe(12 * 3_600_000); + expect(parseDuration('1.5w')).toBe(1.5 * 604_800_000); + expect(parseDuration('250ms')).toBe(250); + }); + + it('rejects durations without a unit, zero, and non-strings', () => { + expect(parseDuration('14')).toBeNull(); + expect(parseDuration('0d')).toBeNull(); + expect(parseDuration(14)).toBeNull(); + expect(parseDuration('14 days')).toBeNull(); + }); + + it('rejects durations that round to nothing or exceed an exact integer', () => { + expect(parseDuration('0.4ms')).toBeNull(); + expect(parseDuration('99999999999999w')).toBeNull(); + }); +}); + +describe('normalizeSyncProfiles', () => { + it('returns no profiles when there is no document', () => { + expect(normalize(undefined)).toEqual({ profiles: [], diagnostics: [] }); + }); + + it('normalizes the list form with defaults and a profile-level scope', () => { + const { profiles, diagnostics } = normalize({ + profiles: { + storefront: { retention: '14d', tables: ['Product'] }, + 'my-orders': { retention: '30d', scope: { customerId: '$token.sub' }, tables: ['Order'] }, + }, + }); + expect(diagnostics).toEqual([]); + expect(profiles).toEqual([ + { + name: 'my-orders', + direction: 'pull', + retention: '30d', + retentionMs: 30 * 86_400_000, + tables: [ + { + database: 'data', + table: 'Order', + scope: { + type: 'match', + conditions: [ + { attribute: 'customerId', operator: 'equals', source: 'claim', claim: 'sub' }, + ], + }, + }, + ], + }, + { + name: 'storefront', + direction: 'pull', + retention: '14d', + retentionMs: 14 * 86_400_000, + tables: [{ database: 'data', table: 'Product', scope: { type: 'all' } }], + }, + ]); + }); + + it('normalizes the map form with per-table databases and scope overrides', () => { + const { profiles, diagnostics } = normalize({ + profiles: { + mixed: { + direction: 'bidirectional', + database: 'coverage', + scope: { email: '$user.email' }, + description: 'customers and their purchases', + tables: { + Customer: null, + Purchase: { scope: { customerId: ['a', 'b', '$$literal'] } }, + Order: { database: 'data', scope: 'all' }, + }, + }, + }, + }); + expect(diagnostics).toEqual([]); + expect(profiles[0]).toMatchObject({ + name: 'mixed', + direction: 'bidirectional', + description: 'customers and their purchases', + }); + expect(profiles[0].tables).toEqual([ + { + database: 'coverage', + table: 'Customer', + scope: { + type: 'match', + conditions: [{ attribute: 'email', operator: 'equals', source: 'user', field: 'email' }], + }, + }, + { + database: 'coverage', + table: 'Purchase', + scope: { + type: 'match', + conditions: [ + { + attribute: 'customerId', + operator: 'in', + source: 'literal', + value: ['a', 'b', '$literal'], + }, + ], + }, + }, + { database: 'data', table: 'Order', scope: { type: 'all' } }, + ]); + }); + + it('treats scalars as literal equality and $$ as an escaped dollar', () => { + const { profiles } = normalize({ + profiles: { + p: { tables: { Order: { scope: { status: '$$open', amount: 5, customerId: true } } } }, + }, + }); + expect(profiles[0].tables[0].scope).toEqual({ + type: 'match', + conditions: [ + { attribute: 'amount', operator: 'equals', source: 'literal', value: 5 }, + { attribute: 'customerId', operator: 'equals', source: 'literal', value: true }, + { attribute: 'status', operator: 'equals', source: 'literal', value: '$open' }, + ], + }); + }); + + it('warns, but keeps the profile, when a scope attribute is not indexed', () => { + const { profiles, diagnostics } = normalize({ + profiles: { p: { scope: { amount: 1 }, tables: ['Order'] } }, + }); + expect(profiles).toHaveLength(1); + expect(diagnostics).toEqual([ + expect.objectContaining({ + level: 'warning', + code: 'SCOPE_ATTRIBUTE_NOT_INDEXED', + profile: 'p', + }), + ]); + }); + + it('does not warn when scoping by the primary key', () => { + expect( + normalize({ profiles: { p: { scope: { id: 'x' }, tables: ['Order'] } } }).diagnostics, + ).toEqual([]); + }); + + /** Every profile below has exactly one error; each must be left out entirely (fail closed). */ + const invalid = /** @type {[string, unknown, string][]} */ ([ + [ + 'a mistyped scope attribute', + { scope: { custmerId: '$token.sub' }, tables: ['Order'] }, + 'SCOPE_ATTRIBUTE_UNKNOWN', + ], + [ + 'a mistyped scope key', + { scopes: { customerId: '$token.sub' }, tables: ['Order'] }, + 'PROFILE_UNKNOWN_KEY', + ], + [ + 'a mistyped table key', + { tables: { Order: { scopes: { customerId: 'x' } } } }, + 'PROFILE_UNKNOWN_KEY', + ], + ['an unknown table', { tables: ['Orders'] }, 'PROFILE_TABLE_UNKNOWN'], + ['a table in another database', { tables: ['Customer'] }, 'PROFILE_TABLE_UNKNOWN'], + ['an empty scope', { scope: {}, tables: ['Order'] }, 'SCOPE_INVALID'], + ['a null scope', { scope: null, tables: ['Order'] }, 'SCOPE_INVALID'], + [ + 'an unknown reference', + { scope: { customerId: '$session.id' }, tables: ['Order'] }, + 'SCOPE_VALUE_INVALID', + ], + [ + 'a reference inside a list', + { scope: { customerId: ['$token.sub'] }, tables: ['Order'] }, + 'SCOPE_VALUE_INVALID', + ], + [ + 'an empty membership list', + { scope: { customerId: [] }, tables: ['Order'] }, + 'SCOPE_VALUE_INVALID', + ], + [ + 'an object value', + { scope: { customerId: { claim: 'sub' } }, tables: ['Order'] }, + 'SCOPE_VALUE_INVALID', + ], + [ + 'a non-scalar scope attribute', + { tables: { Customer: { database: 'coverage', scope: { tags: 'x' } } } }, + 'SCOPE_ATTRIBUTE_NOT_SCALAR', + ], + [ + 'an invalid direction', + { direction: 'sideways', tables: ['Order'] }, + 'PROFILE_DIRECTION_INVALID', + ], + ['an invalid retention', { retention: '14', tables: ['Order'] }, 'PROFILE_RETENTION_INVALID'], + ['no tables', { tables: [] }, 'PROFILE_TABLES_INVALID'], + ['a missing table list', { scope: 'all' }, 'PROFILE_TABLES_INVALID'], + ['a table that is not a name', { tables: [42] }, 'PROFILE_TABLES_INVALID'], + ['a profile that is not a map', 'Order', 'PROFILE_INVALID'], + ]); + it.each(invalid)('drops a profile with %s', (_, definition, code) => { + const { profiles, diagnostics } = normalize({ + profiles: { broken: definition, fine: { tables: ['Report'] } }, + }); + expect(profiles.map((profile) => profile.name)).toEqual(['fine']); + expect(diagnostics).toContainEqual( + expect.objectContaining({ level: 'error', code, profile: 'broken' }), + ); + }); + + it('rejects a document without a profiles map', () => { + const { profiles, diagnostics } = normalize({ storefront: { tables: ['Product'] } }); + expect(profiles).toEqual([]); + expect(diagnostics).toEqual([ + expect.objectContaining({ level: 'error', code: 'PROFILES_INVALID' }), + ]); + }); + + it('rejects listing the same table twice', () => { + const { profiles, diagnostics } = normalize({ + profiles: { p: { tables: ['Order', 'Order'] } }, + }); + expect(profiles).toEqual([]); + expect(diagnostics).toContainEqual( + expect.objectContaining({ code: 'PROFILE_TABLE_DUPLICATE' }), + ); + }); +}); + +describe('loadSyncProfilesFile', () => { + /** @type {string} */ + let directory; + beforeEach(() => { + directory = fs.mkdtempSync(path.join(os.tmpdir(), 'codegen-profiles-')); + }); + afterEach(() => { + fs.rmSync(directory, { recursive: true, force: true }); + }); + + it('parses YAML, including the scope references', async () => { + const file = path.join(directory, 'sync.yaml'); + fs.writeFileSync( + file, + 'profiles:\n my-orders:\n scope:\n customerId: $token.sub\n tables: [Order]\n', + ); + expect(await loadSyncProfilesFile(file)).toEqual({ + profiles: { 'my-orders': { scope: { customerId: '$token.sub' }, tables: ['Order'] } }, + }); + }); + + it('parses JSON by extension and returns undefined for a missing file', async () => { + const file = path.join(directory, 'sync.json'); + fs.writeFileSync(file, '{"profiles":{}}'); + expect(await loadSyncProfilesFile(file)).toEqual({ profiles: {} }); + expect(await loadSyncProfilesFile(path.join(directory, 'absent.yaml'))).toBeUndefined(); + }); + + it('refuses YAML alias expansion bombs', async () => { + const file = path.join(directory, 'bomb.yaml'); + const lines = ['a: &a ["x","x","x","x","x","x","x","x","x","x"]']; + for (let level = 1; level < 12; level++) { + const previous = String.fromCharCode(96 + level); + const next = String.fromCharCode(97 + level); + lines.push(`${next}: &${next} [${Array(10).fill(`*${previous}`).join(',')}]`); + } + fs.writeFileSync(file, `${lines.join('\n')}\n`); + await expect(loadSyncProfilesFile(file)).rejects.toThrow(); + }); +}); diff --git a/utils/client/validateSchemaIR.js b/utils/client/validateSchemaIR.js new file mode 100644 index 0000000..7290af5 --- /dev/null +++ b/utils/client/validateSchemaIR.js @@ -0,0 +1,422 @@ +import { affinityOf, projectionsOf } from './buildSchemaIR.js'; +import { canonicalJSON } from './canonicalHash.js'; +import { hashContracts, hashProfile } from './contractHash.js'; +import { IR_VERSION, SCALAR_TYPES } from './irConstants.js'; +import { RESERVED_TYPE_NAMES } from './naming.js'; + +const IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/; +const HASH = /^[0-9a-f]{64}$/; +const LINE_BREAK_OR_CONTROL = /[\p{Cc}\u2028\u2029]/u; +const AFFINITIES = new Set(['TEXT', 'INTEGER', 'REAL', 'BLOB']); +const SERVER_MANAGED = new Set([null, 'createdTime', 'updatedTime', 'derived']); +const DIRECTIONS = new Set(['pull', 'push', 'bidirectional']); +const CARDINALITIES = new Set(['one', 'many']); + +/** + * Checks that a value is a schema IR this generator can emit from: the supported `irVersion`, the + * expected shapes, unique identifier-safe type names, references that resolve, and hashes that + * match the contents. An IR read from a file is untrusted input, so emitters and the diff call + * this before using one. + * @param {unknown} ir + * @returns {asserts ir is import('./irTypes.js').SchemaIR} + */ +export function assertSchemaIR(ir) { + const fail = (/** @type {string} */ path, /** @type {string} */ problem) => { + throw new Error(`Invalid schema IR at ${path}: ${problem}`); + }; + if (!isObject(ir)) fail('$', 'expected an object'); + const root = /** @type {Record} */ (ir); + if (root.irVersion !== IR_VERSION) { + fail( + '$.irVersion', + `version ${JSON.stringify(root.irVersion)} is not supported (expected ${IR_VERSION})`, + ); + } + if (!isHash(root.schemaHash)) fail('$.schemaHash', 'expected a lowercase hex SHA-256'); + if ( + root.generator !== undefined && + (typeof root.generator !== 'string' || LINE_BREAK_OR_CONTROL.test(root.generator)) + ) { + fail('$.generator', 'expected a single-line string'); + } + for (const key of ['tables', 'types', 'profiles', 'diagnostics']) { + if (!Array.isArray(root[key])) fail(`$.${key}`, 'expected an array'); + } + + /** @type {Map} */ + const tablesByKey = new Map(); + const typeNames = new Set(); + root.types.forEach((/** @type {any} */ type, /** @type {number} */ index) => { + if (!isObject(type) || typeof type.name !== 'string') + fail(`$.types[${index}]`, 'expected a named type'); + if (typeNames.has(type.name)) fail(`$.types[${index}]`, `type "${type.name}" is listed twice`); + typeNames.add(type.name); + }); + const claimed = new Set(); + /** + * @param {string} path + * @param {unknown} typeName + */ + const claimTypeName = (path, typeName) => { + if (typeof typeName !== 'string' || !IDENTIFIER.test(typeName)) + fail(path, 'expected an identifier'); + if (RESERVED_TYPE_NAMES.has(/** @type {string} */ (typeName))) + fail(path, `"${typeName}" is reserved`); + if (claimed.has(typeName)) fail(path, `type name "${typeName}" is used twice`); + claimed.add(typeName); + }; + root.tables.forEach((/** @type {any} */ table, /** @type {number} */ index) => { + if (!isObject(table) || typeof table.database !== 'string' || typeof table.name !== 'string') { + fail(`$.tables[${index}]`, 'expected database and name strings'); + } + const key = `${table.database}\u0000${table.name}`; + if (tablesByKey.has(key)) { + fail(`$.tables[${index}]`, `table ${table.database}.${table.name} is listed twice`); + } + tablesByKey.set(key, table); + }); + + /** + * @param {string} path + * @param {any} type + */ + const checkType = (path, type) => { + if (!isObject(type)) fail(path, 'expected a type'); + switch (type.kind) { + case 'scalar': + if (!SCALAR_TYPES.has(type.scalar)) + fail(path, `unknown scalar ${JSON.stringify(type.scalar)}`); + return; + case 'array': + if (typeof type.elementNullable !== 'boolean') fail(path, 'expected elementNullable'); + checkType(`${path}.element`, type.element); + return; + case 'object': + if (!typeNames.has(type.type)) + fail(path, `object type ${JSON.stringify(type.type)} is not in types`); + return; + case 'record': + if (!tablesByKey.has(`${type.database}\u0000${type.table}`)) { + fail(path, `embedded table ${type.database}.${type.table} is not in tables`); + } + return; + default: + fail(path, `unknown kind ${JSON.stringify(type.kind)}`); + } + }; + + root.tables.forEach((/** @type {any} */ table, /** @type {number} */ index) => { + const path = `$.tables[${index}]`; + claimTypeName(`${path}.typeName`, table.typeName); + if (table.primaryKey !== null && typeof table.primaryKey !== 'string') + fail(`${path}.primaryKey`, 'expected a string or null'); + if (typeof table.sealed !== 'boolean') fail(`${path}.sealed`, 'expected a boolean'); + if (!isHash(table.hash)) fail(`${path}.hash`, 'expected a lowercase hex SHA-256'); + if (!isStringArray(table.profiles)) fail(`${path}.profiles`, 'expected profile names'); + if (!Array.isArray(table.attributes)) fail(`${path}.attributes`, 'expected an array'); + const names = new Set(); + table.attributes.forEach( + (/** @type {any} */ attribute, /** @type {number} */ attributeIndex) => { + const at = `${path}.attributes[${attributeIndex}]`; + if (!isObject(attribute) || typeof attribute.name !== 'string' || attribute.name === '') + fail(at, 'expected a named attribute'); + if (names.has(attribute.name)) fail(at, `attribute "${attribute.name}" is declared twice`); + names.add(attribute.name); + checkType(`${at}.type`, attribute.type); + for (const flag of ['nullable', 'primaryKey', 'indexed', 'readOnly']) { + if (typeof attribute[flag] !== 'boolean') fail(`${at}.${flag}`, 'expected a boolean'); + } + if (!SERVER_MANAGED.has(attribute.serverManaged)) + fail(`${at}.serverManaged`, 'unknown value'); + if ( + attribute.computed !== null && + !( + isObject(attribute.computed) && + isStringOrNull(attribute.computed.from) && + isStringOrNull(attribute.computed.version) + ) + ) { + fail(`${at}.computed`, 'expected { from, version } or null'); + } + }, + ); + if (table.primaryKey !== null && !names.has(table.primaryKey)) + fail(`${path}.primaryKey`, 'is not an attribute'); + if (!Array.isArray(table.relations)) fail(`${path}.relations`, 'expected an array'); + table.relations.forEach((/** @type {any} */ relation, /** @type {number} */ relationIndex) => { + const at = `${path}.relations[${relationIndex}]`; + if ( + !isObject(relation) || + typeof relation.name !== 'string' || + relation.name === '' || + !CARDINALITIES.has(relation.cardinality) || + !isObject(relation.target) || + !isStringOrNull(relation.target.database) || + typeof relation.target.table !== 'string' + ) { + fail(at, 'expected a named relation with a cardinality and a target'); + } + if (names.has(relation.name)) fail(at, `"${relation.name}" is also a stored attribute`); + }); + const projections = table.projections; + if (!isObject(projections)) fail(`${path}.projections`, 'expected an object'); + for (const key of ['record', 'insert', 'patch', 'query']) { + checkProjection(fail, `${path}.projections.${key}`, projections[key], names); + } + if (projections.record.length !== names.size) + fail(`${path}.projections.record`, 'expected every attribute'); + if (projections.upsert !== null) + checkProjection(fail, `${path}.projections.upsert`, projections.upsert, names); + const patchNames = new Set(projections.patch.map((/** @type {any} */ field) => field.name)); + checkNames(fail, `${path}.projections.clearable`, projections.clearable, patchNames); + checkNames(fail, `${path}.projections.unwritable`, projections.unwritable, names); + if ((projections.upsert === null) !== projections.unwritable.length > 0) { + fail(`${path}.projections.upsert`, 'must be null exactly when an attribute is unwritable'); + } + const storage = table.storage; + if (!isObject(storage) || typeof storage.versionColumn !== 'string') + fail(`${path}.storage`, 'expected a version column'); + if (storage.extraColumn !== null && typeof storage.extraColumn !== 'string') + fail(`${path}.storage.extraColumn`, 'expected a string or null'); + if (table.sealed !== (storage.extraColumn === null)) + fail(`${path}.storage.extraColumn`, 'must be null exactly when the table is sealed'); + if ( + names.has(storage.versionColumn) || + (storage.extraColumn !== null && names.has(storage.extraColumn)) + ) { + fail(`${path}.storage`, 'a metadata column collides with an attribute'); + } + if (!Array.isArray(storage.columns) || storage.columns.length !== table.attributes.length) { + fail(`${path}.storage.columns`, 'expected one column per attribute'); + } + storage.columns.forEach((/** @type {any} */ column, /** @type {number} */ columnIndex) => { + if ( + !isObject(column) || + column.name !== table.attributes[columnIndex].name || + !AFFINITIES.has(column.affinity) + ) { + fail( + `${path}.storage.columns[${columnIndex}]`, + 'expected the attribute name and a known affinity', + ); + } + }); + }); + + root.types.forEach((/** @type {any} */ type, /** @type {number} */ index) => { + const path = `$.types[${index}]`; + claimTypeName(`${path}.typeName`, type.typeName); + if (!Array.isArray(type.attributes)) fail(`${path}.attributes`, 'expected an array'); + const names = new Set(); + type.attributes.forEach( + (/** @type {any} */ attribute, /** @type {number} */ attributeIndex) => { + const at = `${path}.attributes[${attributeIndex}]`; + if (!isObject(attribute) || typeof attribute.name !== 'string' || attribute.name === '') + fail(at, 'expected a named attribute'); + if (names.has(attribute.name)) fail(at, `attribute "${attribute.name}" is declared twice`); + names.add(attribute.name); + checkType(`${at}.type`, attribute.type); + if (typeof attribute.nullable !== 'boolean') fail(`${at}.nullable`, 'expected a boolean'); + }, + ); + }); + + const profileNames = new Set(); + root.profiles.forEach((/** @type {any} */ profile, /** @type {number} */ index) => { + const path = `$.profiles[${index}]`; + if (!isObject(profile) || typeof profile.name !== 'string' || profile.name === '') + fail(path, 'expected a named profile'); + if (profileNames.has(profile.name)) fail(path, `profile "${profile.name}" is listed twice`); + profileNames.add(profile.name); + if (profile.description !== undefined && typeof profile.description !== 'string') + fail(`${path}.description`, 'expected a string'); + if (!DIRECTIONS.has(profile.direction)) fail(`${path}.direction`, 'unknown direction'); + if (!isStringOrNull(profile.retention)) fail(`${path}.retention`, 'expected a string or null'); + if ( + profile.retentionMs !== null && + !(Number.isSafeInteger(profile.retentionMs) && profile.retentionMs > 0) + ) { + fail(`${path}.retentionMs`, 'expected a positive integer or null'); + } + if (!isHash(profile.hash)) fail(`${path}.hash`, 'expected a lowercase hex SHA-256'); + if (!isHash(profile.schemaHash)) fail(`${path}.schemaHash`, 'expected a lowercase hex SHA-256'); + if (!Array.isArray(profile.tables) || profile.tables.length === 0) + fail(`${path}.tables`, 'expected at least one table'); + const listed = new Set(); + profile.tables.forEach((/** @type {any} */ entry, /** @type {number} */ entryIndex) => { + const at = `${path}.tables[${entryIndex}]`; + const key = isObject(entry) ? `${entry.database}\u0000${entry.table}` : ''; + const table = tablesByKey.get(key); + if (!table) fail(at, 'does not reference a table in tables'); + if (listed.has(key)) fail(at, 'lists a table twice'); + listed.add(key); + checkScope(fail, `${at}.scope`, entry.scope, table); + }); + }); + + const objectTypes = new Map(root.types.map((/** @type {any} */ type) => [type.name, type])); + root.tables.forEach((/** @type {any} */ table, /** @type {number} */ index) => { + const path = `$.tables[${index}]`; + table.attributes.forEach((/** @type {any} */ attribute, /** @type {number} */ at) => { + if (attribute.readOnly !== (attribute.serverManaged !== null || attribute.computed !== null)) + fail( + `${path}.attributes[${at}].readOnly`, + 'must hold exactly for server-managed and computed attributes', + ); + if (table.storage.columns[at].affinity !== affinityOf(attribute.type)) + fail(`${path}.storage.columns[${at}].affinity`, 'does not match the attribute type'); + }); + if ( + canonicalJSON(table.projections) !== + canonicalJSON(projectionsOf(table, objectTypes, tablesByKey, [])) + ) { + fail(`${path}.projections`, 'do not follow from the attributes'); + } + }); + + const { tableHashes, schemaHash } = hashContracts(root.tables, root.types); + root.tables.forEach((/** @type {any} */ table, /** @type {number} */ index) => { + if (table.hash !== tableHashes.get(`${table.database}\u0000${table.name}`)) + fail(`$.tables[${index}].hash`, 'does not match the table'); + }); + if (root.schemaHash !== schemaHash) fail('$.schemaHash', 'does not match the tables'); + root.profiles.forEach((/** @type {any} */ profile, /** @type {number} */ index) => { + const hashes = hashProfile(profile, tableHashes); + if (profile.hash !== hashes.hash) + fail(`$.profiles[${index}].hash`, 'does not match the profile'); + if (profile.schemaHash !== hashes.schemaHash) + fail(`$.profiles[${index}].schemaHash`, "does not match the profile's tables"); + }); +} + +/** + * The shapes `normalizeSyncProfiles` produces, so an imported IR cannot carry a scope it would + * have refused. + * @param {(path: string, problem: string) => never} fail + * @param {string} path + * @param {any} scope + * @param {any} table + */ +function checkScope(fail, path, scope, table) { + if (isObject(scope) && scope.type === 'all' && Object.keys(scope).length === 1) return; + if ( + !isObject(scope) || + scope.type !== 'match' || + !Array.isArray(scope.conditions) || + scope.conditions.length === 0 + ) { + fail(path, 'expected { type: "all" } or { type: "match" } with conditions'); + } + const scoped = new Set(); + scope.conditions.forEach((/** @type {any} */ condition, /** @type {number} */ index) => { + const at = `${path}.conditions[${index}]`; + const attribute = isObject(condition) + ? table.attributes.find( + (/** @type {any} */ candidate) => candidate.name === condition.attribute, + ) + : undefined; + if (!attribute || attribute.type.kind !== 'scalar') + fail(at, 'expected a scalar attribute of the table'); + if (scoped.has(condition.attribute)) fail(at, `"${condition.attribute}" is scoped twice`); + scoped.add(condition.attribute); + const valid = + condition.operator === 'in' + ? condition.source === 'literal' && + Array.isArray(condition.value) && + condition.value.length > 0 && + condition.value.every(isLiteral) + : condition.operator === 'equals' && + ((condition.source === 'literal' && isLiteral(condition.value)) || + (condition.source === 'claim' && isName(condition.claim)) || + (condition.source === 'user' && isName(condition.field))); + if (!valid) { + fail(at, 'expected an equals condition on a literal, claim or user field, or an in list'); + } + }); +} + +/** + * @param {(path: string, problem: string) => never} fail + * @param {string} path + * @param {unknown} projection + * @param {Set} names + */ +function checkProjection(fail, path, projection, names) { + if (!Array.isArray(projection)) fail(path, 'expected an array'); + const listed = new Set(); + for (const field of /** @type {any[]} */ (projection)) { + if (!isObject(field) || !names.has(field.name) || typeof field.optional !== 'boolean') { + fail(path, 'expected { name, optional } entries naming attributes'); + } + if (listed.has(field.name)) fail(path, `"${field.name}" is listed twice`); + listed.add(field.name); + } +} + +/** + * @param {(path: string, problem: string) => never} fail + * @param {string} path + * @param {unknown} list + * @param {Set} allowed + */ +function checkNames(fail, path, list, allowed) { + if ( + !isStringArray(list) || + new Set(list).size !== list.length || + list.some((name) => !allowed.has(name)) + ) { + fail(path, 'expected distinct attribute names'); + } +} + +/** + * @param {unknown} value + * @returns {value is Record} + */ +function isObject(value) { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +/** + * @param {unknown} value + * @returns {boolean} + */ +function isHash(value) { + return typeof value === 'string' && HASH.test(value); +} + +/** + * @param {unknown} value + * @returns {boolean} + */ +function isStringOrNull(value) { + return value === null || typeof value === 'string'; +} + +/** + * @param {unknown} value + * @returns {value is string[]} + */ +function isStringArray(value) { + return Array.isArray(value) && value.every((item) => typeof item === 'string'); +} + +/** + * @param {unknown} value + * @returns {boolean} + */ +function isName(value) { + return typeof value === 'string' && value !== ''; +} + +/** + * @param {unknown} value + * @returns {boolean} + */ +function isLiteral(value) { + return ( + typeof value === 'string' || + typeof value === 'boolean' || + (typeof value === 'number' && Number.isFinite(value)) + ); +} diff --git a/utils/client/validateSchemaIR.test.js b/utils/client/validateSchemaIR.test.js new file mode 100644 index 0000000..c2a4962 --- /dev/null +++ b/utils/client/validateSchemaIR.test.js @@ -0,0 +1,212 @@ +import { describe, expect, it } from 'vitest'; +import { + coverageTables, + coverageTypes, + spikeProfiles, + spikeTables, +} from '../../test/fixtures/clientSchema.js'; +import { buildSchemaIR } from './buildSchemaIR.js'; +import { assertSchemaIR } from './validateSchemaIR.js'; + +function validIR() { + return /** @type {any} */ ( + JSON.parse( + JSON.stringify( + buildSchemaIR({ + tables: [...spikeTables(), ...coverageTables()], + types: coverageTypes(), + syncProfiles: spikeProfiles(), + }), + ), + ) + ); +} + +describe('assertSchemaIR', () => { + it('accepts what buildSchemaIR produces', () => { + expect(() => assertSchemaIR(validIR())).not.toThrow(); + }); + + /** @type {[string, (ir: any) => void, RegExp][]} */ + const corruptions = [ + ['an unsupported version', (ir) => (ir.irVersion = 2), /irVersion: version 2 is not supported/], + ['a missing table list', (ir) => delete ir.tables, /\$\.tables: expected an array/], + [ + 'a type name that is not an identifier', + (ir) => (ir.tables[0].typeName = '../Escape'), + /typeName: expected an identifier/, + ], + [ + 'a duplicated type name', + (ir) => (ir.tables[1].typeName = ir.tables[0].typeName), + /is used twice/, + ], + [ + 'an object type that is not declared', + (ir) => (ir.types = ir.types.filter((/** @type {any} */ type) => type.name !== 'Address')), + /object type "Address" is not in types/, + ], + [ + 'an unknown scalar', + (ir) => (ir.tables[0].attributes[0].type = { kind: 'scalar', scalar: 'Decimal' }), + /unknown scalar/, + ], + [ + 'an unknown type kind', + (ir) => (ir.tables[0].attributes[0].type = { kind: 'union' }), + /unknown kind/, + ], + [ + 'a duplicated attribute', + (ir) => ir.tables[0].attributes.push(ir.tables[0].attributes[0]), + /declared twice/, + ], + [ + 'a primary key that is not an attribute', + (ir) => (ir.tables[0].primaryKey = 'nope'), + /primaryKey: is not an attribute/, + ], + [ + 'a projection naming an unknown attribute', + (ir) => ir.tables[0].projections.record.push({ name: 'ghost', optional: true }), + /projections\.record/, + ], + [ + 'an upsert that contradicts unwritable', + (ir) => + (ir.tables.find((/** @type {any} */ t) => t.name === 'Attachment').projections.upsert = []), + /must be null exactly when/, + ], + [ + 'an overflow column on a sealed table', + (ir) => (ir.tables.find((/** @type {any} */ t) => t.sealed).storage.extraColumn = '_extra'), + /must be null exactly when the table is sealed/, + ], + [ + 'a metadata column colliding with an attribute', + (ir) => (ir.tables[0].storage.versionColumn = 'id'), + /metadata column collides/, + ], + [ + 'columns out of step with attributes', + (ir) => ir.tables[0].storage.columns.pop(), + /one column per attribute/, + ], + [ + 'an unknown affinity', + (ir) => (ir.tables[0].storage.columns[0].affinity = 'NUMERIC'), + /known affinity/, + ], + [ + 'a profile referencing an unknown table', + (ir) => (ir.profiles[0].tables[0].table = 'Ghost'), + /does not reference a table/, + ], + ['an unknown direction', (ir) => (ir.profiles[0].direction = 'sideways'), /unknown direction/], + [ + 'a generator that would break out of the header comment', + (ir) => (ir.generator = 'x\nimport Evil'), + /generator: expected a single-line string/, + ], + [ + 'a schema hash that is not a SHA-256', + (ir) => (ir.schemaHash = `${ir.schemaHash}\nfun evil() {}`), + /schemaHash: expected a lowercase hex SHA-256/, + ], + ['a table listed twice', (ir) => ir.tables.push({ ...ir.tables[0] }), /is listed twice/], + [ + 'a record projection that leaves out an attribute', + (ir) => ir.tables[0].projections.record.pop(), + /expected every attribute/, + ], + [ + 'a clearable attribute listed twice', + (ir) => { + const { clearable } = ir.tables.find( + (/** @type {any} */ t) => t.projections.clearable.length > 0, + ).projections; + clearable.push(clearable[0]); + }, + /clearable: expected distinct attribute names/, + ], + [ + 'a relationship that shadows a stored attribute', + (ir) => + (ir.tables.find((/** @type {any} */ t) => t.relations.length > 0).relations[0].name = 'id'), + /is also a stored attribute/, + ], + [ + 'a match scope without conditions', + (ir) => (ir.profiles[0].tables[0].scope = { type: 'match', conditions: [] }), + /scope: expected/, + ], + [ + 'a scope on an attribute the table does not store', + (ir) => (ir.profiles[0].tables[0].scope.conditions[0].attribute = 'ghost'), + /expected a scalar attribute of the table/, + ], + [ + 'a scope reference without a field', + (ir) => (ir.profiles[0].tables[0].scope.conditions[0].field = ''), + /expected an equals condition/, + ], + [ + 'a fractional retention', + (ir) => (ir.profiles[0].retentionMs = 1.5), + /retentionMs: expected a positive integer/, + ], + [ + 'a table hash that does not match the table', + (ir) => (ir.tables[0].attributes[1].type = { kind: 'scalar', scalar: 'ID' }), + /tables\[0\]\.hash: does not match the table/, + ], + [ + 'a reserved type name', + (ir) => (ir.tables[0].typeName = 'HarperValue'), + /"HarperValue" is reserved/, + ], + [ + 'a read-only flag that contradicts the attribute', + (ir) => { + const order = ir.tables.find((/** @type {any} */ t) => t.name === 'Order'); + order.attributes.find((/** @type {any} */ a) => a.computed).readOnly = false; + }, + /readOnly: must hold exactly for server-managed and computed attributes/, + ], + [ + 'an affinity that contradicts the attribute type', + (ir) => (ir.tables[0].storage.columns[0].affinity = 'INTEGER'), + /affinity: does not match the attribute type/, + ], + [ + 'projections that do not follow from the attributes', + (ir) => ir.tables[0].projections.insert.pop(), + /projections: do not follow from the attributes/, + ], + [ + 'a schema hash that does not match the tables', + (ir) => (ir.schemaHash = ir.tables[0].hash), + /\$\.schemaHash: does not match the tables/, + ], + [ + 'a profile hash that does not match the profile', + (ir) => (ir.profiles[0].retentionMs += 1), + /profiles\[0\]\.hash: does not match the profile/, + ], + [ + 'a hello-frame hash that does not match the profile tables', + (ir) => (ir.profiles[0].schemaHash = ir.schemaHash), + /profiles\[0\]\.schemaHash: does not match/, + ], + ]; + it.each(corruptions)('rejects %s', (_, corrupt, message) => { + const ir = validIR(); + corrupt(ir); + expect(() => assertSchemaIR(ir)).toThrow(message); + }); + + it('rejects values that are not an IR at all', () => { + expect(() => assertSchemaIR(null)).toThrow(/expected an object/); + expect(() => assertSchemaIR([])).toThrow(/expected an object/); + }); +}); diff --git a/utils/createRegenerationScheduler.js b/utils/createRegenerationScheduler.js new file mode 100644 index 0000000..7840faf --- /dev/null +++ b/utils/createRegenerationScheduler.js @@ -0,0 +1,55 @@ +/** + * Coalesces regeneration requests: requests made while a run is pending or in progress collapse + * into one more run after it, so runs never overlap and an older run can never finish after a + * newer one. Errors go to `onError`; nothing runs after `close()`. + * @param {() => Promise | void} run + * @param {{ delayMs?: number, onError: (error: unknown) => void }} options + */ +export function createRegenerationScheduler(run, { delayMs = 100, onError }) { + let requested = false; + let closed = false; + let running = false; + /** @type {ReturnType | null} */ + let timer = null; + + const drain = async () => { + timer = null; + running = true; + /** @type {{ error: unknown } | undefined} */ + let reporterFailure; + try { + while (requested && !closed) { + requested = false; + try { + await run(); + } catch (error) { + try { + onError(error); + } catch (failure) { + reporterFailure ??= { error: failure }; + } + } + } + } finally { + running = false; + } + // Raised once the queued runs are served, so a broken reporter still surfaces (as an + // unhandled rejection the host logs) without stranding requests made in the meantime. + if (reporterFailure) throw reporterFailure.error; + }; + + return { + schedule() { + if (closed) return; + requested = true; + if (timer || running) return; + timer = setTimeout(drain, delayMs); + }, + /** A run already in progress is not interrupted. */ + close() { + closed = true; + if (timer) clearTimeout(timer); + timer = null; + }, + }; +} diff --git a/utils/createRegenerationScheduler.test.js b/utils/createRegenerationScheduler.test.js new file mode 100644 index 0000000..69133e3 --- /dev/null +++ b/utils/createRegenerationScheduler.test.js @@ -0,0 +1,137 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { createRegenerationScheduler } from './createRegenerationScheduler.js'; + +describe('createRegenerationScheduler', () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + afterEach(() => { + vi.restoreAllMocks(); + vi.useRealTimers(); + }); + + it('coalesces a burst of requests into one run', async () => { + const run = vi.fn(); + const scheduler = createRegenerationScheduler(run, { delayMs: 50, onError: vi.fn() }); + for (let i = 0; i < 10; i++) scheduler.schedule(); + await vi.advanceTimersByTimeAsync(50); + expect(run).toHaveBeenCalledTimes(1); + }); + + it('runs once more after a run that was asked for again while running, never overlapping', async () => { + /** @type {(() => void)[]} */ + const finishers = []; + let active = 0; + let maxActive = 0; + const run = vi.fn( + () => + new Promise((resolve) => { + active++; + maxActive = Math.max(maxActive, active); + finishers.push(() => { + active--; + resolve(undefined); + }); + }), + ); + const scheduler = createRegenerationScheduler(run, { delayMs: 10, onError: vi.fn() }); + scheduler.schedule(); + await vi.advanceTimersByTimeAsync(10); + expect(run).toHaveBeenCalledTimes(1); + scheduler.schedule(); + scheduler.schedule(); + finishers.shift()?.(); + await vi.advanceTimersByTimeAsync(0); + expect(run).toHaveBeenCalledTimes(2); + finishers.shift()?.(); + await vi.advanceTimersByTimeAsync(100); + expect(run).toHaveBeenCalledTimes(2); + expect(maxActive).toBe(1); + }); + + it('reports a failed run and keeps serving requests', async () => { + const onError = vi.fn(); + const run = vi.fn().mockRejectedValueOnce(new Error('boom')).mockResolvedValue(undefined); + const scheduler = createRegenerationScheduler(run, { delayMs: 10, onError }); + scheduler.schedule(); + await vi.advanceTimersByTimeAsync(10); + expect(onError).toHaveBeenCalledWith(expect.objectContaining({ message: 'boom' })); + scheduler.schedule(); + await vi.advanceTimersByTimeAsync(10); + expect(run).toHaveBeenCalledTimes(2); + }); + + it('keeps serving requests after a run that throws synchronously', async () => { + const onError = vi.fn(); + const run = vi + .fn() + .mockImplementationOnce(() => { + throw new Error('sync'); + }) + .mockReturnValue(undefined); + const scheduler = createRegenerationScheduler(run, { delayMs: 10, onError }); + scheduler.schedule(); + await vi.advanceTimersByTimeAsync(10); + expect(onError).toHaveBeenCalledWith(expect.objectContaining({ message: 'sync' })); + scheduler.schedule(); + await vi.advanceTimersByTimeAsync(10); + expect(run).toHaveBeenCalledTimes(2); + }); + + it('folds a request made synchronously by the run into one later run, never overlapping', async () => { + let active = 0; + let maxActive = 0; + /** @type {ReturnType} */ + let scheduler; + const run = vi.fn(async () => { + active++; + maxActive = Math.max(maxActive, active); + if (run.mock.calls.length === 1) scheduler.schedule(); + await new Promise((resolve) => setTimeout(resolve, 50)); + active--; + }); + scheduler = createRegenerationScheduler(run, { delayMs: 10, onError: vi.fn() }); + scheduler.schedule(); + await vi.advanceTimersByTimeAsync(100); + expect(run).toHaveBeenCalledTimes(2); + expect(maxActive).toBe(1); + }); + + it('serves requests queued behind a run whose failure the reporter could not report, then surfaces the reporter failure', async () => { + /** @type {(() => Promise)[]} */ + const timers = []; + vi.spyOn(globalThis, 'setTimeout').mockImplementation( + /** @type {any} */ ((/** @type {() => Promise} */ callback) => timers.push(callback)), + ); + /** @type {ReturnType} */ + let scheduler; + const run = vi + .fn() + .mockImplementationOnce(async () => { + scheduler.schedule(); + throw new Error('boom'); + }) + .mockResolvedValue(undefined); + scheduler = createRegenerationScheduler(run, { + delayMs: 10, + onError: () => { + throw new Error('reporter'); + }, + }); + scheduler.schedule(); + await expect(timers[0]()).rejects.toThrow('reporter'); + expect(run).toHaveBeenCalledTimes(2); + scheduler.schedule(); + expect(timers).toHaveLength(2); + }); + + it('does nothing after close', async () => { + const run = vi.fn(); + const scheduler = createRegenerationScheduler(run, { delayMs: 10, onError: vi.fn() }); + scheduler.schedule(); + scheduler.close(); + scheduler.schedule(); + await vi.advanceTimersByTimeAsync(100); + expect(run).not.toHaveBeenCalled(); + }); +}); diff --git a/utils/generateTablesDTS.js b/utils/generateTablesDTS.js index c64966d..c127115 100644 --- a/utils/generateTablesDTS.js +++ b/utils/generateTablesDTS.js @@ -1,8 +1,7 @@ /** @import { TableMeta } from './tableMeta.js' */ -import fs from 'node:fs'; import path from 'node:path'; -import { getLogger } from './logger.js'; import { safeKey } from './safeKey.js'; +import { writeIfChanged } from './writeIfChanged.js'; /** * @param {string} globalTypesPath @@ -65,12 +64,5 @@ export function generateTablesDTS(globalTypesPath, schemaTypesPath, tables, modu content += `\t};\n`; content += `}\n`; - const outPath = globalTypesPath; - const dir = path.dirname(outPath); - fs.mkdirSync(dir, { recursive: true }); - const existingContent = fs.existsSync(outPath) && fs.readFileSync(outPath, 'utf8'); - if (existingContent !== content) { - fs.writeFileSync(outPath, content, 'utf8'); - getLogger()?.debug?.(`Updated types in ${outPath}`); - } + writeIfChanged(globalTypesPath, content); } diff --git a/utils/regenerateAll.js b/utils/regenerateAll.js index 0e56239..d60bd6f 100644 --- a/utils/regenerateAll.js +++ b/utils/regenerateAll.js @@ -1,3 +1,4 @@ +/** @import { RegenerateClientOptions } from './client/regenerateClient.js' */ import path from 'node:path'; import { collectTables } from './collectTables.js'; import { generateJSDocFromTables } from './generateJSDocFromTables.js'; @@ -5,26 +6,48 @@ import { generateTablesDTS } from './generateTablesDTS.js'; import { generateTSFromTables } from './generateTS.js'; import { writeIfChanged } from './writeIfChanged.js'; +/** + * @param {RegenerateClientOptions | undefined} client + * @returns {client is RegenerateClientOptions} + */ +export function hasClientOutputs(client) { + return Boolean(client?.schemaIR || client?.swift || client?.kotlin); +} + /** * @param {string} globalTypes * @param {string} schemaTypes * @param {string} [jsdoc] - * @param {{ module?: string, includeDatabases?: string[], excludeDatabases?: string[] }} [options] + * @param {{ module?: string, includeDatabases?: string[], excludeDatabases?: string[], baseDirectory?: string, client?: RegenerateClientOptions }} [options] + * `baseDirectory` is what relative output paths resolve against (the application directory); + * client outputs are generated, and their modules loaded, only when one is configured. */ -export function regenerateAll(globalTypes, schemaTypes, jsdoc, options = {}) { - const { module: moduleName, includeDatabases, excludeDatabases } = options; +export async function regenerateAll(globalTypes, schemaTypes, jsdoc, options = {}) { + const { + module: moduleName, + includeDatabases, + excludeDatabases, + baseDirectory = '', + client, + } = options; const list = collectTables({ include: includeDatabases, exclude: excludeDatabases }); + const resolve = (/** @type {string} */ target) => path.resolve(baseDirectory, target); if (jsdoc) { const { jsCode } = generateJSDocFromTables(list, 'HarperDB schema'); - writeIfChanged(path.resolve(jsdoc), jsCode); + writeIfChanged(resolve(jsdoc), jsCode); } const { tsCode, tables } = generateTSFromTables(list, 'HarperDB schema'); if (schemaTypes) { - writeIfChanged(path.resolve(schemaTypes), tsCode); + writeIfChanged(resolve(schemaTypes), tsCode); } if (globalTypes) { - generateTablesDTS(path.resolve(globalTypes), path.resolve(schemaTypes), tables, moduleName); + generateTablesDTS(resolve(globalTypes), resolve(schemaTypes), tables, moduleName); + } + + if (hasClientOutputs(client)) { + const { regenerateClient } = await import('./client/regenerateClient.js'); + await regenerateClient(list, { baseDirectory, ...client }); } } diff --git a/utils/regenerateAll.test.js b/utils/regenerateAll.test.js new file mode 100644 index 0000000..f912aeb --- /dev/null +++ b/utils/regenerateAll.test.js @@ -0,0 +1,70 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { spikeTables } from '../test/fixtures/clientSchema.js'; + +const regenerateClient = vi.hoisted(() => vi.fn()); +vi.mock('./client/regenerateClient.js', () => ({ regenerateClient })); + +const { hasClientOutputs, regenerateAll } = await import('./regenerateAll.js'); + +describe('regenerateAll', () => { + /** @type {string} */ + let directory; + beforeEach(() => { + directory = fs.mkdtempSync(path.join(os.tmpdir(), 'codegen-all-')); + /** @type {any} */ (globalThis).databases = { + data: Object.fromEntries(spikeTables().map((table) => [table.tableName, table])), + }; + }); + afterEach(() => { + delete (/** @type {any} */ (globalThis).databases); + vi.clearAllMocks(); + fs.rmSync(directory, { recursive: true, force: true }); + }); + + it('resolves output paths against the application directory, not the working directory', async () => { + await regenerateAll('schemas/global.d.ts', 'schemas/types.ts', 'schemas/jsdoc.js', { + baseDirectory: directory, + }); + expect(fs.readFileSync(path.join(directory, 'schemas', 'types.ts'), 'utf8')).toContain( + 'export interface Product {', + ); + expect(fs.existsSync(path.join(directory, 'schemas', 'global.d.ts'))).toBe(true); + expect(fs.existsSync(path.join(directory, 'schemas', 'jsdoc.js'))).toBe(true); + }); + + it('does no client work unless a client output is configured', async () => { + await regenerateAll('global.d.ts', 'types.ts', undefined, { + baseDirectory: directory, + client: { syncProfiles: 'sync.yaml' }, + }); + expect(regenerateClient).not.toHaveBeenCalled(); + }); + + it('hands the same filtered tables to client generation', async () => { + /** @type {any} */ (globalThis).databases.other = { Hidden: spikeTables()[0] }; + const client = { swift: 'ios/HarperModels' }; + await regenerateAll('global.d.ts', 'types.ts', undefined, { + baseDirectory: directory, + excludeDatabases: ['other'], + client, + }); + expect(regenerateClient).toHaveBeenCalledTimes(1); + const [tables, options] = regenerateClient.mock.calls[0]; + expect(tables.map((/** @type {any} */ table) => table.tableName)).toEqual([ + 'Product', + 'Order', + 'Report', + ]); + expect(options).toEqual({ baseDirectory: directory, swift: 'ios/HarperModels' }); + }); + + it('detects configured client outputs', () => { + expect(hasClientOutputs(undefined)).toBe(false); + expect(hasClientOutputs({ syncProfiles: 'sync.yaml' })).toBe(false); + expect(hasClientOutputs({ schemaIR: 'ir.json' })).toBe(true); + expect(hasClientOutputs({ kotlin: 'android' })).toBe(true); + }); +}); diff --git a/utils/writeIfChanged.js b/utils/writeIfChanged.js index 455f54d..4457e31 100644 --- a/utils/writeIfChanged.js +++ b/utils/writeIfChanged.js @@ -2,16 +2,113 @@ import fs from 'node:fs'; import path from 'node:path'; import { getLogger } from './logger.js'; +/** What Windows reports when renaming over a file another process holds open. */ +const LOCKED_TARGET_CODES = new Set(['EPERM', 'EACCES', 'EBUSY']); + +/** + * @typedef {Object} StagedFile + * @property {string} target the file the staged content replaces, symlinks resolved + * @property {string} temporaryPath + * @property {string} content + */ + +/** + * Writes content to a temporary file beside the file it will replace, so a failed write (a full + * disk, an interrupted process) leaves the previous file intact. A symlinked target is resolved, + * so committing updates the link's target rather than replacing the link. + * @param {string} filePath + * @param {string} content + * @returns {StagedFile} + */ +export function stageFile(filePath, content) { + const target = resolveTarget(filePath); + const temporaryPath = path.join( + path.dirname(target), + `.${path.basename(target)}.${process.pid}.${Math.random().toString(36).slice(2)}.tmp`, + ); + try { + fs.writeFileSync(temporaryPath, content, 'utf8'); + } catch (error) { + fs.rmSync(temporaryPath, { force: true }); + throw error; + } + return { target, temporaryPath, content }; +} + +/** + * Moves a staged file into place. A target another process holds open cannot be renamed over on + * Windows, so it is overwritten in place instead. + * @param {StagedFile} staged + */ +export function commitStaged(staged) { + try { + fs.renameSync(staged.temporaryPath, staged.target); + } catch (error) { + try { + if (!LOCKED_TARGET_CODES.has(/** @type {NodeJS.ErrnoException} */ (error).code ?? '')) + throw error; + fs.writeFileSync(staged.target, staged.content, 'utf8'); + } finally { + discardStaged(staged); + } + } +} + +/** + * @param {StagedFile} staged + */ +export function discardStaged(staged) { + fs.rmSync(staged.temporaryPath, { force: true }); +} + +/** + * @param {string} filePath + * @param {string} content + */ +export function writeFileAtomic(filePath, content) { + commitStaged(stageFile(filePath, content)); +} + +/** + * @param {string} filePath + * @returns {string | undefined} + */ +export function readIfExists(filePath) { + return fs.existsSync(filePath) ? fs.readFileSync(filePath, 'utf8') : undefined; +} + /** * @param {string} filePath * @param {string} content + * @returns {boolean} whether the file was written */ export function writeIfChanged(filePath, content) { - const dir = path.dirname(filePath); - fs.mkdirSync(dir, { recursive: true }); - const existing = fs.existsSync(filePath) ? fs.readFileSync(filePath, 'utf8') : undefined; - if (existing !== content) { - fs.writeFileSync(filePath, content, 'utf8'); - getLogger()?.debug?.(`Updated types in ${filePath}`); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + if (readIfExists(filePath) === content) return false; + writeFileAtomic(filePath, content); + getLogger()?.debug?.(`Updated types in ${filePath}`); + return true; +} + +/** + * Follows symlinks at the final path component, including one whose target does not exist yet, + * as a plain write would. + * @param {string} filePath + * @returns {string} + */ +function resolveTarget(filePath) { + let target = filePath; + for (let hops = 0; hops < 40; hops++) { + /** @type {string} */ + let link; + try { + link = fs.readlinkSync(target); + } catch { + return target; + } + target = path.resolve(path.dirname(target), link); } + throw Object.assign(new Error(`too many levels of symbolic links: ${filePath}`), { + code: 'ELOOP', + }); } diff --git a/utils/writeIfChanged.test.js b/utils/writeIfChanged.test.js new file mode 100644 index 0000000..a240b78 --- /dev/null +++ b/utils/writeIfChanged.test.js @@ -0,0 +1,85 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { writeFileAtomic, writeIfChanged } from './writeIfChanged.js'; + +describe('writeIfChanged', () => { + /** @type {string} */ + let directory; + beforeEach(() => { + directory = fs.mkdtempSync(path.join(os.tmpdir(), 'codegen-write-')); + }); + afterEach(() => { + vi.restoreAllMocks(); + fs.rmSync(directory, { recursive: true, force: true }); + }); + + it('creates missing directories and reports whether it wrote', () => { + const file = path.join(directory, 'nested', 'types.ts'); + expect(writeIfChanged(file, 'a')).toBe(true); + expect(writeIfChanged(file, 'a')).toBe(false); + expect(writeIfChanged(file, 'b')).toBe(true); + expect(fs.readFileSync(file, 'utf8')).toBe('b'); + }); + + it('keeps the previous content when a write fails part-way', () => { + const file = path.join(directory, 'types.ts'); + fs.writeFileSync(file, 'previous'); + vi.spyOn(fs, 'renameSync').mockImplementation(() => { + throw Object.assign(new Error('no space left on device'), { code: 'ENOSPC' }); + }); + expect(() => writeFileAtomic(file, 'next')).toThrow(/no space left/); + expect(fs.readFileSync(file, 'utf8')).toBe('previous'); + expect(fs.readdirSync(directory)).toEqual(['types.ts']); + }); + + it.skipIf(process.platform === 'win32')( + 'updates the target of a symlink and keeps the link', + () => { + const target = path.join(directory, 'shared', 'types.ts'); + fs.mkdirSync(path.dirname(target)); + fs.writeFileSync(target, 'previous'); + const link = path.join(directory, 'types.ts'); + fs.symlinkSync(target, link); + expect(writeIfChanged(link, 'next')).toBe(true); + expect(fs.lstatSync(link).isSymbolicLink()).toBe(true); + expect(fs.readFileSync(target, 'utf8')).toBe('next'); + }, + ); + + it.skipIf(process.platform === 'win32')( + 'creates the target of a dangling symlink and keeps the link', + () => { + const target = path.join(directory, 'shared', 'types.ts'); + fs.mkdirSync(path.dirname(target)); + const link = path.join(directory, 'types.ts'); + fs.symlinkSync(target, link); + expect(writeIfChanged(link, 'first')).toBe(true); + expect(fs.lstatSync(link).isSymbolicLink()).toBe(true); + expect(fs.readFileSync(target, 'utf8')).toBe('first'); + }, + ); + + it.skipIf(process.platform === 'win32')('refuses a symlink cycle', () => { + const first = path.join(directory, 'a.ts'); + const second = path.join(directory, 'b.ts'); + fs.symlinkSync(second, first); + fs.symlinkSync(first, second); + expect(() => writeFileAtomic(first, 'next')).toThrow( + expect.objectContaining({ code: 'ELOOP' }), + ); + expect(fs.lstatSync(first).isSymbolicLink()).toBe(true); + }); + + it('overwrites in place when the target is locked against renames', () => { + const file = path.join(directory, 'types.ts'); + fs.writeFileSync(file, 'previous'); + vi.spyOn(fs, 'renameSync').mockImplementation(() => { + throw Object.assign(new Error('operation not permitted'), { code: 'EPERM' }); + }); + writeFileAtomic(file, 'next'); + expect(fs.readFileSync(file, 'utf8')).toBe('next'); + expect(fs.readdirSync(directory)).toEqual(['types.ts']); + }); +});