From 03ecc40b7e83a72d1c3a0ce41c6e5a4ab2d33015 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 16 Aug 2026 14:33:04 +0100 Subject: [PATCH 01/21] feat(db): rebuild includes materialization graph --- AGENTS.md | 11 + packages/db/src/collection/changes.ts | 45 + packages/db/src/collection/index.ts | 5 + packages/db/src/collection/subscription.ts | 20 + packages/db/src/collection/sync.ts | 4 + packages/db/src/query/compiler/index.ts | 424 ++++- packages/db/src/query/compiler/joins.ts | 125 +- .../db/src/query/compiler/lazy-targets.ts | 83 +- packages/db/src/query/effect.ts | 180 +- packages/db/src/query/ir.ts | 57 + packages/db/src/query/live/ARCHITECTURE.md | 518 ++++++ .../src/query/live/bucket-facade-adapter.ts | 349 ++++ .../query/live/collection-config-builder.ts | 1501 ++--------------- .../src/query/live/collection-subscriber.ts | 53 +- .../src/query/live/materialized-pipeline.ts | 459 +++++ .../query/live/subset-demand-controller.ts | 115 ++ packages/db/src/query/live/utils.ts | 195 +-- packages/db/src/query/subset-dedupe.ts | 12 +- packages/db/src/types.ts | 5 + .../tests/query/compiler/subqueries.test.ts | 27 +- ...ncludes-optimistic-oracle.property.test.ts | 155 +- .../query/includes-oracle.property.test.ts | 782 +++------ .../query/includes-publication-oracle.test.ts | 117 +- .../query/includes-query-shape-oracle.test.ts | 53 +- .../query/includes-temporal-oracle.test.ts | 122 +- .../includes-work-counter-oracle.test.ts | 70 +- packages/db/tests/query/includes.test.ts | 285 +++- .../tests/query/live-query-collection.test.ts | 8 +- packages/db/tests/query/subset-dedupe.test.ts | 29 + 29 files changed, 3058 insertions(+), 2751 deletions(-) create mode 100644 packages/db/src/query/live/ARCHITECTURE.md create mode 100644 packages/db/src/query/live/bucket-facade-adapter.ts create mode 100644 packages/db/src/query/live/materialized-pipeline.ts create mode 100644 packages/db/src/query/live/subset-demand-controller.ts diff --git a/AGENTS.md b/AGENTS.md index a92ff46761..a802043c4f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,6 +2,17 @@ This guide provides principles and patterns for AI agents contributing to the TanStack DB codebase. These guidelines are derived from PR review patterns and reflect the quality standards expected in this project. +## Required reading: live-query materialization + +Before reading, analyzing, or modifying correlated live-query materialization +code under `packages/db/src/query/live/`, read +`packages/db/src/query/live/ARCHITECTURE.md` in full. Read it before changing +the related includes oracle tests as well. + +Treat that document's component boundaries and normative laws as constraints. +If a change intentionally revises an architectural contract, update the +architecture document in the same pull request. + ## Table of Contents 1. [Type Safety](#type-safety) diff --git a/packages/db/src/collection/changes.ts b/packages/db/src/collection/changes.ts index b5b9dfb83d..8b2aafb6d2 100644 --- a/packages/db/src/collection/changes.ts +++ b/packages/db/src/collection/changes.ts @@ -29,6 +29,11 @@ export class CollectionChangesManager< public changeSubscriptions = new Set() public batchedEvents: Array> = [] public shouldBatchEvents = false + private publicationDeferralDepth = 0 + private deferredPublications: Array<{ + changes: Array> + layoutChanged: boolean + }> = [] private layoutChangeListeners = new Set<() => void>() /** @@ -120,6 +125,44 @@ export class CollectionChangesManager< this.shouldBatchEvents = false } + if (this.publicationDeferralDepth > 0) { + this.deferredPublications.push({ changes: rawEvents, layoutChanged }) + return + } + + this.publishEvents(rawEvents, layoutChanged) + } + + /** + * Defers subscriber delivery while a coherent multi-Collection publication + * installs all of its visible state. State and indexes still commit at their + * normal transaction boundaries. + */ + public deferPublication(): () => void { + this.publicationDeferralDepth++ + let resumed = false + + return () => { + if (resumed) return + resumed = true + if (this.publicationDeferralDepth === 0) return + + this.publicationDeferralDepth-- + if (this.publicationDeferralDepth > 0) return + + const publications = this.deferredPublications + this.deferredPublications = [] + this.publishEvents( + publications.flatMap(({ changes }) => changes), + publications.some(({ layoutChanged }) => layoutChanged), + ) + } + } + + private publishEvents( + rawEvents: Array>, + layoutChanged: boolean, + ): void { if (rawEvents.length === 0 && !layoutChanged) { return } @@ -259,5 +302,7 @@ export class CollectionChangesManager< public cleanup(): void { this.batchedEvents = [] this.shouldBatchEvents = false + this.deferredPublications = [] + this.publicationDeferralDepth = 0 } } diff --git a/packages/db/src/collection/index.ts b/packages/db/src/collection/index.ts index f9bb465fe0..9934b89334 100644 --- a/packages/db/src/collection/index.ts +++ b/packages/db/src/collection/index.ts @@ -447,6 +447,11 @@ export class CollectionImpl< this._sync.markLayoutChange() } + /** Defer subscriber events until a coherent multi-Collection commit ends. */ + public _deferPublication(): () => void { + return this._changes.deferPublication() + } + /** * Register a callback to be executed when the collection first becomes ready * Useful for preloading collections diff --git a/packages/db/src/collection/subscription.ts b/packages/db/src/collection/subscription.ts index 7757be14a6..1a92568940 100644 --- a/packages/db/src/collection/subscription.ts +++ b/packages/db/src/collection/subscription.ts @@ -22,6 +22,7 @@ import type { CollectionImpl } from './index.js' type RequestSnapshotOptions = { where?: BasicExpression + signal?: AbortSignal optimizedOnly?: boolean trackLoadSubsetPromise?: boolean /** Optional orderBy to pass to loadSubset for backend optimization */ @@ -73,6 +74,10 @@ export class CollectionSubscription * We store the exact LoadSubsetOptions we passed to loadSubset to ensure symmetric unload. */ private loadedSubsets: Array = [] + private readonly requestedSubsetWhere = new WeakMap< + LoadSubsetOptions, + BasicExpression + >() // Keep track of the keys we've sent (needed for join and orderBy optimizations) private sentKeys = new Set() @@ -374,6 +379,7 @@ export class CollectionSubscription // don't await it, we will load the data into the collection when it comes in const loadOptions: LoadSubsetOptions = { where: stateOpts.where, + signal: opts?.signal, subscription: this, // Include orderBy and limit if provided so sync layer can optimize the query orderBy: opts?.orderBy, @@ -386,6 +392,7 @@ export class CollectionSubscription // Track this loadSubset call so we can unload it later this.loadedSubsets.push(loadOptions) + if (opts?.where) this.requestedSubsetWhere.set(loadOptions, opts.where) const trackLoadSubsetPromise = opts?.trackLoadSubsetPromise ?? true if (trackLoadSubsetPromise) { @@ -417,6 +424,19 @@ export class CollectionSubscription return true } + /** Release one exact subset request while keeping the subscription alive. */ + releaseSnapshot(where: BasicExpression): void { + const index = this.loadedSubsets.findIndex( + (options) => + options.where === where || + this.requestedSubsetWhere.get(options) === where, + ) + if (index === -1) return + + const [options] = this.loadedSubsets.splice(index, 1) + if (options) this.collection._sync.unloadSubset(options) + } + /** * Sends a snapshot that fulfills the `where` clause and all rows are bigger or equal to the cursor. * Requires a range index to be set with `setOrderByIndex` prior to calling this method. diff --git a/packages/db/src/collection/sync.ts b/packages/db/src/collection/sync.ts index d717106110..3ed19b00f3 100644 --- a/packages/db/src/collection/sync.ts +++ b/packages/db/src/collection/sync.ts @@ -500,6 +500,10 @@ export class CollectionSyncManager< * Returns true if no sync function is configured, if syncMode is 'eager', or if there is no work to do. */ public loadSubset(options: LoadSubsetOptions): Promise | true { + if (options.signal?.aborted) { + return true + } + // Bypass loadSubset when syncMode is 'eager' if (this.syncMode === `eager`) { return true diff --git a/packages/db/src/query/compiler/index.ts b/packages/db/src/query/compiler/index.ts index b7779a21d4..6371a8fcfe 100644 --- a/packages/db/src/query/compiler/index.ts +++ b/packages/db/src/query/compiler/index.ts @@ -5,6 +5,7 @@ import { join as joinOperator, map, reduce, + serializeValue, tap, } from '@tanstack/db-ivm' import { optimizeQuery } from '../optimizer.js' @@ -23,17 +24,18 @@ import { IncludesSubquery, PropRef, Value as ValClass, + collectCollectionSources, getWhereExpression, isExpressionLike, } from '../ir.js' import { ensureIndexForField } from '../../indexes/auto-index.js' -import { inArray } from '../builder/functions.js' +import { deepEquals } from '../../utils.js' import { compileExpression, isCaseWhenConditionTrue, toBooleanPredicate, } from './evaluators.js' -import { processJoins } from './joins.js' +import { processJoins, registerLazyDemandPlan } from './joins.js' import { containsAggregate, processGroupBy } from './group-by.js' import { getLazyLoadTargets } from './lazy-targets.js' import { processOrderBy } from './order-by.js' @@ -54,6 +56,7 @@ import type { Collection } from '../../collection/index.js' import type { KeyedStream, NamespacedAndKeyedStream, + NamespacedRow, ResultStream, } from '../../types.js' import type { QueryCache, QueryMapping, WindowOptions } from './types.js' @@ -62,6 +65,7 @@ export type { WindowOptions } from './types.js' /** Symbol used to tag parent $selected with routing metadata for includes */ export const INCLUDES_ROUTING = Symbol(`includesRouting`) +export const INCLUDES_PUBLIC_KEY = Symbol(`includesPublicKey`) export const FN_SELECT_STATE = Symbol(`fnSelectState`) const SKIP_INCLUDE = Symbol(`skipInclude`) @@ -117,7 +121,7 @@ export interface CompilationResult { /** The compiled query pipeline (D2 stream) */ pipeline: ResultStream - /** Map of source aliases to their WHERE clauses for index optimization */ + /** Map of opaque source IDs to their WHERE clauses for index optimization */ sourceWhereClauses: Map> /** @@ -153,7 +157,7 @@ export interface CompilationResult { * @param collections Mapping of collection IDs to Collection instances * @param subscriptions Mapping of source aliases to CollectionSubscription instances * @param callbacks Mapping of source aliases to lazy loading callbacks - * @param lazySources Set of source aliases that should load data lazily + * @param lazySources Set of source identities that should load data lazily * @param optimizableOrderByCollections Map of collection IDs to order-by optimization info * @param cache Optional cache for compiled subqueries (used internally for recursion) * @param queryMapping Optional mapping from optimized queries to original queries @@ -196,6 +200,7 @@ export function compileQuery( // Create a copy of the inputs map to avoid modifying the original const allInputs = { ...inputs } + bindSourceInputs(rawQuery, allInputs) // Track alias to collection id relationships discovered during compilation. // This includes all user-declared aliases plus inner aliases from subqueries. @@ -243,7 +248,18 @@ export function compileQuery( // The inner join happens BEFORE namespace wrapping / WHERE / SELECT / ORDER BY, // so the child pipeline only processes rows that match parents. let pipeline: NamespacedAndKeyedStream = initialPipeline - if (!isUnionFrom && parentKeyStream && childCorrelationField) { + const childCorrelationAlias = childCorrelationField?.path[0] + const joinsParentAfterJoins = + !isUnionFrom && + parentKeyStream !== undefined && + childCorrelationField !== undefined && + childCorrelationAlias !== mainSource + if ( + !isUnionFrom && + parentKeyStream && + childCorrelationField && + !joinsParentAfterJoins + ) { const mainInput = sources[mainSource]! let filteredMainInput = mainInput // Re-key child input by correlation field: [correlationValue, [childKey, childRow]] @@ -267,7 +283,11 @@ export function compileQuery( }), map(([correlationValue, [childSide, parentSide]]: any) => { const [childKey, childRow] = childSide - const tagged: any = { ...childRow, __correlationKey: correlationValue } + const tagged: any = { + ...childRow, + __correlationKey: correlationValue, + [INCLUDES_PUBLIC_KEY]: childKey, + } if (parentSide != null) { tagged.__parentContext = parentSide } @@ -307,9 +327,46 @@ export function compileQuery( aliasToCollectionId, aliasRemapping, sourceWhereClauses, + parentKeyStream !== undefined && !joinsParentAfterJoins, ) } + // A correlation field owned by a joined source does not exist on the main + // input. Join the fully namespaced child relation with its parent routes + // here, after the source join has made that field available. + if (joinsParentAfterJoins) { + const compiledChildCorrelation = compileExpression(childCorrelationField) + pipeline = pipeline.pipe( + map( + ([key, row]) => + [compiledChildCorrelation(row), [key, row]] as [ + unknown, + [unknown, typeof row], + ], + ), + joinOperator(parentKeyStream, `inner`), + filter(([_correlationValue, [childSide]]) => childSide != null), + map(([correlationValue, [childSide, parentSide]]) => { + const [childKey, row] = childSide as [unknown, NamespacedRow] + const namespaced = { ...row } as Record + namespaced[mainSource] = { + ...namespaced[mainSource], + __correlationKey: correlationValue, + [INCLUDES_PUBLIC_KEY]: childKey, + } + if (parentSide != null) { + Object.assign(namespaced, parentSide) + namespaced.__parentContext = parentSide + } + const effectiveKey = + parentSide != null + ? `${String(childKey)}::${serializeValue(parentSide)}` + : childKey + return [effectiveKey, namespaced] + }), + ) as NamespacedAndKeyedStream + } + // Process the WHERE clause if it exists if (query.where && query.where.length > 0) { // Apply each WHERE condition as a filter (they are ANDed together) @@ -344,6 +401,7 @@ export function compileQuery( const includesRoutingFns: Array<{ fieldName: string getRouting: (nsRow: any) => { + active: boolean correlationKey: unknown parentContext: Record | null } @@ -388,10 +446,11 @@ export function compileQuery( fieldName, getRouting: (nsRow: any) => { if (!matchesConditionalSelectGuards(compiledGuards, nsRow)) { - return { correlationKey: null, parentContext: null } + return { active: false, correlationKey: null, parentContext: null } } return ( nsRow[sourceAlias]?.[INCLUDES_ROUTING]?.[include.fieldName] ?? { + active: false, correlationKey: null, parentContext: null, } @@ -427,10 +486,15 @@ export function compileQuery( fieldName, getRouting: (nsRow: any) => { if (!matchesConditionalSelectGuards(compiledGuards, nsRow)) { - return { correlationKey: null, parentContext: null } + return { + active: false, + correlationKey: null, + parentContext: null, + } } return ( nsRow[INCLUDES_ROUTING]?.[include.fieldName] ?? { + active: false, correlationKey: null, parentContext: null, } @@ -511,7 +575,7 @@ export function compileQuery( // --- Includes lazy loading (mirrors join lazy loading in joins.ts) --- // Resolve the child correlation field to concrete collection targets so // subquery and union child sources can load by branch when it is safe. - const childCorrelationAlias = subquery.childCorrelationField.path[0]! + const childSourceAlias = subquery.childCorrelationField.path[0]! const directChildCollection = subquery.query.from.type === `collectionRef` ? subquery.query.from.collection @@ -519,7 +583,7 @@ export function compileQuery( const lazyTargets = getLazyLoadTargets( subquery.query, subquery.query.from, - childCorrelationAlias, + childSourceAlias, subquery.childCorrelationField, directChildCollection, aliasRemapping, @@ -528,7 +592,7 @@ export function compileQuery( if (lazyTargets.length > 0) { // 1. Mark child source as lazy so CollectionSubscriber skips initial full load for (const target of lazyTargets) { - lazySources.add(target.alias) + lazySources.add(target.sourceId) } // 2. Ensure an index on the correlation field for efficient lookups @@ -539,40 +603,46 @@ export function compileQuery( } } - // 3. Tap parent keys to intercept correlation values and request - // matching child rows on-demand via the child's subscription + const initialKeys = getStaticDemandKeys( + rawQuery, + subquery.correlationField, + ) + const demandPlans = lazyTargets.map((target) => + registerLazyDemandPlan(callbacks, target, initialKeys), + ) + const demandWeights = new Map< + string, + { key: unknown; weight: number } + >() + + // Keep the async demand adapter in sync with the current parent-key + // relation. Retired keys stop participating in readiness immediately. parentKeys = parentKeys.pipe( tap((data: any) => { - const joinKeys = [ - ...new Set( - data - .getInner() - .map( - ([[correlationValue]]: any) => correlationValue as unknown, - ) - .filter((joinKey: unknown) => joinKey != null), - ), - ] - - if (joinKeys.length === 0) { - return - } - - for (const target of lazyTargets) { - const lazySourceSubscription = subscriptions[target.alias] - - if (!lazySourceSubscription) { - continue - } - - if (lazySourceSubscription.hasLoadedInitialState()) { - continue + for (const [[correlationValue], weight] of data.getInner()) { + if (correlationValue == null) continue + const encoded = serializeValue(correlationValue) + const previous = demandWeights.get(encoded) + const nextWeight = (previous?.weight ?? 0) + weight + if (nextWeight === 0) { + demandWeights.delete(encoded) + } else { + demandWeights.set(encoded, { + key: correlationValue, + weight: nextWeight, + }) } + } - const lazyJoinRef = new PropRef(target.path) - lazySourceSubscription.requestSnapshot({ - where: inArray(lazyJoinRef, joinKeys), - }) + const keys = new Set( + [...demandWeights.values()] + .filter(({ weight }) => weight > 0) + .map(({ key: demandedKey }) => demandedKey), + ) + for (let index = 0; index < lazyTargets.length; index++) { + const target = lazyTargets[index]! + const plan = demandPlans[index]! + callbacks[target.sourceId]?.setDemand?.(plan, keys) } }), ) @@ -641,7 +711,11 @@ export function compileQuery( fieldName, getRouting: (nsRow: any) => { if (!matchesConditionalSelectGuards(compiledRoutingGuards, nsRow)) { - return { correlationKey: null, parentContext: null } + return { + active: false, + correlationKey: null, + parentContext: null, + } } const parentContext: Record> = {} for (const proj of compiledProjs) { @@ -658,7 +732,11 @@ export function compileQuery( } target[proj.field[proj.field.length - 1]!] = value } - return { correlationKey: compiledCorr(nsRow), parentContext } + return { + active: true, + correlationKey: compiledCorr(nsRow), + parentContext, + } }, }) } else { @@ -667,9 +745,14 @@ export function compileQuery( fieldName, getRouting: (nsRow: any) => { if (!matchesConditionalSelectGuards(compiledRoutingGuards, nsRow)) { - return { correlationKey: null, parentContext: null } + return { + active: false, + correlationKey: null, + parentContext: null, + } } return { + active: true, correlationKey: compiledCorrelation(nsRow), parentContext: null, } @@ -705,13 +788,17 @@ export function compileQuery( pipeline = pipeline.pipe( map(([key, namespacedRow]) => { const selectResults = query.fnSelect!(namespacedRow) + let selected = selectResults if (selectResults && typeof selectResults === `object`) { + selected = Array.isArray(selectResults) + ? [...selectResults] + : { ...selectResults } const routing = (namespacedRow as any)[INCLUDES_ROUTING] if (routing) { - selectResults[INCLUDES_ROUTING] = routing + selected[INCLUDES_ROUTING] = routing } if (directIncludes.length > 0) { - Object.defineProperty(selectResults, FN_SELECT_STATE, { + Object.defineProperty(selected, FN_SELECT_STATE, { value: { sourceRow: namespacedRow, fnSelect: query.fnSelect!, @@ -725,7 +812,7 @@ export function compileQuery( key, { ...namespacedRow, - $selected: selectResults, + $selected: selected, }, ] as [string, typeof namespacedRow & { $selected: any }] }), @@ -752,21 +839,27 @@ export function compileQuery( ) } - // Tag $selected with routing metadata for includes. - // This lets collection-config-builder extract routing info (correlationKey + parentContext) - // from parent results without depending on the user's select. + // Tag $selected with routing metadata so the materialization graph can route + // children without depending on the user's projection. if (includesRoutingFns.length > 0) { pipeline = pipeline.pipe( map(([key, namespacedRow]: any) => { const routing: Record< string, - { correlationKey: unknown; parentContext: Record | null } + { + active: boolean + correlationKey: unknown + parentContext: Record | null + } > = {} for (const { fieldName, getRouting } of includesRoutingFns) { routing[fieldName] = getRouting(namespacedRow) } - namespacedRow.$selected[INCLUDES_ROUTING] = routing - return [key, namespacedRow] + const selected = Array.isArray(namespacedRow.$selected) + ? [...namespacedRow.$selected] + : { ...namespacedRow.$selected } + selected[INCLUDES_ROUTING] = routing + return [key, { ...namespacedRow, $selected: selected }] }), ) } @@ -832,6 +925,26 @@ export function compileQuery( } } + // Normalize every logical row before DISTINCT and ordering. Those operators + // track visibility by row key, so an insert-before-delete replacement with + // the same key would otherwise keep the old value and hide route or order + // changes. Joined contributors may differ in unselected namespaces; only + // the public value and its route/order inputs must be congruent. + if (!query.select || !containsAggregate(query.select)) { + pipeline = canonicalizeSelectedRows( + pipeline, + query, + mainSource, + parentKeyStream !== undefined, + ) + } + + const keyedSourceWhereClauses = keyWhereClausesBySource( + rawQuery, + sourceWhereClauses, + aliasRemapping, + ) + // Process the DISTINCT clause if it exists if (query.distinct) { pipeline = pipeline.pipe(distinct(([_key, row]) => row.$selected)) @@ -845,7 +958,9 @@ export function compileQuery( parentKeyStream && (query.limit !== undefined || query.offset !== undefined) ? (_key: unknown, row: unknown) => { - const correlationKey = (row as any)?.[mainSource]?.__correlationKey + const correlationKey = + (row as any)?.[mainSource]?.__correlationKey ?? + (row as any)?.__correlationKey const parentContext = (row as any)?.__parentContext if (parentContext != null) { return JSON.stringify([correlationKey, parentContext]) @@ -878,26 +993,32 @@ export function compileQuery( ) // When in includes mode, embed the correlation key and parentContext if (parentKeyStream) { - const correlationKey = (row as any)[mainSource]?.__correlationKey + const correlationKey = + (row as any)[mainSource]?.__correlationKey ?? + (row as any).__correlationKey const parentContext = (row as any).__parentContext ?? null - // Strip internal routing properties that may leak via spread selects - delete finalResults.__correlationKey - delete finalResults.__parentContext + const publicKey = getIncludesPublicKey(row, mainSource, key) + const routedResults = stripInternalCorrelation(finalResults) return [ key, - [finalResults, orderByIndex, correlationKey, parentContext], + [ + routedResults, + orderByIndex, + correlationKey, + parentContext, + publicKey, + ], ] as any } return [key, [finalResults, orderByIndex]] as [unknown, [any, string]] }), ) as ResultStream - const result = resultPipeline // Cache the result before returning (use original query as key) const compilationResult: CompilationResult = { collectionId: mainCollectionId, - pipeline: result, - sourceWhereClauses, + pipeline: resultPipeline, + sourceWhereClauses: keyedSourceWhereClauses, aliasToCollectionId, aliasRemapping, includes: includesResults.length > 0 ? includesResults : undefined, @@ -921,14 +1042,15 @@ export function compileQuery( ) // When in includes mode, embed the correlation key and parentContext if (parentKeyStream) { - const correlationKey = (row as any)[mainSource]?.__correlationKey + const correlationKey = + (row as any)[mainSource]?.__correlationKey ?? + (row as any).__correlationKey const parentContext = (row as any).__parentContext ?? null - // Strip internal routing properties that may leak via spread selects - delete finalResults.__correlationKey - delete finalResults.__parentContext + const publicKey = getIncludesPublicKey(row, mainSource, key) + const routedResults = stripInternalCorrelation(finalResults) return [ key, - [finalResults, undefined, correlationKey, parentContext], + [routedResults, undefined, correlationKey, parentContext, publicKey], ] as any } return [key, [finalResults, undefined]] as [ @@ -938,12 +1060,11 @@ export function compileQuery( }), ) - const result = resultPipeline // Cache the result before returning (use original query as key) const compilationResult: CompilationResult = { collectionId: mainCollectionId, - pipeline: result, - sourceWhereClauses, + pipeline: resultPipeline, + sourceWhereClauses: keyedSourceWhereClauses, aliasToCollectionId, aliasRemapping, includes: includesResults.length > 0 ? includesResults : undefined, @@ -953,6 +1074,92 @@ export function compileQuery( return compilationResult } +function keyWhereClausesBySource( + query: QueryIR, + clauses: Map>, + aliasRemapping: Record, +): Map> { + const sources = collectCollectionSources(query) + const sourceIds = new Set(sources.map(({ sourceId }) => sourceId)) + const result = new Map>() + for (const [key, clause] of clauses) { + if (sourceIds.has(key)) { + result.set(key, clause) + continue + } + + const alias = aliasRemapping[key] ?? key + for (const source of sources) { + if (source.alias === alias) result.set(source.sourceId, clause) + } + } + return result +} + +function bindSourceInputs( + query: QueryIR, + inputs: Record, +): void { + for (const source of collectCollectionSources(query)) { + const input = inputs[source.sourceId] ?? inputs[source.alias] + if (!input) continue + inputs[source.sourceId] = input + inputs[source.alias] = input + } +} + +function canonicalizeSelectedRows( + pipeline: NamespacedAndKeyedStream, + query: QueryIR, + mainSource: string, + isIncludedRelation: boolean, +): NamespacedAndKeyedStream { + const compiledOrder = (query.orderBy ?? []).map(({ expression }) => + compileExpression(expression), + ) + const signature = (row: any) => ({ + value: row.$selected, + routing: row.$selected?.[INCLUDES_ROUTING], + outerCorrelation: isIncludedRelation + ? row[mainSource]?.__correlationKey + : undefined, + parentContext: isIncludedRelation + ? (row.__parentContext ?? row[mainSource]?.__parentContext ?? null) + : undefined, + order: compiledOrder.map((evaluate) => evaluate(row)), + }) + + return pipeline.pipe( + reduce((values: Array<[any, number]>) => { + const totalMultiplicity = values.reduce( + (total, [, multiplicity]) => total + multiplicity, + 0, + ) + if (totalMultiplicity === 0) return [] + if (totalMultiplicity < 0) { + throw new Error(`Query row has negative multiplicity`) + } + + const visible = values.find(([, multiplicity]) => multiplicity > 0)?.[0] + if (!visible) throw new Error(`Query row has no positive contributor`) + const visibleSignature = signature(visible) + + for (const [candidate, multiplicity] of values) { + if ( + multiplicity > 0 && + !deepEquals(visibleSignature, signature(candidate)) + ) { + throw new Error( + `Query contributors with the same row key are not congruent`, + ) + } + } + + return [[visible, 1]] + }), + ) as NamespacedAndKeyedStream +} + /** * Collects aliases used for DIRECT collection references (not subqueries). * Used to validate that subqueries don't reuse parent query collection aliases. @@ -1315,7 +1522,7 @@ function processFrom( } { switch (from.type) { case `collectionRef`: { - const input = allInputs[from.alias] + const input = allInputs[from.sourceId] ?? allInputs[from.alias] if (!input) { throw new CollectionInputNotFoundError( from.alias, @@ -1461,13 +1668,40 @@ function attachVirtualPropsToSelected( return selected } + const result = Array.isArray(selected) ? [...selected] : { ...selected } for (const prop of VIRTUAL_PROP_NAMES) { if (selected[prop] == null && prop in row) { - selected[prop] = row[prop] + result[prop] = row[prop] } } - return selected + return result +} + +function stripInternalCorrelation(selected: any): any { + if ( + !selected || + typeof selected !== `object` || + (!(`__correlationKey` in selected) && + !(`__parentContext` in selected) && + !(INCLUDES_PUBLIC_KEY in selected)) + ) { + return selected + } + + const result = Array.isArray(selected) ? [...selected] : { ...selected } + delete result.__correlationKey + delete result.__parentContext + delete result[INCLUDES_PUBLIC_KEY] + return result +} + +function getIncludesPublicKey( + row: Record, + mainSource: string, + fallback: unknown, +): unknown { + return row[mainSource]?.[INCLUDES_PUBLIC_KEY] ?? fallback } /** @@ -1895,7 +2129,8 @@ function isNestedSelectObject(value: any): value is Record { value != null && typeof value === `object` && !Array.isArray(value) && - !isExpressionLike(value) + !isExpressionLike(value) && + value.__refProxy !== true ) } @@ -2059,3 +2294,44 @@ function matchesConditionalSelectGuards( } export type CompileQueryFn = typeof compileQuery + +function getStaticDemandKeys(query: QueryIR, ref: PropRef): Set { + const constraints: Array> = [] + const visit = (expression: BasicExpression): void => { + if (expression.type !== `func`) return + if (expression.name === `and`) { + expression.args.forEach(visit) + return + } + if (expression.name !== `eq` && expression.name !== `in`) return + + const [left, right] = expression.args + const value = + left?.type === `ref` && + pathsEqual(left.path, ref.path) && + right instanceof ValClass + ? right.value + : right?.type === `ref` && + pathsEqual(right.path, ref.path) && + left instanceof ValClass + ? left.value + : undefined + if (value === undefined) return + constraints.push(new Set(Array.isArray(value) ? value : [value])) + } + + query.where?.forEach((where) => visit(getWhereExpression(where))) + if (constraints.length === 0) return new Set() + return new Set( + [...constraints[0]!].filter((value) => + constraints.slice(1).every((constraint) => constraint.has(value)), + ), + ) +} + +function pathsEqual(left: Array, right: Array): boolean { + return ( + left.length === right.length && + left.every((segment, index) => segment === right[index]) + ) +} diff --git a/packages/db/src/query/compiler/joins.ts b/packages/db/src/query/compiler/joins.ts index 0c37e05f4e..df87cfcfaa 100644 --- a/packages/db/src/query/compiler/joins.ts +++ b/packages/db/src/query/compiler/joins.ts @@ -1,4 +1,10 @@ -import { filter, join as joinOperator, map, tap } from '@tanstack/db-ivm' +import { + filter, + join as joinOperator, + map, + serializeValue, + tap, +} from '@tanstack/db-ivm' import { CollectionInputNotFoundError, InvalidJoinCondition, @@ -7,14 +13,11 @@ import { InvalidJoinConditionSameSourceError, InvalidJoinConditionSourceMismatchError, JoinCollectionNotFoundError, - SubscriptionNotFoundError, UnsupportedJoinSourceTypeError, UnsupportedJoinTypeError, } from '../../errors.js' import { normalizeValue } from '../../utils/comparison.js' import { ensureIndexForField } from '../../indexes/auto-index.js' -import { PropRef } from '../ir.js' -import { inArray } from '../builder/functions.js' import { compileExpression } from './evaluators.js' import { getLazyLoadTargets } from './lazy-targets.js' import type { CompileQueryFn } from './index.js' @@ -36,13 +39,33 @@ import type { import type { QueryCache, QueryMapping, WindowOptions } from './types.js' import type { CollectionSubscription } from '../../collection/subscription.js' -/** Function type for loading specific keys into a lazy collection */ -export type LoadKeysFn = (key: Set) => void +export type LazyDemandPlan = { + id: string + path: Array + initialKeys: Set +} /** Callbacks for managing lazy-loaded collections in optimized joins */ export type LazyCollectionCallbacks = { - loadKeys: LoadKeysFn - loadInitialState: () => void + plans?: Array + setDemand?: (plan: LazyDemandPlan, keys: Set) => void +} + +let nextLazyDemandPlanId = 0 + +export function registerLazyDemandPlan( + callbacks: Record, + target: { sourceId: string; path: Array }, + initialKeys: Set = new Set(), +): LazyDemandPlan { + const plan: LazyDemandPlan = { + id: `lazy-demand-${++nextLazyDemandPlanId}`, + path: target.path, + initialKeys: new Set(initialKeys), + } + const state = (callbacks[target.sourceId] ??= {}) + ;(state.plans ??= []).push(plan) + return plan } /** @@ -69,6 +92,7 @@ export function processJoins( aliasToCollectionId: Record, aliasRemapping: Record, sourceWhereClauses: Map>, + mainSourceIsParentFiltered: boolean, ): NamespacedAndKeyedStream { let resultPipeline = pipeline @@ -93,6 +117,7 @@ export function processJoins( aliasToCollectionId, aliasRemapping, sourceWhereClauses, + mainSourceIsParentFiltered, ) } @@ -123,6 +148,7 @@ function processJoin( aliasToCollectionId: Record, aliasRemapping: Record, sourceWhereClauses: Map>, + mainSourceIsParentFiltered: boolean, ): NamespacedAndKeyedStream { const isCollectionRef = joinClause.from.type === `collectionRef` @@ -171,6 +197,7 @@ function processJoin( joinClause.type, mainCollection, joinedCollection, + mainSourceIsParentFiltered, ) // Analyze which source each expression refers to and swap if necessary @@ -258,9 +285,14 @@ function processJoin( // such that the liveQueryCollection can check it after compilation // to know which source aliases should load data lazily (not initially) for (const target of lazyTargets) { - lazySources.add(target.alias) + lazySources.add(target.sourceId) } + const demandPlans = lazyTargets.map((target) => + registerLazyDemandPlan(callbacks, target), + ) + const demandWeights = new Map() + const activePipeline = activeSource === `main` ? mainPipeline : joinedPipeline @@ -277,55 +309,26 @@ function processJoin( [key: unknown, [originalKey: string, namespacedRow: NamespacedRow]] > = activePipeline.pipe( tap((data) => { - // Deduplicate and filter null keys before requesting snapshot - const joinKeys = [ - ...new Set( - data - .getInner() - .map(([[joinKey]]) => joinKey) - .filter((key) => key != null), - ), - ] - - if (joinKeys.length === 0) { - return - } - - for (const target of lazyTargets) { - const lazySourceSubscription = subscriptions[target.alias] - - if (!lazySourceSubscription) { - throw new SubscriptionNotFoundError( - target.alias, - lazyAlias, - target.collection.id, - Object.keys(subscriptions), - ) - } - - if (lazySourceSubscription.hasLoadedInitialState()) { - // Entire state was already loaded because we deoptimized the join - continue + for (const [[joinKey], weight] of data.getInner()) { + if (joinKey == null) continue + const encoded = serializeValue(joinKey) + const previous = demandWeights.get(encoded) + const nextWeight = (previous?.weight ?? 0) + weight + if (nextWeight === 0) { + demandWeights.delete(encoded) + } else { + demandWeights.set(encoded, { key: joinKey, weight: nextWeight }) } + } - const lazyJoinRef = new PropRef(target.path) - const loaded = lazySourceSubscription.requestSnapshot({ - where: inArray(lazyJoinRef, joinKeys), - optimizedOnly: true, - }) - - if (!loaded) { - // Snapshot wasn't sent because it could not be loaded from the indexes - const collectionId = target.collection.id - const fieldPath = target.path.join(`.`) - console.warn( - `[TanStack DB]${collectionId ? ` [${collectionId}]` : ``} Join requires an index on "${fieldPath}" for efficient loading. ` + - `Falling back to loading all data. ` + - `Consider creating an index on the collection with collection.createIndex((row) => row.${fieldPath}) ` + - `or enable auto-indexing with autoIndex: 'eager' and a defaultIndexType.`, - ) - lazySourceSubscription.requestSnapshot() - } + const keys = new Set( + [...demandWeights.values()] + .filter(({ weight }) => weight > 0) + .map(({ key }) => key), + ) + for (let index = 0; index < lazyTargets.length; index++) { + const target = lazyTargets[index]! + callbacks[target.sourceId]?.setDemand?.(demandPlans[index]!, keys) } }), ) @@ -470,7 +473,7 @@ function processJoinSource( ): { alias: string; input: KeyedStream; collectionId: string } { switch (from.type) { case `collectionRef`: { - const input = allInputs[from.alias] + const input = allInputs[from.sourceId] ?? allInputs[from.alias] if (!input) { throw new CollectionInputNotFoundError( from.alias, @@ -663,6 +666,7 @@ function getActiveAndLazySources( joinType: JoinClause[`type`], leftCollection: Collection, rightCollection: Collection, + mainSourceIsParentFiltered: boolean, ): | { activeSource: `main` | `joined`; lazySource: Collection } | { activeSource: undefined; lazySource: undefined } { @@ -675,6 +679,13 @@ function getActiveAndLazySources( case `right`: return { activeSource: `joined`, lazySource: leftCollection } case `inner`: + // A correlated include has already reduced the main relation to active + // parent routes. Keep that relation active and load the joined side from + // its keys; reversing the join would create an independent demand plan + // that widens the correlated source back to a union of constraints. + if (mainSourceIsParentFiltered) { + return { activeSource: `main`, lazySource: rightCollection } + } // The smallest collection should be the active collection // and the biggest collection should be lazy return leftCollection.size < rightCollection.size diff --git a/packages/db/src/query/compiler/lazy-targets.ts b/packages/db/src/query/compiler/lazy-targets.ts index f8affd1e63..963f62ebff 100644 --- a/packages/db/src/query/compiler/lazy-targets.ts +++ b/packages/db/src/query/compiler/lazy-targets.ts @@ -9,6 +9,7 @@ import type { import type { Collection } from '../../collection/index.js' export type LazyLoadTarget = { + sourceId: string alias: string collection: Collection path: Array @@ -49,6 +50,15 @@ export function getLazyLoadTargets( return [] } + const alias = followRefResult.alias || aliasRemapping[lazyAlias] || lazyAlias + const source = resolveLazySource(rawQuery, lazyFrom, { + alias, + collection: followRefResult.collection, + }) + if (!source) { + return [] + } + // The subscription we drive lazy loading through must be the one for the // collection the join key actually resolves to. When the key traces through a // subquery's select into a *joined* source, that collection differs from the @@ -57,7 +67,8 @@ export function getLazyLoadTargets( // remapping when the key resolves directly to the from source. return [ { - alias: followRefResult.alias || aliasRemapping[lazyAlias] || lazyAlias, + sourceId: source.sourceId, + alias, collection: followRefResult.collection, path: followRefResult.path, }, @@ -149,7 +160,14 @@ function getTargetsFromPropRef( } if (source.type === `collectionRef`) { - return [{ alias: source.alias, collection: source.collection, path }] + return [ + { + sourceId: source.sourceId, + alias: source.alias, + collection: source.collection, + path, + }, + ] } if (source.query.limit || source.query.offset) { @@ -181,13 +199,72 @@ function getSourceFromAlias( return sources.find((source) => source.alias === alias) } +function resolveLazySource( + query: QueryIR, + lazyFrom: From, + target: { alias: string; collection: Collection }, +): CollectionRef | undefined { + // Prefer the lexical source from the user's query. The optimizer may create + // an equivalent CollectionRef with a new source ID, but subscriptions and + // demand callbacks are owned by the original lexical source. + const source = findCollectionSource(query, target.alias, target.collection) + if (source) return source + + if ( + lazyFrom.type === `collectionRef` && + lazyFrom.collection === target.collection + ) { + return lazyFrom + } + + return undefined +} + +function findCollectionSource( + query: QueryIR, + alias: string, + collection: Collection, +): CollectionRef | undefined { + const sources = [ + ...(query.from.type === `unionFrom` + ? query.from.sources + : query.from.type === `unionAll` + ? [] + : [query.from]), + ...(query.join?.map((join) => join.from) ?? []), + ] + + for (const source of sources) { + if ( + source.type === `collectionRef` && + source.alias === alias && + source.collection === collection + ) { + return source + } + if (source.type === `queryRef`) { + const nested = findCollectionSource(source.query, alias, collection) + if (nested) return nested + } + } + + if (query.from.type === `unionAll`) { + for (const branch of query.from.queries) { + const nested = findCollectionSource(branch, alias, collection) + if (nested) return nested + } + } + + return undefined +} + function dedupeLazyLoadTargets( targets: Array, ): Array { const seen = new Set() const deduped: Array = [] for (const target of targets) { - const key = `${target.alias}:${target.path.join(`.`)}` + const key = `${target.sourceId}:${target.path.join(`.`)}` if (!seen.has(key)) { seen.add(key) deduped.push(target) diff --git a/packages/db/src/query/effect.ts b/packages/db/src/query/effect.ts index 0237519212..101e718d61 100644 --- a/packages/db/src/query/effect.ts +++ b/packages/db/src/query/effect.ts @@ -7,11 +7,12 @@ import { normalizeOrderByPaths, } from './compiler/expressions.js' import { getCollectionBuilder } from './live/collection-registry.js' +import { SubsetDemandController } from './live/subset-demand-controller.js' import { buildQueryFromConfig, computeOrderedLoadCursor, computeSubscriptionOrderByHints, - extractCollectionAliases, + extractCollectionSources, extractCollectionsFromQuery, filterDuplicateInserts, sendChangesToInput, @@ -25,6 +26,10 @@ import type { InitialQueryBuilder, QueryBuilder } from './builder/index.js' import type { Context } from './builder/types.js' import type { BasicExpression, QueryIR } from './ir.js' import type { OrderByOptimizationInfo } from './compiler/order-by.js' +import type { + LazyCollectionCallbacks, + LazyDemandPlan, +} from './compiler/joins.js' import type { ChangeMessage, KeyedStream, ResultStream } from '../types.js' // --------------------------------------------------------------------------- @@ -314,14 +319,15 @@ interface EffectPipelineRunnerConfig< * Sets up the IVM graph, subscribes to source collections, runs the graph * when changes arrive, and classifies output multiplicities into DeltaEvents. * - * Unlike CollectionConfigBuilder, this does NOT: - * - Create or write to a collection (no materialisation) - * - Manage ordering, windowing, or lazy loading + * Unlike CollectionConfigBuilder, this does not publish results to a + * Collection. */ class EffectPipelineRunner { private readonly query: QueryIR private readonly collections: Record> - private readonly collectionByAlias: Record> + private readonly collectionSources: ReturnType< + typeof extractCollectionSources + > private graph: D2 | undefined private inputs: Record> | undefined @@ -333,8 +339,12 @@ class EffectPipelineRunner { // The join compiler captures these references and reads them later when // the graph runs, so they must be populated before the first graph run. private readonly subscriptions: Record = {} - private readonly lazySourcesCallbacks: Record = {} + private readonly lazySourcesCallbacks: Record< + string, + LazyCollectionCallbacks + > = {} private readonly lazySources = new Set() + private readonly demand = new SubsetDemandController() // OrderBy optimization info populated by the compiler when limit is present private readonly optimizableOrderByCollections: Record< string, @@ -348,8 +358,11 @@ class EffectPipelineRunner { // Subscription management private readonly unsubscribeCallbacks = new Set<() => void>() - // Duplicate insert prevention per alias - private readonly sentToD2KeysByAlias = new Map>() + // Duplicate insert prevention per lexical source + private readonly sentToD2KeysBySource = new Map< + string, + Set + >() // Output accumulator private pendingChanges: Map> = new Map() @@ -361,7 +374,7 @@ class EffectPipelineRunner { // Scheduler integration private subscribedToAllCollections = false private readonly builderDependencies = new Set() - private readonly aliasDependencies: Record> = {} + private readonly sourceDependencies: Record> = {} // Reentrance guard private isGraphRunning = false @@ -384,17 +397,7 @@ class EffectPipelineRunner { // Extract source collections this.collections = extractCollectionsFromQuery(this.query) - const aliasesById = extractCollectionAliases(this.query) - - // Build alias → collection map - this.collectionByAlias = {} - for (const [collectionId, aliases] of aliasesById.entries()) { - const collection = this.collections[collectionId] - if (!collection) continue - for (const alias of aliases) { - this.collectionByAlias[alias] = collection - } - } + this.collectionSources = extractCollectionSources(this.query) // Compile the pipeline this.compilePipeline() @@ -404,8 +407,8 @@ class EffectPipelineRunner { private compilePipeline(): void { this.graph = new D2() this.inputs = Object.fromEntries( - Object.keys(this.collectionByAlias).map((alias) => [ - alias, + this.collectionSources.map((source) => [ + source.sourceId, this.graph!.newInput(), ]), ) @@ -441,9 +444,7 @@ class EffectPipelineRunner { /** Subscribe to source collections and start processing */ start(): void { - // Use compiled aliases as the source of truth - const compiledAliases = Object.entries(this.compiledAliasToCollectionId) - if (compiledAliases.length === 0) { + if (this.collectionSources.length === 0) { // Nothing to subscribe to return } @@ -467,38 +468,38 @@ class EffectPipelineRunner { Array>> >() - for (const [alias, collectionId] of compiledAliases) { - const collection = - this.collectionByAlias[alias] ?? this.collections[collectionId]! + for (const source of this.collectionSources) { + const { sourceId, alias, collection } = source + const collectionId = collection.id - // Initialise per-alias duplicate tracking - this.sentToD2KeysByAlias.set(alias, new Set()) + // Initialise per-source duplicate tracking + this.sentToD2KeysBySource.set(sourceId, new Set()) // Discover dependencies: if source collection is itself a live query // collection, its builder must run first during transaction flushes. const dependencyBuilder = getCollectionBuilder(collection) if (dependencyBuilder) { - this.aliasDependencies[alias] = [dependencyBuilder] + this.sourceDependencies[sourceId] = [dependencyBuilder] this.builderDependencies.add(dependencyBuilder) } else { - this.aliasDependencies[alias] = [] + this.sourceDependencies[sourceId] = [] } // Get where clause for this alias (for predicate push-down) - const whereClause = this.sourceWhereClauses?.get(alias) + const whereClause = this.sourceWhereClauses?.get(sourceId) const whereExpression = whereClause ? normalizeExpressionPaths(whereClause, alias) : undefined // Initialise buffer for this alias const buffer: Array>> = [] - pendingBuffers.set(alias, buffer) + pendingBuffers.set(sourceId, buffer) // Lazy aliases (marked by the join compiler) should NOT load initial state // eagerly — the join tap operator will load exactly the rows it needs on demand. // For on-demand collections, eager loading would trigger a full server fetch // for data that should be lazily loaded based on join keys. - const isLazy = this.lazySources.has(alias) + const isLazy = this.lazySources.has(sourceId) // Check if this alias has orderBy optimization (cursor-based loading) const orderByInfo = this.getOrderByInfoForAlias(alias) @@ -507,19 +508,19 @@ class EffectPipelineRunner { // delete+insert and track the biggest sent value for cursor positioning. const changeCallback = orderByInfo ? (changes: Array>) => { - if (pendingBuffers.has(alias)) { - pendingBuffers.get(alias)!.push(changes) + if (pendingBuffers.has(sourceId)) { + pendingBuffers.get(sourceId)!.push(changes) } else { - this.trackSentValues(alias, changes, orderByInfo.comparator) + this.trackSentValues(sourceId, changes, orderByInfo.comparator) const split = [...splitUpdates(changes)] - this.handleSourceChanges(alias, split) + this.handleSourceChanges(sourceId, split) } } : (changes: Array>) => { - if (pendingBuffers.has(alias)) { - pendingBuffers.get(alias)!.push(changes) + if (pendingBuffers.has(sourceId)) { + pendingBuffers.get(sourceId)!.push(changes) } else { - this.handleSourceChanges(alias, changes) + this.handleSourceChanges(sourceId, changes) } } @@ -538,7 +539,18 @@ class EffectPipelineRunner { ) // Store subscription immediately so the join compiler can find it - this.subscriptions[alias] = subscription + this.subscriptions[sourceId] = subscription + + const lazyCallbacks = this.lazySourcesCallbacks[sourceId] + if (lazyCallbacks) { + lazyCallbacks.setDemand = (plan: LazyDemandPlan, keys: Set) => + this.setDemand(subscription, plan, keys) + for (const plan of lazyCallbacks.plans ?? []) { + if (plan.initialKeys.size > 0) { + lazyCallbacks.setDemand(plan, plan.initialKeys) + } + } + } // For ordered aliases with an index, trigger the initial limited snapshot. // This loads only the top N rows rather than the entire collection. @@ -548,7 +560,7 @@ class EffectPipelineRunner { this.unsubscribeCallbacks.add(() => { subscription.unsubscribe() - delete this.subscriptions[alias] + delete this.subscriptions[sourceId] }) // Listen for status changes on source collections @@ -600,22 +612,25 @@ class EffectPipelineRunner { // switches that alias to direct-processing mode. Any new callbacks that // fire during the drain (e.g. from requestLimitedSnapshot) will go // through handleSourceChanges directly instead of being lost. - for (const [alias] of pendingBuffers) { - const buffer = pendingBuffers.get(alias)! - pendingBuffers.delete(alias) + for (const [sourceId] of pendingBuffers) { + const buffer = pendingBuffers.get(sourceId)! + pendingBuffers.delete(sourceId) + const source = this.collectionSources.find( + (candidate) => candidate.sourceId === sourceId, + )! - const orderByInfo = this.getOrderByInfoForAlias(alias) + const orderByInfo = this.getOrderByInfoForAlias(source.alias) // Drain all buffered batches. Since we deleted the alias from // pendingBuffers above, any new changes arriving during drain go // through handleSourceChanges directly (not back into this buffer). for (const changes of buffer) { if (orderByInfo) { - this.trackSentValues(alias, changes, orderByInfo.comparator) + this.trackSentValues(sourceId, changes, orderByInfo.comparator) const split = [...splitUpdates(changes)] - this.sendChangesToD2(alias, split) + this.sendChangesToD2(sourceId, split) } else { - this.sendChangesToD2(alias, changes) + this.sendChangesToD2(sourceId, changes) } } } @@ -635,11 +650,19 @@ class EffectPipelineRunner { /** Handle incoming changes from a source collection */ private handleSourceChanges( - alias: string, + sourceId: string, changes: Array>, ): void { - this.sendChangesToD2(alias, changes) - this.scheduleGraphRun(alias) + this.sendChangesToD2(sourceId, changes) + this.scheduleGraphRun(sourceId) + } + + private setDemand( + subscription: CollectionSubscription, + plan: LazyDemandPlan, + keys: Set, + ): void { + this.demand.setDemand(subscription, plan, keys) } /** @@ -652,15 +675,15 @@ class EffectPipelineRunner { * Dependencies are discovered from source collections that are themselves * live query collections, ensuring parent queries run before effects. */ - private scheduleGraphRun(alias?: string): void { + private scheduleGraphRun(sourceId?: string): void { const contextId = getActiveTransaction()?.id // Collect dependencies for this schedule call const deps = new Set(this.builderDependencies) - if (alias) { - const aliasDeps = this.aliasDependencies[alias] - if (aliasDeps) { - for (const dep of aliasDeps) { + if (sourceId) { + const sourceDeps = this.sourceDependencies[sourceId] + if (sourceDeps) { + for (const dep of sourceDeps) { deps.add(dep) } } @@ -699,20 +722,20 @@ class EffectPipelineRunner { } /** - * Send changes to the D2 input for the given alias. + * Send changes to the D2 input for the given lexical source. * Returns the number of multiset entries sent. */ private sendChangesToD2( - alias: string, + sourceId: string, changes: Array>, ): number { if (this.disposed || !this.inputs || !this.graph) return 0 - const input = this.inputs[alias] + const input = this.inputs[sourceId] if (!input) return 0 - // Filter duplicates per alias - const sentKeys = this.sentToD2KeysByAlias.get(alias)! + // Filter duplicates per lexical source + const sentKeys = this.sentToD2KeysBySource.get(sourceId)! const filtered = filterDuplicateInserts(changes, sentKeys) return sendChangesToInput(input, filtered) @@ -900,19 +923,23 @@ class EffectPipelineRunner { */ private loadNextItems(orderByInfo: OrderByOptimizationInfo, n: number): void { const { alias } = orderByInfo - const subscription = this.subscriptions[alias] + const source = this.collectionSources.find( + (candidate) => candidate.alias === alias, + ) + if (!source) return + const subscription = this.subscriptions[source.sourceId] if (!subscription) return const cursor = computeOrderedLoadCursor( orderByInfo, - this.biggestSentValue.get(alias), - this.lastLoadRequestKey.get(alias), + this.biggestSentValue.get(source.sourceId), + this.lastLoadRequestKey.get(source.sourceId), alias, n, ) if (!cursor) return // Duplicate request — skip - this.lastLoadRequestKey.set(alias, cursor.loadRequestKey) + this.lastLoadRequestKey.set(source.sourceId, cursor.loadRequestKey) subscription.requestLimitedSnapshot({ orderBy: cursor.normalizedOrderBy, @@ -938,20 +965,20 @@ class EffectPipelineRunner { * Used for cursor-based pagination in loadNextItems. */ private trackSentValues( - alias: string, + sourceId: string, changes: Array>, comparator: (a: any, b: any) => number, ): void { - const sentKeys = this.sentToD2KeysByAlias.get(alias) ?? new Set() + const sentKeys = this.sentToD2KeysBySource.get(sourceId) ?? new Set() const result = trackBiggestSentValue( changes, - this.biggestSentValue.get(alias), + this.biggestSentValue.get(sourceId), sentKeys, comparator, ) - this.biggestSentValue.set(alias, result.biggest) + this.biggestSentValue.set(sourceId, result.biggest) if (result.shouldResetLoadKey) { - this.lastLoadRequestKey.delete(alias) + this.lastLoadRequestKey.delete(sourceId) } } @@ -964,9 +991,10 @@ class EffectPipelineRunner { // Immediately unsubscribe from sources and clear cheap state this.unsubscribeCallbacks.forEach((fn) => fn()) this.unsubscribeCallbacks.clear() - this.sentToD2KeysByAlias.clear() + this.sentToD2KeysBySource.clear() this.pendingChanges.clear() this.lazySources.clear() + this.demand.clear() this.builderDependencies.clear() this.biggestSentValue.clear() this.lastLoadRequestKey.clear() @@ -976,8 +1004,8 @@ class EffectPipelineRunner { for (const key of Object.keys(this.lazySourcesCallbacks)) { delete this.lazySourcesCallbacks[key] } - for (const key of Object.keys(this.aliasDependencies)) { - delete this.aliasDependencies[key] + for (const key of Object.keys(this.sourceDependencies)) { + delete this.sourceDependencies[key] } for (const key of Object.keys(this.optimizableOrderByCollections)) { delete this.optimizableOrderByCollections[key] diff --git a/packages/db/src/query/ir.ts b/packages/db/src/query/ir.ts index 0cfb15ea50..60e61d35a2 100644 --- a/packages/db/src/query/ir.ts +++ b/packages/db/src/query/ir.ts @@ -74,6 +74,8 @@ export type Limit = number export type Offset = number +let nextCollectionSourceId = 0 + /* Expressions */ abstract class BaseExpression { @@ -84,11 +86,17 @@ abstract class BaseExpression { export class CollectionRef extends BaseExpression { public type = `collectionRef` as const + /** Opaque runtime identity; aliases are lexical names only. */ + public readonly sourceId!: string constructor( public collection: CollectionImpl, public alias: string, ) { super() + Object.defineProperty(this, `sourceId`, { + value: `source-${++nextCollectionSourceId}`, + enumerable: false, + }) } } @@ -254,6 +262,55 @@ export function isExpressionLike(value: any): boolean { return false } +/** Returns each lexical Collection source in a query tree once. */ +export function collectCollectionSources(query: QueryIR): Array { + const sources: Array = [] + const seen = new Set() + + const visitSource = (source: QueryIR[`from`]): void => { + if (source.type === `collectionRef`) { + if (!seen.has(source.sourceId)) { + seen.add(source.sourceId) + sources.push(source) + } + } else if (source.type === `queryRef`) { + visitQuery(source.query) + } else if (source.type === `unionFrom`) { + source.sources.forEach(visitSource) + } else { + source.queries.forEach(visitQuery) + } + } + + const visitSelectValue = (value: any): void => { + if (value instanceof IncludesSubquery) { + visitQuery(value.query) + } else if (value instanceof ConditionalSelect) { + value.branches.forEach((branch) => visitSelectValue(branch.value)) + if (value.defaultValue !== undefined) { + visitSelectValue(value.defaultValue) + } + } else if ( + value !== null && + typeof value === `object` && + !Array.isArray(value) && + !isExpressionLike(value) && + value.__refProxy !== true + ) { + Object.values(value).forEach(visitSelectValue) + } + } + + const visitQuery = (current: QueryIR): void => { + visitSource(current.from) + current.join?.forEach(({ from }) => visitSource(from)) + if (current.select) Object.values(current.select).forEach(visitSelectValue) + } + + visitQuery(query) + return sources +} + /** * Helper functions for working with Where clauses */ diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md new file mode 100644 index 0000000000..d794ea7a1c --- /dev/null +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -0,0 +1,518 @@ +# Live-query materialization architecture + +This document defines the architecture for correlated live-query +materialization in `@tanstack/db`. It follows +[RFC #1658](https://github.com/TanStack/db/issues/1658). + +The central rule is simple: + +> Keep relation contents, routes, nested materialization, and propagation in +> one D2 graph. Use custom state only at asynchronous source and public +> Collection boundaries. + +The correlated-materialization oracle suites listed below are green behavioral +contracts for this design. Suites for adjacent planner and query-db ownership +boundaries may also contain exact classifiers for defects outside this graph. + +## Scope + +This architecture covers: + +- compiled identities for sources, relations, and materialization edges; +- weighted contributions and public-key reduction; +- correlated routes and ordered bucket contents; +- nested inline and Collection-valued materialization; +- lazy and progressive source demand; +- coherent publication to public Collections; +- the boundaries with query-db ownership and physical query planning. + +It does not define new public APIs. Optimistic transactions are another source +of weighted input changes; they do not have a separate routing model. + +## One relational graph + +Correlated materialization is part of the compiled D2 graph, not a second +incremental engine around its output. + +```text +raw weighted query rows + | + v +public-key reduction + | + v +CanonicalRow(base row, order, outgoing parameters) + | + +------------------------------+ + | | + v | +Route(bucket, cell) | + | | + +--> distinct --> ActiveBucket-+--> async demand adapter + | | | + | v | + | child rows --> BucketValue + | | + +------------------------------+ + | + v + CellValue(cell, value) + | + v + CanonicalRow + outgoing CellValues + | + v + MaterializedRow + | + v + one normal root Collection transaction +``` + +Canonical base rows flow down to derive correlation routes and source demand. +Fully materialized child rows flow up into their parents. Because the include +graph is acyclic, these streams form one acyclic D2 graph even though demand +and results move in opposite conceptual directions. + +The graph owns the data plane. A small adapter owns asynchronous demand. The +normal Collection transaction boundary owns public publication. + +## Identity + +Aliases are lexical query-language names. They are not runtime identities. +Compilation assigns opaque IDs to the plan: + +```ts +type SourceId = Brand +type RelationNodeId = Brand +type MaterializationEdgeId = Brand +``` + +Alias text may remain as debug metadata. Alpha-renaming an alias cannot change +the compiled graph or its result. + +A `CanonicalCorrelationKey` is the canonical tuple of every evaluated +parent-dependent value that can affect the child plan. This includes values +used by filters, joins, ordering, limits, and nullable predicates, not only the +obvious foreign-key equality. + +A bucket key identifies one such correlated partition at one relation node: + +```ts +type BucketKey = readonly [ + relationNodeId: RelationNodeId, + correlationKey: CanonicalCorrelationKey, +] +``` + +Correlation equality must use the same value semantics as query predicates. +Implementations use canonical values, interned handles, or nested maps; they do +not reconstruct array or object keys and expect JavaScript `Map` identity to +match. + +A materialization cell identifies one include field on one parent-row +occurrence: + +```ts +type MaterializationCellId = readonly [ + containingBucket: BucketKey | 'root', + parentPublicKey: PublicKey, + edgeId: MaterializationEdgeId, +] +``` + +The containing bucket prevents equal child keys in separate correlated +contexts from colliding. + +Only work that crosses an asynchronous boundary needs a generation token. A +live-query graph generation invalidates work from an old graph. A demand +generation invalidates an old load for the same bucket. Synchronous route rows +inside D2 do not need their own lifecycle objects or generations. + +## Weighted relations and public keys + +D2 multisets are the source of truth. A row with positive weight contributes; +a row with negative weight retracts the same contribution. + +Internal contribution identity is independent of the user-visible Collection +key. When several internal rows collapse to one public key, a keyed D2 +reduction retains all contributors and derives at most one canonical row: + +```ts +type CanonicalRow = { + publicKey: PublicKey + value: Row + order: OrderKey | undefined + outgoingParameters: ReadonlyMap< + MaterializationEdgeId, + CanonicalCorrelationKey + > +} +``` + +```text +raw weighted rows + -> reduce by [containing bucket, public key] + -> CanonicalRow + +-> derive route rows + +-> compose with materialized include values +``` + +For every affected public key, the reduction compares its complete before and +after state and emits no change, one replacement, or one removal. It does not +infer the previous state from `collection.has()`. + +Routes derive only from canonical rows. Raw contributors never create routes +that must later be reconciled. The same boundary applies recursively: roots +reduce by root public key, while child rows reduce by their containing bucket +and child public key. + +All positive contributors collapsed under one public key must be congruent on: + +- the visible value; +- the total order key; +- every outgoing correlation input. + +Query aggregation occurs upstream in the query graph. This reduction only +preserves multiplicity while collapsing congruent contributors under the +public Collection key. Incongruent contributors are a duplicate-key invariant +error; flush order never chooses a winner. A zero aggregate removes the public +row. A negative aggregate is an invariant violation. + +This is a specialized use of the existing D2 keyed reduction. It is not a +separate contribution-ledger subsystem. + +## Routes and buckets are relations + +For each materialization edge, the compiler produces these keyed relations: + +```ts +type RouteRow = readonly [bucketKey: BucketKey, cellId: MaterializationCellId] + +type ActiveBucket = readonly [bucketKey: BucketKey] + +type BucketRow = readonly [ + bucketKey: BucketKey, + child: readonly [publicKey: PublicKey, row: Row, order: OrderKey | undefined], +] + +type BucketValue = readonly [bucketKey: BucketKey, value: Value] + +type CellValue = readonly [cellId: MaterializationCellId, value: Value] +``` + +A route move is an ordinary weighted batch: + +```text +-1 [old bucket, cell] ++1 [new bucket, cell] +``` + +Distinct route keys produce `ActiveBucket`. For inline modes, child rows are +ordered and reduced once per active bucket into exactly one `BucketValue`. +Routes then join with bucket values to fan the same immutable logical value out +as a `CellValue`: + +```text +Route(bucket, cell) -> distinct -> ActiveBucket(bucket) + | +ActiveBucket + BucketRow -> reduce ------+-> BucketValue(bucket, value) + | +Route(bucket, cell) -------------------------------+ + v + CellValue(cell, value) +``` + +The bucket-value reduction belongs to the materialization edge because two +edges may apply different materialization modes to the same child relation. +Computing it before fan-out means ordering and materialization happen once per +unique bucket rather than once per parent. + +`ActiveBucket` also seeds the empty value. Every active inline bucket therefore +has exactly one value even when it has no child rows: + +- `array`: `[]`; +- `singleton`: `undefined`; +- `concat`: `""`. + +A null or otherwise unsatisfiable correlation may route to an active empty +bucket without creating source demand. This preserves the materialization +mode's empty value instead of relying on a placeholder or a missing join path. + +D2's retained join indexes provide the required lifecycle behavior: + +- adding a route joins it with the bucket's existing value; +- removing a route retracts only that cell's value; +- moving a route retracts the old rows and adds the new rows in one graph run; +- several cells may consume one bucket without recomputing its value; +- changing a bucket value reaches every current route; +- a departed route receives no later value changes. + +Root rows and nested rows use the same relation shape and operators. There is +no special root routing path. + +The implementation must not recreate these semantics with route registries, +reverse indexes, drained buffers, or per-depth snapshots outside D2. Existing +retained operator state is the first implementation choice. Add a reusable +arrangement only if counters show that the compiler duplicates indexes or +state; arrangements are a physical optimization, not part of correctness. + +The total-materialization law is: + +> Every active inline materialization cell has exactly one canonical value, +> including when its bucket contains no rows. + +## Nested materialization + +The compiler builds each include from the materialized output relation of its +child: + +```text +child base rows + + child include values + -> child materialized rows + -> rows in the parent's bucket relation + -> parent include value +``` + +A descendant update therefore becomes an ordinary change to the child's +materialized row and propagates through the same joins and reductions at every +depth. There are no depth-specific flush passes, dirty-cell registries, or +manual relation revisions. + +Inline modes are reductions over the rows in one active bucket: + +- `array`: total-order the rows and return their values; +- `singleton`: choose the first row under the total order; +- `concat`: total-order the rows and concatenate their scalar values. + +A total order is the query's order keys followed by a deterministic stable +tie-breaker, normally the child public key. An order-only change is a +bucket-value change for arrays, singletons, concatenation, and Collection +layout. + +A bare child query is a Collection-valued include. It exposes one stable public +Collection facade per active bucket in that edge: + +```text +BucketRow -> BucketFacade(bucket, Collection) +Route + BucketFacade -> CellValue(cell, Collection) +``` + +Parents sharing a bucket share its facade. Child changes update that Collection +without re-emitting every parent, and moving a route changes the parent field to +the destination bucket's facade. A facade is never retargeted to another +bucket. The adapter retains a facade only while at least one parent route uses +its bucket. When the last route leaves, it retracts the facade's rows and drops +its strong reference. An external holder may keep that empty Collection alive, +but a later active interval gets a new facade. Inline modes do not create child +Collections. + +Composition is pure. It constructs a new result along changed paths and does +not mutate a previously published row or use public routing metadata: + +```ts +compose( + baseRow: BaseRow, + includeValues: ReadonlyMap, +): MaterializedRow +``` + +## Demand plane + +Demand is derived from data, but it performs asynchronous side effects outside +D2: + +```text +ActiveBucket(bucket, demand parameters) + -> group by [source, parameterized child plan] + -> current demanded parameter set + -> demand adapter + -> source loadSubset / release + -> source deltas return to D2 inputs +``` + +The adapter treats demand as coverage, not as one request per bucket: + +```ts +type DemandPlanId = Brand + +type DemandSet = readonly [ + planId: DemandPlanId, + parameters: CanonicalSet, +] +``` + +One request may cover many buckets, and the adapter may coalesce or reuse +requests according to the compiled demand plan. Its semantic contract is: + +> Every active, satisfiable bucket must be covered by a settled current demand +> request before initial preload completes. + +A request may remain in flight after some covered buckets become inactive. +Those buckets no longer participate in readiness and cannot receive rows +through routes that no longer exist. Sharing source work never merges the route +rows themselves. + +The source contract stays abstract: a demand request eventually establishes +one coherent baseline and identifies when that baseline is complete. Each +request receives an `AbortSignal`. Replacing or releasing its demand aborts +that signal, and the source must check it before installing fetched rows. An +aborted request cannot install or settle after its graph or request generation +becomes obsolete. Buffering, snapshot tokens, shape offsets, Collection +transactions, and local indexes are source-specific ways to satisfy that +contract; they are not materializer state. + +This project uses a single graph-run order rather than multi-dimensional +timely-dataflow frontiers. Do not introduce a general timestamp or frontier +framework unless a source contract proves that the generation and up-to-date +protocol cannot express its ordering. + +**Initial readiness:** preload is complete when every demand currently +reachable from the initial query graph is covered by a settled request. Demand +that is no longer reachable does not block completion. An empty outer relation +has no child demand, but its root demand must still settle. Later readiness +transitions follow the existing Collection contract until an executable test +defines another public behavior. + +Pending demand does not hide the parent row. An active empty bucket gives it +the current canonical bucket value, and available partial source rows produce +the current partial materialization when the source supports progressive +delivery. Later source rows enter D2 as ordinary deltas and recompute the +parent. “Fully composed” means that every include field has its canonical value +for the graph's current input state; it does not mean that asynchronous demand +has settled. + +## Coherent publication + +D2 runs until the whole materialization graph has no pending synchronous work +for its currently available inputs. Only fully materialized canonical root +deltas cross into the public Collection. + +For each scheduled graph turn: + +1. enqueue all currently committed input deltas into their D2 inputs; +2. run D2 until it has no pending synchronous work; +3. consolidate the already canonical final-output deltas; +4. install child-facade state through normal Collection transactions while + deferring their subscriber delivery; +5. apply direct root insert, update, and delete writes through one normal + Collection transaction; +6. release the deferred child-facade events after every synchronous read can + see the complete root and facade state; +7. allow dependent live-query graphs to run through the existing + transaction-scoped scheduler. + +The Collection boundary performs no identity reconciliation, routing, +materialization, or multiplicity interpretation. The canonical root relation +has already done that work. + +The public Collection is an output, never scratch state. Placeholder rows, +in-place include repair, and forced secondary events are forbidden. + +Installed state, synchronous reads, change-event payloads, and downstream +queries must all observe the same fully materialized commit. The facade adapter +may defer event delivery across its Collection transactions, but it must not +defer state or index installation. Routing and identity remain inside D2. + +## External boundaries + +### Query-db ownership + +Row ownership in `@tanstack/query-db-collection` is separate. Eager retention, +active query acquisition, and persisted retention are distinct owner tokens. +The live-query graph publishes coherent rows but does not own query-db cache or +listener lifetime. + +### Physical planning and work + +Correct relation state does not prove efficient work. When an applicable index +exists, irrelevant correlated rows must not cause scans of unrelated rows or +activate unrelated downstream routes. Relation rows, indexed keys, active +demands, materialization cells, and public facades are the relevant space +units. Inline materialization must not create recursive Collection machinery. + +## Normative laws + +1. **Alpha-renaming:** lexical alias names cannot change results. +2. **Contribution conservation:** a public row exists exactly when its reduced + supporting weight and collision policy produce one. +3. **Batch partition:** equivalent valid split and atomic deliveries converge. +4. **Route relation:** current route rows joined with current bucket values + equal current materialization-cell values. +5. **Total materialization:** every active inline cell has exactly one value, + including its mode's empty value when its bucket has no rows. +6. **Stale demand:** an obsolete graph or demand generation can neither publish + rows nor settle current readiness. +7. **Nested propagation:** every materialized relation consumes the fully + materialized output relation of its children. +8. **Publication:** reads, events, and downstream queries observe the same + complete graph result. +9. **Initial demand:** preload completes when every initially reachable demand + is covered; obsolete demand does not block it. +10. **Ownership:** a query-db row exists exactly while an explicit owner + remains. +11. **Work:** irrelevant rows do not cause unrelated scans or activate unrelated + routes when an applicable index exists. +12. **Space:** state scales with retained D2 relation/index rows, active demands, + materialization cells, visible rows, and required Collection facades. + +## Glossary + +- **Relation:** an internal weighted multiset maintained by D2, not a public + TanStack Collection. +- **Weighted delta:** a positive or negative change to a relation row. +- **Data plane:** the D2 graph that joins, reduces, orders, and materializes + relations. +- **Demand plane:** the async adapter that starts and releases source loads. +- **Bucket key:** the canonical identity of one correlated child partition. +- **Bucket relation:** child rows partitioned by bucket key. +- **Active bucket:** a bucket referenced by at least one current route; it seeds + empty materialization values and contributes to source demand. +- **Bucket value:** the one inline value reduced from an active bucket's rows. +- **Route relation:** weighted links from bucket keys to materialization cells. +- **Materialization cell:** one include field on one parent-row occurrence. +- **Arrangement:** retained relation state indexed for efficient keyed access + and reuse. +- **Reduction:** deriving one visible value from weighted rows sharing a key. +- **Hydration:** establishing an initial snapshot before forwarding later + changes. +- **Generation:** a token that rejects obsolete asynchronous work. +- **Collection facade:** a stable public Collection view shared by the parents + routed to one active bucket. +- **Coherent commit:** one publication in which state, events, and consumers see + the same fully materialized result. + +## Executable contracts + +| Contract | Test suite | +| --------------------------------------------------------------------------- | ------------------------------------------------------------------------- | +| State equivalence, route lifecycle, transition history, and batch partition | `packages/db/tests/query/includes-oracle.property.test.ts` | +| Joined multiplicity, alias identity, and null-key normalization | `packages/db/tests/query/includes-query-shape-oracle.test.ts` | +| Demand, cancellation, and progressive timing | `packages/db/tests/query/includes-temporal-oracle.test.ts` | +| Optimistic confirmation, rollback, and later reactivity | `packages/db/tests/query/includes-optimistic-oracle.property.test.ts` | +| Coherent layered publication | `packages/db/tests/query/includes-publication-oracle.test.ts` | +| Correlated physical work | `packages/db/tests/query/includes-work-counter-oracle.test.ts` | +| Query-db ownership | `packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts` | +| Reachable nested shape | `packages/query-db-collection/tests/includes-work-counter-oracle.test.ts` | + +Each oracle identifies the first divergent checkpoint and compares either the +whole result or one exact structural difference. Correlated-materialization +scenarios use direct assertions. A boundary suite may retain an exact +expected-failure guard for a planner or ownership defect that this graph does +not own. + +## Implementation discipline + +- Express relation state with existing D2 inputs, joins, reductions, grouping, + ordering, and consolidation before adding custom state. +- Keep route and bucket rows in the same graph as parent and child query rows. +- Add a reusable indexed D2 primitive only when existing operators cannot share + or expose required retained state. +- Keep asynchronous demand state outside D2 and make its generation boundary + explicit. +- Never use a public Collection, emitted event, or materialized row as internal + routing or contribution state. +- Add a reduced oracle trace before adding any special lifecycle branch. +- Measure retained relation rows, active demands, and public facades so the + simpler architecture remains a space improvement as well as a correctness + improvement. diff --git a/packages/db/src/query/live/bucket-facade-adapter.ts b/packages/db/src/query/live/bucket-facade-adapter.ts new file mode 100644 index 0000000000..cbe2840ea9 --- /dev/null +++ b/packages/db/src/query/live/bucket-facade-adapter.ts @@ -0,0 +1,349 @@ +import { output, serializeValue } from '@tanstack/db-ivm' +import { createCollection } from '../../collection/index.js' +import { FN_SELECT_STATE, INCLUDES_ROUTING } from '../compiler/index.js' +import { BUCKET_FACADE_REF } from './materialized-pipeline.js' +import type { Collection } from '../../collection/index.js' +import type { SyncConfig } from '../../types.js' +import type { + BucketFacadeCompilation, + BucketFacadeRef, + BucketRow, +} from './materialized-pipeline.js' + +type FacadeSync = Parameters[`sync`]>[0] + +type PendingRow = { + deletes: number + inserts: number + value: BucketRow +} + +type FacadeEntry = { + collection: Collection + sync: FacadeSync | undefined + keys: WeakMap + order: WeakMap + currentOrder: Map +} + +/** + * The only stateful boundary outside the materialization graph. It turns inert + * bucket references into stable public Collection facades and applies the + * graph's canonical bucket-row deltas to those facades. + */ +export class BucketFacadeAdapter { + private readonly pending = new Map< + string, + Map> + >() + private readonly pendingActivity = new Map>() + private readonly activeBuckets = new Map>() + private readonly entries = new Map>() + private readonly resolvedValues = new WeakMap() + + constructor( + private readonly parentId: string, + private readonly compilations: Array, + onMessages: (count: number) => void, + ) { + for (const compilation of compilations) { + compilation.rows.pipe( + output((data) => { + const messages = data.getInner() + onMessages(messages.length) + for (const [[bucketKey, row], multiplicity] of messages) { + this.accumulate(compilation.edgeId, bucketKey, row, multiplicity) + } + }), + ) + compilation.activeBuckets.pipe( + output((data) => { + const messages = data.getInner() + onMessages(messages.length) + for (const [[bucketKey], multiplicity] of messages) { + this.accumulateActivity(compilation.edgeId, bucketKey, multiplicity) + } + }), + ) + } + } + + hasPendingChanges(): boolean { + return this.pending.size > 0 || this.pendingActivity.size > 0 + } + + flush(): () => void { + const deferredEntries = new Set() + const resumePublications: Array<() => void> = [] + const deferPublication = (entry: FacadeEntry) => { + if (deferredEntries.has(entry)) return + deferredEntries.add(entry) + resumePublications.push(entry.collection._deferPublication()) + } + let resumed = false + const resume = () => { + if (resumed) return + resumed = true + for (const resumePublication of resumePublications) { + resumePublication() + } + } + + // Compilations are child-first, so nested facade references resolve before + // their containing rows are written to the next facade. + try { + for (const compilation of this.compilations) { + const activity = this.pendingActivity.get(compilation.edgeId) + const active = this.getActiveBuckets(compilation.edgeId) + for (const [bucketKey, multiplicity] of activity ?? []) { + if (multiplicity > 0) active.add(bucketKey) + } + + const buckets = this.pending.get(compilation.edgeId) + for (const [bucketKey, changes] of buckets ?? []) { + const existing = this.entries.get(compilation.edgeId)?.get(bucketKey) + if (!active.has(bucketKey) && !existing) continue + const entry = this.getEntry(compilation.edgeId, bucketKey) + const sync = entry.sync + if (!sync || changes.size === 0) continue + + deferPublication(entry) + sync.begin() + for (const change of changes.values()) { + this.applyChange(entry, sync, change, compilation.hasOrderBy) + } + sync.commit() + } + this.pending.delete(compilation.edgeId) + + for (const [bucketKey, multiplicity] of activity ?? []) { + if (multiplicity >= 0) continue + active.delete(bucketKey) + this.retireEntry(compilation.edgeId, bucketKey, deferPublication) + } + this.pendingActivity.delete(compilation.edgeId) + } + } catch (error) { + resume() + throw error + } + + return resume + } + + resolve(value: T): T { + return this.resolveValue(value) as T + } + + cleanup(): void { + for (const byBucket of this.entries.values()) { + for (const entry of byBucket.values()) { + void entry.collection.cleanup() + } + } + this.entries.clear() + this.pending.clear() + this.pendingActivity.clear() + this.activeBuckets.clear() + } + + private accumulate( + edgeId: string, + bucketKey: string, + row: BucketRow, + multiplicity: number, + ): void { + let buckets = this.pending.get(edgeId) + if (!buckets) { + buckets = new Map() + this.pending.set(edgeId, buckets) + } + let rows = buckets.get(bucketKey) + if (!rows) { + rows = new Map() + buckets.set(bucketKey, rows) + } + + const key = serializeValue(row.publicKey) + const change = rows.get(key) ?? { + deletes: 0, + inserts: 0, + value: row, + } + if (multiplicity < 0) { + change.deletes += -multiplicity + } else if (multiplicity > 0) { + change.inserts += multiplicity + change.value = row + } + rows.set(key, change) + } + + private accumulateActivity( + edgeId: string, + bucketKey: string, + multiplicity: number, + ): void { + let activity = this.pendingActivity.get(edgeId) + if (!activity) { + activity = new Map() + this.pendingActivity.set(edgeId, activity) + } + activity.set(bucketKey, (activity.get(bucketKey) ?? 0) + multiplicity) + } + + private getActiveBuckets(edgeId: string): Set { + let active = this.activeBuckets.get(edgeId) + if (!active) { + active = new Set() + this.activeBuckets.set(edgeId, active) + } + return active + } + + private retireEntry( + edgeId: string, + bucketKey: string, + deferPublication: (entry: FacadeEntry) => void, + ): void { + const byBucket = this.entries.get(edgeId) + const entry = byBucket?.get(bucketKey) + if (!entry) return + + const sync = entry.sync + const keys = [...entry.collection.keys()] + if (sync && keys.length > 0) { + deferPublication(entry) + sync.begin() + for (const key of keys) sync.write({ type: `delete`, key }) + sync.commit() + } + byBucket!.delete(bucketKey) + if (byBucket!.size === 0) this.entries.delete(edgeId) + } + + private getEntry(edgeId: string, bucketKey: string): FacadeEntry { + let byBucket = this.entries.get(edgeId) + if (!byBucket) { + byBucket = new Map() + this.entries.set(edgeId, byBucket) + } + const existing = byBucket.get(bucketKey) + if (existing) return existing + + const keys = new WeakMap() + const order = new WeakMap() + let sync: FacadeSync | undefined + const collection = createCollection({ + id: `__bucket-facade:${this.parentId}:${edgeId}:${bucketKey}`, + getKey: (row) => keys.get(row)!, + compare: (left, right) => { + const leftOrder = order.get(left) + const rightOrder = order.get(right) + if (leftOrder === rightOrder) return 0 + if (leftOrder === undefined) return 1 + if (rightOrder === undefined) return -1 + return leftOrder < rightOrder ? -1 : 1 + }, + sync: { + rowUpdateMode: `full`, + sync: (methods) => { + sync = methods + return () => { + sync = undefined + } + }, + }, + startSync: true, + gcTime: 0, + }) + const entry: FacadeEntry = { + collection, + get sync() { + return sync + }, + keys, + order, + currentOrder: new Map(), + } + byBucket.set(bucketKey, entry) + return entry + } + + private applyChange( + entry: FacadeEntry, + sync: FacadeSync, + change: PendingRow, + hasOrderBy: boolean, + ): void { + const key = change.value.publicKey as string | number + const previousOrder = entry.currentOrder.get(key) + const nextOrder = change.value.order + const orderChanged = sync.collection.has(key) && previousOrder !== nextOrder + const resolvedRow = this.resolve(change.value.value) + const row = + orderChanged && sync.collection.get(key) === resolvedRow + ? { ...resolvedRow } + : resolvedRow + entry.keys.set(row, key) + if (nextOrder !== undefined) { + entry.order.set(row, nextOrder) + } + + if (change.inserts > change.deletes) { + sync.write({ + type: sync.collection.has(key) ? `update` : `insert`, + value: row, + }) + } else if (change.inserts === change.deletes && sync.collection.has(key)) { + sync.write({ type: `update`, value: row }) + } else if (change.deletes > 0) { + sync.write({ type: `delete`, key }) + entry.currentOrder.delete(key) + return + } + + entry.currentOrder.set(key, nextOrder) + if (hasOrderBy && orderChanged) sync.collection._markLayoutChange() + } + + private resolveValue(value: unknown): unknown { + if (value !== null && typeof value === `object`) { + const cached = this.resolvedValues.get(value) + if (cached !== undefined) return cached + } + if (isBucketFacadeRef(value)) { + const { edgeId, bucketKey } = value[BUCKET_FACADE_REF] + const facade = this.getEntry(edgeId, bucketKey).collection + this.resolvedValues.set(value, facade) + return facade + } + if (Array.isArray(value)) { + const result: Array = [] + this.resolvedValues.set(value, result) + result.push(...value.map((item) => this.resolveValue(item))) + return result + } + if (!isPlainObject(value)) return value + + const result: Record = {} + this.resolvedValues.set(value, result) + for (const key of Reflect.ownKeys(value)) { + if (key === INCLUDES_ROUTING || key === FN_SELECT_STATE) continue + result[key] = this.resolveValue(value[key]) + } + return result + } +} + +function isBucketFacadeRef(value: unknown): value is BucketFacadeRef { + return ( + value !== null && typeof value === `object` && BUCKET_FACADE_REF in value + ) +} + +function isPlainObject(value: unknown): value is Record { + if (value === null || typeof value !== `object`) return false + const prototype = Object.getPrototypeOf(value) + return prototype === Object.prototype || prototype === null +} diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index e497150e50..3df211e096 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -1,10 +1,5 @@ -import { D2, output, serializeValue } from '@tanstack/db-ivm' -import { - FN_SELECT_STATE, - INCLUDES_ROUTING, - compileQuery, -} from '../compiler/index.js' -import { createCollection } from '../../collection/index.js' +import { D2, output } from '@tanstack/db-ivm' +import { compileQuery } from '../compiler/index.js' import { MissingAliasInputsError, SetWindowRequiresOrderByError, @@ -15,24 +10,22 @@ import { deepEquals } from '../../utils.js' import { CollectionSubscriber } from './collection-subscriber.js' import { getCollectionBuilder } from './collection-registry.js' import { LIVE_QUERY_INTERNAL } from './internal.js' +import { materializeCompilation } from './materialized-pipeline.js' +import { BucketFacadeAdapter } from './bucket-facade-adapter.js' import { buildQueryFromConfig, - extractCollectionAliases, extractCollectionFromSource, + extractCollectionSources, extractCollectionsFromQuery, } from './utils.js' import type { LiveQueryInternalUtils } from './internal.js' -import type { - IncludesCompilationResult, - WindowOptions, -} from '../compiler/index.js' +import type { WindowOptions } from '../compiler/index.js' import type { SchedulerContextId } from '../../scheduler.js' import type { CollectionSubscription } from '../../collection/subscription.js' import type { RootStreamBuilder } from '@tanstack/db-ivm' import type { OrderByOptimizationInfo } from '../compiler/order-by.js' import type { Collection } from '../../collection/index.js' import type { - ChangeMessage, CollectionConfigSingleRowOption, KeyedStream, ResultStream, @@ -41,12 +34,7 @@ import type { UtilsRecord, } from '../../types.js' import type { Context, GetResult } from '../builder/types.js' -import type { - BasicExpression, - IncludesMaterialization, - PropRef, - QueryIR, -} from '../ir.js' +import type { BasicExpression, QueryIR } from '../ir.js' import type { LazyCollectionCallbacks } from '../compiler/joins.js' import type { Changes, @@ -92,6 +80,9 @@ export class CollectionConfigBuilder< private readonly id: string readonly query: QueryIR private readonly collections: Record> + private readonly collectionSources: ReturnType< + typeof extractCollectionSources + > private readonly collectionByAlias: Record> // Populated during compilation with all aliases (including subquery inner aliases) private compiledAliasToCollectionId: Record = {} @@ -128,7 +119,7 @@ export class CollectionConfigBuilder< private maybeRunGraphFn: (() => void) | undefined - private readonly aliasDependencies: Record< + private readonly sourceDependencies: Record< string, Array> > = {} @@ -156,15 +147,21 @@ export class CollectionConfigBuilder< public sourceWhereClausesCache: | Map> | undefined - private includesCache: Array | undefined + private bucketFacadesCache: + | ReturnType[`facades`] + | undefined - // Map of source alias to subscription + // Map of opaque source ID to subscription readonly subscriptions: Record = {} - // Map of source aliases to functions that load keys for that lazy source + // Map of opaque source ID to demand callbacks for that lazy source lazySourcesCallbacks: Record = {} - // Set of source aliases that are lazy (don't load initial state) + // Set of opaque source IDs that are lazy (don't load initial state) readonly lazySources = new Set() - // Set of collection IDs that include an optimizable ORDER BY clause + private readonly activeDemands = new Map< + string, + { generation: number; settled: boolean } + >() + // Map of collection IDs to optimizable ORDER BY state optimizableOrderByCollections: Record = {} constructor( @@ -184,19 +181,13 @@ export class CollectionConfigBuilder< } : undefined this.collections = extractCollectionsFromQuery(this.query) - const collectionAliasesById = extractCollectionAliases(this.query) - - // Build a reverse lookup map from alias to collection instance. - // This enables self-join support where the same collection can be referenced - // multiple times with different aliases (e.g., { employee: col, manager: col }) - this.collectionByAlias = {} - for (const [collectionId, aliases] of collectionAliasesById.entries()) { - const collection = this.collections[collectionId] - if (!collection) continue - for (const alias of aliases) { - this.collectionByAlias[alias] = collection - } - } + this.collectionSources = extractCollectionSources(this.query) + this.collectionByAlias = Object.fromEntries( + this.collectionSources.map(({ alias, collection }) => [ + alias, + collection, + ]), + ) // Create compare function for ordering if the query has orderBy if (this.query.orderBy && this.query.orderBy.length > 0) { @@ -338,8 +329,25 @@ export class CollectionConfigBuilder< throw new Error(`Unknown source alias "${alias}"`) } - isLazyAlias(alias: string): boolean { - return this.lazySources.has(alias) + isLazySource(sourceId: string): boolean { + return this.lazySources.has(sourceId) + } + + beginDemand(planId: string): number { + const generation = (this.activeDemands.get(planId)?.generation ?? 0) + 1 + this.activeDemands.set(planId, { generation, settled: false }) + return generation + } + + settleDemand(planId: string, generation: number): void { + const demand = this.activeDemands.get(planId) + if (!demand || demand.generation !== generation || demand.settled) return + demand.settled = true + this.maybeRunGraphFn?.() + } + + retireDemand(planId: string): void { + this.activeDemands.delete(planId) } // The callback function is called after the graph has run. @@ -430,7 +438,7 @@ export class CollectionConfigBuilder< * @param options - Optional scheduling configuration * @param options.contextId - Transaction ID to group work; defaults to active transaction * @param options.jobId - Unique identifier for this job; defaults to this builder instance - * @param options.alias - Source alias that triggered this schedule; adds alias-specific dependencies + * @param options.sourceId - Source that triggered this schedule; adds its dependencies * @param options.dependencies - Explicit dependency list; overrides auto-discovered dependencies */ scheduleGraphRun( @@ -438,7 +446,7 @@ export class CollectionConfigBuilder< options?: { contextId?: SchedulerContextId jobId?: unknown - alias?: string + sourceId?: string dependencies?: Array> }, ) { @@ -452,10 +460,10 @@ export class CollectionConfigBuilder< } const deps = new Set(this.builderDependencies) - if (options?.alias) { - const aliasDeps = this.aliasDependencies[options.alias] - if (aliasDeps) { - for (const dep of aliasDeps) { + if (options?.sourceId) { + const sourceDeps = this.sourceDependencies[options.sourceId] + if (sourceDeps) { + for (const dep of sourceDeps) { deps.add(dep) } } @@ -673,10 +681,11 @@ export class CollectionConfigBuilder< this.inputsCache = undefined this.pipelineCache = undefined this.sourceWhereClausesCache = undefined - this.includesCache = undefined + this.bucketFacadesCache = undefined // Reset lazy source alias state this.lazySources.clear() + this.activeDemands.clear() this.optimizableOrderByCollections = {} this.lazySourcesCallbacks = {} @@ -700,8 +709,8 @@ export class CollectionConfigBuilder< private compileBasePipeline() { this.graphCache = new D2() this.inputsCache = Object.fromEntries( - Object.keys(this.collectionByAlias).map((alias) => [ - alias, + this.collectionSources.map((source) => [ + source.sourceId, this.graphCache!.newInput(), ]), ) @@ -725,19 +734,17 @@ export class CollectionConfigBuilder< }, ) - this.pipelineCache = compilation.pipeline + const materialized = materializeCompilation(compilation, this.config.getKey) + this.pipelineCache = materialized.pipeline this.sourceWhereClausesCache = compilation.sourceWhereClauses this.compiledAliasToCollectionId = compilation.aliasToCollectionId - this.includesCache = compilation.includes + this.bucketFacadesCache = materialized.facades - // Defensive check: verify all compiled aliases have corresponding inputs - // This should never happen since all aliases come from user declarations, - // but catch it early if the assumption is violated in the future. - const missingAliases = Object.keys(this.compiledAliasToCollectionId).filter( - (alias) => !Object.hasOwn(this.inputsCache!, alias), - ) - if (missingAliases.length > 0) { - throw new MissingAliasInputsError(missingAliases) + const missingSources = this.collectionSources + .map((source) => source.sourceId) + .filter((sourceId) => !Object.hasOwn(this.inputsCache!, sourceId)) + if (missingSources.length > 0) { + throw new MissingAliasInputsError(missingSources) } } @@ -776,74 +783,54 @@ export class CollectionConfigBuilder< }), ) - // Set up includes output routing and child collection lifecycle - const includesState = this.setupIncludesOutput( - this.includesCache, - syncState, + const bucketFacades = new BucketFacadeAdapter( + this.id, + this.bucketFacadesCache ?? [], + (count) => { + syncState.messagesCount += count + }, ) + syncState.unsubscribeCallbacks.add(() => bucketFacades.cleanup()) // Flush pending changes and reset the accumulator. // Called at the end of each graph run to commit all accumulated changes. syncState.flushPendingChanges = () => { const hasParentChanges = pendingChanges.size > 0 - const hasChildChanges = hasPendingIncludesChanges(includesState) + const hasChildChanges = bucketFacades.hasPendingChanges() if (!hasParentChanges && !hasChildChanges) { return } - let changesToApply = pendingChanges - - // When a custom getKey is provided, multiple D2 internal keys may map - // to the same user-visible key. Re-accumulate by custom key so that a - // retract + insert for the same logical row merges into an UPDATE - // instead of a separate DELETE and INSERT that can race. - if (this.config.getKey) { - const merged = new Map>() - for (const [, changes] of pendingChanges) { - const customKey = this.config.getKey(changes.value) - const existing = merged.get(customKey) - if (existing) { - existing.inserts += changes.inserts - existing.deletes += changes.deletes - // Keep the value from the insert side (the new value) - if (changes.inserts > 0) { - existing.value = changes.value - if (changes.orderByIndex !== undefined) { - existing.orderByIndex = changes.orderByIndex - } + const resumeFacadePublications = bucketFacades.flush() + try { + const changesToApply: Map> = new Map( + [...pendingChanges].map(([key, changes]) => { + const resolved: Changes = { + ...changes, + value: bucketFacades.resolve(changes.value), } - // Keep the retracted (old) side for order-only-move detection. - if (changes.deletes > 0) { - existing.previousValue = changes.previousValue - existing.previousOrderByIndex = changes.previousOrderByIndex + if (changes.previousValue !== undefined) { + resolved.previousValue = bucketFacades.resolve( + changes.previousValue, + ) } - } else { - merged.set(customKey, { ...changes }) - } - } - changesToApply = merged - } + return [key, resolved] + }), + ) - // 1. Flush parent changes - if (hasParentChanges) { - begin() - changesToApply.forEach(this.applyChanges.bind(this, config)) - if (hasOrderOnlyMove(changesToApply)) { - markLayoutChange(config.collection) + if (hasParentChanges) { + begin() + changesToApply.forEach(this.applyChanges.bind(this, config)) + if (hasOrderOnlyMove(changesToApply)) { + markLayoutChange(config.collection) + } + commit() } - commit() + } finally { + resumeFacadePublications() } pendingChanges = new Map() - - // 2. Process includes: create/dispose child Collections, route child changes - flushIncludesState( - includesState, - config.collection, - this.id, - hasParentChanges ? changesToApply : null, - config, - ) } graph.finalize() @@ -856,92 +843,6 @@ export class CollectionConfigBuilder< return syncState as FullSyncState } - /** - * Sets up output callbacks for includes child pipelines. - * Each includes entry gets its own output callback that accumulates child changes, - * and a child registry that maps correlation key → child Collection. - */ - private setupIncludesOutput( - includesEntries: Array | undefined, - syncState: SyncState, - ): Array { - if (!includesEntries || includesEntries.length === 0) { - return [] - } - - return includesEntries.map((entry) => { - const state: IncludesOutputState = { - fieldName: entry.fieldName, - resultPath: entry.resultPath, - childCorrelationField: entry.childCorrelationField, - hasOrderBy: entry.hasOrderBy, - materialization: entry.materialization, - scalarField: entry.scalarField, - childRegistry: new Map(), - pendingChildChanges: new Map(), - correlationToParentKeys: new Map(), - } - - // Attach output callback on the child pipeline - entry.pipeline.pipe( - output((data) => { - const messages = data.getInner() - syncState.messagesCount += messages.length - - for (const [[childKey, tupleData], multiplicity] of messages) { - const [childResult, _orderByIndex, correlationKey, parentContext] = - tupleData as unknown as [ - any, - string | undefined, - unknown, - Record | null, - ] - - const routingKey = computeRoutingKey(correlationKey, parentContext) - - // Accumulate by [routingKey, childKey] - let byChild = state.pendingChildChanges.get(routingKey) - if (!byChild) { - byChild = new Map() - state.pendingChildChanges.set(routingKey, byChild) - } - - const existing = byChild.get(childKey) || { - deletes: 0, - inserts: 0, - value: childResult, - orderByIndex: _orderByIndex, - } - - if (multiplicity < 0) { - existing.deletes += Math.abs(multiplicity) - existing.previousValue = childResult - existing.previousOrderByIndex = _orderByIndex - } else if (multiplicity > 0) { - existing.inserts += multiplicity - existing.value = childResult - if (_orderByIndex !== undefined) { - existing.orderByIndex = _orderByIndex - } - } - - byChild.set(childKey, existing) - } - }), - ) - - // Set up shared buffers for nested includes (e.g., comments inside issues) - if (entry.childCompilationResult.includes) { - state.nestedSetups = setupNestedPipelines( - entry.childCompilationResult.includes, - syncState, - ) - } - - return state - }) - } - private applyChanges( config: SyncMethods, changes: { @@ -1038,7 +939,10 @@ export class CollectionConfigBuilder< } const subscribedToAll = this.currentSyncState?.subscribedToAllCollections - const allReady = this.allCollectionsReady() + const allReady = this.allRequiredSourcesReady() + const allDemandsSettled = [...this.activeDemands.values()].every( + (demand) => demand.settled, + ) const isLoading = this.liveQueryCollection?.isLoadingSubset // Mark ready when: // 1. All subscriptions are set up (subscribedToAllCollections) @@ -1046,7 +950,7 @@ export class CollectionConfigBuilder< // 3. The live query collection is not loading subset data // This prevents marking the live query ready before its data is processed // (fixes issue where useLiveQuery returns isReady=true with empty data) - if (subscribedToAll && allReady && !isLoading) { + if (subscribedToAll && allReady && allDemandsSettled && !isLoading) { markReady() } } @@ -1064,48 +968,44 @@ export class CollectionConfigBuilder< this.liveQueryCollection?._lifecycle.setStatus(`error`) } - private allCollectionsReady() { - return Object.values(this.collections).every((collection) => - collection.isReady(), + private allRequiredSourcesReady() { + return this.collectionSources.every( + (source) => + this.lazySources.has(source.sourceId) || source.collection.isReady(), ) } /** - * Creates per-alias subscriptions enabling self-join support. - * Each alias gets its own subscription with independent filters, even for the same collection. + * Creates one subscription per lexical collection source. + * Each source gets independent filters, even when aliases or collections repeat. * Example: `{ employee: col, manager: col }` creates two separate subscriptions. */ private subscribeToAllCollections( config: SyncMethods, syncState: FullSyncState, ) { - // Use compiled aliases as the source of truth - these include all aliases from the query - // including those from subqueries, which may not be in collectionByAlias - const compiledAliases = Object.entries(this.compiledAliasToCollectionId) - if (compiledAliases.length === 0) { + if (this.collectionSources.length === 0) { throw new Error( - `Compiler returned no alias metadata for query '${this.id}'. This should not happen; please report.`, + `Query '${this.id}' has no collection sources. This should not happen; please report.`, ) } - // Create a separate subscription for each alias, enabling self-joins where the same - // collection can be used multiple times with different filters and subscriptions - const loaders = compiledAliases.map(([alias, collectionId]) => { - // Try collectionByAlias first (for declared aliases), fall back to collections (for subquery aliases) - const collection = - this.collectionByAlias[alias] ?? this.collections[collectionId]! + const loaders = this.collectionSources.map((source) => { + const { sourceId, alias, collection } = source + const collectionId = collection.id const dependencyBuilder = getCollectionBuilder(collection) if (dependencyBuilder && dependencyBuilder !== this) { - this.aliasDependencies[alias] = [dependencyBuilder] + this.sourceDependencies[sourceId] = [dependencyBuilder] this.builderDependencies.add(dependencyBuilder) } else { - this.aliasDependencies[alias] = [] + this.sourceDependencies[sourceId] = [] } // CollectionSubscriber handles the actual subscription to the source collection // and feeds data into the D2 graph inputs for this specific alias const collectionSubscriber = new CollectionSubscriber( + sourceId, alias, collectionId, collection, @@ -1119,9 +1019,18 @@ export class CollectionConfigBuilder< syncState.unsubscribeCallbacks.add(statusUnsubscribe) const subscription = collectionSubscriber.subscribe() - // Store subscription by alias (not collection ID) to support lazy loading - // which needs to look up subscriptions by their query alias - this.subscriptions[alias] = subscription + this.subscriptions[sourceId] = subscription + + const lazyCallbacks = this.lazySourcesCallbacks[sourceId] + if (lazyCallbacks) { + lazyCallbacks.setDemand = (plan, keys) => + collectionSubscriber.setDemand(subscription, plan, keys) + for (const plan of lazyCallbacks.plans ?? []) { + if (plan.initialKeys.size > 0) { + lazyCallbacks.setDemand(plan, plan.initialKeys) + } + } + } // Create a callback for loading more data if needed (used by OrderBy optimization) const loadMore = collectionSubscriber.loadMoreIfNeeded.bind( @@ -1177,1160 +1086,6 @@ function createOrderByComparator( } } -type SnapshotRow = { - value: any - orderByIndex: string | undefined - /** Net multiplicity (inserts − deletes) currently materialized for this row */ - count: number -} - -type NestedRouteIndex = Map< - unknown, - Map>> -> - -type NestedRouteReverseIndex = Map< - IncludesOutputState, - Map> -> - -type NestedRouteChildToNested = Map< - IncludesOutputState, - Map> -> - -/** - * Shared buffer setup for a single nested includes level. - * Pipeline output writes into the buffer; during flush the buffer is drained - * into per-entry states via the routing index. - */ -type NestedIncludesSetup = { - compilationResult: IncludesCompilationResult - /** Shared buffer: nestedCorrelationKey → Map */ - buffer: Map>> - /** - * Cumulative net-present grandchild rows per nested correlation key. The - * buffer holds only deltas since the last drain and is cleared once drained, - * so a parent group that starts referencing an existing correlation key - * *after* the rows were already drained (the pipeline does not re-emit them) - * would otherwise see nothing. The snapshot lets such late-arriving parent - * groups be seeded with the rows their siblings already received. - */ - snapshot: Map> - /** - * Shared route store for this shared buffer. Routes target concrete - * IncludesOutputState instances so one emitted child row can fan out across - * every per-entry state that references the same nested correlation key. - */ - routingIndex: NestedRouteIndex - routingReverseIndex: NestedRouteReverseIndex - routingChildToNested: NestedRouteChildToNested - /** For 3+ levels of nesting */ - nestedSetups?: Array -} - -/** - * State tracked per includes entry for output routing and child lifecycle - */ -type IncludesOutputState = { - fieldName: string - resultPath: Array - childCorrelationField: PropRef - /** Whether the child query has an ORDER BY clause */ - hasOrderBy: boolean - /** How the child result is materialized on the parent row */ - materialization: IncludesMaterialization - /** Internal field used to unwrap scalar child selects */ - scalarField?: string - /** Maps correlation key value → child Collection entry */ - childRegistry: Map - /** Pending child changes: correlationKey → Map */ - pendingChildChanges: Map>> - /** Reverse index: correlation key → Set of parent collection keys */ - correlationToParentKeys: Map> - /** Shared nested pipeline setups (one per nested includes level) */ - nestedSetups?: Array -} - -type ChildCollectionEntry = { - collection: Collection - syncMethods: SyncMethods | null - resultKeys: WeakMap - orderByIndices: WeakMap | null - /** Per-entry nested includes states (one per nested includes level) */ - includesStates?: Array -} - -function materializesInline(state: IncludesOutputState): boolean { - return state.materialization !== `collection` -} - -function materializeIncludedValue( - state: IncludesOutputState, - entry: ChildCollectionEntry | undefined, -): unknown { - if (!entry) { - if (state.materialization === `array`) { - return [] - } - if (state.materialization === `concat`) { - return `` - } - // `singleton` and `collection` both fall through to undefined when no - // child entry exists for the parent's correlation key. - return undefined - } - - if (state.materialization === `collection`) { - return entry.collection - } - - const rows = [...entry.collection.toArray] - const values = state.scalarField - ? rows.map((row) => row?.[state.scalarField!]) - : rows - - if (state.materialization === `array`) { - return values - } - - if (state.materialization === `singleton`) { - // findOne() doesn't currently push LIMIT 1 to the IR, so the child - // Collection may hold more than one row; pick the first deterministically. - return values[0] - } - - return values.map((value) => String(value ?? ``)).join(``) -} - -/** - * Sets up shared buffers for nested includes pipelines. - * Instead of writing directly into a single shared IncludesOutputState, - * each nested pipeline writes into a buffer that is later drained per-entry. - */ -function setupNestedPipelines( - includes: Array, - syncState: SyncState, -): Array { - return includes.map((entry) => { - const buffer: Map>> = new Map() - - // Attach output callback that writes into the shared buffer - entry.pipeline.pipe( - output((data) => { - const messages = data.getInner() - syncState.messagesCount += messages.length - - for (const [[childKey, tupleData], multiplicity] of messages) { - const [childResult, _orderByIndex, correlationKey, parentContext] = - tupleData as unknown as [ - any, - string | undefined, - unknown, - Record | null, - ] - - const routingKey = computeRoutingKey(correlationKey, parentContext) - - let byChild = buffer.get(routingKey) - if (!byChild) { - byChild = new Map() - buffer.set(routingKey, byChild) - } - - const existing = byChild.get(childKey) || { - deletes: 0, - inserts: 0, - value: childResult, - orderByIndex: _orderByIndex, - } - - if (multiplicity < 0) { - existing.deletes += Math.abs(multiplicity) - existing.previousValue = childResult - existing.previousOrderByIndex = _orderByIndex - } else if (multiplicity > 0) { - existing.inserts += multiplicity - existing.value = childResult - if (_orderByIndex !== undefined) { - existing.orderByIndex = _orderByIndex - } - } - - byChild.set(childKey, existing) - } - }), - ) - - const setup: NestedIncludesSetup = { - compilationResult: entry, - buffer, - snapshot: new Map(), - routingIndex: new Map(), - routingReverseIndex: new Map(), - routingChildToNested: new Map(), - } - - // Recursively set up deeper levels - if (entry.childCompilationResult.includes) { - setup.nestedSetups = setupNestedPipelines( - entry.childCompilationResult.includes, - syncState, - ) - } - - return setup - }) -} - -/** - * Creates fresh per-entry IncludesOutputState array from NestedIncludesSetup array. - * Each entry gets its own isolated state for nested includes. - */ -function createPerEntryIncludesStates( - setups: Array, -): Array { - return setups.map((setup) => { - const state: IncludesOutputState = { - fieldName: setup.compilationResult.fieldName, - resultPath: setup.compilationResult.resultPath, - childCorrelationField: setup.compilationResult.childCorrelationField, - hasOrderBy: setup.compilationResult.hasOrderBy, - materialization: setup.compilationResult.materialization, - scalarField: setup.compilationResult.scalarField, - childRegistry: new Map(), - pendingChildChanges: new Map(), - correlationToParentKeys: new Map(), - } - - if (setup.nestedSetups) { - state.nestedSetups = setup.nestedSetups - } - - return state - }) -} - -function cloneSnapshotValue(value: T): T { - if (value == null || typeof value !== `object`) { - return value - } - - return (Array.isArray(value) ? [...value] : { ...value }) as T -} - -/** - * Folds a drained delta into a nested setup's cumulative snapshot, tracking the - * net multiplicity per child row and dropping rows (and empty keys) once their - * net count reaches zero. - */ -function accumulateSnapshot( - setup: NestedIncludesSetup, - nestedCorrelationKey: unknown, - childChanges: Map>, -): void { - let snap = setup.snapshot.get(nestedCorrelationKey) - if (!snap) { - snap = new Map() - setup.snapshot.set(nestedCorrelationKey, snap) - } - - for (const [childKey, changes] of childChanges) { - let row = snap.get(childKey) - if (!row) { - row = { - value: cloneSnapshotValue(changes.value), - orderByIndex: changes.orderByIndex, - count: 0, - } - snap.set(childKey, row) - } - row.count += changes.inserts - changes.deletes - if (changes.inserts > 0) { - row.value = cloneSnapshotValue(changes.value) - if (changes.orderByIndex !== undefined) { - row.orderByIndex = changes.orderByIndex - } - } - if (row.count <= 0) { - snap.delete(childKey) - } - } - - if (snap.size === 0) { - setup.snapshot.delete(nestedCorrelationKey) - } -} - -/** - * Seeds a parent group's per-entry state with the rows already materialized for - * a nested correlation key. Used when a parent group starts referencing a key - * whose rows were drained (and cleared from the buffer) in an earlier flush, so - * the pipeline will not re-emit them. - */ -function seedParentFromSnapshot( - state: IncludesOutputState, - setupIndex: number, - parentCorrelationKey: unknown, - nestedCorrelationKey: unknown, -): void { - const setup = state.nestedSetups![setupIndex]! - const snap = setup.snapshot.get(nestedCorrelationKey) - if (!snap || snap.size === 0) return - - const entry = state.childRegistry.get(parentCorrelationKey) - if (!entry || !entry.includesStates) return - - const entryState = entry.includesStates[setupIndex]! - let byChild = entryState.pendingChildChanges.get(nestedCorrelationKey) - if (!byChild) { - byChild = new Map() - entryState.pendingChildChanges.set(nestedCorrelationKey, byChild) - } - for (const [childKey, row] of snap) { - if (byChild.has(childKey)) continue - byChild.set(childKey, { - deletes: 0, - inserts: row.count, - value: cloneSnapshotValue(row.value), - orderByIndex: row.orderByIndex, - }) - } -} - -/** - * Drains shared buffers into per-entry states using the routing index. - * Returns the set of parent correlation keys that had changes routed to them. - */ -function drainNestedBuffers(state: IncludesOutputState): Set { - const dirtyCorrelationKeys = new Set() - - if (!state.nestedSetups) return dirtyCorrelationKeys - - for (const setup of state.nestedSetups) { - const toDelete: Array = [] - - for (const [nestedCorrelationKey, childChanges] of setup.buffer) { - const stateRoutes = setup.routingIndex.get(nestedCorrelationKey) - if (stateRoutes === undefined || stateRoutes.size === 0) { - // Unroutable — parent not yet seen; keep in buffer - continue - } - - // A single nested correlation key can map to multiple parent groups when - // sibling parents share the same correlation value, and at depth 4+ those - // parents may live in different per-entry states. Fan the buffered changes - // out to every ready target before clearing the shared buffer entry. - let routedToAny = false - for (const [targetState, parentRoutes] of stateRoutes) { - const targetSetupIndex = targetState.nestedSetups?.indexOf(setup) ?? -1 - if (targetSetupIndex < 0) continue - - for (const parentCorrelationKey of parentRoutes.keys()) { - const entry = targetState.childRegistry.get(parentCorrelationKey) - if (!entry || !entry.includesStates) { - continue - } - - // Route changes into this entry's per-entry state at the same setup. - const entryState = entry.includesStates[targetSetupIndex]! - for (const [childKey, changes] of childChanges) { - let byChild = - entryState.pendingChildChanges.get(nestedCorrelationKey) - if (!byChild) { - byChild = new Map() - entryState.pendingChildChanges.set(nestedCorrelationKey, byChild) - } - const existing = byChild.get(childKey) - if (existing) { - existing.inserts += changes.inserts - existing.deletes += changes.deletes - if (changes.inserts > 0) { - existing.value = changes.value - if (changes.orderByIndex !== undefined) { - existing.orderByIndex = changes.orderByIndex - } - } - } else { - byChild.set(childKey, { ...changes }) - } - } - - if (targetState === state) { - dirtyCorrelationKeys.add(parentCorrelationKey) - } - routedToAny = true - } - } - - if (routedToAny) { - // Fold the drained delta into the cumulative snapshot so a parent group - // that starts referencing this nested key later can be seeded with it. - accumulateSnapshot(setup, nestedCorrelationKey, childChanges) - toDelete.push(nestedCorrelationKey) - } - } - - for (const key of toDelete) { - setup.buffer.delete(key) - } - } - - return dirtyCorrelationKeys -} - -/** - * Updates the routing index after processing child changes. - * Maps nested correlation keys to parent correlation keys so that - * grandchild changes can be routed to the correct per-entry state. - */ -/** - * Removes a single child row's reference to a nested routing key from a parent - * group's route, dropping the parent (and the nested key, and the reverse-index - * entry) once no child row in the group references the key anymore. - */ -function removeChildKeyFromRoute( - setup: NestedIncludesSetup, - state: IncludesOutputState, - correlationKey: unknown, - nestedRoutingKey: unknown, - childKey: unknown, -): void { - const stateRoutes = setup.routingIndex.get(nestedRoutingKey) - const parents = stateRoutes?.get(state) - const childKeys = parents?.get(correlationKey) - if (!parents || !childKeys) return - - childKeys.delete(childKey) - // Only drop the parent group from the route once its last child row - // referencing this nested key is gone — a surviving sibling in the same - // parent group must keep receiving grandchild changes. - if (childKeys.size === 0) { - parents.delete(correlationKey) - if (parents.size === 0) { - stateRoutes!.delete(state) - if (stateRoutes!.size === 0) { - setup.routingIndex.delete(nestedRoutingKey) - } - } - // The reverse index tracks parent → nested keys at group granularity, so - // only drop the entry when no child row in this parent group references the - // nested key anymore. - const reverse = setup.routingReverseIndex.get(state) - const reverseSet = reverse?.get(correlationKey) - if (reverseSet) { - reverseSet.delete(nestedRoutingKey) - if (reverseSet.size === 0) { - reverse!.delete(correlationKey) - if (reverse!.size === 0) { - setup.routingReverseIndex.delete(state) - } - } - } - } -} - -function updateRoutingIndex( - state: IncludesOutputState, - correlationKey: unknown, - childChanges: Map>, -): void { - if (!state.nestedSetups) return - - for (let i = 0; i < state.nestedSetups.length; i++) { - const setup = state.nestedSetups[i]! - let childToNested = setup.routingChildToNested.get(state) - if (!childToNested) { - childToNested = new Map() - setup.routingChildToNested.set(state, childToNested) - } - for (const [childKey, change] of childChanges) { - if (change.inserts > 0) { - // Read the nested routing key from the INCLUDES_ROUTING stamp. - // Must use the composite routing key (not raw correlationKey) to match - // how nested buffers are keyed by computeRoutingKey. - const nestedRouting = - change.value[INCLUDES_ROUTING]?.[setup.compilationResult.fieldName] - const nestedCorrelationKey = nestedRouting?.correlationKey - const nestedParentContext = nestedRouting?.parentContext ?? null - const nestedRoutingKey = computeRoutingKey( - nestedCorrelationKey, - nestedParentContext, - ) - - // An update (inserts > 0 && deletes > 0) can change a child row's nested - // correlation key (e.g. a price range's regionId changes). The change - // only carries the NEW key, so drop the row's previous reference for - // THIS setup using the recorded mapping before re-routing it. - // - // This relies on the compiler stamping the FULL INCLUDES_ROUTING map on - // every emitted row (one entry per nested include field), so for an - // unrelated nested include the recomputed nestedRoutingKey equals the - // recorded one and the guard below is a no-op — a change to one nested - // include never disturbs the recorded key of another on the same row. - const perParent = childToNested.get(correlationKey) - const prevNestedKey = perParent?.get(childKey) - if (prevNestedKey !== undefined && prevNestedKey !== nestedRoutingKey) { - removeChildKeyFromRoute( - setup, - state, - correlationKey, - prevNestedKey, - childKey, - ) - perParent!.delete(childKey) - } - - if (nestedCorrelationKey != null) { - let stateRoutes = setup.routingIndex.get(nestedRoutingKey) - if (!stateRoutes) { - stateRoutes = new Map() - setup.routingIndex.set(nestedRoutingKey, stateRoutes) - } - let parents = stateRoutes.get(state) - if (!parents) { - parents = new Map() - stateRoutes.set(state, parents) - } - let childKeys = parents.get(correlationKey) - // The parent group is "new" for this nested key only when no child row - // in it referenced the key before; that's the case that needs seeding. - const isNewParent = !childKeys || childKeys.size === 0 - if (!childKeys) { - childKeys = new Set() - parents.set(correlationKey, childKeys) - } - childKeys.add(childKey) - let reverse = setup.routingReverseIndex.get(state) - if (!reverse) { - reverse = new Map() - setup.routingReverseIndex.set(state, reverse) - } - let reverseSet = reverse.get(correlationKey) - if (!reverseSet) { - reverseSet = new Set() - reverse.set(correlationKey, reverseSet) - } - reverseSet.add(nestedRoutingKey) - - // Record the row's current nested key for this setup so a later update - // that changes it can release the old reference. Reuse perParent when - // it already exists to avoid a second lookup. - let recorded = perParent - if (!recorded) { - recorded = new Map() - childToNested.set(correlationKey, recorded) - } - recorded.set(childKey, nestedRoutingKey) - - // If this parent group is newly associated with a nested key whose - // rows were already drained (and cleared from the buffer) in an - // earlier flush, the pipeline will not re-emit them. Seed this parent - // from the cumulative snapshot so it receives the same rows its - // siblings already have. - if (isNewParent) { - seedParentFromSnapshot(state, i, correlationKey, nestedRoutingKey) - } - } else if (perParent && perParent.size === 0) { - // The row no longer has a nested key (cleared via update) and held no - // others — drop the now-empty per-parent record. - childToNested.delete(correlationKey) - } - } else if (change.deletes > 0 && change.inserts === 0) { - // Remove from routing index - const nestedRouting2 = - change.value[INCLUDES_ROUTING]?.[setup.compilationResult.fieldName] - const nestedCorrelationKey = nestedRouting2?.correlationKey - const nestedParentContext2 = nestedRouting2?.parentContext ?? null - const nestedRoutingKey = computeRoutingKey( - nestedCorrelationKey, - nestedParentContext2, - ) - - if (nestedCorrelationKey != null) { - removeChildKeyFromRoute( - setup, - state, - correlationKey, - nestedRoutingKey, - childKey, - ) - } - const perParent = childToNested.get(correlationKey) - if (perParent) { - perParent.delete(childKey) - if (perParent.size === 0) childToNested.delete(correlationKey) - } - } - } - } -} - -/** - * Cleans routing index entries when a parent is deleted. - * Uses the reverse index to find and remove all nested routing entries. - */ -function cleanRoutingIndexOnDelete( - state: IncludesOutputState, - correlationKey: unknown, -): void { - if (!state.nestedSetups) return - - // The whole parent group is gone, so drop it from every nested setup's route - // (along with all the child keys it tracked); other sibling parent groups may - // still reference the same nested correlation key. - for (const setup of state.nestedSetups) { - const reverseIndex = setup.routingReverseIndex.get(state) - const nestedKeys = reverseIndex?.get(correlationKey) - if (!nestedKeys) continue - for (const nestedKey of nestedKeys) { - const stateRoutes = setup.routingIndex.get(nestedKey) - const parents = stateRoutes?.get(state) - if (parents) { - parents.delete(correlationKey) - if (parents.size === 0) { - stateRoutes!.delete(state) - if (stateRoutes!.size === 0) { - setup.routingIndex.delete(nestedKey) - } - } - } - } - reverseIndex!.delete(correlationKey) - if (reverseIndex!.size === 0) { - setup.routingReverseIndex.delete(state) - } - - const childToNested = setup.routingChildToNested.get(state) - if (childToNested) { - childToNested.delete(correlationKey) - if (childToNested.size === 0) { - setup.routingChildToNested.delete(state) - } - } - } -} - -/** - * Recursively checks whether any nested buffer has pending changes. - */ -function hasNestedBufferChanges(setups: Array): boolean { - for (const setup of setups) { - if (setup.buffer.size > 0) return true - if (setup.nestedSetups && hasNestedBufferChanges(setup.nestedSetups)) - return true - } - return false -} - -/** - * Computes a composite routing key from correlation key and parent context. - * When parentContext is null (no parent filters), returns the raw correlationKey - * for zero behavioral change on existing queries. - */ -function computeRoutingKey( - correlationKey: unknown, - parentContext: Record | null, -): unknown { - if (parentContext == null) return correlationKey - return JSON.stringify([correlationKey, parentContext]) -} - -/** - * Creates a child Collection entry for includes subqueries. - * The child Collection is a full-fledged Collection instance that starts syncing immediately. - */ -function createChildCollectionEntry( - parentId: string, - fieldName: string, - correlationKey: unknown, - hasOrderBy: boolean, - nestedSetups?: Array, -): ChildCollectionEntry { - const resultKeys = new WeakMap() - const orderByIndices = hasOrderBy ? new WeakMap() : null - let syncMethods: SyncMethods | null = null - - const compare = orderByIndices - ? createOrderByComparator(orderByIndices) - : undefined - - const collection = createCollection({ - id: `__child-collection:${parentId}-${fieldName}-${serializeValue(correlationKey)}`, - getKey: (item: any) => resultKeys.get(item) as string | number, - compare, - sync: { - rowUpdateMode: `full`, - sync: (methods) => { - syncMethods = methods - return () => { - syncMethods = null - } - }, - }, - startSync: true, - gcTime: 0, - }) - - const entry: ChildCollectionEntry = { - collection, - get syncMethods() { - return syncMethods - }, - resultKeys, - orderByIndices, - } - - if (nestedSetups) { - entry.includesStates = createPerEntryIncludesStates(nestedSetups) - } - - return entry -} - -/** - * Flushes includes state using a bottom-up per-entry approach. - * Five phases ensure correct ordering: - * 1. Parent INSERTs — create child entries with per-entry nested states - * 2. Child changes — apply to child Collections, update routing index - * 3. Drain nested buffers — route buffered grandchild changes to per-entry states - * 4. Flush per-entry states — recursively flush nested includes on each entry - * 5. Parent DELETEs — clean up child entries and routing index - */ -function flushIncludesState( - includesState: Array, - parentCollection: Collection, - parentId: string, - parentChanges: Map> | null, - parentSyncMethods: SyncMethods | null, -): void { - for (const state of includesState) { - // Phase 1: Parent INSERTs — ensure a child Collection exists for every parent - if (parentChanges) { - for (const [parentKey, changes] of parentChanges) { - if (changes.inserts > 0) { - const parentResult = changes.value - // Extract routing info from INCLUDES_ROUTING symbol (set by compiler) - const routing = parentResult[INCLUDES_ROUTING]?.[state.fieldName] - const correlationKey = routing?.correlationKey - const parentContext = routing?.parentContext ?? null - const routingKey = computeRoutingKey(correlationKey, parentContext) - - if (correlationKey != null) { - // Ensure child Collection exists for this routing key - if (!state.childRegistry.has(routingKey)) { - const entry = createChildCollectionEntry( - parentId, - state.fieldName, - routingKey, - state.hasOrderBy, - state.nestedSetups, - ) - state.childRegistry.set(routingKey, entry) - } - // Update reverse index: routing key → parent keys - let parentKeys = state.correlationToParentKeys.get(routingKey) - if (!parentKeys) { - parentKeys = new Set() - state.correlationToParentKeys.set(routingKey, parentKeys) - } - parentKeys.add(parentKey) - - const childValue = materializeIncludedValue( - state, - state.childRegistry.get(routingKey), - ) - setIncludedValue(parentResult, state.resultPath, childValue) - - // Parent rows may already be materialized in the live collection by the - // time includes state is flushed, so update the stored row as well. - const storedParent = parentCollection.get(parentKey as any) - if (storedParent && storedParent !== parentResult) { - setIncludedValue(storedParent, state.resultPath, childValue) - } - } - } - } - } - - // Track affected correlation keys for inline materializations before clearing child changes. - const affectedCorrelationKeys = materializesInline(state) - ? new Set(state.pendingChildChanges.keys()) - : null - - // Phase 2: Child changes — apply to child Collections - // Track which entries had child changes and capture their childChanges maps - const entriesWithChildChanges = new Map< - unknown, - { entry: ChildCollectionEntry; childChanges: Map> } - >() - if (state.pendingChildChanges.size > 0) { - for (const [correlationKey, childChanges] of state.pendingChildChanges) { - // Ensure child Collection exists for this correlation key - let entry = state.childRegistry.get(correlationKey) - if (!entry) { - entry = createChildCollectionEntry( - parentId, - state.fieldName, - correlationKey, - state.hasOrderBy, - state.nestedSetups, - ) - state.childRegistry.set(correlationKey, entry) - } - - if (state.materialization === `collection`) { - attachChildCollectionToParent( - parentCollection, - state.resultPath, - correlationKey, - state.correlationToParentKeys, - entry.collection, - ) - } - - // Apply child changes to the child Collection - if (entry.syncMethods) { - entry.syncMethods.begin() - for (const [childKey, change] of childChanges) { - entry.resultKeys.set(change.value, childKey) - if (entry.orderByIndices && change.orderByIndex !== undefined) { - entry.orderByIndices.set(change.value, change.orderByIndex) - } - const key = entry.syncMethods.collection.getKeyFromItem( - change.value, - ) - const childAlreadyExists = entry.syncMethods.collection.has(key) - - if (change.inserts > 0 && change.deletes === 0) { - entry.syncMethods.write({ - value: change.value, - type: childAlreadyExists ? `update` : `insert`, - }) - } else if ( - change.inserts > change.deletes || - (change.inserts === change.deletes && childAlreadyExists) - ) { - entry.syncMethods.write({ value: change.value, type: `update` }) - } else if (change.deletes > 0) { - entry.syncMethods.write({ value: change.value, type: `delete` }) - } - } - if (hasOrderOnlyMove(childChanges)) { - markLayoutChange(entry.syncMethods.collection) - } - entry.syncMethods.commit() - } - - // Update routing index for nested includes - updateRoutingIndex(state, correlationKey, childChanges) - - entriesWithChildChanges.set(correlationKey, { entry, childChanges }) - } - state.pendingChildChanges.clear() - } - - // Phase 3: Drain nested buffers — route buffered grandchild changes to per-entry states - const dirtyFromBuffers = drainNestedBuffers(state) - - // Phase 4: Flush per-entry states - // First: entries that had child changes in Phase 2 - for (const [, { entry, childChanges }] of entriesWithChildChanges) { - if (entry.includesStates) { - flushIncludesState( - entry.includesStates, - entry.collection, - entry.collection.id, - childChanges, - entry.syncMethods, - ) - } - } - // Then: entries that only had buffer-routed changes (no child changes at this level) - for (const correlationKey of dirtyFromBuffers) { - if (entriesWithChildChanges.has(correlationKey)) continue - const entry = state.childRegistry.get(correlationKey) - if (entry?.includesStates) { - flushIncludesState( - entry.includesStates, - entry.collection, - entry.collection.id, - null, - entry.syncMethods, - ) - } - } - // Finally: entries with deep nested buffer changes (grandchild-or-deeper buffers - // have pending data, but neither this level nor the immediate child level changed). - // Without this pass, changes at depth 3+ are stranded because drainNestedBuffers - // only drains one level and Phase 4 only flushes entries dirty from Phase 2/3. - const deepBufferDirty = new Set() - if (state.nestedSetups) { - for (const [correlationKey, entry] of state.childRegistry) { - if (entriesWithChildChanges.has(correlationKey)) continue - if (dirtyFromBuffers.has(correlationKey)) continue - if ( - entry.includesStates && - hasPendingIncludesChanges(entry.includesStates) - ) { - flushIncludesState( - entry.includesStates, - entry.collection, - entry.collection.id, - null, - entry.syncMethods, - ) - deepBufferDirty.add(correlationKey) - } - } - } - - // For inline materializations: re-emit affected parents with updated snapshots. - // We mutate items in-place (so collection.get() reflects changes immediately) - // and emit UPDATE events directly. We bypass the sync methods because - // commitPendingTransactions compares previous vs new visible state using - // deepEquals, but in-place mutation means both sides reference the same - // object, so the comparison always returns true and suppresses the event. - const inlineReEmitKeys = materializesInline(state) - ? new Set([ - ...(affectedCorrelationKeys || []), - ...dirtyFromBuffers, - ...deepBufferDirty, - ]) - : null - if (parentSyncMethods && inlineReEmitKeys && inlineReEmitKeys.size > 0) { - const events: Array> = [] - for (const correlationKey of inlineReEmitKeys) { - const parentKeys = state.correlationToParentKeys.get(correlationKey) - if (!parentKeys) continue - const entry = state.childRegistry.get(correlationKey) - for (const parentKey of parentKeys) { - const item = parentCollection.get(parentKey as any) - if (item) { - // Capture previous value before in-place mutation - const previousValue = cloneForIncludesUpdate(item, state.resultPath) - setIncludedValue( - item, - state.resultPath, - materializeIncludedValue(state, entry), - ) - const nextValue = cloneForIncludesUpdate(item, state.resultPath) - events.push({ - type: `update`, - key: parentKey as any, - value: nextValue, - previousValue, - }) - } - } - } - if (events.length > 0) { - // Emit directly — the in-place mutation already updated the data in - // syncedData, so we only need to notify subscribers. - const changesManager = (parentCollection as any)._changes as { - emitEvents: ( - changes: Array>, - forceEmit?: boolean, - ) => void - } - changesManager.emitEvents(events, true) - } - } - - // Phase 5: Parent DELETEs — dispose child Collections and clean up - if (parentChanges) { - for (const [parentKey, changes] of parentChanges) { - if (changes.deletes > 0 && changes.inserts === 0) { - const routing = changes.value[INCLUDES_ROUTING]?.[state.fieldName] - const correlationKey = routing?.correlationKey - const parentContext = routing?.parentContext ?? null - const routingKey = computeRoutingKey(correlationKey, parentContext) - if (correlationKey != null) { - // Clean up reverse index first, only delete child collection - // when the last parent referencing it is removed - const parentKeys = state.correlationToParentKeys.get(routingKey) - if (parentKeys) { - parentKeys.delete(parentKey) - if (parentKeys.size === 0) { - cleanRoutingIndexOnDelete(state, routingKey) - state.childRegistry.delete(routingKey) - state.correlationToParentKeys.delete(routingKey) - } - } - } - } - } - } - } - - // Clean up the internal routing stamp from parent/child results - if (parentChanges) { - for (const [, changes] of parentChanges) { - delete changes.value[INCLUDES_ROUTING] - } - } -} - -/** - * Checks whether any includes state has pending changes that need to be flushed. - * Checks direct pending child changes and shared nested buffers. - */ -function hasPendingIncludesChanges( - states: Array, -): boolean { - for (const state of states) { - if (state.pendingChildChanges.size > 0) return true - if (state.nestedSetups && hasNestedBufferChanges(state.nestedSetups)) - return true - for (const entry of state.childRegistry.values()) { - if ( - entry.includesStates && - hasPendingIncludesChanges(entry.includesStates) - ) - return true - } - } - return false -} - -/** - * Attaches a child Collection to parent rows that match a given correlation key. - * Uses the reverse index to look up parent keys directly instead of scanning. - */ -function attachChildCollectionToParent( - parentCollection: Collection, - resultPath: Array, - correlationKey: unknown, - correlationToParentKeys: Map>, - childCollection: Collection, -): void { - const parentKeys = correlationToParentKeys.get(correlationKey) - if (!parentKeys) return - - for (const parentKey of parentKeys) { - const item = parentCollection.get(parentKey as any) - if (item) { - setIncludedValue(item, resultPath, childCollection) - } - } -} - -function setIncludedValue( - target: Record, - path: Array, - value: unknown, -): void { - const state = getFnSelectState(target) - if (!state) { - setNestedValue(target, path, value) - return - } - - setNestedValue(state.sourceRow, path, value) - refreshFnSelectResult(target, state) -} - -function getFnSelectState(target: Record): - | { - sourceRow: Record - fnSelect: (row: Record) => any - } - | undefined { - return (target as Record)[FN_SELECT_STATE] as - | { - sourceRow: Record - fnSelect: (row: Record) => any - } - | undefined -} - -function refreshFnSelectResult( - target: Record, - state: { - sourceRow: Record - fnSelect: (row: Record) => any - }, -): void { - const targetRecord = target as Record - const sourceRecord = state.sourceRow as Record - const routing = - targetRecord[INCLUDES_ROUTING] ?? sourceRecord[INCLUDES_ROUTING] - const nextValue = state.fnSelect(state.sourceRow) - if (!nextValue || typeof nextValue !== `object`) { - return - } - - for (const key of Object.keys(target)) { - delete target[key] - } - Object.assign(target, nextValue) - - if (routing) { - targetRecord[INCLUDES_ROUTING] = routing - } - Object.defineProperty(target, FN_SELECT_STATE, { - value: state, - enumerable: true, - configurable: true, - }) -} - -function setNestedValue( - target: Record, - path: Array, - value: unknown, -): void { - if (path.length === 0) { - return - } - - let cursor = target - for (let i = 0; i < path.length - 1; i++) { - const segment = path[i]! - const next = cursor[segment] - if (next == null || typeof next !== `object`) { - cursor[segment] = {} - } - cursor = cursor[segment] - } - cursor[path[path.length - 1]!] = value -} - -function cloneForIncludesUpdate>( - target: T, - path: Array, -): T { - return getFnSelectState(target) - ? { ...target } - : clonePathForUpdate(target, path) -} - -function clonePathForUpdate>( - target: T, - path: Array, -): T { - const root = { ...target } - let sourceCursor: any = target - let cloneCursor: any = root - - for (let i = 0; i < path.length - 1; i++) { - const segment = path[i]! - const sourceValue = sourceCursor?.[segment] - if (sourceValue == null || typeof sourceValue !== `object`) { - return root - } - - const clonedValue = Array.isArray(sourceValue) - ? [...sourceValue] - : { ...sourceValue } - cloneCursor[segment] = clonedValue - sourceCursor = sourceValue - cloneCursor = clonedValue - } - - return root -} - function accumulateChanges( acc: Map>, [[key, tupleData], multiplicity]: [ diff --git a/packages/db/src/query/live/collection-subscriber.ts b/packages/db/src/query/live/collection-subscriber.ts index 9cfbf5852a..143e77359b 100644 --- a/packages/db/src/query/live/collection-subscriber.ts +++ b/packages/db/src/query/live/collection-subscriber.ts @@ -10,6 +10,7 @@ import { splitUpdates, trackBiggestSentValue, } from './utils.js' +import { SubsetDemandController } from './subset-demand-controller.js' import type { Collection } from '../../collection/index.js' import type { ChangeMessage, @@ -20,6 +21,7 @@ import type { BasicExpression } from '../ir.js' import type { OrderByOptimizationInfo } from '../compiler/order-by.js' import type { CollectionConfigBuilder } from './collection-config-builder.js' import type { CollectionSubscription } from '../../collection/subscription.js' +import type { LazyDemandPlan } from '../compiler/joins.js' const loadMoreCallbackSymbol = Symbol.for( `@tanstack/db.collection-config-builder`, @@ -52,8 +54,10 @@ export class CollectionSubscriber< // used by loadNextItems for subsequent requestLimitedSnapshot calls) private orderedLoadSubsetResult?: (result: Promise | true) => void private pendingOrderedLoadPromise: Promise | undefined + private readonly demand = new SubsetDemandController() constructor( + private sourceId: string, private alias: string, private collectionId: string, private collection: Collection, @@ -61,7 +65,7 @@ export class CollectionSubscriber< ) {} subscribe(): CollectionSubscription { - const whereClause = this.getWhereClauseForAlias() + const whereClause = this.getWhereClause() if (whereClause) { const whereExpression = normalizeExpressionPaths(whereClause, this.alias) @@ -90,6 +94,7 @@ export class CollectionSubscriber< // Used as a fallback for status transitions not covered by direct tracking // (e.g., truncate-triggered reloads that call trackLoadSubsetPromise directly). const onStatusChange = (event: SubscriptionStatusChangeEvent) => { + if (this.collectionConfigBuilder.isLazySource(this.sourceId)) return const subscription = event.subscription as CollectionSubscription if (event.status === `loadingSubset`) { this.ensureLoadingPromise(subscription) @@ -113,9 +118,9 @@ export class CollectionSubscriber< trackLoadResult, ) } else { - // If the source alias is lazy then we should not include the initial state - const includeInitialState = !this.collectionConfigBuilder.isLazyAlias( - this.alias, + // Lazy sources load only the subsets demanded by the compiled graph. + const includeInitialState = !this.collectionConfigBuilder.isLazySource( + this.sourceId, ) subscription = this.subscribeToMatchingChanges( @@ -127,7 +132,10 @@ export class CollectionSubscriber< // Check current status after subscribing - if status is 'loadingSubset', track it. // The onStatusChange listener will catch the transition to 'ready'. - if (subscription.status === `loadingSubset`) { + if ( + !this.collectionConfigBuilder.isLazySource(this.sourceId) && + subscription.status === `loadingSubset` + ) { this.ensureLoadingPromise(subscription) } @@ -139,6 +147,7 @@ export class CollectionSubscriber< deferred.resolve() } + this.demand.clear() subscription.unsubscribe() } // currentSyncState is always defined when subscribe() is called @@ -149,6 +158,32 @@ export class CollectionSubscriber< return subscription } + setDemand( + subscription: CollectionSubscription, + plan: LazyDemandPlan, + keys: Set, + ): void { + const update = this.demand.setDemand(subscription, plan, keys) + if (!update.changed) return + + if (update.empty) { + this.collectionConfigBuilder.retireDemand(plan.id) + return + } + + const generation = this.collectionConfigBuilder.beginDemand(plan.id) + const pending = update.loadResults.filter( + (result): result is Promise => result instanceof Promise, + ) + if (pending.length > 0) { + void Promise.allSettled(pending).then(() => + this.collectionConfigBuilder.settleDemand(plan.id, generation), + ) + } else { + this.collectionConfigBuilder.settleDemand(plan.id, generation) + } + } + private sendChangesToPipeline( changes: Iterable>, callback?: () => boolean, @@ -162,7 +197,7 @@ export class CollectionSubscriber< // currentSyncState and input are always defined when this method is called // (only called from active subscriptions during a sync session) const input = - this.collectionConfigBuilder.currentSyncState!.inputs[this.alias]! + this.collectionConfigBuilder.currentSyncState!.inputs[this.sourceId]! const sentChanges = sendChangesToInput(input, filteredChanges) // Do not provide the callback that loads more data @@ -174,7 +209,7 @@ export class CollectionSubscriber< // because we need to mark the collection as ready if it's not already // and that's only done in `scheduleGraphRun` this.collectionConfigBuilder.scheduleGraphRun(dataLoader, { - alias: this.alias, + sourceId: this.sourceId, }) } @@ -411,13 +446,13 @@ export class CollectionSubscriber< }) } - private getWhereClauseForAlias(): BasicExpression | undefined { + private getWhereClause(): BasicExpression | undefined { const sourceWhereClausesCache = this.collectionConfigBuilder.sourceWhereClausesCache if (!sourceWhereClausesCache) { return undefined } - return sourceWhereClausesCache.get(this.alias) + return sourceWhereClausesCache.get(this.sourceId) } private getOrderByInfo(): OrderByOptimizationInfo | undefined { diff --git a/packages/db/src/query/live/materialized-pipeline.ts b/packages/db/src/query/live/materialized-pipeline.ts new file mode 100644 index 0000000000..95c45246f9 --- /dev/null +++ b/packages/db/src/query/live/materialized-pipeline.ts @@ -0,0 +1,459 @@ +import { + distinct, + filter, + join, + map, + reduce, + serializeValue, +} from '@tanstack/db-ivm' +import { FN_SELECT_STATE, INCLUDES_ROUTING } from '../compiler/index.js' +import { VIRTUAL_PROP_NAMES } from '../../virtual-props.js' +import { deepEquals } from '../../utils.js' +import type { + CompilationResult, + IncludesCompilationResult, +} from '../compiler/index.js' +import type { IncludesMaterialization } from '../ir.js' +import type { IStreamBuilder } from '@tanstack/db-ivm' +import type { ResultStream } from '../../types.js' + +type ResultTuple = [ + value: Record, + order: string | undefined, + correlationKey?: unknown, + parentContext?: Record | null, + routing?: IncludesRouting, + publicKey?: unknown, +] + +type IncludesRouting = Record + +type IncludeRoute = { + active: boolean + correlationKey: unknown + parentContext: Record | null +} + +type FnSelectState = { + sourceRow: Record + fnSelect: (row: any) => unknown +} + +type CanonicalResult = { + publicKey: unknown + tuple: ResultTuple +} + +export type BucketRow = { + publicKey: unknown + value: Record + order: string | undefined +} + +export type BucketFacadeCompilation = { + edgeId: string + rows: IStreamBuilder<[string, BucketRow]> + activeBuckets: IStreamBuilder<[string, true]> + hasOrderBy: boolean +} + +export const BUCKET_FACADE_REF = Symbol(`bucketFacadeRef`) + +export type BucketFacadeRef = { + [BUCKET_FACADE_REF]: { + edgeId: string + bucketKey: string + } +} + +export type MaterializedCompilation = { + pipeline: ResultStream + facades: Array +} + +let nextBucketFacadeEdgeId = 0 + +/** + * Compiles inline includes into the same D2 graph as their parent relation. + * Collection-valued includes become inert bucket references. The public facade + * adapter resolves those references after the graph reaches quiescence. + */ +export function materializeCompilation( + compilation: CompilationResult, + getRootKey?: (row: any) => unknown, +): MaterializedCompilation { + const built = new WeakMap() + const materialized = materializeRelation(compilation, getRootKey, built) + return { + ...materialized, + facades: dedupeFacades(materialized.facades), + } +} + +function materializeRelation( + compilation: CompilationResult, + getKey: ((row: any) => unknown) | undefined, + built: WeakMap, +): MaterializedCompilation { + const cached = built.get(compilation) + if (cached) return cached + + let pipeline = canonicalizeByPublicKey( + exposeRouting(compilation.pipeline), + getKey, + ) + const facades: Array = [] + + for (const include of compilation.includes ?? []) { + const child = materializeRelation( + include.childCompilationResult, + undefined, + built, + ) + facades.push(...child.facades) + + const bucketRows = createBucketRows(child.pipeline) + if (include.materialization === `collection`) { + const edgeId = `bucket-facade-${++nextBucketFacadeEdgeId}` + const activeBuckets = createActiveBuckets(pipeline, include) + facades.push({ + edgeId, + rows: bucketRows, + activeBuckets, + hasOrderBy: include.hasOrderBy, + }) + pipeline = attachCollectionInclude(pipeline, include, edgeId) + } else { + pipeline = attachInlineInclude(pipeline, bucketRows, include) + } + } + + const result = { pipeline, facades } + built.set(compilation, result) + return result +} + +function dedupeFacades( + facades: Array, +): Array { + return [...new Map(facades.map((facade) => [facade.edgeId, facade])).values()] +} + +function exposeRouting(pipeline: ResultStream): ResultStream { + return pipeline.pipe( + map(([key, rawTuple]) => { + const tuple = rawTuple as ResultTuple + return [ + key, + [ + tuple[0], + tuple[1], + tuple[2], + tuple[3], + tuple[0][INCLUDES_ROUTING], + tuple[4], + ], + ] + }), + ) as unknown as ResultStream +} + +function canonicalizeByPublicKey( + pipeline: ResultStream, + getKey: ((row: any) => unknown) | undefined, +): ResultStream { + return pipeline.pipe( + map(([internalKey, rawTuple]) => { + const tuple = rawTuple as ResultTuple + const publicKey = getKey ? getKey(tuple[0]) : (tuple[5] ?? internalKey) + return [serializeValue(publicKey), { publicKey, tuple }] as [ + string, + CanonicalResult, + ] + }), + reduce((values: Array<[CanonicalResult, number]>) => { + const totalMultiplicity = values.reduce( + (total, [, multiplicity]) => total + multiplicity, + 0, + ) + if (totalMultiplicity === 0) return [] + if (totalMultiplicity < 0) { + throw new Error(`Canonical query row has negative multiplicity`) + } + + const visible = values.find(([, multiplicity]) => multiplicity > 0)?.[0] + if (!visible) { + throw new Error(`Canonical query row has no positive contributor`) + } + + for (const [candidate, multiplicity] of values) { + if (multiplicity <= 0) continue + assertCongruentContributors(visible, candidate) + } + + return [[visible, 1]] + }), + map(([_serializedKey, { publicKey, tuple }]) => [publicKey, tuple]), + ) as ResultStream +} + +function assertCongruentContributors( + left: CanonicalResult, + right: CanonicalResult, +): void { + const [leftValue, leftOrder, leftCorrelation, leftContext, leftRouting] = + left.tuple + const [rightValue, rightOrder, rightCorrelation, rightContext, rightRouting] = + right.tuple + + if ( + leftOrder !== rightOrder || + !deepEquals(leftValue, rightValue) || + !deepEquals(leftCorrelation, rightCorrelation) || + !deepEquals(leftContext, rightContext) || + !deepEquals(leftRouting, rightRouting) + ) { + throw new Error( + `Query contributors for public key ${serializeValue(left.publicKey)} are not congruent`, + ) + } +} + +function attachInlineInclude( + parentPipeline: ResultStream, + bucketRows: IStreamBuilder<[string, BucketRow]>, + include: IncludesCompilationResult, +): ResultStream { + const bucketValues = bucketRows.pipe( + reduce((values: Array<[BucketRow, number]>) => { + const rows: Array = [] + for (const [row, multiplicity] of values) { + if (multiplicity < 0) { + throw new Error( + `Materialization bucket row has negative multiplicity`, + ) + } + for (let index = 0; index < multiplicity; index++) rows.push(row) + } + if (rows.length === 0) return [] + + rows.sort(compareBucketRows) + return [[materializeRows(rows, include), 1]] + }), + ) + const routedParents = parentPipeline.pipe( + map(([parentKey, rawTuple]) => { + const tuple = rawTuple as ResultTuple + const routing = getIncludeRoute(tuple, include.fieldName) + return [ + routing?.active !== true + ? `inactive:${serializeValue(parentKey)}` + : routeKey(routing.correlationKey, routing.parentContext), + { parentKey, tuple }, + ] as [string, { parentKey: unknown; tuple: ResultTuple }] + }), + join(bucketValues, `left`), + map(([_bucketKey, [parent, bucketValue]]) => { + const [value, order, correlationKey, parentContext, routing] = + parent!.tuple + const edgeRouting = routing?.[include.fieldName] + if (edgeRouting?.active !== true) { + return [ + parent!.parentKey, + [value, order, correlationKey, parentContext, routing], + ] + } + const materialized = + bucketValue ?? emptyMaterializedValue(include.materialization) + return [ + parent!.parentKey, + [ + setMaterializedInclude(value, include.resultPath, materialized), + order, + correlationKey, + parentContext, + routing, + ], + ] + }), + ) + // A route move can make the join emit matched and empty-bucket deltas for + // the same parent key in one graph turn. Reduce those deltas back to the one + // canonical parent row before the next include or the public output sees it. + return canonicalizeByPublicKey(routedParents as ResultStream, undefined) +} + +function createBucketRows( + childPipeline: ResultStream, +): IStreamBuilder<[string, BucketRow]> { + return childPipeline.pipe( + map(([publicKey, rawTuple]) => { + const [value, order, correlationKey, parentContext] = + rawTuple as ResultTuple + return [ + routeKey(correlationKey, parentContext), + { publicKey, value, order }, + ] as [string, BucketRow] + }), + ) +} + +function attachCollectionInclude( + parentPipeline: ResultStream, + include: IncludesCompilationResult, + edgeId: string, +): ResultStream { + const routedParents = parentPipeline.pipe( + map(([parentKey, rawTuple]) => { + const tuple = rawTuple as ResultTuple + const routing = getIncludeRoute(tuple, include.fieldName) + if (routing?.active !== true) return [parentKey, tuple] + const facade = createBucketFacadeRef( + edgeId, + routeKey(routing.correlationKey, routing.parentContext), + ) + return [ + parentKey, + [ + setMaterializedInclude(tuple[0], include.resultPath, facade), + tuple[1], + tuple[2], + tuple[3], + tuple[4], + ], + ] + }), + ) + return canonicalizeByPublicKey(routedParents as ResultStream, undefined) +} + +function createActiveBuckets( + parentPipeline: ResultStream, + include: IncludesCompilationResult, +): IStreamBuilder<[string, true]> { + return parentPipeline.pipe( + map(([parentKey, rawTuple]) => { + const tuple = rawTuple as ResultTuple + const routing = getIncludeRoute(tuple, include.fieldName) + const bucketKey = + routing?.active === true + ? routeKey(routing.correlationKey, routing.parentContext) + : undefined + return [parentKey, bucketKey] as [unknown, string | undefined] + }), + filter(([, bucketKey]) => bucketKey !== undefined), + distinct(([, bucketKey]) => bucketKey), + map(([, bucketKey]) => [bucketKey!, true] as [string, true]), + ) +} + +function createBucketFacadeRef( + edgeId: string, + bucketKey: string, +): BucketFacadeRef { + return { [BUCKET_FACADE_REF]: { edgeId, bucketKey } } +} + +function getIncludeRoute( + tuple: ResultTuple, + fieldName: string, +): IncludeRoute | undefined { + return tuple[4]?.[fieldName] +} + +function routeKey( + correlationKey: unknown, + parentContext: Record | null | undefined, +): string { + return serializeValue([correlationKey ?? null, parentContext ?? null]) +} + +function compareBucketRows(left: BucketRow, right: BucketRow): number { + if (left.order !== right.order) { + if (left.order === undefined) return 1 + if (right.order === undefined) return -1 + return left.order < right.order ? -1 : 1 + } + + const leftKey = serializeValue(left.publicKey) + const rightKey = serializeValue(right.publicKey) + return leftKey < rightKey ? -1 : leftKey > rightKey ? 1 : 0 +} + +function materializeRows( + rows: Array, + include: IncludesCompilationResult, +): unknown { + const scalarField = include.scalarField + const values = scalarField + ? rows.map(({ value }) => value[scalarField]) + : rows.map(({ value }) => value) + + if (include.materialization === `array`) return values + if (include.materialization === `singleton`) return values[0] + return values.map((value) => String(value ?? ``)).join(``) +} + +function emptyMaterializedValue( + materialization: IncludesMaterialization, +): unknown { + if (materialization === `array`) return [] + if (materialization === `concat`) return `` + if (materialization === `singleton`) return undefined + throw new Error(`Collection includes require a bucket facade`) +} + +function setNestedValue( + source: Record, + path: Array, + value: unknown, +): Record { + const root = { ...source } + let target = root + let current: Record | null | undefined = source + + for (let index = 0; index < path.length - 1; index++) { + const part = path[index]! + const currentChild: any = current?.[part] + const next = Array.isArray(currentChild) + ? [...currentChild] + : { ...(currentChild ?? {}) } + target[part] = next + target = next + current = currentChild + } + + target[path[path.length - 1]!] = value + return root +} + +function setMaterializedInclude( + value: Record, + path: Array, + materialized: unknown, +): Record { + const state = value[FN_SELECT_STATE] as FnSelectState | undefined + if (!state) return setNestedValue(value, path, materialized) + + const sourceRow = setNestedValue(state.sourceRow, path, materialized) + const selectedValue = state.fnSelect(sourceRow) + if (!selectedValue || typeof selectedValue !== `object`) { + throw new Error(`fn.select must return an object when it projects includes`) + } + + const selected: Record = Array.isArray(selectedValue) + ? [...selectedValue] + : { ...selectedValue } + for (const property of VIRTUAL_PROP_NAMES) { + if (property in value && !(property in selected)) { + selected[property] = value[property] + } + } + selected[INCLUDES_ROUTING] = value[INCLUDES_ROUTING] + Object.defineProperty(selected, FN_SELECT_STATE, { + value: { sourceRow, fnSelect: state.fnSelect }, + enumerable: true, + configurable: true, + }) + return selected +} diff --git a/packages/db/src/query/live/subset-demand-controller.ts b/packages/db/src/query/live/subset-demand-controller.ts new file mode 100644 index 0000000000..267750e123 --- /dev/null +++ b/packages/db/src/query/live/subset-demand-controller.ts @@ -0,0 +1,115 @@ +import { serializeValue } from '@tanstack/db-ivm' +import { inArray } from '../builder/functions.js' +import { PropRef } from '../ir.js' +import type { CollectionSubscription } from '../../collection/subscription.js' +import type { LazyDemandPlan } from '../compiler/joins.js' +import type { BasicExpression } from '../ir.js' + +type DemandSegment = { + keys: Map + where: BasicExpression + abortController: AbortController +} + +type DemandState = { + keys: Map + segments: Array +} + +export type DemandUpdate = { + changed: boolean + empty: boolean + loadResults: Array | true> +} + +/** + * Keeps lazy subset requests aligned with the current relation of demanded + * keys. Additions load only new coverage. Removals release and rebuild only + * request segments that covered a removed key. + */ +export class SubsetDemandController { + private readonly states = new Map() + + setDemand( + subscription: CollectionSubscription, + plan: LazyDemandPlan, + keys: Set, + ): DemandUpdate { + const nextKeys = canonicalizeKeys(keys) + const previous = this.states.get(plan.id) + if (previous && equalKeySets(previous.keys, nextKeys)) { + return { changed: false, empty: nextKeys.size === 0, loadResults: [] } + } + + const loadResults: Array | true> = [] + const segments: Array = [] + + for (const segment of previous?.segments ?? []) { + if ([...segment.keys.keys()].every((key) => nextKeys.has(key))) { + segments.push(segment) + continue + } + + const retained = new Map( + [...segment.keys].filter(([key]) => nextKeys.has(key)), + ) + if (retained.size > 0) { + segments.push(requestSegment(subscription, plan, retained, loadResults)) + } + segment.abortController.abort() + subscription.releaseSnapshot(segment.where) + } + + const added = new Map( + [...nextKeys].filter(([key]) => !previous?.keys.has(key)), + ) + if (added.size > 0) { + segments.push(requestSegment(subscription, plan, added, loadResults)) + } + + if (nextKeys.size === 0) { + this.states.delete(plan.id) + } else { + this.states.set(plan.id, { keys: nextKeys, segments }) + } + + return { changed: true, empty: nextKeys.size === 0, loadResults } + } + + clear(): void { + for (const state of this.states.values()) { + for (const segment of state.segments) segment.abortController.abort() + } + this.states.clear() + } +} + +function canonicalizeKeys(keys: Set): Map { + return new Map([...keys].map((key) => [serializeValue(key), key])) +} + +function equalKeySets( + left: Map, + right: Map, +): boolean { + return ( + left.size === right.size && [...left.keys()].every((key) => right.has(key)) + ) +} + +function requestSegment( + subscription: CollectionSubscription, + plan: LazyDemandPlan, + keys: Map, + loadResults: Array | true>, +): DemandSegment { + const where = inArray(new PropRef(plan.path), [...keys.values()]) + const abortController = new AbortController() + subscription.requestSnapshot({ + where, + signal: abortController.signal, + trackLoadSubsetPromise: false, + onLoadSubsetResult: (result) => loadResults.push(result), + }) + return { keys, where, abortController } +} diff --git a/packages/db/src/query/live/utils.ts b/packages/db/src/query/live/utils.ts index c7f701124f..f45177a322 100644 --- a/packages/db/src/query/live/utils.ts +++ b/packages/db/src/query/live/utils.ts @@ -2,7 +2,7 @@ import { MultiSet, serializeValue } from '@tanstack/db-ivm' import { UnsupportedRootScalarSelectError } from '../../errors.js' import { normalizeOrderByPaths } from '../compiler/expressions.js' import { buildQuery, getQueryIR } from '../builder/index.js' -import { ConditionalSelect, IncludesSubquery, isExpressionLike } from '../ir.js' +import { collectCollectionSources, isExpressionLike } from '../ir.js' import type { MultiSetArray, RootStreamBuilder } from '@tanstack/db-ivm' import type { Collection } from '../../collection/index.js' import type { ChangeMessage } from '../../types.js' @@ -17,90 +17,17 @@ import type { OrderByOptimizationInfo } from '../compiler/order-by.js' * Maps collections by their ID (not alias) as expected by the compiler. */ export function extractCollectionsFromQuery( - query: any, + query: QueryIR, ): Record> { - const collections: Record = {} - - // Helper function to recursively extract collections from a query or source - function extractFromSource(source: any) { - if (source.type === `collectionRef`) { - collections[source.collection.id] = source.collection - } else if (source.type === `queryRef`) { - // Recursively extract from subquery - extractFromQuery(source.query) - } else if (source.type === `unionFrom`) { - for (const childSource of source.sources) { - extractFromSource(childSource) - } - } else if (source.type === `unionAll`) { - for (const branch of source.queries) { - extractFromQuery(branch) - } - } - } - - // Helper function to recursively extract collections from a query - function extractFromQuery(q: any) { - // Extract from FROM clause - if (q.from) { - extractFromSource(q.from) - } - - // Extract from JOIN clauses - if (q.join && Array.isArray(q.join)) { - for (const joinClause of q.join) { - if (joinClause.from) { - extractFromSource(joinClause.from) - } - } - } - - // Extract from SELECT (for IncludesSubquery) - if (q.select) { - extractFromSelect(q.select) - } - } - - function extractFromSelect(select: any) { - for (const [key, value] of Object.entries(select)) { - if (typeof key === `string` && key.startsWith(`__SPREAD_SENTINEL__`)) { - continue - } - if (value instanceof IncludesSubquery) { - extractFromQuery(value.query) - } else if (value instanceof ConditionalSelect) { - extractFromConditionalSelect(value) - } else if (isNestedSelectObject(value)) { - extractFromSelect(value) - } - } - } - - function extractFromConditionalSelect(conditional: ConditionalSelect) { - for (const branch of conditional.branches) { - extractFromSelectValue(branch.value) - } - if (conditional.defaultValue !== undefined) { - extractFromSelectValue(conditional.defaultValue) - } - } - - function extractFromSelectValue(value: any) { - if (value instanceof IncludesSubquery) { - extractFromQuery(value.query) - } else if (value instanceof ConditionalSelect) { - extractFromConditionalSelect(value) - } else if (isNestedSelectObject(value)) { - extractFromSelect(value) - } + const collections: Record> = {} + for (const source of collectCollectionSources(query)) { + collections[source.collection.id] = source.collection } - - // Start extraction from the root query - extractFromQuery(query) - return collections } +export { collectCollectionSources as extractCollectionSources } + /** * Helper function to extract the collection that is referenced in the query's FROM clause. * The FROM clause may refer directly to a collection or indirectly to a subquery. @@ -126,119 +53,11 @@ export function extractCollectionFromSource( ) } -/** - * Extracts all aliases used for each collection across the entire query tree. - * - * Traverses the QueryIR recursively to build a map from collection ID to all aliases - * that reference that collection. This is essential for self-join support, where the - * same collection may be referenced multiple times with different aliases. - * - * For example, given a query like: - * ```ts - * q.from({ employee: employeesCollection }) - * .join({ manager: employeesCollection }, ({ employee, manager }) => - * eq(employee.managerId, manager.id) - * ) - * ``` - * - * This function would return: - * ``` - * Map { "employees" => Set { "employee", "manager" } } - * ``` - * - * @param query - The query IR to extract aliases from - * @returns A map from collection ID to the set of all aliases referencing that collection - */ -export function extractCollectionAliases( - query: QueryIR, -): Map> { - const aliasesById = new Map>() - - function recordAlias(source: any) { - if (!source) return - - if (source.type === `collectionRef`) { - const { id } = source.collection - const existing = aliasesById.get(id) - if (existing) { - existing.add(source.alias) - } else { - aliasesById.set(id, new Set([source.alias])) - } - } else if (source.type === `queryRef`) { - traverse(source.query) - } else if (source.type === `unionFrom`) { - for (const childSource of source.sources) { - recordAlias(childSource) - } - } else if (source.type === `unionAll`) { - for (const branch of source.queries) { - traverse(branch) - } - } - } - - function traverseSelect(select: any) { - for (const [key, value] of Object.entries(select)) { - if (typeof key === `string` && key.startsWith(`__SPREAD_SENTINEL__`)) { - continue - } - if (value instanceof IncludesSubquery) { - traverse(value.query) - } else if (value instanceof ConditionalSelect) { - traverseConditionalSelect(value) - } else if (isNestedSelectObject(value)) { - traverseSelect(value) - } - } - } - - function traverseConditionalSelect(conditional: ConditionalSelect) { - for (const branch of conditional.branches) { - traverseSelectValue(branch.value) - } - if (conditional.defaultValue !== undefined) { - traverseSelectValue(conditional.defaultValue) - } - } - - function traverseSelectValue(value: any) { - if (value instanceof IncludesSubquery) { - traverse(value.query) - } else if (value instanceof ConditionalSelect) { - traverseConditionalSelect(value) - } else if (isNestedSelectObject(value)) { - traverseSelect(value) - } - } - - function traverse(q?: QueryIR) { - if (!q) return - - recordAlias(q.from) - - if (q.join) { - for (const joinClause of q.join) { - recordAlias(joinClause.from) - } - } - - if (q.select) { - traverseSelect(q.select) - } - } - - traverse(query) - - return aliasesById -} - /** * Check if a value is a nested select object (plain object, not an expression) */ function isNestedSelectObject(obj: any): boolean { if (obj === null || typeof obj !== `object`) return false - if (obj instanceof IncludesSubquery) return false if (isExpressionLike(obj)) return false // Ref proxies from spread operations if (obj.__refProxy) return false diff --git a/packages/db/src/query/subset-dedupe.ts b/packages/db/src/query/subset-dedupe.ts index 3dc5cc2bc8..b1fcead473 100644 --- a/packages/db/src/query/subset-dedupe.ts +++ b/packages/db/src/query/subset-dedupe.ts @@ -112,8 +112,11 @@ export class DeduplicatedLoadSubset { // Check against in-flight calls using the same subset logic as resolved calls // This prevents duplicate requests when concurrent calls have subset relationships - const matchingInflight = this.inflightCalls.find((inflight) => - isPredicateSubset(options, inflight.options), + const matchingInflight = this.inflightCalls.find( + (inflight) => + !inflight.options.signal?.aborted && + inflight.options.signal === options.signal && + isPredicateSubset(options, inflight.options), ) if (matchingInflight !== undefined) { @@ -162,7 +165,10 @@ export class DeduplicatedLoadSubset { // Only update tracking if this request is still from the current generation // If reset() was called, the generation will have incremented and we should // not repopulate the state that was just cleared - if (capturedGeneration === this.generation) { + if ( + capturedGeneration === this.generation && + !trackingOptions.signal?.aborted + ) { this.updateTracking(trackingOptions) } return result diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index 6087e234ec..c26c8139b2 100644 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -302,6 +302,11 @@ export type LoadSubsetOptions = { * The sync layer can use this instead of `cursor` if it prefers offset-based pagination. */ offset?: number + /** + * Aborted when this exact subset request is no longer current. Async sync + * adapters must check the signal before installing fetched rows. + */ + signal?: AbortSignal /** * The subscription that triggered the load. * Advanced sync implementations can use this for: diff --git a/packages/db/tests/query/compiler/subqueries.test.ts b/packages/db/tests/query/compiler/subqueries.test.ts index 07ba9921a2..3bf3a6f99f 100644 --- a/packages/db/tests/query/compiler/subqueries.test.ts +++ b/packages/db/tests/query/compiler/subqueries.test.ts @@ -212,10 +212,7 @@ describe(`Query2 Subqueries`, () => { }) describe(`Subqueries in JOIN clause`, () => { - const dummyCallbacks = { - loadKeys: (_: any) => {}, - loadInitialState: () => {}, - } + const dummyCallbacks = {} it(`supports subquery in join clause`, () => { // Create a subquery for active users @@ -301,11 +298,18 @@ describe(`Query2 Subqueries`, () => { ) const { pipeline } = compilation - // Since we're doing a left join, the right-side source should be handled lazily. - // For subquery-backed joins, lazy loading is marked on the concrete source - // alias that has a subscription (`user`), not the outer QueryRef alias - // (`activeUser`). - expect(lazySources).contains(`user`) + // Since we're doing a left join, the concrete lexical source inside the + // right-side subquery should be handled lazily. Aliases are query-language + // names; the compiler tracks runtime demand by the source's opaque ID. + const activeUserJoin = builtQuery.join![0]!.from + expect(activeUserJoin.type).toBe(`queryRef`) + if (activeUserJoin.type === `queryRef`) { + const activeUserSource = activeUserJoin.query.from + expect(activeUserSource.type).toBe(`collectionRef`) + if (activeUserSource.type === `collectionRef`) { + expect(lazySources).contains(activeUserSource.sourceId) + } + } const messages: Array> = [] pipeline.pipe( @@ -378,10 +382,7 @@ describe(`Query2 Subqueries`, () => { user: usersSubscription, } - const dummyCallbacks = { - loadKeys: (_: any) => {}, - loadInitialState: () => {}, - } + const dummyCallbacks = {} // Compile the query const graph = new D2() diff --git a/packages/db/tests/query/includes-optimistic-oracle.property.test.ts b/packages/db/tests/query/includes-optimistic-oracle.property.test.ts index ea9b852486..24e599018f 100644 --- a/packages/db/tests/query/includes-optimistic-oracle.property.test.ts +++ b/packages/db/tests/query/includes-optimistic-oracle.property.test.ts @@ -11,9 +11,7 @@ import { mockSyncCollectionOptions, withExpectedRejection, } from '../utils.js' -import { expectAssertionFailure } from '../expected-failure.js' import { runTrace } from '../trace-runner.js' -import type { AssertionDifference } from '../expected-failure.js' import type { TraceDriver, TraceProjection } from '../trace-runner.js' type RootRow = { @@ -74,20 +72,6 @@ type OracleNode = RootRow & { children?: Array } -type RelationshipNode = Record & { - id: number - children?: unknown -} - -function isRelationshipNode(value: unknown): value is RelationshipNode { - return ( - typeof value === `object` && - value !== null && - `id` in value && - typeof value.id === `number` - ) -} - type ControlledCollection = ReturnType< typeof createControlledCollection > @@ -334,53 +318,6 @@ function stripVirtualProperties(value: unknown): unknown { ) } -function replaceDirectChildren( - value: unknown, - parentId: number, - children: ReadonlyArray, -): unknown | undefined { - if (!Array.isArray(value)) return undefined - - let replacements = 0 - const visit = (entries: ReadonlyArray): Array => - entries.map((entry) => { - if (!isRelationshipNode(entry)) return entry - if (entry.id === parentId) { - replacements += 1 - return { ...entry, children } - } - if (!Array.isArray(entry.children)) return entry - return { ...entry, children: visit(entry.children) } - }) - - const replaced = visit(value) - return replacements === 1 ? replaced : undefined -} - -function matchesExactly(actual: unknown, expected: unknown): boolean { - try { - expect(actual).toEqual(expected) - return true - } catch { - return false - } -} - -function classifyRetainedDetachedGrandchild( - { actual, expected }: AssertionDifference, - retainedChildren: ReadonlyArray, -) { - const expectedWithOnlyKnownDefect = replaceDirectChildren( - expected, - 11, - retainedChildren, - ) - return ( - expectedWithOnlyKnownDefect !== undefined && - matchesExactly(actual, expectedWithOnlyKnownDefect) - ) -} - function updateModel( model: Map, changes: ReadonlyArray>, @@ -556,18 +493,6 @@ function fixture(routes: RouteValues) { return { roots, levels } } -function retainedDetachedChildren(routes: RouteValues): Array { - return [ - { - id: 21, - group: routes.original + 1000, - value: 210, - position: 0, - children: [], - }, - ] -} - async function expectHistoryMatches( routes: RouteValues, steps: ReadonlyArray, @@ -588,58 +513,6 @@ const routeValuesArbitrary: fc.Arbitrary = fc.record({ }) describe(`optimistic relationship-transition oracle`, () => { - fcTest(`known-defect classifier rejects collateral corruption`, () => { - const routes: RouteValues = { - rootA: 10, - rootB: 100, - rootC: 200, - original: 300, - optimistic: 400, - authoritative: 500, - } - const expected = [ - { - id: 1, - group: routes.rootA, - value: 10, - position: 0, - children: [ - { - id: 11, - group: routes.optimistic, - value: 110, - position: 0, - children: [], - }, - ], - }, - ] - const actual = [ - { - id: 1, - group: routes.rootA, - value: 999, - position: 0, - children: [ - { - id: 11, - group: routes.optimistic, - value: 110, - position: 0, - children: retainedDetachedChildren(routes), - }, - ], - }, - ] - - expect( - classifyRetainedDetachedGrandchild( - { actual, expected }, - retainedDetachedChildren(routes), - ), - ).toBe(false) - }) - fcTest( `rejects optimistic handles the sync mock cannot settle independently`, () => { @@ -658,29 +531,17 @@ describe(`optimistic relationship-transition oracle`, () => { ) fcTest.prop([routeValuesArbitrary], { numRuns: 12 })( - `known defect: an optimistic rekey detaches its old descendants immediately`, + `an optimistic rekey detaches its old descendants immediately`, async (routes) => { - await expectAssertionFailure( - async () => { - await expectHistoryMatches(routes, [ - { - type: `optimistic`, - handle: `rekey`, - level: 1, - id: 11, - patch: { group: routes.optimistic }, - }, - ]) - }, + await expectHistoryMatches(routes, [ { - checkpoint: 1, - classify: (difference) => - classifyRetainedDetachedGrandchild( - difference, - retainedDetachedChildren(routes), - ), + type: `optimistic`, + handle: `rekey`, + level: 1, + id: 11, + patch: { group: routes.optimistic }, }, - )() + ]) }, ) diff --git a/packages/db/tests/query/includes-oracle.property.test.ts b/packages/db/tests/query/includes-oracle.property.test.ts index c0bce07f2b..c0c0f71109 100644 --- a/packages/db/tests/query/includes-oracle.property.test.ts +++ b/packages/db/tests/query/includes-oracle.property.test.ts @@ -1,4 +1,3 @@ -import { isDeepStrictEqual } from 'node:util' import { fc, test as fcTest } from '@fast-check/vitest' import { describe, expect } from 'vitest' import { @@ -15,9 +14,7 @@ import { mockSyncCollectionOptions, withExpectedRejection, } from '../utils.js' -import { expectAssertionFailure } from '../expected-failure.js' import { runTrace } from '../trace-runner.js' -import type { AssertionDifference } from '../expected-failure.js' import type { TraceCheckpoint, TraceDriver, @@ -101,111 +98,6 @@ function hasDirectChild(value: unknown, parentId: number, childId: number) { ) } -function classifyUnexpectedSharedRoute( - { actual, expected }: AssertionDifference, - unexpectedParentId: number, - expectedParentId: number, - childId: number, -): boolean { - const expectedParent = findRelationshipNode(expected, expectedParentId) - const expectedWithSharedRoute = replaceDirectChildren( - expected, - unexpectedParentId, - Array.isArray(expectedParent?.children) ? expectedParent.children : [], - ) - return ( - !hasDirectChild(expected, unexpectedParentId, childId) && - hasDirectChild(expected, expectedParentId, childId) && - isDeepStrictEqual(actual, expectedWithSharedRoute) - ) -} - -function classifyMissingReplacementChild( - { actual, expected }: AssertionDifference, - replacementRowId: number, - childId: number, -): boolean { - const expectedWithoutChild = removeDirectChild(expected, replacementRowId, [ - childId, - ]) - return ( - findRelationshipNode(expected, replacementRowId) !== undefined && - hasDirectChild(expected, replacementRowId, childId) && - isDeepStrictEqual(actual, expectedWithoutChild) - ) -} - -function classifyMissingSharedRouteSnapshot( - { actual, expected }: AssertionDifference, - enteringParentId: number, - existingParentId: number, - childIds: number | ReadonlyArray, -): boolean { - const expectedChildIds = Array.isArray(childIds) ? childIds : [childIds] - const expectedWithoutSnapshot = removeDirectChild( - expected, - enteringParentId, - expectedChildIds, - ) - return ( - expectedChildIds.every( - (childId) => - hasDirectChild(expected, enteringParentId, childId) && - hasDirectChild(actual, existingParentId, childId) && - hasDirectChild(expected, existingParentId, childId), - ) && isDeepStrictEqual(actual, expectedWithoutSnapshot) - ) -} - -function removeDirectChild( - value: unknown, - parentId: number, - childIds: ReadonlyArray, -): unknown { - if (Array.isArray(value)) { - return value.map((entry) => removeDirectChild(entry, parentId, childIds)) - } - if (typeof value !== `object` || value === null) return value - - const isParent = isRelationshipNode(value) && value.id === parentId - return Object.fromEntries( - Object.entries(value).map(([key, entry]) => [ - key, - key === `children` && isParent && Array.isArray(entry) - ? entry - .filter( - (child) => - !isRelationshipNode(child) || !childIds.includes(child.id), - ) - .map((child) => removeDirectChild(child, parentId, childIds)) - : removeDirectChild(entry, parentId, childIds), - ]), - ) -} - -function replaceDirectChildren( - value: unknown, - parentId: number, - children: ReadonlyArray, -): unknown { - if (Array.isArray(value)) { - return value.map((entry) => - replaceDirectChildren(entry, parentId, children), - ) - } - if (typeof value !== `object` || value === null) return value - - const isParent = isRelationshipNode(value) && value.id === parentId - return Object.fromEntries( - Object.entries(value).map(([key, entry]) => [ - key, - key === `children` && isParent - ? children - : replaceDirectChildren(entry, parentId, children), - ]), - ) -} - type RelationshipProjectionNode = { id: number children?: Array @@ -277,8 +169,8 @@ function levelArbitrary( function actionArbitrary(depth: IncludeDepth): fc.Arbitrary { return levelArbitrary(depth).chain((level) => fc.record({ - // Root delete/reinsert has a deterministic expected-failure trace below. - // Keep the green fuzz corpus from rediscovering the same defect class. + // Root delete/reinsert has a focused history matrix below. Keep this + // unconstrained corpus small enough to shrink to one clear action. type: level === 0 ? fc.constantFrom( @@ -1515,14 +1407,10 @@ function visibleRelationshipScenarioArbitrary( scalarNoiseArbitrary(`before`, branchArbitrary), { maxLength: 4 }, ), - afterValues: fc.array( - fc.record({ - level: levelArbitrary(depth), - value: fc.integer({ min: -3, max: 3 }), - position: fc.integer({ min: -2, max: 2 }), - }), - { minLength: 1, maxLength: 5 }, - ), + afterValues: fc.array(scalarNoiseArbitrary(`after`, branchArbitrary), { + minLength: 1, + maxLength: 5, + }), }) .map( ({ @@ -1536,18 +1424,10 @@ function visibleRelationshipScenarioArbitrary( extraBeforeNoise, afterValues, }) => { - const stableBranch = otherBranch(sourceBranch) - // Updating two descendant levels after a reparent exposes a separate - // known defect, captured for every failing depth/level below. Keep the - // generated corpus green by updating only the branch that did not move. const connectedNoise: Array = [ beforeNoise, ...extraBeforeNoise, - ...afterValues.map((entry) => ({ - ...entry, - side: `after` as const, - branch: stableBranch, - })), + ...afterValues, ] const options = { depth, @@ -1633,7 +1513,7 @@ function createTransitionHistoryScenario({ }: TransitionHistoryScenarioOptions): TransitionHistoryScenario { // Fresh keys keep this matrix green through both transitions. Separate // state-aware families below reuse retired routes and replace existing rows, - // so those known failures remain shrinkable without masking later steps. + // so those histories remain shrinkable without masking later steps. const insertedRows = [ { id: 3_000, @@ -2398,14 +2278,11 @@ function createInitiallySharedRoutePrefix( function createMergeIntoSharedRouteScenarios( parentLevel: 0 | 1 | 2, enteringRow: 0 | 1, -): ClassifiedHistoryScenario { +): HistoryScenarioPair { const depth = (parentLevel + 1) as IncludeDepth const branches = transitionHistoryBranches - const childLevel = depth const existingRow = otherBranch(enteringRow) const sharedRoute = branches[existingRow].groupBase + parentLevel - const enteringParentId = rowAt(branches, enteringRow, parentLevel) - const childId = rowAt(branches, existingRow, childLevel) const candidate = createRouteLifecycleScenario({ depth, branches, @@ -2431,14 +2308,6 @@ function createMergeIntoSharedRouteScenarios( ), }, candidate, - candidateCheckpoint: candidate.steps.length, - classify: (difference) => - classifyMissingSharedRouteSnapshot( - difference, - enteringParentId, - rowAt(branches, existingRow, parentLevel), - childId, - ), } } @@ -2488,7 +2357,7 @@ function createSharedRouteLastSubscriberScenario( function createSnapshotOnResubscribeScenarios( parentLevel: 0 | 1 | 2, -): Pick { +): HistoryScenarioPair { const depth = (parentLevel + 1) as IncludeDepth const branches = transitionHistoryBranches const childLevel = depth @@ -2537,7 +2406,7 @@ function createSnapshotOnResubscribeScenarios( function createInitiallySharedRouteResubscribeScenario( parentLevel: 0 | 1 | 2, -): ClassifiedHistoryScenario { +): HistoryScenarioPair { const depth = (parentLevel + 1) as IncludeDepth const branches = transitionHistoryBranches const childLevel = depth @@ -2587,23 +2456,13 @@ function createInitiallySharedRouteResubscribeScenario( return { control: createSharedRouteLastSubscriberScenario(parentLevel), candidate: scenario, - candidateCheckpoint: scenario.steps.length, - classify: (difference) => - classifyUnexpectedSharedRoute( - difference, - rowAt(branches, 0, parentLevel), - rowAt(branches, 1, parentLevel), - childId, - ), } } -type ClassifiedHistoryScenario = { +type HistoryScenarioPair = { control: FullRowBatchScenario greenVariants?: ReadonlyArray candidate: FullRowBatchScenario - candidateCheckpoint: number - classify: (difference: AssertionDifference) => boolean } type RekeyRouteReuseOptions = { @@ -2628,7 +2487,6 @@ function createRekeyRouteReuseFixture({ prefix: Array rekey: FullRowBatchStep reuse: FullRowBatchStep - classify: (difference: AssertionDifference) => boolean } { const targetLevel = (depth - 1) as IncludeDepth assertDisjointRelationshipKeys(depth, branches, { @@ -2666,44 +2524,30 @@ function createRekeyRouteReuseFixture({ ], } - const retiredRowId = source.idBase + targetLevel - const childId = source.idBase + targetLevel + 1 - return { prefix, rekey, reuse: insertOldRoute, - classify: (difference) => - classifyUnexpectedSharedRoute( - difference, - retiredRowId, - insertedId, - childId, - ), } } function createRekeyRouteReuseScenarios( options: RekeyRouteReuseOptions, -): ClassifiedHistoryScenario { +): HistoryScenarioPair { const { depth } = options - const { prefix, rekey, reuse, classify } = - createRekeyRouteReuseFixture(options) + const { prefix, rekey, reuse } = createRekeyRouteReuseFixture(options) return { control: { depth, steps: [...prefix, reuse] }, candidate: { depth, steps: [...prefix, rekey, reuse] }, - candidateCheckpoint: prefix.length + 2, - classify, } } function createIntraBatchRekeyRouteReuseScenarios( options: RekeyRouteReuseOptions, -): ClassifiedHistoryScenario { +): HistoryScenarioPair { const { depth } = options - const { prefix, rekey, reuse, classify } = - createRekeyRouteReuseFixture(options) + const { prefix, rekey, reuse } = createRekeyRouteReuseFixture(options) if (rekey.level === 0 || rekey.level !== reuse.level) { throw new Error(`Intra-batch route reuse must share a child level`) } @@ -2723,8 +2567,6 @@ function createIntraBatchRekeyRouteReuseScenarios( { level: rekey.level, changes: [...rekey.changes, ...reuse.changes] }, ], }, - candidateCheckpoint: prefix.length + 1, - classify, } } @@ -2733,8 +2575,8 @@ function rekeyRouteReuseScenarioArbitrary( sourceBranch: 0 | 1, createScenario: ( options: RekeyRouteReuseOptions, - ) => ClassifiedHistoryScenario = createRekeyRouteReuseScenarios, -): fc.Arbitrary { + ) => HistoryScenarioPair = createRekeyRouteReuseScenarios, +): fc.Arbitrary { return fc .record({ ...generatedBranchArbitraries, @@ -2789,7 +2631,7 @@ function createMovedChildReplacementScenarios({ insertedId, insertedValue, insertedPosition, -}: MovedChildReplacementOptions): ClassifiedHistoryScenario { +}: MovedChildReplacementOptions): HistoryScenarioPair { if (targetLevel + 2 > depth) { throw new Error(`Child replacement needs a visible grandchild`) } @@ -2865,8 +2707,6 @@ function createMovedChildReplacementScenarios({ }, ], }) - const grandchildId = source.idBase + targetLevel + 2 - return { control: { depth, steps: [...prefix, ...replaceChild] }, greenVariants: [ @@ -2874,9 +2714,6 @@ function createMovedChildReplacementScenarios({ atomicReplacement(`insert-first`), ], candidate: { depth, steps: [...prefix, reparent, ...replaceChild] }, - candidateCheckpoint: prefix.length + 3, - classify: (difference) => - classifyMissingReplacementChild(difference, insertedId, grandchildId), } } @@ -2884,7 +2721,7 @@ function movedChildReplacementMirrorsArbitrary( depth: 3 | 4, targetLevel: IncludeDepth, sourceBranch: 0 | 1, -): fc.Arbitrary> { +): fc.Arbitrary> { return fc .record({ ...generatedBranchArbitraries, @@ -2958,7 +2795,7 @@ function createRelationshipBatchShapeScenarios({ replacementPosition, movedPosition, ...branchOptions -}: RelationshipBatchShapeOptions): ClassifiedHistoryScenario { +}: RelationshipBatchShapeOptions): HistoryScenarioPair { const depth = 3 const branches = createGeneratedBranches(branchOptions) assertDisjointRelationshipKeys(depth, branches, { @@ -3050,13 +2887,6 @@ function createRelationshipBatchShapeScenarios({ ...replacementSteps, ], }, - candidateCheckpoint: prefix.length + 1 + replacementSteps.length, - classify: (difference) => - classifyMissingReplacementChild( - difference, - replacementChild.id, - source.idBase + depth, - ), } } @@ -3073,7 +2903,7 @@ const relationshipBatchFixtureArbitrary: fc.Arbitrary< type RelationshipBatchShapeCell = { shape: RelationshipBatchShape - scenarios: ClassifiedHistoryScenario + scenarios: HistoryScenarioPair } function createRelationshipBatchShapeMatrix( @@ -3110,23 +2940,9 @@ function createRelationshipBatchShapeMatrix( return cells } -function isKnownRelationshipBatchFailure( - shape: RelationshipBatchShape, -): boolean { - // Split delete-then-insert with a new public id is the known moved-subtree - // replacement defect: the replacement arrives, but its handed-off route - // omits the existing grandchild. - return ( - shape.delivery === `split` && - shape.order === `delete-insert` && - shape.publicId === `new` && - shape.route === `handoff` - ) -} - function createSharedRouteLifetimeScenarios( parentLevel: 0 | 1, -): ClassifiedHistoryScenario { +): HistoryScenarioPair { if (parentLevel === 0) { const departed = batchRoot(100, 600, 100, 0) const remaining = batchRoot(1_100, 600, 1_100, 1) @@ -3159,14 +2975,6 @@ function createSharedRouteLifetimeScenarios( return { control: { depth: 1, steps: controlSteps }, candidate: { depth: 1, steps: candidateSteps }, - candidateCheckpoint: candidateSteps.length, - classify: (difference) => - classifyUnexpectedSharedRoute( - difference, - departed.id, - remaining.id, - child.id, - ), } } @@ -3204,14 +3012,6 @@ function createSharedRouteLifetimeScenarios( return { control: { depth: 2, steps: controlSteps }, candidate: { depth: 2, steps: candidateSteps }, - candidateCheckpoint: candidateSteps.length, - classify: (difference) => - classifyUnexpectedSharedRoute( - difference, - departed.id, - remaining.id, - child.id, - ), } } @@ -3226,31 +3026,11 @@ async function expectFullRowBatchScenarioMatches({ }) } -async function expectClassifiedHistoryFailure({ +async function expectHistoryScenarioPairMatches({ control, greenVariants = [], candidate, - candidateCheckpoint, - classify, -}: ClassifiedHistoryScenario): Promise { - await expectFullRowBatchScenarioMatches(control) - for (const greenVariant of greenVariants) { - await expectFullRowBatchScenarioMatches(greenVariant) - } - await expectAssertionFailure( - () => expectFullRowBatchScenarioMatches(candidate), - { checkpoint: candidateCheckpoint, classify }, - )() -} - -async function expectClassifiedHistoryMatches({ - control, - greenVariants = [], - candidate, -}: Pick< - ClassifiedHistoryScenario, - `control` | `greenVariants` | `candidate` ->): Promise { +}: HistoryScenarioPair): Promise { await expectFullRowBatchScenarioMatches(control) for (const greenVariant of greenVariants) { await expectFullRowBatchScenarioMatches(greenVariant) @@ -3758,13 +3538,9 @@ const transitionHistoryBranches = [ { idBase: 1_100, groupBase: 1_600 }, ] as const -// The first rekey is correct on its own. Reusing its old correlation key for a -// new visible row then resurrects the detached descendant under the old row. const { control: rekeyRouteReuseControl, candidate: rekeyRouteResurrectionScenario, - candidateCheckpoint: rekeyRouteResurrectionCheckpoint, - classify: classifyRekeyRouteResurrection, } = createRekeyRouteReuseScenarios({ depth: 2, sourceBranch: 0, @@ -3775,13 +3551,9 @@ const { insertedPosition: 0, }) -// The reparent and delete are each correct. Replacing the moved row's child -// under the same correlation key then loses the existing grandchild snapshot. const { control: childReplacementControl, candidate: movedSubtreeChildReplacementScenario, - candidateCheckpoint: movedSubtreeChildReplacementCheckpoint, - classify: classifyMovedSubtreeChildReplacement, } = createMovedChildReplacementScenarios({ depth: 3, targetLevel: 1, @@ -3793,57 +3565,6 @@ const { }) describe(`includes recompute oracle`, () => { - fcTest( - `shared-route snapshot classification rejects extra corruption`, - () => { - const expected = [ - { id: 1, value: 10, children: [{ id: 3, value: 30 }] }, - { id: 2, value: 20, children: [{ id: 3, value: 30 }] }, - ] - const actual = [ - { id: 1, value: 11, children: [] }, - { id: 2, value: 20, children: [{ id: 3, value: 31 }] }, - ] - - expect( - classifyMissingSharedRouteSnapshot({ actual, expected }, 1, 2, 3), - ).toBe(false) - }, - ) - - fcTest( - `unexpected shared-child classification rejects extra corruption`, - () => { - const expected = [ - { id: 1, value: 10, children: [] }, - { id: 2, value: 20, children: [{ id: 3, value: 30 }] }, - ] - const actual = [ - { id: 1, value: 11, children: [{ id: 3, value: 30 }] }, - { id: 2, value: 20, children: [{ id: 3, value: 31 }] }, - ] - - expect(classifyUnexpectedSharedRoute({ actual, expected }, 1, 2, 3)).toBe( - false, - ) - }, - ) - - fcTest(`missing replacement classification rejects extra corruption`, () => { - const expected = [ - { id: 1, value: 10, children: [{ id: 3, value: 30 }] }, - { id: 2, value: 20, children: [] }, - ] - const actual = [ - { id: 1, value: 11, children: [] }, - { id: 2, value: 21, children: [] }, - ] - - expect(classifyMissingReplacementChild({ actual, expected }, 1, 3)).toBe( - false, - ) - }) - fcTest(`rejects subscriber lifecycle labels on reparent transitions`, () => { expect(() => createRouteLifecycleScenario({ @@ -3930,7 +3651,7 @@ describe(`includes recompute oracle`, () => { }) fcTest( - `discovered trace: a root entering a live route misses its ordered snapshot`, + `a root entering a live route receives its ordered snapshot`, async () => { const branches = transitionHistoryBranches const prefix = createConnectedBatchBranches(1, branches) @@ -3971,19 +3692,7 @@ describe(`includes recompute oracle`, () => { ], } - await expectAssertionFailure( - () => expectFullRowBatchScenarioMatches(scenario), - { - checkpoint: 3, - classify: (difference) => - classifyMissingSharedRouteSnapshot( - difference, - branches[1].idBase, - branches[0].idBase, - [3_000, branches[0].idBase + 1], - ), - }, - )() + await expectFullRowBatchScenarioMatches(scenario) }, ) @@ -4015,17 +3724,10 @@ describe(`includes recompute oracle`, () => { for (const parentLevel of [0, 1, 2] as const) { for (const enteringRow of [0, 1] as const) { - // Only roots currently miss an existing shared-route snapshot; nested - // subscribers receive the snapshot and remain green controls. - const expectsFailure = parentLevel === 0 fcTest( - expectsFailure - ? `discovered trace: root ${enteringRow} entering a live shared route receives its snapshot` - : `matches recomputation when level-${parentLevel} row ${enteringRow} enters a live shared route`, + `matches recomputation when level-${parentLevel} row ${enteringRow} enters a live shared route`, () => - (expectsFailure - ? expectClassifiedHistoryFailure - : expectClassifiedHistoryMatches)( + expectHistoryScenarioPairMatches( createMergeIntoSharedRouteScenarios(parentLevel, enteringRow), ), ) @@ -4042,19 +3744,15 @@ describe(`includes recompute oracle`, () => { fcTest( `matches recomputation after a level-${parentLevel} route resubscribes`, () => - expectClassifiedHistoryMatches( + expectHistoryScenarioPairMatches( createSnapshotOnResubscribeScenarios(parentLevel), ), ) fcTest( - parentLevel === 0 - ? `discovered trace: an initially shared root route retires, changes, and resubscribes` - : `matches recomputation when an initially shared level-${parentLevel} route retires, changes, and resubscribes`, + `matches recomputation when an initially shared level-${parentLevel} route retires, changes, and resubscribes`, () => - (parentLevel === 0 - ? expectClassifiedHistoryFailure - : expectClassifiedHistoryMatches)( + expectHistoryScenarioPairMatches( createInitiallySharedRouteResubscribeScenario(parentLevel), ), ) @@ -4245,16 +3943,10 @@ describe(`includes recompute oracle`, () => { }) for (const materialization of [`array`, `concat`] as const) { - fcTest( - `discovered trace: ${materialization} follows an intra-batch child hand-off`, - expectAssertionFailure( - async () => { - await expectFlatMaterializationScenarioMatches( - materialization, - intraBatchChildHandOffScenario, - ) - }, - { checkpoint: 3 }, + fcTest(`${materialization} follows an intra-batch child hand-off`, () => + expectFlatMaterializationScenarioMatches( + materialization, + intraBatchChildHandOffScenario, ), ) } @@ -4265,34 +3957,20 @@ describe(`includes recompute oracle`, () => { [4, 2], ] as const) { fcTest( - `discovered trace: later updates propagate through a reparented subtree at depth ${depth}, level ${targetLevel}`, - expectAssertionFailure( - () => - expectFullRowBatchScenarioMatches( - createReparentedSubtreeUpdateScenario(depth, targetLevel), - ), - { checkpoint: depth + 4 }, - ), + `later updates propagate through a reparented subtree at depth ${depth}, level ${targetLevel}`, + () => + expectFullRowBatchScenarioMatches( + createReparentedSubtreeUpdateScenario(depth, targetLevel), + ), ) } - fcTest( - `discovered trace: rekeying a row detaches two descendant levels`, - expectAssertionFailure( - () => expectFullRowBatchScenarioMatches(minimalRekeyScenario), - { checkpoint: 5 }, - ), + fcTest(`rekeying a row detaches two descendant levels`, () => + expectFullRowBatchScenarioMatches(minimalRekeyScenario), ) - fcTest( - `discovered trace: reusing a rekeyed row's old route does not resurrect its child`, - expectAssertionFailure( - () => expectFullRowBatchScenarioMatches(rekeyRouteResurrectionScenario), - { - checkpoint: rekeyRouteResurrectionCheckpoint, - classify: classifyRekeyRouteResurrection, - }, - ), + fcTest(`reusing a rekeyed row's old route does not resurrect its child`, () => + expectFullRowBatchScenarioMatches(rekeyRouteResurrectionScenario), ) fcTest( @@ -4300,16 +3978,8 @@ describe(`includes recompute oracle`, () => { () => expectFullRowBatchScenarioMatches(rekeyRouteReuseControl), ) - fcTest( - `discovered trace: replacing a moved subtree child retains its grandchild`, - expectAssertionFailure( - () => - expectFullRowBatchScenarioMatches(movedSubtreeChildReplacementScenario), - { - checkpoint: movedSubtreeChildReplacementCheckpoint, - classify: classifyMovedSubtreeChildReplacement, - }, - ), + fcTest(`replacing a moved subtree child retains its grandchild`, () => + expectFullRowBatchScenarioMatches(movedSubtreeChildReplacementScenario), ) fcTest( @@ -4323,8 +3993,8 @@ describe(`includes recompute oracle`, () => { numRuns: 4, seed: 1726 + depth * 10 + sourceBranch, })( - `discovered histories: reusing a retired route at depth ${depth}, branch ${sourceBranch}`, - expectClassifiedHistoryFailure, + `reuses a retired route at depth ${depth}, branch ${sourceBranch}`, + expectHistoryScenarioPairMatches, ) fcTest.prop( @@ -4340,17 +4010,17 @@ describe(`includes recompute oracle`, () => { seed: 1733 + depth * 10 + sourceBranch, }, )( - `discovered histories: intra-batch rekey then retired-route reuse at depth ${depth}, branch ${sourceBranch}`, - expectClassifiedHistoryFailure, + `handles intra-batch rekey then retired-route reuse at depth ${depth}, branch ${sourceBranch}`, + expectHistoryScenarioPairMatches, ) } } for (const parentLevel of [0, 1] as const) { fcTest( - `discovered trace: a departed level-${parentLevel} shared-route subscriber receives later child updates`, + `a departed level-${parentLevel} shared-route subscriber ignores later child updates`, () => - expectClassifiedHistoryFailure( + expectHistoryScenarioPairMatches( createSharedRouteLifetimeScenarios(parentLevel), ), ) @@ -4372,23 +4042,10 @@ describe(`includes recompute oracle`, () => { seed: 1727 + depth * 100 + targetLevel * 10 + sourceBranch, }, )( - `classifies forward/reverse delivery mirrors when replacing a moved child at depth ${depth}, level ${targetLevel}, source ${sourceBranch}`, + `matches forward/reverse delivery mirrors when replacing a moved child at depth ${depth}, level ${targetLevel}, source ${sourceBranch}`, async (scenarios) => { for (const deliveryOrder of branchDeliveryOrders) { - const deliveredSource = deliveredBranchIndex( - sourceBranch, - deliveryOrder, - ) - // At depth 4, level 2, replacement stays green only when the - // moved branch was delivered second. The mirrors share one - // generated fixture, so delivery order is the only difference. - const expectsFailure = - depth !== 4 || targetLevel !== 2 || deliveredSource !== 1 - await ( - expectsFailure - ? expectClassifiedHistoryFailure - : expectClassifiedHistoryMatches - )(scenarios[deliveryOrder]) + await expectHistoryScenarioPairMatches(scenarios[deliveryOrder]) } }, ) @@ -4409,7 +4066,7 @@ describe(`includes recompute oracle`, () => { numRuns: 4, seed: 1738 + publicIdIndex * 100 + routeIndex * 10 + updateIndex, })( - `classifies the split/atomic replacement matrix for ${publicId} public id, ${route} route, ${ancestorUpdate}`, + `matches the split/atomic replacement matrix for ${publicId} public id, ${route} route, ${ancestorUpdate}`, async (fixture) => { const cells = createRelationshipBatchShapeMatrix( fixture, @@ -4429,18 +4086,10 @@ describe(`includes recompute oracle`, () => { } for (const { - shape, - scenarios: { control, candidate, candidateCheckpoint, classify }, + scenarios: { control, candidate }, } of cells) { await expectFullRowBatchScenarioMatches(control) - if (isKnownRelationshipBatchFailure(shape)) { - await expectAssertionFailure( - () => expectFullRowBatchScenarioMatches(candidate), - { checkpoint: candidateCheckpoint, classify }, - )() - } else { - await expectFullRowBatchScenarioMatches(candidate) - } + await expectFullRowBatchScenarioMatches(candidate) } }, ) @@ -4476,10 +4125,6 @@ describe(`includes recompute oracle`, () => { ] for (const transition of transitions) { for (let targetLevel = 1; targetLevel <= depth; targetLevel++) { - // Incremental routing fails to fully detach a rekeyed row when two or - // more descendant include levels still hang below it. - const expectsFailure = - transition === `rekey` && targetLevel + 2 <= depth fcTest.prop( [ visibleRelationshipScenarioArbitrary( @@ -4493,9 +4138,7 @@ describe(`includes recompute oracle`, () => { seed: 1721 + depth + targetLevel, }, )( - expectsFailure - ? `discovered trace: a visible rekey at depth ${depth}, level ${targetLevel}` - : `matches recomputation for a visible ${transition} at depth ${depth}, level ${targetLevel}`, + `matches recomputation for a visible ${transition} at depth ${depth}, level ${targetLevel}`, async (scenarios) => { for (const scenario of [ scenarios.transitionOnly, @@ -4511,14 +4154,7 @@ describe(`includes recompute oracle`, () => { ) expect(result).not.toEqual(beforeTransition) - if (expectsFailure) { - await expectAssertionFailure( - () => expectFullRowBatchScenarioMatches(scenario), - { checkpoint: scenario.transitionStepIndex + 1 }, - )() - } else { - await expectFullRowBatchScenarioMatches(scenario) - } + await expectFullRowBatchScenarioMatches(scenario) } }, ) @@ -4573,24 +4209,18 @@ describe(`includes recompute oracle`, () => { } } - fcTest( - `discovered seed: nested scalar materialization follows a reference update`, - expectAssertionFailure( - async () => { - await runTrace({ - steps: [ - { type: `insert`, insert: `root-1` }, - { type: `insert`, insert: `middle-1` }, - { type: `insert`, insert: `shared-1` }, - { type: `insert`, insert: `leaf-1` }, - { type: `redirectMiddle`, id: 1, sharedId: 2 }, - ], - driver: createMaterializeTraceDriver(false), - projection: materializeProjection, - }) - }, - { checkpoint: 5 }, - ), + fcTest(`nested scalar materialization follows a reference update`, () => + runTrace({ + steps: [ + { type: `insert`, insert: `root-1` }, + { type: `insert`, insert: `middle-1` }, + { type: `insert`, insert: `shared-1` }, + { type: `insert`, insert: `leaf-1` }, + { type: `redirectMiddle`, id: 1, sharedId: 2 }, + ], + driver: createMaterializeTraceDriver(false), + projection: materializeProjection, + }), ) fcTest(`matches recomputation for full-row sync batches`, async () => { @@ -4601,12 +4231,8 @@ describe(`includes recompute oracle`, () => { }) }) - fcTest( - `discovered seed: a reinserted parent drops its old shared route`, - expectAssertionFailure( - () => expectFullRowBatchScenarioMatches(fullRowSharedRoutingSeed), - { checkpoint: 4 }, - ), + fcTest(`a reinserted parent drops its old shared route`, () => + expectFullRowBatchScenarioMatches(fullRowSharedRoutingSeed), ) fcTest(`supports repeated optimistic rollbacks in one history`, async () => { @@ -4878,154 +4504,156 @@ describe(`includes recompute oracle`, () => { expectScenarioMatches, ) - // These known failures must reject with the oracle's assertion mismatch. - // A fixed bug or an unrelated runtime error makes the matching test fail. fcTest.prop([fc.constant(sharedMaterializeSeed)], { numRuns: 1, seed: 1685, })( - `known seed: shared scalar materialization preserves the deepest row`, - expectAssertionFailure(expectMaterializeScenarioMatches, { checkpoint: 6 }), + `shared scalar materialization preserves the deepest row`, + expectMaterializeScenarioMatches, ) fcTest.prop([fc.constant(`correlation-key-update`)], { numRuns: 1, seed: 1658, - })( - `discovered seed: parent correlation-key update rematerializes children`, - expectAssertionFailure( - async () => { - const roots = createControlledCollection( - `correlation-seed-roots`, - ) - const children = createControlledCollection( - `correlation-seed-children`, + })(`parent correlation-key update rematerializes children`, async () => { + const roots = createControlledCollection(`correlation-seed-roots`) + const children = createControlledCollection( + `correlation-seed-children`, + ) + const live = createLiveQueryCollection((q) => + q.from({ root: roots.collection }).select(({ root }) => ({ + id: root.id, + group: root.group, + children: toArray( + q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, root.group)) + .select(({ child }) => ({ id: child.id })), + ), + })), + ) + + try { + await live.preload() + children.write(`insert`, { + id: 1, + parentGroup: 0, + group: 0, + value: 0, + position: 0, + }) + children.write(`insert`, { + id: 2, + parentGroup: 1, + group: 0, + value: 0, + position: 0, + }) + roots.write(`insert`, { id: 1, group: 1, value: 0, position: 0 }) + roots.write(`update`, { id: 1, group: 0, value: 0, position: 0 }) + + expect(stripVirtualProperties(live.toArray)).toEqual([ + { id: 1, group: 0, children: [{ id: 1 }] }, + ]) + } finally { + await live.cleanup() + await Promise.all([ + roots.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }) + + fcTest.prop([fc.constant(`#1454`)], { numRuns: 1, seed: 1454 })( + `alpha-renaming a duplicate sibling alias preserves results`, + async () => { + const roots = createControlledCollection(`alias-seed-roots`, [ + { id: 1, group: 1, value: 0, position: 0 }, + ]) + const issues = createControlledCollection(`alias-seed-issues`, [ + { + id: 10, + parentGroup: 1, + group: 10, + value: 10, + position: 0, + }, + { + id: 11, + parentGroup: 1, + group: 11, + value: 99, + position: 1, + }, + ]) + const tags = createControlledCollection(`alias-seed-tags`, [ + { + id: 20, + parentGroup: 1, + group: 20, + value: 20, + position: 0, + }, + { + id: 21, + parentGroup: 1, + group: 21, + value: 99, + position: 1, + }, + ]) + + try { + const uniqueAliases = await queryOnce((q) => + q.from({ root: roots.collection }).select(({ root }) => ({ + id: root.id, + issues: toArray( + q + .from({ issue: issues.collection }) + .where(({ issue }) => eq(issue.parentGroup, root.group)) + .where(({ issue }) => eq(issue.value, 10)) + .select(({ issue }) => ({ id: issue.id })), + ), + tags: toArray( + q + .from({ tag: tags.collection }) + .where(({ tag }) => eq(tag.parentGroup, root.group)) + .where(({ tag }) => eq(tag.value, 20)) + .select(({ tag }) => ({ id: tag.id })), + ), + })), ) - const live = createLiveQueryCollection((q) => + const duplicateAliases = await queryOnce((q) => q.from({ root: roots.collection }).select(({ root }) => ({ id: root.id, - group: root.group, - children: toArray( + issues: toArray( q - .from({ child: children.collection }) - .where(({ child }) => eq(child.parentGroup, root.group)) - .select(({ child }) => ({ id: child.id })), + .from({ item: issues.collection }) + .where(({ item }) => eq(item.parentGroup, root.group)) + .where(({ item }) => eq(item.value, 10)) + .select(({ item }) => ({ id: item.id })), + ), + tags: toArray( + q + .from({ item: tags.collection }) + .where(({ item }) => eq(item.parentGroup, root.group)) + .where(({ item }) => eq(item.value, 20)) + .select(({ item }) => ({ id: item.id })), ), })), ) - try { - await live.preload() - children.write(`insert`, { - id: 1, - parentGroup: 0, - group: 0, - value: 0, - position: 0, - }) - children.write(`insert`, { - id: 2, - parentGroup: 1, - group: 0, - value: 0, - position: 0, - }) - roots.write(`insert`, { id: 1, group: 1, value: 0, position: 0 }) - roots.write(`update`, { id: 1, group: 0, value: 0, position: 0 }) - - expect(stripVirtualProperties(live.toArray)).toEqual([ - { id: 1, group: 0, children: [{ id: 1 }] }, - ]) - } finally { - await live.cleanup() - await Promise.all([ - roots.collection.cleanup(), - children.collection.cleanup(), - ]) - } - }, - { message: /children/ }, - ), - ) - - fcTest.prop([fc.constant(`#1454`)], { numRuns: 1, seed: 1454 })( - `known seed: alpha-renaming a duplicate sibling alias preserves results`, - expectAssertionFailure( - async () => { - const roots = createControlledCollection(`alias-seed-roots`, [ - { id: 1, group: 1, value: 0, position: 0 }, - ]) - const issues = createControlledCollection( - `alias-seed-issues`, - [ - { - id: 10, - parentGroup: 1, - group: 10, - value: 10, - position: 0, - }, - ], + expect(stripVirtualProperties(duplicateAliases)).toEqual( + stripVirtualProperties(uniqueAliases), ) - const tags = createControlledCollection(`alias-seed-tags`, [ - { - id: 20, - parentGroup: 1, - group: 20, - value: 20, - position: 0, - }, + } finally { + await Promise.all([ + roots.collection.cleanup(), + issues.collection.cleanup(), + tags.collection.cleanup(), ]) - - try { - const uniqueAliases = await queryOnce((q) => - q.from({ root: roots.collection }).select(({ root }) => ({ - id: root.id, - issues: toArray( - q - .from({ issue: issues.collection }) - .where(({ issue }) => eq(issue.parentGroup, root.group)) - .select(({ issue }) => ({ id: issue.id })), - ), - tags: toArray( - q - .from({ tag: tags.collection }) - .where(({ tag }) => eq(tag.parentGroup, root.group)) - .select(({ tag }) => ({ id: tag.id })), - ), - })), - ) - const duplicateAliases = await queryOnce((q) => - q.from({ root: roots.collection }).select(({ root }) => ({ - id: root.id, - issues: toArray( - q - .from({ item: issues.collection }) - .where(({ item }) => eq(item.parentGroup, root.group)) - .select(({ item }) => ({ id: item.id })), - ), - tags: toArray( - q - .from({ item: tags.collection }) - .where(({ item }) => eq(item.parentGroup, root.group)) - .select(({ item }) => ({ id: item.id })), - ), - })), - ) - - expect(stripVirtualProperties(duplicateAliases)).toEqual( - stripVirtualProperties(uniqueAliases), - ) - } finally { - await Promise.all([ - roots.collection.cleanup(), - issues.collection.cleanup(), - tags.collection.cleanup(), - ]) - } - }, - { message: /deeply equal/ }, - ), + } + }, ) fcTest.prop([fc.constant(`#1444`)], { numRuns: 1, seed: 1444 })( diff --git a/packages/db/tests/query/includes-publication-oracle.test.ts b/packages/db/tests/query/includes-publication-oracle.test.ts index 27d7c81342..584b117bee 100644 --- a/packages/db/tests/query/includes-publication-oracle.test.ts +++ b/packages/db/tests/query/includes-publication-oracle.test.ts @@ -7,14 +7,12 @@ import { eq, materialize, } from '../../src/query/index.js' -import { expectAssertionFailure } from '../expected-failure.js' -import { TraceAssertionError, runTrace } from '../trace-runner.js' +import { runTrace } from '../trace-runner.js' import { flushPromises, mockSyncCollectionOptions, withExpectedRejection, } from '../utils.js' -import type { AssertionDifference } from '../expected-failure.js' import type { TraceDriver, TraceProjection } from '../trace-runner.js' type ParentRow = { @@ -257,71 +255,6 @@ const publicationProjection: TraceProjection< }, } -function sameValue(left: unknown, right: unknown): boolean { - try { - expect(left).toEqual(right) - return true - } catch { - return false - } -} - -// #1713 is specifically publication of Q1's compiled null placeholder to Q2: -// Q1 has already patched its materialization, while Q2 persists the placeholder. -function classifyDroppedQ2Materialization({ - actual, - expected, -}: AssertionDifference): boolean { - if ( - typeof actual !== `object` || - actual === null || - typeof expected !== `object` || - expected === null || - !(`q1` in actual) || - !(`q2` in actual) || - !(`q1` in expected) || - !(`q2` in expected) || - !Array.isArray(actual.q2) || - !Array.isArray(expected.q2) - ) { - return false - } - - const expectedWithDroppedChildren = expected.q2.map((row) => - typeof row === `object` && row !== null - ? { ...row, children: null, otherChildren: null } - : row, - ) - - return ( - sameValue(actual.q1, expected.q1) && - sameValue(actual.q2, expectedWithDroppedChildren) - ) -} - -function expectDroppedQ2FailureAt(error: unknown, checkpoint: number): void { - expect(error).toMatchObject({ - name: `TraceAssertionError`, - checkpoint, - cause: { name: `AssertionError` }, - }) - if ( - !(error instanceof TraceAssertionError) || - typeof error.cause !== `object` || - error.cause === null || - !(`actual` in error.cause) || - !(`expected` in error.cause) - ) { - throw error - } - expect( - classifyDroppedQ2Materialization({ - actual: error.cause.actual, - expected: error.cause.expected, - }), - ).toBe(true) -} - async function settleRollback( rejectSync: (error: Error) => void, persisted: Promise, @@ -396,13 +329,7 @@ function createPublicationDriver( context.sources.parents.write(`update`, nextParent) context.model.parents.set(nextParent.id, { ...nextParent }) - let publicationFailure: unknown - try { - checkpoint() - } catch (error) { - publicationFailure = error - } - expectDroppedQ2FailureAt(publicationFailure, 1) + checkpoint() const nextChild = { ...child, value: action.childValue } context.sources.children.write(`update`, nextChild) @@ -507,22 +434,6 @@ async function expectPublicationMatches( }) } -async function expectDroppedQ2Materialization( - action: PublicationAction, - checkpointOptimistic = false, - q1Shape: Q1Shape = `direct`, - q2Shape: Q2Shape = `passThrough`, -): Promise { - await expectAssertionFailure( - () => - expectPublicationMatches(action, checkpointOptimistic, q1Shape, q2Shape), - { - checkpoint: 1, - classify: classifyDroppedQ2Materialization, - }, - )() -} - const q2Shapes = [`passThrough`, `where`, `orderBy`, `select`] as const const q1Shapes = [`direct`, `joined`] as const @@ -539,9 +450,9 @@ describe(`layered-query publication oracle`, () => { for (const q1Shape of q1Shapes) { for (const q2Shape of q2Shapes) { fcTest.prop([changedValueArbitrary], { numRuns: 12 })( - `classifies #1713 through a ${q1Shape} Q1 and ${q2Shape} Q2`, + `publishes #1713 updates through a ${q1Shape} Q1 and ${q2Shape} Q2`, async (value) => { - await expectDroppedQ2Materialization( + await expectPublicationMatches( { type: `parentScalar`, value }, false, q1Shape, @@ -565,9 +476,9 @@ describe(`layered-query publication oracle`, () => { ) fcTest.prop([changedValueArbitrary], { numRuns: 8 })( - `classifies optimistic publication before confirmation through a ${q1Shape} Q1 and ${q2Shape} Q2`, + `publishes optimistic state before confirmation through a ${q1Shape} Q1 and ${q2Shape} Q2`, async (value) => { - await expectDroppedQ2Materialization( + await expectPublicationMatches( { type: `optimisticConfirm`, value }, true, q1Shape, @@ -577,9 +488,9 @@ describe(`layered-query publication oracle`, () => { ) fcTest.prop([changedValueArbitrary], { numRuns: 8 })( - `classifies publication after optimistic confirmation through a ${q1Shape} Q1 and ${q2Shape} Q2`, + `publishes state after optimistic confirmation through a ${q1Shape} Q1 and ${q2Shape} Q2`, async (value) => { - await expectDroppedQ2Materialization( + await expectPublicationMatches( { type: `optimisticConfirm`, value }, false, q1Shape, @@ -612,20 +523,14 @@ describe(`layered-query publication oracle`, () => { ])( `compares atomic parent replacements at both query layers`, async (row) => { - // Changing the route rebuilds the materialization before publication and - // is green. A same-route replacement republishes the null placeholder. - const assertion = - row.group === 10 - ? expectDroppedQ2Materialization - : expectPublicationMatches - await assertion({ type: `atomicReplace`, ...row }) + await expectPublicationMatches({ type: `atomicReplace`, ...row }) }, ) fcTest.prop([changedValueArbitrary])( - `classifies stale publication after optimistic rollback`, + `publishes restored state after optimistic rollback`, async (value) => { - await expectDroppedQ2Materialization({ + await expectPublicationMatches({ type: `optimisticRollback`, value, }) diff --git a/packages/db/tests/query/includes-query-shape-oracle.test.ts b/packages/db/tests/query/includes-query-shape-oracle.test.ts index 479bd74b22..4a6f4b2bec 100644 --- a/packages/db/tests/query/includes-query-shape-oracle.test.ts +++ b/packages/db/tests/query/includes-query-shape-oracle.test.ts @@ -7,7 +7,6 @@ import { eq, materialize, } from '../../src/query/index.js' -import { expectAssertionFailure } from '../expected-failure.js' import { runTrace } from '../trace-runner.js' import { mockSyncCollectionOptions } from '../utils.js' import type { TraceDriver, TraceProjection } from '../trace-runner.js' @@ -389,17 +388,13 @@ describe(`includes query-shape recompute oracle`, () => { numRuns: 12, seed: 1703, })( - `discovered trace: deleting one joined contributor preserves remaining multiplicity (#1703)`, + `deleting one joined contributor preserves remaining multiplicity (#1703)`, async (childCount) => { - await expectAssertionFailure( - () => - runTrace({ - steps: [1], - driver: createMultiplicityDriver(childCount), - projection: multiplicityProjection, - }), - { checkpoint: 1 }, - )() + await runTrace({ + steps: [1], + driver: createMultiplicityDriver(childCount), + projection: multiplicityProjection, + }) }, ) @@ -422,21 +417,13 @@ describe(`includes query-shape recompute oracle`, () => { ], { numRuns: 12, seed: 1704 }, )( - `discovered trace: materialization follows correlation through a joined alias (#1704)`, + `materialization follows correlation through a joined alias (#1704)`, async ({ correlationId, productionId }) => { - await expectAssertionFailure( - () => - runTrace({ - steps: [], - driver: createCorrelationDriver( - `joined`, - correlationId, - productionId, - ), - projection: correlationProjection, - }), - { checkpoint: 0 }, - )() + await runTrace({ + steps: [], + driver: createCorrelationDriver(`joined`, correlationId, productionId), + projection: correlationProjection, + }) }, ) @@ -454,17 +441,13 @@ describe(`includes query-shape recompute oracle`, () => { numRuns: 12, seed: 1706, })( - `discovered trace: findOne maps a null correlation key to undefined (#1706)`, + `findOne maps a null correlation key to undefined (#1706)`, async (postId) => { - await expectAssertionFailure( - () => - runTrace({ - steps: [], - driver: createNullableDriver([], [{ id: postId, authorId: null }]), - projection: nullableProjection, - }), - { checkpoint: 0 }, - )() + await runTrace({ + steps: [], + driver: createNullableDriver([], [{ id: postId, authorId: null }]), + projection: nullableProjection, + }) }, ) diff --git a/packages/db/tests/query/includes-temporal-oracle.test.ts b/packages/db/tests/query/includes-temporal-oracle.test.ts index b3cd0a3bfb..e954a9d4ba 100644 --- a/packages/db/tests/query/includes-temporal-oracle.test.ts +++ b/packages/db/tests/query/includes-temporal-oracle.test.ts @@ -7,8 +7,8 @@ import { eq, toArray, } from '../../src/query/index.js' -import { expectAssertionFailure } from '../expected-failure.js' import { runTrace } from '../trace-runner.js' +import { flushPromises } from '../utils.js' import type { Collection } from '../../src/collection/index.js' import type { Deferred } from '../../src/deferred.js' import type { LoadSubsetOptions } from '../../src/types.js' @@ -276,6 +276,7 @@ type DemandCancellationContext = { function createRemovablePost(): { collection: Collection remove: () => void + add: () => void } { const post: Post = { id: 1, @@ -285,6 +286,9 @@ function createRemovablePost(): { let remove: () => void = () => { throw new Error(`Post collection has not started`) } + let add: () => void = () => { + throw new Error(`Post collection has not started`) + } const collection = createCollection({ id: nextCollectionId(`temporal-removable-post`), getKey: (row) => row.id, @@ -299,10 +303,15 @@ function createRemovablePost(): { write({ type: `delete`, value: post }) commit() } + add = () => { + begin() + write({ type: `insert`, value: post }) + commit() + } }, }, }) - return { collection, remove: () => remove() } + return { collection, remove: () => remove(), add: () => add() } } function createDemandCancellationDriver(): TraceDriver< @@ -400,6 +409,88 @@ async function expectObsoleteDemandDoesNotBlockReadiness(): Promise { }) } +async function expectObsoleteDemandCannotPublishAfterReactivation(): Promise { + const { collection: posts, remove, add } = createRemovablePost() + const requests: Array<{ + deferred: Deferred + outcome: Promise + signal: AbortSignal | undefined + }> = [] + const comments = createCollection({ + id: nextCollectionId(`temporal-generation-comments`), + getKey: (comment) => comment.id, + syncMode: `on-demand`, + sync: { + sync: ({ begin, write, commit, markReady }) => ({ + loadSubset: (options) => { + const requestIndex = requests.length + const deferred = createDeferred() + const signal = options.signal + const outcome = deferred.promise.then(() => { + if (signal?.aborted) return + begin() + write({ + type: `insert`, + value: + requestIndex === 0 + ? { id: 100, postId: 1, body: `obsolete` } + : { id: 200, postId: 1, body: `current` }, + }) + commit() + markReady() + }) + requests.push({ deferred, outcome, signal }) + return outcome + }, + }), + }, + }) + const live = createLiveQueryCollection((q) => + q.from({ post: posts }).select(({ post }) => ({ + id: post.id, + comments: toArray( + q + .from({ comment: comments }) + .where(({ comment }) => eq(comment.postId, post.id)) + .select(({ comment }) => ({ + id: comment.id, + body: comment.body, + })), + ), + })), + ) + + const preload = live.preload() + try { + await flushPromises() + expect(requests).toHaveLength(1) + + remove() + await preload + expect(live.size).toBe(0) + + add() + await flushPromises() + expect(requests).toHaveLength(2) + + requests[1]!.deferred.resolve() + await requests[1]!.outcome + await flushPromises() + expect(live.get(1)?.comments).toEqual([{ id: 200, body: `current` }]) + + requests[0]!.deferred.resolve() + await requests[0]!.outcome + await flushPromises() + expect(live.get(1)?.comments).toEqual([{ id: 200, body: `current` }]) + expect(requests[0]!.signal?.aborted).toBe(true) + } finally { + for (const request of requests) request.deferred.resolve() + await Promise.allSettled(requests.map(({ outcome }) => outcome)) + await live.cleanup() + await Promise.all([posts.cleanup(), comments.cleanup()]) + } +} + type FastPathEvent = { phase: `fast` | `late` keys: Array @@ -632,12 +723,8 @@ async function expectProgressiveTraceMatches( } describe(`includes temporal oracle`, () => { - it( - `discovered trace: an empty outer does not wait for an undemanded child`, - expectAssertionFailure(() => expectReadinessMatches([]), { - checkpoint: 0, - }), - ) + it(`an empty outer does not wait for an undemanded child`, () => + expectReadinessMatches([])) it(`loads a demanded child before becoming ready`, async () => { await expectReadinessMatches([ @@ -647,20 +734,19 @@ describe(`includes temporal oracle`, () => { }) it( - `discovered trace: obsolete child demand does not block readiness`, - expectAssertionFailure(expectObsoleteDemandDoesNotBlockReadiness, { - checkpoint: 1, - }), + `obsolete child demand does not block readiness`, + expectObsoleteDemandDoesNotBlockReadiness, + ) + + it( + `obsolete child demand cannot publish after the route is reactivated`, + expectObsoleteDemandCannotPublishAfterReactivation, ) it(`loads a direct progressive subset inside the fast-path window`, async () => { await expectProgressiveTraceMatches(`direct`) }) - it( - `discovered trace: a nested progressive subset loads inside the fast-path window`, - expectAssertionFailure(() => expectProgressiveTraceMatches(`nested`), { - checkpoint: 0, - }), - ) + it(`a nested progressive subset loads inside the fast-path window`, () => + expectProgressiveTraceMatches(`nested`)) }) diff --git a/packages/db/tests/query/includes-work-counter-oracle.test.ts b/packages/db/tests/query/includes-work-counter-oracle.test.ts index 873ea1de00..46649e3830 100644 --- a/packages/db/tests/query/includes-work-counter-oracle.test.ts +++ b/packages/db/tests/query/includes-work-counter-oracle.test.ts @@ -8,8 +8,6 @@ import { eq, materialize, } from '../../src/query/index.js' -import { expectAssertionFailure } from '../expected-failure.js' -import { TraceAssertionError } from '../trace-runner.js' import type { Collection } from '../../src/collection/index.js' let nextCollectionId = 0 @@ -346,44 +344,6 @@ function expectedResult({ ] } -function assertEqualSourceWork( - actual: SourceWork, - expected: SourceWork, -): Promise { - try { - expect(actual).toEqual(expected) - return Promise.resolve() - } catch (error) { - return Promise.reject(new TraceAssertionError(1, error)) - } -} - -function isExactWorkCount(value: unknown, expected: WorkCount): boolean { - return ( - typeof value === `object` && - value !== null && - `delivered` in value && - value.delivered === expected.delivered && - `examined` in value && - value.examined === expected.examined - ) -} - -function isExactSourceWork(value: unknown, expected: SourceWork): boolean { - return ( - typeof value === `object` && - value !== null && - `terms` in value && - isExactWorkCount(value.terms, expected.terms) && - `meanings` in value && - isExactWorkCount(value.meanings, expected.meanings) && - `groups` in value && - isExactWorkCount(value.groups, expected.groups) && - `links` in value && - isExactWorkCount(value.links, expected.links) - ) -} - const joinedBaselineWork: SourceWork = { terms: { delivered: 3, examined: 3 }, meanings: { delivered: 1, examined: 1 }, @@ -401,7 +361,7 @@ const joinFreeBaselineWork: SourceWork = { let joinedBaselineObservation: WorkObservation let joinFreeBaselineObservation: WorkObservation -async function expectKnownCorrelatedJoinDefect( +async function expectCorrelatedJoinWorkBound( fillerCount: number, ): Promise { const baseline = joinedBaselineObservation @@ -418,25 +378,7 @@ async function expectKnownCorrelatedJoinDefect( expect(scaled.result).toEqual(baseline.result) expect(baseline.sourceWork).toEqual(joinedBaselineWork) - const knownLinkWork = { - delivered: baseline.sourceWork.links.delivered + fillerCount, - // Once irrelevant rows exist, the defective route scans the whole - // collection and then reads the two selected rows through the index. - examined: baseline.sourceWork.links.examined + fillerCount + 2, - } - const knownScaledWork: SourceWork = { - // The full left scan also activates one extra indexed target route. - terms: { delivered: 4, examined: 4 }, - meanings: baseline.sourceWork.meanings, - groups: baseline.sourceWork.groups, - links: knownLinkWork, - } - await expectAssertionFailure(assertEqualSourceWork, { - checkpoint: 1, - classify: ({ actual, expected }) => - isExactSourceWork(actual, knownScaledWork) && - isExactSourceWork(expected, baseline.sourceWork), - })(scaled.sourceWork, baseline.sourceWork) + expect(scaled.sourceWork).toEqual(baseline.sourceWork) } describe(`includes deterministic work-counter oracle`, () => { @@ -450,16 +392,16 @@ describe(`includes deterministic work-counter oracle`, () => { }) it.each([1, 2, 3])( - `pins the #1709 defect formula at the small filler boundary (%i)`, - expectKnownCorrelatedJoinDefect, + `pins the #1709 work bound at the small filler boundary (%i)`, + expectCorrelatedJoinWorkBound, ) fcTest.prop([fc.integer({ min: 1, max: 24 })], { numRuns: 6, seed: 1709, })( - `known work defect: a join defeats correlated source pushdown (#1709)`, - expectKnownCorrelatedJoinDefect, + `a join preserves correlated source pushdown (#1709)`, + expectCorrelatedJoinWorkBound, ) fcTest.prop([fc.integer({ min: 1, max: 24 })], { diff --git a/packages/db/tests/query/includes.test.ts b/packages/db/tests/query/includes.test.ts index 2eba6f122a..9a28f2472f 100644 --- a/packages/db/tests/query/includes.test.ts +++ b/packages/db/tests/query/includes.test.ts @@ -326,6 +326,38 @@ describe(`includes subqueries`, () => { }, ]) }) + + it(`gives an active null correlation an empty child Collection`, async () => { + const parents = createCollection( + mockSyncCollectionOptions<{ id: number; groupId: number | null }>({ + id: `includes-null-correlation-parents`, + getKey: (parent) => parent.id, + initialData: [{ id: 1, groupId: null }], + }), + ) + const children = createCollection( + mockSyncCollectionOptions<{ id: number; groupId: number }>({ + id: `includes-null-correlation-children`, + getKey: (child) => child.id, + initialData: [{ id: 10, groupId: 1 }], + }), + ) + const collection = createLiveQueryCollection((q) => + q.from({ parent: parents }).select(({ parent }) => ({ + id: parent.id, + items: q + .from({ child: children }) + .where(({ child }) => eq(child.groupId, parent.groupId)) + .select(({ child }) => ({ id: child.id })), + })), + ) + + await collection.preload() + + const items = collection.get(1)!.items + expect(items).toBeDefined() + expect(plainRows(items)).toEqual([]) + }) }) describe(`reactivity`, () => { @@ -394,7 +426,8 @@ describe(`includes subqueries`, () => { const collection = buildIncludesQuery() await collection.preload() - expect(childItems((collection.get(1) as any).issues)).toHaveLength(2) + const originalIssues = (collection.get(1) as any).issues + expect(childItems(originalIssues)).toHaveLength(2) // Remove project Alpha projects.utils.begin() @@ -405,6 +438,7 @@ describe(`includes subqueries`, () => { projects.utils.commit() expect(collection.get(1)).toBeUndefined() + expect(childItems(originalIssues)).toEqual([]) // Re-add project Alpha — should get a fresh child collection projects.utils.begin() @@ -416,6 +450,8 @@ describe(`includes subqueries`, () => { const alpha = collection.get(1) as any expect(alpha).toMatchObject({ id: 1, name: `Alpha Reborn` }) + expect(alpha.issues).not.toBe(originalIssues) + expect(childItems(originalIssues)).toEqual([]) expect(childItems(alpha.issues)).toEqual([ { id: 10, title: `Bug in Alpha` }, { id: 11, title: `Feature for Alpha` }, @@ -822,6 +858,37 @@ describe(`includes subqueries`, () => { { id: 11, title: `Feature for Alpha` }, ]) }) + + it(`order-only child changes update Collection layout`, async () => { + const collection = createLiveQueryCollection((q) => + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + issues: q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .orderBy(({ i }) => i.title, `asc`) + .select(({ i }) => ({ id: i.id })), + })), + ) + + await collection.preload() + expect(plainRows((collection.get(1) as any).issues)).toEqual([ + { id: 10 }, + { id: 11 }, + ]) + + issues.utils.begin() + issues.utils.write({ + type: `update`, + value: { id: 11, projectId: 1, title: `A Feature for Alpha` }, + }) + issues.utils.commit() + + expect(plainRows((collection.get(1) as any).issues)).toEqual([ + { id: 11 }, + { id: 10 }, + ]) + }) }) describe(`ordered child queries with limit`, () => { @@ -1060,8 +1127,9 @@ describe(`includes subqueries`, () => { await collection.preload() // Both Frontend and Backend share departmentId 100 - expect(childItems((collection.get(1) as any).members)).toHaveLength(2) - expect(childItems((collection.get(2) as any).members)).toHaveLength(2) + const sharedMembers = (collection.get(1) as any).members + expect((collection.get(2) as any).members).toBe(sharedMembers) + expect(childItems(sharedMembers)).toHaveLength(2) // Delete the Frontend team teams.utils.begin() @@ -1074,10 +1142,211 @@ describe(`includes subqueries`, () => { expect(collection.get(1)).toBeUndefined() // Backend should still have its child collection with all members + expect((collection.get(2) as any).members).toBe(sharedMembers) expect(childItems((collection.get(2) as any).members)).toEqual([ { id: 10, name: `Alice` }, { id: 11, name: `Bob` }, ]) + + // Rejoining the still-active route reuses its shared facade. + teams.utils.begin() + teams.utils.write({ type: `insert`, value: sampleTeams[0]! }) + teams.utils.commit() + expect((collection.get(1) as any).members).toBe(sharedMembers) + }) + + it(`publishes a parent route move and its child facades coherently`, async () => { + type Parent = { id: number; groupId: number } + type Child = { id: number; groupId: number } + + const parents = createCollection( + mockSyncCollectionOptions({ + id: `coherent-route-parents`, + getKey: (parent) => parent.id, + initialData: [{ id: 1, groupId: 1 }], + }), + ) + const children = createCollection( + mockSyncCollectionOptions({ + id: `coherent-route-children`, + getKey: (child) => child.id, + initialData: [ + { id: 10, groupId: 1 }, + { id: 20, groupId: 2 }, + ], + }), + ) + const collection = createLiveQueryCollection((q) => + q.from({ parent: parents }).select(({ parent }) => ({ + id: parent.id, + groupId: parent.groupId, + children: q + .from({ child: children }) + .where(({ child }) => eq(child.groupId, parent.groupId)) + .select(({ child }) => ({ id: child.id })), + })), + ) + + await collection.preload() + + const oldFacade = collection.get(1)!.children + const observations: Array<{ + groupId: number + sameFacade: boolean + oldRows: Array<{ id: number }> + currentRows: Array<{ id: number }> + }> = [] + const facadeSubscription = oldFacade.subscribeChanges( + () => { + const current = collection.get(1)! + observations.push({ + groupId: current.groupId, + sameFacade: current.children === oldFacade, + oldRows: plainRows(oldFacade), + currentRows: plainRows(current.children), + }) + }, + { includeInitialState: false }, + ) + const rootObservations: Array<{ + groupId: number + oldRows: Array<{ id: number }> + currentRows: Array<{ id: number }> + }> = [] + const rootSubscription = collection.subscribeChanges( + () => { + const current = collection.get(1)! + rootObservations.push({ + groupId: current.groupId, + oldRows: plainRows(oldFacade), + currentRows: plainRows(current.children), + }) + }, + { includeInitialState: false }, + ) + + try { + parents.utils.begin() + parents.utils.write({ + type: `update`, + value: { id: 1, groupId: 2 }, + previousValue: { id: 1, groupId: 1 }, + }) + parents.utils.commit() + + expect(observations).toEqual([ + { + groupId: 2, + sameFacade: false, + oldRows: [], + currentRows: [{ id: 20 }], + }, + ]) + expect(rootObservations).toEqual([ + { + groupId: 2, + oldRows: [], + currentRows: [{ id: 20 }], + }, + ]) + } finally { + facadeSubscription.unsubscribe() + rootSubscription.unsubscribe() + } + }) + + it(`retires a shared facade after all parent routes publish`, async () => { + type Parent = { id: number; groupId: number } + type Child = { id: number; groupId: number } + + const initialParents: Array = [ + { id: 1, groupId: 1 }, + { id: 2, groupId: 1 }, + ] + const parents = createCollection( + mockSyncCollectionOptions({ + id: `coherent-shared-route-parents`, + getKey: (parent) => parent.id, + initialData: initialParents, + }), + ) + const children = createCollection( + mockSyncCollectionOptions({ + id: `coherent-shared-route-children`, + getKey: (child) => child.id, + initialData: [ + { id: 10, groupId: 1 }, + { id: 20, groupId: 2 }, + { id: 30, groupId: 3 }, + ], + }), + ) + const collection = createLiveQueryCollection((q) => + q.from({ parent: parents }).select(({ parent }) => ({ + id: parent.id, + groupId: parent.groupId, + children: q + .from({ child: children }) + .where(({ child }) => eq(child.groupId, parent.groupId)) + .select(({ child }) => ({ id: child.id })), + })), + ) + + await collection.preload() + + const sharedFacade = collection.get(1)!.children + expect(collection.get(2)!.children).toBe(sharedFacade) + const observations: Array<{ + oldRows: Array<{ id: number }> + parents: Array<{ + id: number + groupId: number + rows: Array<{ id: number }> + }> + }> = [] + const subscription = sharedFacade.subscribeChanges( + () => { + observations.push({ + oldRows: plainRows(sharedFacade), + parents: [1, 2].map((id) => { + const current = collection.get(id)! + return { + id, + groupId: current.groupId, + rows: plainRows(current.children), + } + }), + }) + }, + { includeInitialState: false }, + ) + + try { + parents.utils.begin() + parents.utils.write({ + type: `update`, + value: { id: 1, groupId: 2 }, + previousValue: initialParents[0]!, + }) + parents.utils.write({ + type: `update`, + value: { id: 2, groupId: 3 }, + previousValue: initialParents[1]!, + }) + parents.utils.commit() + + expect(observations).toEqual([ + { + oldRows: [], + parents: [ + { id: 1, groupId: 2, rows: [{ id: 20 }] }, + { id: 2, groupId: 3, rows: [{ id: 30 }] }, + ], + }, + ]) + } finally { + subscription.unsubscribe() + } }) it(`correlation field does not need to be in the parent select`, async () => { @@ -2288,6 +2557,10 @@ describe(`includes subqueries`, () => { ], }, ]) + + const alphaIssues = collection.get(1)!.issues + expect([...alphaIssues.keys()]).toEqual([10]) + expect(alphaIssues.get(10)?.$key).toBe(10) }) it(`reacts to parent field change`, async () => { @@ -2593,6 +2866,12 @@ describe(`includes subqueries`, () => { items: [], }, ]) + + const aliceItems = collection.get(1)!.items + const bobItems = collection.get(2)!.items + expect([...aliceItems.keys()]).toEqual([10]) + expect(aliceItems.get(10)?.$key).toBe(10) + expect([...bobItems.keys()]).toEqual([]) }) it(`shared correlation key with parent filter + orderBy + limit`, async () => { diff --git a/packages/db/tests/query/live-query-collection.test.ts b/packages/db/tests/query/live-query-collection.test.ts index a2613c0104..85478e95cc 100644 --- a/packages/db/tests/query/live-query-collection.test.ts +++ b/packages/db/tests/query/live-query-collection.test.ts @@ -2407,14 +2407,12 @@ describe(`createLiveQueryCollection`, () => { commentsOptions.utils.commit() await new Promise((resolve) => setTimeout(resolve, 10)) } catch (error: any) { - expect(error.message).toContain(`already exists in the collection`) - expect(error.message).toContain(`custom getKey`) - expect(error.message).toContain(`joined queries`) - expect(error.message).toContain(`composite key`) + expect(error.message).toContain(`public key "user1"`) + expect(error.message).toContain(`not congruent`) return } - throw new Error(`Expected DuplicateKeySyncError to be thrown`) + throw new Error(`Expected duplicate public-key invariant to be thrown`) }) }) diff --git a/packages/db/tests/query/subset-dedupe.test.ts b/packages/db/tests/query/subset-dedupe.test.ts index 02af82eb77..081d892c4d 100644 --- a/packages/db/tests/query/subset-dedupe.test.ts +++ b/packages/db/tests/query/subset-dedupe.test.ts @@ -1099,6 +1099,35 @@ describe(`createDeduplicatedLoadSubset`, () => { expect(onDeduplicate).toHaveBeenCalledTimes(1) expect(onDeduplicate).toHaveBeenCalledWith(subsetOptions) }) + + it(`does not share in-flight work across independent cancellation owners`, async () => { + const pending: Array<() => void> = [] + const loadSubset = vi.fn( + () => + new Promise((resolve) => { + pending.push(resolve) + }), + ) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const firstController = new AbortController() + const secondController = new AbortController() + + const first = deduplicated.loadSubset({ + where: gt(ref(`age`), val(10)), + signal: firstController.signal, + }) + const second = deduplicated.loadSubset({ + where: gt(ref(`age`), val(20)), + signal: secondController.signal, + }) + + expect(loadSubset).toHaveBeenCalledTimes(2) + expect(second).not.toBe(first) + + firstController.abort() + for (const resolve of pending) resolve() + await Promise.all([first, second]) + }) }) describe(`limited queries with different where clauses`, () => { From 9ca5d01f433fe49391620aac75ab28894556ddbe Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 16 Aug 2026 14:36:45 +0100 Subject: [PATCH 02/21] chore: add includes materialization changeset --- .changeset/fix-includes-materialization.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/fix-includes-materialization.md diff --git a/.changeset/fix-includes-materialization.md b/.changeset/fix-includes-materialization.md new file mode 100644 index 0000000000..ce759d556d --- /dev/null +++ b/.changeset/fix-includes-materialization.md @@ -0,0 +1,5 @@ +--- +'@tanstack/db': patch +--- + +Rebuild correlated include materialization as one D2 graph, fixing stale or missing nested results across route changes, batching, lazy loading, optimistic updates, and layered queries. Add abortable subset demand and coherent publication for Collection-valued includes. From 058233decb31c13089eaa08af49167a711a40868 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 16 Aug 2026 22:28:58 +0100 Subject: [PATCH 03/21] fix(db): await eager include sources --- packages/db/src/query/live/collection-config-builder.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index 3df211e096..4cb40ece87 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -971,7 +971,11 @@ export class CollectionConfigBuilder< private allRequiredSourcesReady() { return this.collectionSources.every( (source) => - this.lazySources.has(source.sourceId) || source.collection.isReady(), + // Only on-demand sources settle through route demand. Eager + // loadSubset calls return immediately, so they must reach ready. + (this.lazySources.has(source.sourceId) && + source.collection.config.syncMode === `on-demand`) || + source.collection.isReady(), ) } From 4d27d0a55c8f8a4ebce31ac72b10df0892970f85 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 17 Aug 2026 14:18:30 +0100 Subject: [PATCH 04/21] fix(db): tighten materialization boundaries --- packages/db/src/collection/changes.ts | 26 +- packages/db/src/collection/index.ts | 3 +- packages/db/src/query/compiler/index.ts | 21 +- packages/db/src/query/live/ARCHITECTURE.md | 31 +- .../src/query/live/bucket-facade-adapter.ts | 143 +++++- .../query/live/collection-config-builder.ts | 38 +- .../src/query/live/collection-subscriber.ts | 11 +- .../src/query/live/materialized-pipeline.ts | 70 ++- .../query/live/subset-demand-controller.ts | 78 ++- packages/db/src/types.ts | 5 +- .../query/compiler/subquery-caching.test.ts | 34 ++ .../tests/query/includes-lazy-loading.test.ts | 101 +++- .../query/includes-temporal-oracle.test.ts | 304 +++++++++++- packages/db/tests/query/includes.test.ts | 467 ++++++++++++++++++ .../db/tests/query/validate-aliases.test.ts | 17 + 15 files changed, 1248 insertions(+), 101 deletions(-) diff --git a/packages/db/src/collection/changes.ts b/packages/db/src/collection/changes.ts index 8b2aafb6d2..f7802c0e98 100644 --- a/packages/db/src/collection/changes.ts +++ b/packages/db/src/collection/changes.ts @@ -13,6 +13,11 @@ import type { CollectionImpl } from './index.js' import type { CollectionStateManager } from './state.js' import type { WithVirtualProps } from '../virtual-props.js' +export type PublicationDeferral = { + publish: () => void + discard: () => void +} + export class CollectionChangesManager< TOutput extends object = Record, TKey extends string | number = string | number, @@ -30,6 +35,7 @@ export class CollectionChangesManager< public batchedEvents: Array> = [] public shouldBatchEvents = false private publicationDeferralDepth = 0 + private discardDeferredPublications = false private deferredPublications: Array<{ changes: Array> layoutChanged: boolean @@ -138,25 +144,35 @@ export class CollectionChangesManager< * installs all of its visible state. State and indexes still commit at their * normal transaction boundaries. */ - public deferPublication(): () => void { + public deferPublication(): PublicationDeferral { this.publicationDeferralDepth++ - let resumed = false + let closed = false - return () => { - if (resumed) return - resumed = true + const close = (discard: boolean) => { + if (closed) return + closed = true if (this.publicationDeferralDepth === 0) return + this.discardDeferredPublications ||= discard this.publicationDeferralDepth-- if (this.publicationDeferralDepth > 0) return const publications = this.deferredPublications this.deferredPublications = [] + if (this.discardDeferredPublications) { + this.discardDeferredPublications = false + return + } this.publishEvents( publications.flatMap(({ changes }) => changes), publications.some(({ layoutChanged }) => layoutChanged), ) } + + return { + publish: () => close(false), + discard: () => close(true), + } } private publishEvents( diff --git a/packages/db/src/collection/index.ts b/packages/db/src/collection/index.ts index 9934b89334..51500f9537 100644 --- a/packages/db/src/collection/index.ts +++ b/packages/db/src/collection/index.ts @@ -13,6 +13,7 @@ import { CollectionSyncManager } from './sync' import { CollectionIndexesManager } from './indexes' import { CollectionMutationsManager } from './mutations' import { CollectionEventsManager } from './events.js' +import type { PublicationDeferral } from './changes' import type { CollectionSubscription } from './subscription' import type { AllCollectionEvents, @@ -448,7 +449,7 @@ export class CollectionImpl< } /** Defer subscriber events until a coherent multi-Collection commit ends. */ - public _deferPublication(): () => void { + public _deferPublication(): PublicationDeferral { return this._changes.deferPublication() } diff --git a/packages/db/src/query/compiler/index.ts b/packages/db/src/query/compiler/index.ts index 6371a8fcfe..7331786142 100644 --- a/packages/db/src/query/compiler/index.ts +++ b/packages/db/src/query/compiler/index.ts @@ -179,7 +179,8 @@ export function compileQuery( childCorrelationField?: PropRef, ): CompilationResult { // Check if the original raw query has already been compiled - const cachedResult = cache.get(rawQuery) + const cachedResult = + parentKeyStream === undefined ? cache.get(rawQuery) : undefined if (cachedResult) { return cachedResult } @@ -292,9 +293,7 @@ export function compileQuery( tagged.__parentContext = parentSide } const effectiveKey = - parentSide != null - ? `${String(childKey)}::${JSON.stringify(parentSide)}` - : childKey + parentSide != null ? serializeValue([childKey, parentSide]) : childKey return [effectiveKey, tagged] }), ) @@ -359,9 +358,7 @@ export function compileQuery( namespaced.__parentContext = parentSide } const effectiveKey = - parentSide != null - ? `${String(childKey)}::${serializeValue(parentSide)}` - : childKey + parentSide != null ? serializeValue([childKey, parentSide]) : childKey return [effectiveKey, namespaced] }), ) as NamespacedAndKeyedStream @@ -1023,7 +1020,7 @@ export function compileQuery( aliasRemapping, includes: includesResults.length > 0 ? includesResults : undefined, } - cache.set(rawQuery, compilationResult) + if (parentKeyStream === undefined) cache.set(rawQuery, compilationResult) return compilationResult } else if (query.limit !== undefined || query.offset !== undefined) { @@ -1069,7 +1066,7 @@ export function compileQuery( aliasRemapping, includes: includesResults.length > 0 ? includesResults : undefined, } - cache.set(rawQuery, compilationResult) + if (parentKeyStream === undefined) cache.set(rawQuery, compilationResult) return compilationResult } @@ -1236,6 +1233,12 @@ function validateQueryStructure( } } } + + if (query.select) { + for (const { subquery } of extractIncludesFromSelect(query.select)) { + validateQueryStructure(subquery.query, combinedAliases) + } + } } /** diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index d794ea7a1c..780bb5b1aa 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -78,8 +78,10 @@ normal Collection transaction boundary owns public publication. ## Identity -Aliases are lexical query-language names. They are not runtime identities. -Compilation assigns opaque IDs to the plan: +Aliases are lexical query-language names. They are not runtime identities. The +query builder requires collection aliases to be unique across one query tree, +including includes; shadowing is rejected before compilation. Compilation then +assigns opaque IDs to the accepted plan: ```ts type SourceId = Brand @@ -87,8 +89,8 @@ type RelationNodeId = Brand type MaterializationEdgeId = Brand ``` -Alias text may remain as debug metadata. Alpha-renaming an alias cannot change -the compiled graph or its result. +Alias text may remain as debug metadata. Renaming an accepted alias to another +unused name cannot change the compiled graph or its result. A `CanonicalCorrelationKey` is the canonical tuple of every evaluated parent-dependent value that can affect the child plan. This includes values @@ -355,12 +357,13 @@ rows themselves. The source contract stays abstract: a demand request eventually establishes one coherent baseline and identifies when that baseline is complete. Each -request receives an `AbortSignal`. Replacing or releasing its demand aborts -that signal, and the source must check it before installing fetched rows. An -aborted request cannot install or settle after its graph or request generation -becomes obsolete. Buffering, snapshot tokens, shape offsets, Collection -transactions, and local indexes are source-specific ways to satisfy that -contract; they are not materializer state. +request receives an `AbortSignal`. Cancellation is cooperative at this source +boundary. Core guarantees that an obsolete request cannot settle current +readiness; the source must honor the signal immediately before installing a +baseline or later request-scoped result. Core cannot prevent an arbitrary +adapter from writing after it ignores that signal. Buffering, snapshot tokens, +shape offsets, Collection transactions, and local indexes are source-specific +ways to satisfy that contract; they are not materializer state. This project uses a single graph-run order rather than multi-dimensional timely-dataflow frontiers. Do not introduce a general timestamp or frontier @@ -433,7 +436,8 @@ units. Inline materialization must not create recursive Collection machinery. ## Normative laws -1. **Alpha-renaming:** lexical alias names cannot change results. +1. **Alpha-renaming:** changing any accepted alias to another unused name cannot + change results; alias shadowing within a query tree is rejected. 2. **Contribution conservation:** a public row exists exactly when its reduced supporting weight and collision policy produce one. 3. **Batch partition:** equivalent valid split and atomic deliveries converge. @@ -441,8 +445,9 @@ units. Inline materialization must not create recursive Collection machinery. equal current materialization-cell values. 5. **Total materialization:** every active inline cell has exactly one value, including its mode's empty value when its bucket has no rows. -6. **Stale demand:** an obsolete graph or demand generation can neither publish - rows nor settle current readiness. +6. **Stale demand:** an obsolete graph or demand generation cannot settle + current readiness, and a conforming source cannot publish its request-scoped + rows after cancellation. 7. **Nested propagation:** every materialized relation consumes the fully materialized output relation of its children. 8. **Publication:** reads, events, and downstream queries observe the same diff --git a/packages/db/src/query/live/bucket-facade-adapter.ts b/packages/db/src/query/live/bucket-facade-adapter.ts index cbe2840ea9..39b131e65c 100644 --- a/packages/db/src/query/live/bucket-facade-adapter.ts +++ b/packages/db/src/query/live/bucket-facade-adapter.ts @@ -4,6 +4,7 @@ import { FN_SELECT_STATE, INCLUDES_ROUTING } from '../compiler/index.js' import { BUCKET_FACADE_REF } from './materialized-pipeline.js' import type { Collection } from '../../collection/index.js' import type { SyncConfig } from '../../types.js' +import type { PublicationDeferral } from '../../collection/changes.js' import type { BucketFacadeCompilation, BucketFacadeRef, @@ -26,6 +27,24 @@ type FacadeEntry = { currentOrder: Map } +type FacadeSnapshot = { + activeBuckets: Map> + entries: Map> + rows: Map< + FacadeEntry, + Array<{ + key: string | number + value: object + order: string | undefined + }> + > +} + +export type FacadePublication = { + publish: () => void + rollback: () => void +} + /** * The only stateful boundary outside the materialization graph. It turns inert * bucket references into stable public Collection facades and applies the @@ -39,7 +58,7 @@ export class BucketFacadeAdapter { private readonly pendingActivity = new Map>() private readonly activeBuckets = new Map>() private readonly entries = new Map>() - private readonly resolvedValues = new WeakMap() + private resolvedValues = new WeakMap() constructor( private readonly parentId: string, @@ -72,21 +91,14 @@ export class BucketFacadeAdapter { return this.pending.size > 0 || this.pendingActivity.size > 0 } - flush(): () => void { + flush(): FacadePublication { + const snapshot = this.snapshot() const deferredEntries = new Set() - const resumePublications: Array<() => void> = [] + const publications: Array = [] const deferPublication = (entry: FacadeEntry) => { if (deferredEntries.has(entry)) return deferredEntries.add(entry) - resumePublications.push(entry.collection._deferPublication()) - } - let resumed = false - const resume = () => { - if (resumed) return - resumed = true - for (const resumePublication of resumePublications) { - resumePublication() - } + publications.push(entry.collection._deferPublication()) } // Compilations are child-first, so nested facade references resolve before @@ -95,8 +107,12 @@ export class BucketFacadeAdapter { for (const compilation of this.compilations) { const activity = this.pendingActivity.get(compilation.edgeId) const active = this.getActiveBuckets(compilation.edgeId) + const newBaselines: Array = [] for (const [bucketKey, multiplicity] of activity ?? []) { - if (multiplicity > 0) active.add(bucketKey) + if (multiplicity > 0 && !active.has(bucketKey)) { + active.add(bucketKey) + newBaselines.push(this.getEntry(compilation.edgeId, bucketKey)) + } } const buckets = this.pending.get(compilation.edgeId) @@ -116,6 +132,8 @@ export class BucketFacadeAdapter { } this.pending.delete(compilation.edgeId) + for (const entry of newBaselines) entry.sync?.markReady() + for (const [bucketKey, multiplicity] of activity ?? []) { if (multiplicity >= 0) continue active.delete(bucketKey) @@ -124,11 +142,24 @@ export class BucketFacadeAdapter { this.pendingActivity.delete(compilation.edgeId) } } catch (error) { - resume() + for (const publication of publications) publication.publish() throw error } - return resume + let closed = false + return { + publish: () => { + if (closed) return + closed = true + for (const publication of publications) publication.publish() + }, + rollback: () => { + if (closed) return + closed = true + this.restore(snapshot, deferredEntries) + for (const publication of publications) publication.discard() + }, + } } resolve(value: T): T { @@ -179,6 +210,88 @@ export class BucketFacadeAdapter { rows.set(key, change) } + private snapshot(): FacadeSnapshot { + const rows = new Map< + FacadeEntry, + Array<{ + key: string | number + value: object + order: string | undefined + }> + >() + for (const byBucket of this.entries.values()) { + for (const entry of byBucket.values()) { + rows.set( + entry, + [...entry.collection._state.syncedData].map(([key, value]) => ({ + key, + value, + order: entry.currentOrder.get(key), + })), + ) + } + } + return { + activeBuckets: new Map( + [...this.activeBuckets].map(([edgeId, buckets]) => [ + edgeId, + new Set(buckets), + ]), + ), + entries: new Map( + [...this.entries].map(([edgeId, byBucket]) => [ + edgeId, + new Map(byBucket), + ]), + ), + rows, + } + } + + private restore( + snapshot: FacadeSnapshot, + changedEntries: Set, + ): void { + const previousEntries = new Set( + [...snapshot.entries.values()].flatMap((byBucket) => [ + ...byBucket.values(), + ]), + ) + const currentEntries = new Set( + [...this.entries.values()].flatMap((byBucket) => [...byBucket.values()]), + ) + + for (const entry of changedEntries) { + if (!previousEntries.has(entry)) continue + const sync = entry.sync + if (!sync) continue + sync.begin() + sync.truncate() + entry.currentOrder.clear() + for (const row of snapshot.rows.get(entry) ?? []) { + entry.keys.set(row.value, row.key) + if (row.order !== undefined) entry.order.set(row.value, row.order) + entry.currentOrder.set(row.key, row.order) + sync.write({ type: `insert`, value: row.value }) + } + sync.commit() + } + + this.entries.clear() + for (const [edgeId, byBucket] of snapshot.entries) { + this.entries.set(edgeId, new Map(byBucket)) + } + this.activeBuckets.clear() + for (const [edgeId, buckets] of snapshot.activeBuckets) { + this.activeBuckets.set(edgeId, new Set(buckets)) + } + this.resolvedValues = new WeakMap() + + for (const entry of currentEntries) { + if (!previousEntries.has(entry)) void entry.collection.cleanup() + } + } + private accumulateActivity( edgeId: string, bucketKey: string, diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index 4cb40ece87..f395aa787f 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -161,6 +161,7 @@ export class CollectionConfigBuilder< string, { generation: number; settled: boolean } >() + private readonly demandGenerations = new Map() // Map of collection IDs to optimizable ORDER BY state optimizableOrderByCollections: Record = {} @@ -334,7 +335,8 @@ export class CollectionConfigBuilder< } beginDemand(planId: string): number { - const generation = (this.activeDemands.get(planId)?.generation ?? 0) + 1 + const generation = (this.demandGenerations.get(planId) ?? 0) + 1 + this.demandGenerations.set(planId, generation) this.activeDemands.set(planId, { generation, settled: false }) return generation } @@ -346,6 +348,13 @@ export class CollectionConfigBuilder< this.maybeRunGraphFn?.() } + failDemand(planId: string, generation: number, error: unknown): void { + const demand = this.activeDemands.get(planId) + if (!demand || demand.generation !== generation) return + const message = error instanceof Error ? error.message : String(error) + this.transitionToError(`Subset demand '${planId}' failed: ${message}`) + } + retireDemand(planId: string): void { this.activeDemands.delete(planId) } @@ -685,6 +694,7 @@ export class CollectionConfigBuilder< // Reset lazy source alias state this.lazySources.clear() + this.demandGenerations.clear() this.activeDemands.clear() this.optimizableOrderByCollections = {} this.lazySourcesCallbacks = {} @@ -802,7 +812,10 @@ export class CollectionConfigBuilder< return } - const resumeFacadePublications = bucketFacades.flush() + const facadePublication = bucketFacades.flush() + const rootPublication = hasParentChanges + ? config.collection._deferPublication() + : undefined try { const changesToApply: Map> = new Map( [...pendingChanges].map(([key, changes]) => { @@ -827,10 +840,27 @@ export class CollectionConfigBuilder< } commit() } - } finally { - resumeFacadePublications() + } catch (error) { + pendingChanges = new Map() + rootPublication?.discard() + facadePublication.rollback() + throw error } pendingChanges = new Map() + + let publicationError: unknown + for (const publish of [ + rootPublication?.publish, + facadePublication.publish, + ]) { + if (!publish) continue + try { + publish() + } catch (error) { + publicationError ??= error + } + } + if (publicationError !== undefined) throw publicationError } graph.finalize() diff --git a/packages/db/src/query/live/collection-subscriber.ts b/packages/db/src/query/live/collection-subscriber.ts index 143e77359b..bfe1ef3e34 100644 --- a/packages/db/src/query/live/collection-subscriber.ts +++ b/packages/db/src/query/live/collection-subscriber.ts @@ -172,12 +172,11 @@ export class CollectionSubscriber< } const generation = this.collectionConfigBuilder.beginDemand(plan.id) - const pending = update.loadResults.filter( - (result): result is Promise => result instanceof Promise, - ) - if (pending.length > 0) { - void Promise.allSettled(pending).then(() => - this.collectionConfigBuilder.settleDemand(plan.id, generation), + if (update.ready instanceof Promise) { + void update.ready.then( + () => this.collectionConfigBuilder.settleDemand(plan.id, generation), + (error) => + this.collectionConfigBuilder.failDemand(plan.id, generation, error), ) } else { this.collectionConfigBuilder.settleDemand(plan.id, generation) diff --git a/packages/db/src/query/live/materialized-pipeline.ts b/packages/db/src/query/live/materialized-pipeline.ts index 95c45246f9..c918a0f07b 100644 --- a/packages/db/src/query/live/materialized-pipeline.ts +++ b/packages/db/src/query/live/materialized-pipeline.ts @@ -71,6 +71,13 @@ export type MaterializedCompilation = { facades: Array } +type RelationScope = `root` | `child` + +type BuiltRelations = WeakMap< + CompilationResult, + Partial> +> + let nextBucketFacadeEdgeId = 0 /** @@ -82,8 +89,13 @@ export function materializeCompilation( compilation: CompilationResult, getRootKey?: (row: any) => unknown, ): MaterializedCompilation { - const built = new WeakMap() - const materialized = materializeRelation(compilation, getRootKey, built) + const built: BuiltRelations = new WeakMap() + const materialized = materializeRelation( + compilation, + getRootKey, + built, + `root`, + ) return { ...materialized, facades: dedupeFacades(materialized.facades), @@ -93,14 +105,16 @@ export function materializeCompilation( function materializeRelation( compilation: CompilationResult, getKey: ((row: any) => unknown) | undefined, - built: WeakMap, + built: BuiltRelations, + scope: RelationScope, ): MaterializedCompilation { - const cached = built.get(compilation) + const cached = built.get(compilation)?.[scope] if (cached) return cached let pipeline = canonicalizeByPublicKey( exposeRouting(compilation.pipeline), getKey, + scope, ) const facades: Array = [] @@ -109,6 +123,7 @@ function materializeRelation( include.childCompilationResult, undefined, built, + `child`, ) facades.push(...child.facades) @@ -122,14 +137,14 @@ function materializeRelation( activeBuckets, hasOrderBy: include.hasOrderBy, }) - pipeline = attachCollectionInclude(pipeline, include, edgeId) + pipeline = attachCollectionInclude(pipeline, include, edgeId, scope) } else { - pipeline = attachInlineInclude(pipeline, bucketRows, include) + pipeline = attachInlineInclude(pipeline, bucketRows, include, scope) } } const result = { pipeline, facades } - built.set(compilation, result) + built.set(compilation, { ...built.get(compilation), [scope]: result }) return result } @@ -161,15 +176,17 @@ function exposeRouting(pipeline: ResultStream): ResultStream { function canonicalizeByPublicKey( pipeline: ResultStream, getKey: ((row: any) => unknown) | undefined, + scope: RelationScope, ): ResultStream { return pipeline.pipe( map(([internalKey, rawTuple]) => { const tuple = rawTuple as ResultTuple const publicKey = getKey ? getKey(tuple[0]) : (tuple[5] ?? internalKey) - return [serializeValue(publicKey), { publicKey, tuple }] as [ - string, - CanonicalResult, - ] + const relationKey = + scope === `root` + ? serializeValue([`root`, publicKey]) + : serializeValue([routeKey(tuple[2], tuple[3]), publicKey]) + return [relationKey, { publicKey, tuple }] as [string, CanonicalResult] }), reduce((values: Array<[CanonicalResult, number]>) => { const totalMultiplicity = values.reduce( @@ -193,7 +210,10 @@ function canonicalizeByPublicKey( return [[visible, 1]] }), - map(([_serializedKey, { publicKey, tuple }]) => [publicKey, tuple]), + map(([relationKey, { publicKey, tuple }]) => [ + scope === `root` ? publicKey : relationKey, + tuple, + ]), ) as ResultStream } @@ -223,6 +243,7 @@ function attachInlineInclude( parentPipeline: ResultStream, bucketRows: IStreamBuilder<[string, BucketRow]>, include: IncludesCompilationResult, + scope: RelationScope, ): ResultStream { const bucketValues = bucketRows.pipe( reduce((values: Array<[BucketRow, number]>) => { @@ -254,13 +275,13 @@ function attachInlineInclude( }), join(bucketValues, `left`), map(([_bucketKey, [parent, bucketValue]]) => { - const [value, order, correlationKey, parentContext, routing] = + const [value, order, correlationKey, parentContext, routing, publicKey] = parent!.tuple const edgeRouting = routing?.[include.fieldName] if (edgeRouting?.active !== true) { return [ parent!.parentKey, - [value, order, correlationKey, parentContext, routing], + [value, order, correlationKey, parentContext, routing, publicKey], ] } const materialized = @@ -273,6 +294,7 @@ function attachInlineInclude( correlationKey, parentContext, routing, + publicKey, ], ] }), @@ -280,19 +302,23 @@ function attachInlineInclude( // A route move can make the join emit matched and empty-bucket deltas for // the same parent key in one graph turn. Reduce those deltas back to the one // canonical parent row before the next include or the public output sees it. - return canonicalizeByPublicKey(routedParents as ResultStream, undefined) + return canonicalizeByPublicKey( + routedParents as ResultStream, + undefined, + scope, + ) } function createBucketRows( childPipeline: ResultStream, ): IStreamBuilder<[string, BucketRow]> { return childPipeline.pipe( - map(([publicKey, rawTuple]) => { - const [value, order, correlationKey, parentContext] = + map(([internalKey, rawTuple]) => { + const [value, order, correlationKey, parentContext, , publicKey] = rawTuple as ResultTuple return [ routeKey(correlationKey, parentContext), - { publicKey, value, order }, + { publicKey: publicKey ?? internalKey, value, order }, ] as [string, BucketRow] }), ) @@ -302,6 +328,7 @@ function attachCollectionInclude( parentPipeline: ResultStream, include: IncludesCompilationResult, edgeId: string, + scope: RelationScope, ): ResultStream { const routedParents = parentPipeline.pipe( map(([parentKey, rawTuple]) => { @@ -320,11 +347,16 @@ function attachCollectionInclude( tuple[2], tuple[3], tuple[4], + tuple[5], ], ] }), ) - return canonicalizeByPublicKey(routedParents as ResultStream, undefined) + return canonicalizeByPublicKey( + routedParents as ResultStream, + undefined, + scope, + ) } function createActiveBuckets( diff --git a/packages/db/src/query/live/subset-demand-controller.ts b/packages/db/src/query/live/subset-demand-controller.ts index 267750e123..0dee57e068 100644 --- a/packages/db/src/query/live/subset-demand-controller.ts +++ b/packages/db/src/query/live/subset-demand-controller.ts @@ -9,6 +9,8 @@ type DemandSegment = { keys: Map where: BasicExpression abortController: AbortController + ready: Promise | true + state: `pending` | `settled` | `failed` } type DemandState = { @@ -19,7 +21,7 @@ type DemandState = { export type DemandUpdate = { changed: boolean empty: boolean - loadResults: Array | true> + ready: Promise | true } /** @@ -37,34 +39,38 @@ export class SubsetDemandController { ): DemandUpdate { const nextKeys = canonicalizeKeys(keys) const previous = this.states.get(plan.id) - if (previous && equalKeySets(previous.keys, nextKeys)) { - return { changed: false, empty: nextKeys.size === 0, loadResults: [] } + const hasFailedCoverage = previous?.segments.some( + (segment) => + segment.state === `failed` && intersects(segment.keys, nextKeys), + ) + if ( + previous && + equalKeySets(previous.keys, nextKeys) && + !hasFailedCoverage + ) { + return { changed: false, empty: nextKeys.size === 0, ready: true } } - const loadResults: Array | true> = [] const segments: Array = [] for (const segment of previous?.segments ?? []) { - if ([...segment.keys.keys()].every((key) => nextKeys.has(key))) { + if (segment.state !== `failed` && intersects(segment.keys, nextKeys)) { segments.push(segment) continue } - const retained = new Map( - [...segment.keys].filter(([key]) => nextKeys.has(key)), - ) - if (retained.size > 0) { - segments.push(requestSegment(subscription, plan, retained, loadResults)) - } segment.abortController.abort() subscription.releaseSnapshot(segment.where) } + const coveredKeys = new Set( + segments.flatMap((segment) => [...segment.keys.keys()]), + ) const added = new Map( - [...nextKeys].filter(([key]) => !previous?.keys.has(key)), + [...nextKeys].filter(([key]) => !coveredKeys.has(key)), ) if (added.size > 0) { - segments.push(requestSegment(subscription, plan, added, loadResults)) + segments.push(requestSegment(subscription, plan, added)) } if (nextKeys.size === 0) { @@ -73,7 +79,18 @@ export class SubsetDemandController { this.states.set(plan.id, { keys: nextKeys, segments }) } - return { changed: true, empty: nextKeys.size === 0, loadResults } + const activeSegments = segments.filter((segment) => + intersects(segment.keys, nextKeys), + ) + const pending = activeSegments + .map((segment) => segment.ready) + .filter((ready): ready is Promise => ready instanceof Promise) + return { + changed: true, + empty: nextKeys.size === 0, + ready: + pending.length > 0 ? Promise.all(pending).then(() => undefined) : true, + } } clear(): void { @@ -97,19 +114,46 @@ function equalKeySets( ) } +function intersects( + left: Map, + right: Map, +): boolean { + return [...left.keys()].some((key) => right.has(key)) +} + function requestSegment( subscription: CollectionSubscription, plan: LazyDemandPlan, keys: Map, - loadResults: Array | true>, ): DemandSegment { const where = inArray(new PropRef(plan.path), [...keys.values()]) const abortController = new AbortController() + const load = { ready: true as Promise | true } subscription.requestSnapshot({ where, signal: abortController.signal, trackLoadSubsetPromise: false, - onLoadSubsetResult: (result) => loadResults.push(result), + onLoadSubsetResult: (result) => { + load.ready = result + }, }) - return { keys, where, abortController } + const ready = load.ready + const segment: DemandSegment = { + keys, + where, + abortController, + ready, + state: ready instanceof Promise ? `pending` : `settled`, + } + if (ready instanceof Promise) { + void ready.then( + () => { + segment.state = `settled` + }, + () => { + segment.state = `failed` + }, + ) + } + return segment } diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index c26c8139b2..49cf2a88f4 100644 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -303,8 +303,9 @@ export type LoadSubsetOptions = { */ offset?: number /** - * Aborted when this exact subset request is no longer current. Async sync - * adapters must check the signal before installing fetched rows. + * Aborted when this exact subset request is no longer current. Cancellation + * is cooperative: async sync adapters must check the signal immediately + * before installing a baseline or later request-scoped rows. */ signal?: AbortSignal /** diff --git a/packages/db/tests/query/compiler/subquery-caching.test.ts b/packages/db/tests/query/compiler/subquery-caching.test.ts index 769cd7e7a0..991e78c0d4 100644 --- a/packages/db/tests/query/compiler/subquery-caching.test.ts +++ b/packages/db/tests/query/compiler/subquery-caching.test.ts @@ -263,6 +263,40 @@ describe(`Subquery Caching`, () => { expect(sharedCache.has(subquery)).toBe(true) }) + it(`does not reuse a correlated query across parent streams`, () => { + const usersCollection = createMockCollection(`users`) + const query: QueryIR = { + from: new CollectionRef(usersCollection, `u`), + select: { id: new PropRef([`u`, `id`]) }, + } + const graph = new D2() + const userInput = graph.newInput<[number, any]>() + const firstParents = graph.newInput<[number, any]>() + const secondParents = graph.newInput<[number, any]>() + const cache = new WeakMap() + const compileWithParents = (parents: typeof firstParents) => + compileQuery( + query, + { u: userInput }, + { users: usersCollection }, + {}, + {}, + new Set(), + {}, + () => {}, + cache, + new WeakMap(), + parents, + new PropRef([`u`, `id`]), + ) + + const first = compileWithParents(firstParents) + const second = compileWithParents(secondParents) + + expect(second).not.toBe(first) + expect(cache.has(query)).toBe(false) + }) + it(`should use cache to avoid recompilation in nested subqueries`, () => { const usersCollection = createMockCollection(`users`) diff --git a/packages/db/tests/query/includes-lazy-loading.test.ts b/packages/db/tests/query/includes-lazy-loading.test.ts index 8e8eccace3..f2ed8f5bc3 100644 --- a/packages/db/tests/query/includes-lazy-loading.test.ts +++ b/packages/db/tests/query/includes-lazy-loading.test.ts @@ -8,7 +8,11 @@ import { } from '../../src/query/index.js' import { createCollection } from '../../src/collection/index.js' import { extractSimpleComparisons } from '../../src/query/expression-helpers.js' -import { flushPromises, stripVirtualProps } from '../utils.js' +import { + flushPromises, + mockSyncCollectionOptions, + stripVirtualProps, +} from '../utils.js' import type { LoadSubsetOptions } from '../../src/types.js' /** @@ -127,6 +131,85 @@ describe(`includes lazy loading`, () => { ]) }) + it(`targets the joined source in a lazy self-join`, async () => { + type SelfItem = { + id: number + peerId: number + rootId: number + label: string + } + const roots = createCollection( + mockSyncCollectionOptions<{ id: number }>({ + id: `includes-lazy-self-join-roots`, + getKey: (root) => root.id, + initialData: [{ id: 1 }], + }), + ) + const rows: Array = [ + { id: 1, peerId: 2, rootId: 999, label: `left` }, + { id: 2, peerId: 0, rootId: 1, label: `right` }, + ] + const installed = new Set() + const items = createCollection({ + id: `includes-lazy-self-join-items`, + getKey: (item) => item.id, + syncMode: `on-demand`, + sync: { + sync: ({ begin, write, commit, markReady }) => ({ + loadSubset: (options) => { + const rootIds = new Set( + extractSimpleComparisons(options.where).flatMap((comparison) => + comparison.field[0] === `rootId` && + comparison.operator === `in` && + Array.isArray(comparison.value) + ? comparison.value + : [], + ), + ) + begin() + for (const row of rows) { + if ( + installed.has(row.id) || + (rootIds.size > 0 && !rootIds.has(row.rootId)) + ) { + continue + } + installed.add(row.id) + write({ type: `insert`, value: row }) + } + commit() + markReady() + return Promise.resolve() + }, + }), + }, + }) + + const live = createLiveQueryCollection((q) => + q.from({ root: roots }).select(({ root }) => ({ + id: root.id, + matches: toArray( + q + .from({ left: items }) + .join({ right: items }, ({ left, right }) => + eq(left.peerId, right.id), + ) + .where(({ right }) => eq(right.rootId, root.id)) + .select(({ left, right }) => ({ + left: left.label, + right: right.label, + })), + ), + })), + ) + + await live.preload() + expect(stripVirtualProps(live.get(1))).toMatchObject({ + id: 1, + matches: [{ left: `left`, right: `right` }], + }) + }) + it(`should produce correct query results with lazy-loaded includes`, async () => { const roots = createRootsCollection() const { collection: items } = createItemsCollectionWithTracking() @@ -434,12 +517,12 @@ describe(`includes child where clauses in loadSubset`, () => { * through to the child collection's loadSubset/queryFn. */ - type Root = { + type FilterRoot = { id: number name: string } - type Item = { + type FilterItem = { id: number rootId: number status: string @@ -447,12 +530,12 @@ describe(`includes child where clauses in loadSubset`, () => { title: string } - const sampleRoots: Array = [ + const filterRoots: Array = [ { id: 1, name: `Root A` }, { id: 2, name: `Root B` }, ] - const sampleItems: Array = [ + const filterItems: Array = [ { id: 10, rootId: 1, status: `active`, priority: 3, title: `A1 active` }, { id: 11, @@ -466,13 +549,13 @@ describe(`includes child where clauses in loadSubset`, () => { ] function createRootsCollection() { - return createCollection({ + return createCollection({ id: `child-where-roots`, getKey: (r) => r.id, sync: { sync: ({ begin, write, commit, markReady }) => { begin() - for (const root of sampleRoots) { + for (const root of filterRoots) { write({ type: `insert`, value: root }) } commit() @@ -485,14 +568,14 @@ describe(`includes child where clauses in loadSubset`, () => { function createItemsCollectionWithTracking() { const loadSubsetCalls: Array = [] - const collection = createCollection({ + const collection = createCollection({ id: `child-where-items`, getKey: (item) => item.id, syncMode: `on-demand`, sync: { sync: ({ begin, write, commit, markReady }) => { begin() - for (const item of sampleItems) { + for (const item of filterItems) { write({ type: `insert`, value: item }) } commit() diff --git a/packages/db/tests/query/includes-temporal-oracle.test.ts b/packages/db/tests/query/includes-temporal-oracle.test.ts index e954a9d4ba..38e7b27b01 100644 --- a/packages/db/tests/query/includes-temporal-oracle.test.ts +++ b/packages/db/tests/query/includes-temporal-oracle.test.ts @@ -1,7 +1,8 @@ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { createCollection } from '../../src/collection/index.js' import { createDeferred } from '../../src/deferred.js' import { extractSimpleComparisons } from '../../src/query/expression-helpers.js' +import { DeduplicatedLoadSubset } from '../../src/query/subset-dedupe.js' import { createLiveQueryCollection, eq, @@ -491,6 +492,290 @@ async function expectObsoleteDemandCannotPublishAfterReactivation(): Promise, + options: { markReadyInitially?: boolean } = {}, +): { + collection: Collection + write: (type: `insert` | `delete`, post: Post) => void + markReady: () => void +} { + let writePost: (type: `insert` | `delete`, post: Post) => void = () => { + throw new Error(`Post collection has not started`) + } + let markPostsReady: () => void = () => { + throw new Error(`Post collection has not started`) + } + const collection = createCollection({ + id: nextCollectionId(`temporal-mutable-posts`), + getKey: (post) => post.id, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + for (const post of initial) write({ type: `insert`, value: post }) + commit() + if (options.markReadyInitially !== false) markReady() + writePost = (type, post) => { + begin() + write({ type, value: post }) + commit() + } + markPostsReady = markReady + }, + }, + }) + return { + collection, + write: (type, post) => writePost(type, post), + markReady: () => markPostsReady(), + } +} + +function createPendingComments(): { + collection: Collection + requests: Array<{ + deferred: Deferred + outcome: Promise + keys: Array + signal: AbortSignal | undefined + }> +} { + const requests: Array<{ + deferred: Deferred + outcome: Promise + keys: Array + signal: AbortSignal | undefined + }> = [] + const collection = createCollection({ + id: nextCollectionId(`temporal-pending-coverage-comments`), + getKey: (comment) => comment.id, + syncMode: `on-demand`, + sync: { + sync: ({ markReady }) => ({ + loadSubset: (options) => { + const deferred = createDeferred() + const outcome = deferred.promise.then(() => { + if (!options.signal?.aborted) markReady() + }) + requests.push({ + deferred, + outcome, + keys: correlationKeys([options], `postId`), + signal: options.signal, + }) + return outcome + }, + }), + }, + }) + return { collection, requests } +} + +function createPostsWithCommentsLive( + posts: Collection, + comments: Collection, +) { + return createLiveQueryCollection((q) => + q.from({ post: posts }).select(({ post }) => ({ + id: post.id, + comments: toArray( + q + .from({ comment: comments }) + .where(({ comment }) => eq(comment.postId, post.id)), + ), + })), + ) +} + +async function expectRetainedDemandBlocksReadiness(): Promise { + const firstPost = { id: 1, authorId: `selected`, title: `one` } + const secondPost = { id: 2, authorId: `selected`, title: `two` } + const posts = createMutablePosts([firstPost]) + const { collection: comments, requests } = createPendingComments() + const live = createPostsWithCommentsLive(posts.collection, comments) + const preload: PreloadState = { preloadSettled: false } + startPreload(live, preload) + + try { + await flushPromises() + expect(requests.map(({ keys }) => keys)).toEqual([[1]]) + + posts.write(`insert`, secondPost) + await flushPromises() + expect(requests.map(({ keys }) => keys)).toEqual([[1], [2]]) + + requests[1]!.deferred.resolve() + await requests[1]!.outcome + await flushPromises() + expect(preload.preloadSettled).toBe(false) + expect(live.isReady()).toBe(false) + + requests[0]!.deferred.resolve() + await requests[0]!.outcome + await finishPreload(preload) + expect(live.isReady()).toBe(true) + } finally { + for (const request of requests) request.deferred.resolve() + await Promise.allSettled(requests.map(({ outcome }) => outcome)) + await live.cleanup() + await Promise.all([posts.collection.cleanup(), comments.cleanup()]) + } +} + +async function expectObsoleteDemandCannotSettleReactivatedDemand(): Promise { + const post = { id: 1, authorId: `selected`, title: `one` } + const posts = createMutablePosts([post], { markReadyInitially: false }) + const { collection: comments, requests } = createPendingComments() + const live = createPostsWithCommentsLive(posts.collection, comments) + const preload: PreloadState = { preloadSettled: false } + startPreload(live, preload) + + try { + await flushPromises() + expect(requests).toHaveLength(1) + + posts.write(`delete`, post) + posts.write(`insert`, post) + await flushPromises() + expect(requests).toHaveLength(2) + expect(requests[0]!.signal?.aborted).toBe(true) + + requests[0]!.deferred.resolve() + await requests[0]!.outcome + posts.markReady() + await flushPromises() + expect(preload.preloadSettled).toBe(false) + expect(live.isReady()).toBe(false) + + requests[1]!.deferred.resolve() + await requests[1]!.outcome + await finishPreload(preload) + } finally { + for (const request of requests) request.deferred.resolve() + await Promise.allSettled(requests.map(({ outcome }) => outcome)) + await live.cleanup() + await Promise.all([posts.collection.cleanup(), comments.cleanup()]) + } +} + +async function expectRejectedDemandEntersError(): Promise { + const posts = createMutablePosts([ + { id: 1, authorId: `selected`, title: `one` }, + ]) + let loadCount = 0 + let shouldReject = true + const comments = createCollection({ + id: nextCollectionId(`temporal-rejected-comments`), + getKey: (comment) => comment.id, + syncMode: `on-demand`, + sync: { + sync: ({ markReady }) => ({ + loadSubset: () => { + loadCount += 1 + if (shouldReject) { + return Promise.reject(new Error(`child load failed`)) + } + markReady() + return true + }, + }), + }, + }) + const live = createPostsWithCommentsLive(posts.collection, comments) + const preload: PreloadState = { preloadSettled: false } + const consoleError = vi.spyOn(console, `error`).mockImplementation(() => {}) + startPreload(live, preload) + + try { + await flushPromises() + expect(loadCount).toBe(1) + expect(live.status).toBe(`error`) + expect(preload.preloadSettled).toBe(false) + + await live.cleanup() + await preload.preloadOutcome + shouldReject = false + await live.preload() + expect(loadCount).toBe(2) + expect(live.isReady()).toBe(true) + } finally { + await live.cleanup() + await preload.preloadOutcome + await Promise.all([posts.collection.cleanup(), comments.cleanup()]) + consoleError.mockRestore() + } +} + +async function expectPartialShrinkRetainsCoverage(): Promise { + const firstPost = { id: 1, authorId: `selected`, title: `one` } + const secondPost = { id: 2, authorId: `selected`, title: `two` } + const posts = createMutablePosts([firstPost, secondPost]) + const initialLoad = createDeferred() + const installed = new Map() + let begin: () => void + let write: (change: { type: `insert` | `delete`; value: Comment }) => void + let commit: () => void + let markReady: () => void + let deduped: DeduplicatedLoadSubset + const unloads: Array> = [] + const comments = createCollection({ + id: nextCollectionId(`temporal-shrink-comments`), + getKey: (comment) => comment.id, + syncMode: `on-demand`, + sync: { + sync: (methods) => { + ;({ begin, write, commit, markReady } = methods) + deduped = new DeduplicatedLoadSubset({ + loadSubset: (options) => + initialLoad.promise.then(() => { + const keys = correlationKeys([options], `postId`) + begin() + for (const postId of keys) { + const comment = { id: postId * 100, postId, body: `${postId}` } + installed.set(postId, comment) + write({ type: `insert`, value: comment }) + } + commit() + markReady() + }), + }) + return { + loadSubset: (options) => deduped.loadSubset(options), + unloadSubset: (options) => { + const keys = correlationKeys([options], `postId`) + unloads.push(keys) + begin() + for (const postId of keys) { + const comment = installed.get(postId) + if (comment) write({ type: `delete`, value: comment }) + installed.delete(postId) + } + commit() + }, + } + }, + }, + }) + const live = createPostsWithCommentsLive(posts.collection, comments) + const preload = live.preload() + + try { + await flushPromises() + initialLoad.resolve() + await preload + expect(live.get(1)?.comments).toHaveLength(1) + + posts.write(`delete`, secondPost) + await flushPromises() + expect(live.get(1)?.comments).toHaveLength(1) + expect(unloads).toEqual([]) + } finally { + initialLoad.resolve() + await live.cleanup() + await Promise.all([posts.collection.cleanup(), comments.cleanup()]) + } +} + type FastPathEvent = { phase: `fast` | `late` keys: Array @@ -743,6 +1028,23 @@ describe(`includes temporal oracle`, () => { expectObsoleteDemandCannotPublishAfterReactivation, ) + it( + `retained pending demand blocks readiness after demand expands`, + expectRetainedDemandBlocksReadiness, + ) + + it( + `obsolete demand cannot settle a reactivated demand incarnation`, + expectObsoleteDemandCannotSettleReactivatedDemand, + ) + + it(`rejected demand enters error`, expectRejectedDemandEntersError) + + it( + `partially shrinking demand retains established coverage`, + expectPartialShrinkRetainsCoverage, + ) + it(`loads a direct progressive subset inside the fast-path window`, async () => { await expectProgressiveTraceMatches(`direct`) }) diff --git a/packages/db/tests/query/includes.test.ts b/packages/db/tests/query/includes.test.ts index 9a28f2472f..692646e210 100644 --- a/packages/db/tests/query/includes.test.ts +++ b/packages/db/tests/query/includes.test.ts @@ -6,11 +6,13 @@ import { count, createLiveQueryCollection, eq, + gte, materialize, toArray, } from '../../src/query/index.js' import { createCollection } from '../../src/collection/index.js' import { CleanupQueue } from '../../src/collection/cleanup-queue.js' +import { BTreeIndex } from '../../src/indexes/btree-index.js' import { localOnlyCollectionOptions } from '../../src/local-only.js' import { flushPromises, @@ -357,6 +359,14 @@ describe(`includes subqueries`, () => { const items = collection.get(1)!.items expect(items).toBeDefined() expect(plainRows(items)).toEqual([]) + expect(items.isReady()).toBe(true) + let preloadSettled = false + const preload = items.preload().then(() => { + preloadSettled = true + }) + await flushPromises() + expect(preloadSettled).toBe(true) + await preload }) }) @@ -1255,6 +1265,204 @@ describe(`includes subqueries`, () => { } }) + it(`replays existing bucket rows when their parent route becomes active`, async () => { + type Parent = { id: number; groupId: number } + type Child = { id: number; groupId: number } + + const parents = createCollection( + mockSyncCollectionOptions({ + id: `late-route-parents`, + getKey: (parent) => parent.id, + initialData: [], + }), + ) + const children = createCollection( + mockSyncCollectionOptions({ + id: `late-route-children`, + getKey: (child) => child.id, + initialData: [{ id: 10, groupId: 1 }], + }), + ) + const collection = createLiveQueryCollection((q) => + q.from({ parent: parents }).select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children }) + .where(({ child }) => eq(child.groupId, parent.groupId)) + .select(({ child }) => ({ id: child.id })), + })), + ) + + await collection.preload() + parents.utils.begin() + parents.utils.write({ + type: `insert`, + value: { id: 1, groupId: 1 }, + }) + parents.utils.commit() + + expect(childItems(collection.get(1)!.children)).toEqual([{ id: 10 }]) + }) + + it(`does not publish facade changes when root publication fails`, async () => { + type Parent = { id: number; groupId: number } + type Child = { id: number; groupId: number } + + const parents = createCollection( + mockSyncCollectionOptions({ + id: `failed-publication-parents`, + getKey: (parent) => parent.id, + initialData: [{ id: 1, groupId: 1 }], + }), + ) + const children = createCollection( + mockSyncCollectionOptions({ + id: `failed-publication-children`, + getKey: (child) => child.id, + initialData: [ + { id: 10, groupId: 1 }, + { id: 20, groupId: 2 }, + ], + }), + ) + let keyReads = 0 + let failAt: number | undefined + const collection = createLiveQueryCollection({ + query: (q) => + q.from({ parent: parents }).select(({ parent }) => ({ + id: parent.id, + groupId: parent.groupId, + children: q + .from({ child: children }) + .where(({ child }) => eq(child.groupId, parent.groupId)) + .select(({ child }) => ({ id: child.id })), + })), + getKey: (row) => { + keyReads += 1 + if (keyReads === failAt) throw new Error(`root publication failed`) + return row.id + }, + }) + + await collection.preload() + const oldFacade = collection.get(1)!.children + const facadeChanges = vi.fn() + const subscription = oldFacade.subscribeChanges(facadeChanges, { + includeInitialState: false, + }) + + try { + keyReads = 0 + failAt = 3 + expect(() => { + parents.utils.begin() + parents.utils.write({ + type: `update`, + previousValue: { id: 1, groupId: 1 }, + value: { id: 1, groupId: 2 }, + }) + parents.utils.commit() + }).toThrow(`root publication failed`) + + expect(collection.get(1)!.groupId).toBe(1) + expect(collection.get(1)!.children).toBe(oldFacade) + expect(childItems(oldFacade)).toEqual([{ id: 10 }]) + expect(facadeChanges).not.toHaveBeenCalled() + } finally { + subscription.unsubscribe() + } + }) + + it(`publishes coherent includes through immediate ordered load-more passes`, async () => { + type Parent = { id: number; groupId: number; rank: number } + type Child = { id: number; groupId: number } + + const sourceRows: Array = [ + { id: 1, groupId: 1, rank: 1 }, + { id: 2, groupId: 2, rank: 2 }, + { id: 3, groupId: 3, rank: 3 }, + ] + let nextRow = 0 + let loadCount = 0 + const parents = createCollection({ + id: `ordered-publication-parents`, + getKey: (parent) => parent.id, + syncMode: `on-demand`, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ begin, write, commit, markReady }) => ({ + loadSubset: () => { + loadCount += 1 + const row = sourceRows[nextRow++] + if (row) { + begin() + write({ type: `insert`, value: row }) + commit() + } + markReady() + return true + }, + }), + }, + }) + const children = createCollection( + mockSyncCollectionOptions({ + id: `ordered-publication-children`, + getKey: (child) => child.id, + initialData: [ + { id: 10, groupId: 1 }, + { id: 20, groupId: 2 }, + { id: 30, groupId: 3 }, + ], + }), + ) + const collection = createLiveQueryCollection((q) => + q + .from({ parent: parents }) + .orderBy(({ parent }) => parent.rank) + .limit(3) + .select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children }) + .where(({ child }) => eq(child.groupId, parent.groupId)) + .select(({ child }) => ({ id: child.id })), + })), + ) + const observations: Array< + Array<{ id: number; childIds: Array }> + > = [] + const readObservation = () => + [...collection.values()].map((parent) => ({ + id: parent.id, + childIds: plainRows(parent.children).map((child) => child.id), + })) + const subscription = collection.subscribeChanges( + () => { + observations.push(readObservation()) + }, + { includeInitialState: false }, + ) + + try { + await collection.preload() + expect(loadCount).toBe(3) + for (const observation of observations) { + for (const parent of observation) { + expect(parent.childIds).toEqual([parent.id * 10]) + } + } + expect(readObservation()).toEqual([ + { id: 1, childIds: [10] }, + { id: 2, childIds: [20] }, + { id: 3, childIds: [30] }, + ]) + } finally { + subscription.unsubscribe() + } + }) + it(`retires a shared facade after all parent routes publish`, async () => { type Parent = { id: number; groupId: number } type Child = { id: number; groupId: number } @@ -2874,6 +3082,265 @@ describe(`includes subqueries`, () => { expect([...bobItems.keys()]).toEqual([]) }) + it(`keeps the same child public key in distinct parent-context buckets`, async () => { + type ScoreParent = { id: number; groupId: number; minimumScore: number } + type ScoreChild = { + id: number + groupId: number + score: number + label: string + } + + const parents = createCollection( + mockSyncCollectionOptions({ + id: `same-child-context-parents`, + getKey: (parent) => parent.id, + initialData: [ + { id: 1, groupId: 1, minimumScore: 10 }, + { id: 2, groupId: 1, minimumScore: 20 }, + ], + }), + ) + const children = createCollection( + mockSyncCollectionOptions({ + id: `same-child-context-children`, + getKey: (child) => child.id, + initialData: [{ id: 7, groupId: 1, score: 30, label: `seven` }], + }), + ) + + const collection = createLiveQueryCollection((q) => + q.from({ parent: parents }).select(({ parent }) => ({ + id: parent.id, + children: materialize( + q + .from({ child: children }) + .where(({ child }) => eq(child.groupId, parent.groupId)) + .where(({ child }) => gte(child.score, parent.minimumScore)) + .select(({ child }) => ({ id: child.id, score: child.score })), + ), + firstChild: materialize( + q + .from({ firstChild: children }) + .where(({ firstChild }) => eq(firstChild.groupId, parent.groupId)) + .where(({ firstChild }) => + gte(firstChild.score, parent.minimumScore), + ) + .select(({ firstChild }) => ({ id: firstChild.id })) + .findOne(), + ), + labels: concat( + toArray( + q + .from({ labelChild: children }) + .where(({ labelChild }) => + eq(labelChild.groupId, parent.groupId), + ) + .where(({ labelChild }) => + gte(labelChild.score, parent.minimumScore), + ) + .select(({ labelChild }) => labelChild.label), + ), + ), + })), + ) + + await collection.preload() + + expect(toTree(collection)).toEqual([ + { + id: 1, + children: [{ id: 7, score: 30 }], + firstChild: { id: 7 }, + labels: `seven`, + }, + { + id: 2, + children: [{ id: 7, score: 30 }], + firstChild: { id: 7 }, + labels: `seven`, + }, + ]) + + parents.utils.begin() + parents.utils.write({ + type: `update`, + previousValue: { id: 2, groupId: 1, minimumScore: 20 }, + value: { id: 2, groupId: 2, minimumScore: 20 }, + }) + parents.utils.commit() + children.utils.begin() + children.utils.write({ + type: `update`, + previousValue: { id: 7, groupId: 1, score: 30, label: `seven` }, + value: { id: 7, groupId: 2, score: 30, label: `seven` }, + }) + children.utils.commit() + + expect(toTree(collection)).toEqual([ + { id: 1, children: [], firstChild: undefined, labels: `` }, + { + id: 2, + children: [{ id: 7, score: 30 }], + firstChild: { id: 7 }, + labels: `seven`, + }, + ]) + }) + + it(`uses canonical identity for non-JSON parent context values`, async () => { + type TaggedParent = { id: number; groupId: number; tag: bigint } + type TaggedChild = { + id: number | string + groupId: number + tag: bigint + metadataId: number + } + type Metadata = { id: number; groupId: number; tag: bigint } + + const parents = createCollection( + mockSyncCollectionOptions({ + id: `bigint-context-parents`, + getKey: (parent) => parent.id, + initialData: [{ id: 1, groupId: 1, tag: 1n }], + }), + ) + const children = createCollection( + mockSyncCollectionOptions({ + id: `bigint-context-children`, + getKey: (child) => child.id, + initialData: [ + { id: 1, groupId: 1, tag: 1n, metadataId: 101 }, + { id: `1`, groupId: 1, tag: 1n, metadataId: 102 }, + ], + }), + ) + const metadataRows = createCollection( + mockSyncCollectionOptions({ + id: `bigint-context-metadata`, + getKey: (row) => row.id, + initialData: [ + { id: 101, groupId: 1, tag: 1n }, + { id: 102, groupId: 1, tag: 1n }, + ], + }), + ) + + const collection = createLiveQueryCollection((q) => + q.from({ parent: parents }).select(({ parent }) => ({ + id: parent.id, + children: materialize( + q + .from({ child: children }) + .where(({ child }) => eq(child.groupId, parent.groupId)) + .where(({ child }) => eq(child.tag, parent.tag)) + .select(({ child }) => ({ id: child.id })), + ), + joinedChildren: materialize( + q + .from({ joinedChild: children }) + .join( + { metadata: metadataRows }, + ({ joinedChild, metadata }) => + eq(joinedChild.metadataId, metadata.id), + `inner`, + ) + .where(({ metadata }) => eq(metadata.groupId, parent.groupId)) + .where(({ metadata }) => eq(metadata.tag, parent.tag)) + .select(({ joinedChild }) => ({ id: joinedChild.id })), + ), + })), + ) + + await collection.preload() + const [row] = toTree(collection) + const typedIds = (values: Array<{ id: number | string }>) => + values.map(({ id }) => `${typeof id}:${id}`).sort() + expect(row!.id).toBe(1) + expect(typedIds(row!.children)).toEqual([`number:1`, `string:1`]) + expect(typedIds(row!.joinedChildren)).toEqual([`number:1`, `string:1`]) + }) + + it(`keeps relation-local identity through Collection and nested includes`, async () => { + type Parent = { id: number; groupId: number; minimumScore: number } + type Child = { id: number; groupId: number; score: number } + type Note = { id: number; childId: number } + + const parents = createCollection( + mockSyncCollectionOptions({ + id: `nested-context-parents`, + getKey: (parent) => parent.id, + initialData: [ + { id: 1, groupId: 1, minimumScore: 10 }, + { id: 2, groupId: 1, minimumScore: 20 }, + ], + }), + ) + const children = createCollection( + mockSyncCollectionOptions({ + id: `nested-context-children`, + getKey: (child) => child.id, + initialData: [{ id: 7, groupId: 1, score: 30 }], + }), + ) + const notes = createCollection( + mockSyncCollectionOptions({ + id: `nested-context-notes`, + getKey: (note) => note.id, + initialData: [{ id: 70, childId: 7 }], + }), + ) + + const collection = createLiveQueryCollection((q) => + q.from({ parent: parents }).select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children }) + .where(({ child }) => eq(child.groupId, parent.groupId)) + .where(({ child }) => gte(child.score, parent.minimumScore)) + .select(({ child }) => ({ + id: child.id, + notes: materialize( + q + .from({ note: notes }) + .where(({ note }) => eq(note.childId, child.id)) + .select(({ note }) => ({ id: note.id })), + ), + })), + })), + ) + + await collection.preload() + + const firstChildren = collection.get(1)!.children + const secondChildren = collection.get(2)!.children + expect([...firstChildren.keys()]).toEqual([7]) + expect([...secondChildren.keys()]).toEqual([7]) + expect(stripVirtualProps(firstChildren.get(7))).toEqual({ + id: 7, + notes: [{ id: 70 }], + }) + expect(stripVirtualProps(secondChildren.get(7))).toEqual({ + id: 7, + notes: [{ id: 70 }], + }) + + children.utils.begin() + children.utils.write({ + type: `update`, + previousValue: { id: 7, groupId: 1, score: 30 }, + value: { id: 7, groupId: 1, score: 15 }, + }) + children.utils.commit() + + expect([...firstChildren.keys()]).toEqual([7]) + expect([...secondChildren.keys()]).toEqual([]) + expect(stripVirtualProps(firstChildren.get(7))).toEqual({ + id: 7, + notes: [{ id: 70 }], + }) + }) + it(`shared correlation key with parent filter + orderBy + limit`, async () => { // Regression: grouped ordering for limit must use the composite routing // key, not the raw correlation key. Otherwise two parents that share the diff --git a/packages/db/tests/query/validate-aliases.test.ts b/packages/db/tests/query/validate-aliases.test.ts index e09635de9b..bdfa2955fd 100644 --- a/packages/db/tests/query/validate-aliases.test.ts +++ b/packages/db/tests/query/validate-aliases.test.ts @@ -81,6 +81,23 @@ describe(`Alias validation in subqueries`, () => { }).toThrow(/Subquery uses alias "vote"/) }) + test(`should throw DuplicateAliasInSubqueryError when an include reuses a parent alias`, () => { + expect(() => { + createLiveQueryCollection({ + startSync: true, + query: (q) => + q.from({ lock: locksCollection }).select(({ lock: parentLock }) => ({ + _id: parentLock._id, + votes: q + .from({ lock: votesCollection }) + .where(({ lock: childLock }) => + eq(childLock.lockId, parentLock._id), + ), + })), + }) + }).toThrow(/Subquery uses alias "lock"/) + }) + test(`should allow subqueries when all collection aliases are unique`, () => { const query = createLiveQueryCollection({ startSync: true, From 55fa747cd576891f5c1b50f92f28bc8aed5de5bb Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 17 Aug 2026 15:46:43 +0100 Subject: [PATCH 05/21] test(db): expose materialization boundary gaps --- packages/db/tests/effect.test.ts | 40 + .../db/tests/query/compiler/basic.test.ts | 28 + ...ncludes-collection-oracle.property.test.ts | 731 ++++++++++++++++++ .../query/includes-temporal-oracle.test.ts | 36 + packages/db/tests/query/join-subquery.test.ts | 59 ++ packages/db/tests/query/subset-dedupe.test.ts | 30 + .../tests/load-hooks.test.ts | 7 + 7 files changed, 931 insertions(+) create mode 100644 packages/db/tests/query/includes-collection-oracle.property.test.ts diff --git a/packages/db/tests/effect.test.ts b/packages/db/tests/effect.test.ts index 5c557087c8..193ece5819 100644 --- a/packages/db/tests/effect.test.ts +++ b/packages/db/tests/effect.test.ts @@ -1437,6 +1437,46 @@ describe(`createEffect`, () => { }) describe(`source error handling`, () => { + it(`reports a rejected lazy subset load and disposes the effect`, async () => { + const users = createUsersCollection([sampleUsers[0]!]) + const issues = createCollection({ + id: `effect-rejected-lazy-issues`, + getKey: (issue) => issue.id, + syncMode: `on-demand`, + sync: { + sync: () => ({ + loadSubset: () => Promise.reject(new Error(`lazy load failed`)), + }), + }, + }) + const sourceErrors: Array = [] + const effect = createEffect({ + query: (q) => + q + .from({ user: users }) + .leftJoin({ issue: issues }, ({ user, issue }) => + eq(user.id, issue.userId), + ) + .select(({ user, issue }) => ({ + id: user.id, + issueId: issue.id, + })), + onBatch: () => {}, + onSourceError: (error) => sourceErrors.push(error), + }) + + try { + await flushPromises() + expect(sourceErrors).toEqual([ + expect.objectContaining({ message: `lazy load failed` }), + ]) + expect(effect.disposed).toBe(true) + } finally { + await effect.dispose() + await Promise.all([users.cleanup(), issues.cleanup()]) + } + }) + it(`should auto-dispose when source collection is cleaned up`, async () => { const users = createUsersCollection() const events: Array> = [] diff --git a/packages/db/tests/query/compiler/basic.test.ts b/packages/db/tests/query/compiler/basic.test.ts index 66b7a15149..2ff6b56adb 100644 --- a/packages/db/tests/query/compiler/basic.test.ts +++ b/packages/db/tests/query/compiler/basic.test.ts @@ -1,6 +1,7 @@ import { describe, expect, test } from 'vitest' import { D2, MultiSet, output } from '@tanstack/db-ivm' import { compileQuery } from '../../../src/query/compiler/index.js' +import { materializeCompilation } from '../../../src/query/live/materialized-pipeline.js' import { CollectionRef, Func, PropRef, Value } from '../../../src/query/ir.js' import type { QueryIR } from '../../../src/query/ir.js' import type { CollectionImpl } from '../../../src/collection/index.js' @@ -30,6 +31,33 @@ const sampleUsers: Array = [ describe(`Query2 Compiler`, () => { describe(`Basic Compilation`, () => { + test(`queries without includes keep their compiled pipeline`, () => { + const usersCollection = { id: `users` } as CollectionImpl + const query: QueryIR = { + from: new CollectionRef(usersCollection, `users`), + } + const graph = new D2() + const input = graph.newInput<[number, User]>() + const compilation = compileQuery( + query, + { users: input }, + { users: usersCollection }, + {}, + {}, + new Set(), + {}, + () => {}, + ) + + const materialized = materializeCompilation( + compilation, + (user: User) => user.id, + ) + + expect(materialized.pipeline).toBe(compilation.pipeline) + expect(materialized.facades).toEqual([]) + }) + test(`compiles a simple FROM query`, () => { // Create a mock collection const usersCollection = { diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts new file mode 100644 index 0000000000..4e67a4298f --- /dev/null +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -0,0 +1,731 @@ +import { fc, test as fcTest } from '@fast-check/vitest' +import { describe, expect } from 'vitest' +import { createCollection } from '../../src/collection/index.js' +import { + createLiveQueryCollection, + eq, + materialize, +} from '../../src/query/index.js' +import { expectAssertionFailure } from '../expected-failure.js' +import { runTrace } from '../trace-runner.js' +import { + flushPromises, + mockSyncCollectionOptions, + withExpectedRejection, +} from '../utils.js' +import type { Collection } from '../../src/collection/index.js' +import type { ChangeMessage } from '../../src/types.js' +import type { TraceDriver, TraceProjection } from '../trace-runner.js' + +type ParentRow = { + id: number + group: number +} + +type ChildRow = { + id: number + parentGroup: number + value: number +} + +type CollectionAction = + | { type: `putParent`; row: ParentRow } + | { type: `deleteParent`; id: number } + | { type: `putChild`; row: ChildRow } + | { type: `deleteChild`; id: number } + +type ProjectedParent = { + id: number + group: number + childrenReady: boolean + children: Array +} + +type CollectionObservation = { + rows: Array + publications: Array> +} + +type ControlledCollection = { + collection: Collection + write: (type: `insert` | `update` | `delete`, value: T) => void + writeBatch: ( + changes: ReadonlyArray<{ + type: `insert` | `update` | `delete` + value: T + }>, + ) => void + resolveSync: () => void + rejectSync: (error: Error) => void +} + +type CollectionContext = { + parents: ControlledCollection + children: ControlledCollection + live: ReturnType + model: { + parents: Map + children: Map + } + publications: Array> + subscription?: { unsubscribe: () => void } +} + +let nextCollectionOracleId = 0 + +function createControlledCollection( + name: string, + initialData: ReadonlyArray, +): ControlledCollection { + const options = mockSyncCollectionOptions({ + id: `${name}-${nextCollectionOracleId++}`, + getKey: (row) => row.id, + initialData: initialData.map((row) => ({ ...row })), + }) + options.sync.rowUpdateMode = `full` + const collection = createCollection(options) + const writeBatch: ControlledCollection[`writeBatch`] = (changes) => { + options.utils.begin() + for (const change of changes) { + options.utils.write({ + type: change.type, + value: { ...change.value }, + }) + } + options.utils.commit() + } + + return { + collection, + write(type, value) { + writeBatch([{ type, value }]) + }, + writeBatch, + resolveSync: options.utils.resolveSync, + rejectSync: options.utils.rejectSync, + } +} + +function createCollectionQuery( + parents: Collection, + children: Collection, +) { + return createLiveQueryCollection((q) => + q + .from({ parent: parents }) + .orderBy(({ parent }) => parent.id) + .select(({ parent }) => ({ + id: parent.id, + group: parent.group, + children: q + .from({ child: children }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ child }) => child.id) + .select(({ child }) => ({ + id: child.id, + parentGroup: child.parentGroup, + value: child.value, + })), + })), + ) +} + +type IncludedChildCollection = ReturnType< + typeof createCollectionQuery +>[`toArray`][number][`children`] + +function projectLive( + live: ReturnType, +): Array { + return [...live.values()].map((parent) => ({ + id: parent.id, + group: parent.group, + childrenReady: parent.children.isReady(), + children: [...parent.children.values()] + .map(({ id, parentGroup, value }) => ({ id, parentGroup, value })) + .sort((left, right) => left.id - right.id), + })) +} + +function recompute(context: CollectionContext): Array { + return [...context.model.parents.values()] + .sort((left, right) => left.id - right.id) + .map((parent) => ({ + ...parent, + childrenReady: true, + children: [...context.model.children.values()] + .filter((child) => child.parentGroup === parent.group) + .sort((left, right) => left.id - right.id) + .map((child) => ({ ...child })), + })) +} + +function createCollectionDriver( + initialParents: ReadonlyArray, + initialChildren: ReadonlyArray, +): TraceDriver { + return { + setup() { + const parents = createControlledCollection( + `collection-oracle-parents`, + initialParents, + ) + const children = createControlledCollection( + `collection-oracle-children`, + initialChildren, + ) + return { + parents, + children, + live: createCollectionQuery(parents.collection, children.collection), + model: { + parents: new Map(initialParents.map((row) => [row.id, { ...row }])), + children: new Map(initialChildren.map((row) => [row.id, { ...row }])), + }, + publications: [], + } + }, + async start(context) { + await context.live.preload() + context.subscription = context.live.subscribeChanges( + () => context.publications.push(projectLive(context.live)), + { includeInitialState: false }, + ) + }, + apply(action, context) { + context.publications = [] + switch (action.type) { + case `putParent`: { + const type = context.model.parents.has(action.row.id) + ? `update` + : `insert` + context.model.parents.set(action.row.id, { ...action.row }) + context.parents.write(type, action.row) + return + } + case `deleteParent`: { + const previous = context.model.parents.get(action.id) + if (!previous) return + context.model.parents.delete(action.id) + context.parents.write(`delete`, previous) + return + } + case `putChild`: { + const type = context.model.children.has(action.row.id) + ? `update` + : `insert` + context.model.children.set(action.row.id, { ...action.row }) + context.children.write(type, action.row) + return + } + case `deleteChild`: { + const previous = context.model.children.get(action.id) + if (!previous) return + context.model.children.delete(action.id) + context.children.write(`delete`, previous) + } + } + }, + async cleanup(context) { + context.subscription?.unsubscribe() + await Promise.all([ + context.live.cleanup(), + context.parents.collection.cleanup(), + context.children.collection.cleanup(), + ]) + }, + } +} + +const collectionProjection: TraceProjection< + CollectionContext, + CollectionObservation +> = { + observe: (context) => ({ + rows: projectLive(context.live), + publications: context.publications, + }), + recompute: (context) => { + const rows = recompute(context) + return { + rows, + publications: context.publications.map(() => rows), + } + }, + assertEqual(observed, expected) { + expect(observed).toEqual(expected) + return undefined + }, +} + +const actionArbitrary: fc.Arbitrary = fc.oneof( + fc.record({ + type: fc.constant(`putParent` as const), + row: fc.record({ + id: fc.integer({ min: 0, max: 3 }), + group: fc.integer({ min: -3, max: 3 }), + }), + }), + fc.record({ + type: fc.constant(`deleteParent` as const), + id: fc.integer({ min: 0, max: 3 }), + }), + fc.record({ + type: fc.constant(`putChild` as const), + row: fc.record({ + id: fc.integer({ min: 10, max: 14 }), + parentGroup: fc.integer({ min: -3, max: 3 }), + value: fc.integer({ min: -5, max: 5 }), + }), + }), + fc.record({ + type: fc.constant(`deleteChild` as const), + id: fc.integer({ min: 10, max: 14 }), + }), +) + +const collectionScenarioArbitrary = fc.record({ + parentGroup: fc.integer({ min: -3, max: 3 }), + childValue: fc.integer({ min: -5, max: 5 }), + actions: fc.array(actionArbitrary, { minLength: 1, maxLength: 16 }), +}) + +describe(`Collection-valued includes oracle`, () => { + fcTest.prop([collectionScenarioArbitrary], { numRuns: 30 })( + `matches recomputation and publishes coherent snapshots across generated relationship histories`, + ({ parentGroup, childValue, actions }) => + runTrace({ + steps: actions, + driver: createCollectionDriver( + [{ id: 0, group: parentGroup }], + [{ id: 10, parentGroup, value: childValue }], + ), + projection: collectionProjection, + }), + ) + + fcTest(`replays a dormant bucket when its first parent route activates`, () => + runTrace({ + steps: [{ type: `putParent`, row: { id: 1, group: 7 } }], + driver: createCollectionDriver( + [], + [{ id: 10, parentGroup: 7, value: 1 }], + ), + projection: collectionProjection, + }), + ) + + fcTest( + `discovered trace: retiring a route cleans up its facade`, + async () => { + let retiredFacade: IncludedChildCollection | undefined + const driver = createCollectionDriver( + [{ id: 1, group: 1 }], + [{ id: 10, parentGroup: 1, value: 1 }], + ) + const lifecycleDriver: TraceDriver = + { + ...driver, + apply(action, context, checkpoint) { + retiredFacade ??= context.live.get(1)?.children + return driver.apply(action, context, checkpoint) + }, + } + const projection: TraceProjection< + CollectionContext, + { rows: Array; retiredStatus: string | undefined } + > = { + observe: (context) => ({ + rows: projectLive(context.live), + retiredStatus: retiredFacade?.status, + }), + recompute: (context) => ({ + rows: recompute(context), + retiredStatus: + context.model.parents.size === 0 + ? `cleaned-up` + : retiredFacade?.status, + }), + assertEqual(observed, expected) { + expect(observed).toEqual(expected) + return undefined + }, + } + + await expectAssertionFailure( + () => + runTrace({ + steps: [{ type: `deleteParent`, id: 1 }], + driver: lifecycleDriver, + projection, + }), + { + checkpoint: 1, + classify: ({ actual, expected }) => + JSON.stringify(actual) === + JSON.stringify({ rows: [], retiredStatus: `ready` }) && + JSON.stringify(expected) === + JSON.stringify({ rows: [], retiredStatus: `cleaned-up` }), + }, + )() + }, + ) + + fcTest( + `discovered trace: a delete event preserves the published facade identity`, + async () => { + let publishedFacade: IncludedChildCollection | undefined + let previousFacadeMatched = true + const driver = createCollectionDriver( + [{ id: 1, group: 1 }], + [{ id: 10, parentGroup: 1, value: 1 }], + ) + const eventDriver: TraceDriver = { + ...driver, + async start(context) { + await driver.start?.(context) + publishedFacade = context.live.get(1)?.children + context.subscription = context.live.subscribeChanges( + (changes: Array>) => { + for (const change of changes) { + if (change.type === `delete`) { + previousFacadeMatched = + change.previousValue?.children === publishedFacade + } + } + }, + { includeInitialState: false }, + ) + }, + } + const projection: TraceProjection< + CollectionContext, + { previousFacadeMatched: boolean } + > = { + observe: () => ({ previousFacadeMatched }), + recompute: () => ({ previousFacadeMatched: true }), + assertEqual(observed, expected) { + expect(observed).toEqual(expected) + return undefined + }, + } + + await expectAssertionFailure( + () => + runTrace({ + steps: [{ type: `deleteParent`, id: 1 }], + driver: eventDriver, + projection, + }), + { + checkpoint: 1, + classify: ({ actual, expected }) => + JSON.stringify(actual) === + JSON.stringify({ previousFacadeMatched: false }) && + JSON.stringify(expected) === + JSON.stringify({ previousFacadeMatched: true }), + }, + )() + }, + ) + + fcTest( + `discovered trace: facade public keys survive row cloning`, + async () => { + const driver = createCollectionDriver( + [{ id: 1, group: 1 }], + [{ id: 10, parentGroup: 1, value: 1 }], + ) + const projection: TraceProjection< + CollectionContext, + { clonedKey: unknown }, + { clonedKey: number } + > = { + observe(context) { + const facade = context.live.get(1)!.children + const row = facade.get(10)! + return { clonedKey: facade.getKeyFromItem({ ...row }) } + }, + recompute: () => ({ clonedKey: 10 }), + assertEqual(observed, expected) { + expect(observed).toEqual(expected) + return undefined + }, + } + + await expectAssertionFailure( + () => runTrace({ steps: [], driver, projection }), + { + checkpoint: 0, + classify: ({ actual, expected }) => + JSON.stringify(actual) === + JSON.stringify({ clonedKey: undefined }) && + JSON.stringify(expected) === JSON.stringify({ clonedKey: 10 }), + }, + )() + }, + ) + + fcTest( + `facade application failure leaves no partial state or publication`, + async () => { + const driver = createCollectionDriver( + [{ id: 1, group: 1 }], + [ + { id: 10, parentGroup: 1, value: 1 }, + { id: 20, parentGroup: 1, value: 2 }, + ], + ) + const context = await driver.setup() + await driver.start?.(context) + const facade = context.live.get(1)!.children + const changes: Array = [] + const subscription = facade.subscribeChanges( + (batch) => changes.push(...batch), + { includeInitialState: false }, + ) + const internalConfig = ( + facade._state as unknown as { + config: typeof facade.config + } + ).config + const originalGetKey = internalConfig.getKey + internalConfig.getKey = (row) => { + if (row.id === 20) throw new Error(`facade key failed`) + return originalGetKey(row) + } + + try { + expect(() => + context.children.writeBatch([ + { + type: `update`, + value: { id: 10, parentGroup: 1, value: 10 }, + }, + { + type: `update`, + value: { id: 20, parentGroup: 1, value: 20 }, + }, + ]), + ).toThrow(`facade key failed`) + expect(projectLive(context.live)).toEqual([ + { + id: 1, + group: 1, + childrenReady: true, + children: [ + { id: 10, parentGroup: 1, value: 1 }, + { id: 20, parentGroup: 1, value: 2 }, + ], + }, + ]) + expect(changes).toEqual([]) + } finally { + internalConfig.getKey = originalGetKey + subscription.unsubscribe() + await driver.cleanup(context) + } + }, + ) + + fcTest(`facade events observe the matching root publication`, async () => { + const driver = createCollectionDriver( + [{ id: 1, group: 1 }], + [ + { id: 10, parentGroup: 1, value: 1 }, + { id: 20, parentGroup: 2, value: 2 }, + ], + ) + const context = await driver.setup() + await driver.start?.(context) + const oldFacade = context.live.get(1)!.children + const callbackSnapshots: Array<{ + group: number | undefined + usesOldFacade: boolean + rows: Array + }> = [] + const subscription = oldFacade.subscribeChanges( + () => { + const current = context.live.get(1) + callbackSnapshots.push({ + group: current?.group, + usesOldFacade: current?.children === oldFacade, + rows: current + ? [...current.children.keys()].filter( + (key): key is number => typeof key === `number`, + ) + : [], + }) + }, + { includeInitialState: false }, + ) + + try { + context.parents.write(`update`, { id: 1, group: 2 }) + expect(callbackSnapshots).toEqual([ + { group: 2, usesOldFacade: false, rows: [20] }, + ]) + } finally { + subscription.unsubscribe() + await driver.cleanup(context) + } + }) + + fcTest( + `shared facades remain active until their last parent departs`, + async () => { + const driver = createCollectionDriver( + [ + { id: 1, group: 1 }, + { id: 2, group: 1 }, + ], + [{ id: 10, parentGroup: 1, value: 1 }], + ) + const context = await driver.setup() + await driver.start?.(context) + const sharedFacade = context.live.get(1)!.children + + try { + expect(context.live.get(2)!.children).toBe(sharedFacade) + context.parents.write(`delete`, { id: 1, group: 1 }) + expect(context.live.get(2)!.children).toBe(sharedFacade) + expect([...sharedFacade.keys()]).toEqual([10]) + expect(sharedFacade.status).toBe(`ready`) + } finally { + await driver.cleanup(context) + } + }, + ) + + fcTest( + `discovered trace: a matched null singleton remains null`, + expectAssertionFailure( + async () => { + type NullableChild = { id: number; parentGroup: number; value: null } + const parents = createControlledCollection(`nullable-oracle-parents`, [ + { id: 1, group: 1 }, + ]) + const children = createControlledCollection( + `nullable-oracle-children`, + [{ id: 10, parentGroup: 1, value: null }], + ) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => ({ + id: parent.id, + value: materialize( + q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .select(({ child }) => child.value) + .findOne(), + ), + })), + ) + + try { + await live.preload() + expect(live.get(1)!.value).toBeNull() + } finally { + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }, + { message: `expected undefined to be null` }, + ), + ) + + fcTest.prop( + [ + fc.record({ + group: fc.integer({ min: -10, max: 10 }), + insertedId: fc.integer({ min: 10, max: 100 }), + confirmedId: fc.integer({ min: 101, max: 200 }), + value: fc.integer({ min: -10, max: 10 }), + }), + ], + { numRuns: 20 }, + )( + `matches recomputation through optimistic child insert and delete confirmation and rollback`, + async ({ group, insertedId, confirmedId, value }) => { + type OptimisticAction = + | { type: `insert`; row: ChildRow; settlement: `confirm` | `rollback` } + | { type: `delete`; id: number; settlement: `confirm` | `rollback` } + const base = createCollectionDriver([{ id: 1, group }], []) + const driver: TraceDriver = { + ...base, + async apply(action, context, checkpoint) { + if (action.type === `insert`) { + const transaction = context.children.collection.insert({ + ...action.row, + }) + context.model.children.set(action.row.id, { ...action.row }) + checkpoint() + if (action.settlement === `confirm`) { + context.children.write(`insert`, action.row) + context.children.resolveSync() + await transaction.isPersisted.promise + } else { + context.model.children.delete(action.row.id) + const message = `rollback insert` + const persisted = transaction.isPersisted.promise.catch( + () => undefined, + ) + await withExpectedRejection(message, async () => { + context.children.rejectSync(new Error(message)) + await persisted + await flushPromises() + }) + } + return + } + + const previous = context.model.children.get(action.id) + if (!previous) throw new Error(`Missing optimistic delete row`) + const transaction = context.children.collection.delete(action.id) + context.model.children.delete(action.id) + checkpoint() + if (action.settlement === `confirm`) { + context.children.write(`delete`, previous) + context.children.resolveSync() + await transaction.isPersisted.promise + } else { + context.model.children.set(action.id, previous) + const message = `rollback delete` + const persisted = transaction.isPersisted.promise.catch( + () => undefined, + ) + await withExpectedRejection(message, async () => { + context.children.rejectSync(new Error(message)) + await persisted + await flushPromises() + }) + } + }, + } + const rolledBack = { + id: insertedId, + parentGroup: group, + value, + } + const confirmed = { + id: confirmedId, + parentGroup: group, + value: value + 1, + } + + await runTrace({ + steps: [ + { type: `insert`, row: rolledBack, settlement: `rollback` }, + { type: `insert`, row: confirmed, settlement: `confirm` }, + { type: `delete`, id: confirmedId, settlement: `rollback` }, + { type: `delete`, id: confirmedId, settlement: `confirm` }, + ], + driver, + projection: collectionProjection, + }) + }, + ) +}) diff --git a/packages/db/tests/query/includes-temporal-oracle.test.ts b/packages/db/tests/query/includes-temporal-oracle.test.ts index 38e7b27b01..97b1fa8edc 100644 --- a/packages/db/tests/query/includes-temporal-oracle.test.ts +++ b/packages/db/tests/query/includes-temporal-oracle.test.ts @@ -706,6 +706,37 @@ async function expectRejectedDemandEntersError(): Promise { } } +async function expectSynchronousEmptyDemandIsReady(): Promise { + const posts = createMutablePosts([ + { id: 1, authorId: `selected`, title: `one` }, + ]) + let loadCount = 0 + const comments = createCollection({ + id: nextCollectionId(`temporal-empty-comments`), + getKey: (comment) => comment.id, + syncMode: `on-demand`, + sync: { + sync: () => ({ + loadSubset: () => { + loadCount += 1 + return true + }, + }), + }, + }) + const live = createPostsWithCommentsLive(posts.collection, comments) + + try { + await live.preload() + expect(loadCount).toBe(1) + expect(live.isReady()).toBe(true) + expect(live.get(1)?.comments).toEqual([]) + } finally { + await live.cleanup() + await Promise.all([posts.collection.cleanup(), comments.cleanup()]) + } +} + async function expectPartialShrinkRetainsCoverage(): Promise { const firstPost = { id: 1, authorId: `selected`, title: `one` } const secondPost = { id: 2, authorId: `selected`, title: `two` } @@ -1040,6 +1071,11 @@ describe(`includes temporal oracle`, () => { it(`rejected demand enters error`, expectRejectedDemandEntersError) + it( + `a synchronous empty demand can establish ready coverage`, + expectSynchronousEmptyDemandIsReady, + ) + it( `partially shrinking demand retains established coverage`, expectPartialShrinkRetainsCoverage, diff --git a/packages/db/tests/query/join-subquery.test.ts b/packages/db/tests/query/join-subquery.test.ts index eac49d6491..3468dedd3c 100644 --- a/packages/db/tests/query/join-subquery.test.ts +++ b/packages/db/tests/query/join-subquery.test.ts @@ -953,3 +953,62 @@ describe(`Lazy join: subquery whose join key resolves to an indexed collection`, ) }) }) + +describe(`Lazy join without a usable index`, () => { + test(`warns when demand falls back to a full local scan`, async () => { + type Team = { id: string } + type Member = { id: string; teamId: string } + const teams = createCollection( + mockSyncCollectionOptions({ + id: `lazy-fallback-teams`, + getKey: (team) => team.id, + initialData: [{ id: `t1` }], + }), + ) + const members = createCollection( + mockSyncCollectionOptions({ + id: `lazy-fallback-members`, + getKey: (member) => member.id, + initialData: [{ id: `m1`, teamId: `t1` }], + syncMode: `on-demand`, + autoIndex: `off`, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + write({ type: `insert`, value: { id: `m1`, teamId: `t1` } }) + commit() + markReady() + return { loadSubset: () => true } + }, + }, + }), + ) + const warnSpy = vi.spyOn(console, `warn`).mockImplementation(() => {}) + const live = createLiveQueryCollection((q) => + q + .from({ team: teams }) + .leftJoin({ member: members }, ({ team, member }) => + eq(team.id, member.teamId), + ) + .select(({ team, member }) => ({ + id: team.id, + memberId: member.id, + })), + ) + + try { + await live.preload() + expect(live.toArray.map(stripVirtualProps)).toEqual([ + { id: `t1`, memberId: `m1` }, + ]) + expect(warnSpy).toHaveBeenCalledWith( + expect.stringContaining( + `[lazy-fallback-members] Join requires an index on "teamId"`, + ), + ) + } finally { + warnSpy.mockRestore() + await Promise.all([live.cleanup(), teams.cleanup(), members.cleanup()]) + } + }) +}) diff --git a/packages/db/tests/query/subset-dedupe.test.ts b/packages/db/tests/query/subset-dedupe.test.ts index 081d892c4d..c11c3ac188 100644 --- a/packages/db/tests/query/subset-dedupe.test.ts +++ b/packages/db/tests/query/subset-dedupe.test.ts @@ -45,6 +45,36 @@ function not(expression: BasicExpression): Func { } describe(`createDeduplicatedLoadSubset`, () => { + it(`keeps independently abortable in-flight requests isolated`, async () => { + const releases: Array<() => void> = [] + let callCount = 0 + const deduplicated = new DeduplicatedLoadSubset({ + loadSubset: () => { + callCount += 1 + return new Promise((resolve) => releases.push(resolve)) + }, + }) + const first = new AbortController() + const second = new AbortController() + const where = gt(ref(`age`), val(10)) + + const firstLoad = deduplicated.loadSubset({ where, signal: first.signal }) + const secondLoad = deduplicated.loadSubset({ where, signal: second.signal }) + expect(callCount).toBe(2) + + first.abort() + for (const release of releases) release() + await Promise.all([firstLoad, secondLoad]) + + expect( + deduplicated.loadSubset({ + where, + signal: new AbortController().signal, + }), + ).toBe(true) + expect(callCount).toBe(2) + }) + it(`should call underlying loadSubset on first call`, async () => { let callCount = 0 const mockLoadSubset = () => { diff --git a/packages/powersync-db-collection/tests/load-hooks.test.ts b/packages/powersync-db-collection/tests/load-hooks.test.ts index 6a3f5cb1e0..9c7a8b4f7d 100644 --- a/packages/powersync-db-collection/tests/load-hooks.test.ts +++ b/packages/powersync-db-collection/tests/load-hooks.test.ts @@ -81,6 +81,8 @@ describe(`Sync Streams`, () => { const onLoadSubsetMock = vi.fn() const onUnloadSubsetMock = vi.fn() + const unloadedRequests: Array = [] + let nextRequest = 0 const collection = createCollection( powerSyncCollectionOptions({ @@ -89,9 +91,12 @@ describe(`Sync Streams`, () => { syncMode: `on-demand`, onLoadSubset: () => { onLoadSubsetMock() + nextRequest += 1 + const request = nextRequest return () => { onUnloadSubsetMock() + unloadedRequests.push(request) } }, }), @@ -158,6 +163,7 @@ describe(`Sync Streams`, () => { await vi.waitFor( () => { expect(onUnloadSubsetMock).toHaveBeenCalledTimes(1) + expect(unloadedRequests).toEqual([1]) }, { timeout: 2000 }, ) @@ -168,6 +174,7 @@ describe(`Sync Streams`, () => { await vi.waitFor( () => { expect(onUnloadSubsetMock).toHaveBeenCalledTimes(2) + expect(unloadedRequests).toEqual([1, 2]) }, { timeout: 2000 }, ) From 1dd6429ffde28b71e9d07f82c20c36fc15a4242a Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 17 Aug 2026 16:04:02 +0100 Subject: [PATCH 06/21] test(db): cover limited facade activation --- packages/db/tests/query/includes.test.ts | 54 ++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/packages/db/tests/query/includes.test.ts b/packages/db/tests/query/includes.test.ts index 692646e210..6bd0eb7c93 100644 --- a/packages/db/tests/query/includes.test.ts +++ b/packages/db/tests/query/includes.test.ts @@ -1304,6 +1304,60 @@ describe(`includes subqueries`, () => { expect(childItems(collection.get(1)!.children)).toEqual([{ id: 10 }]) }) + it(`replays existing bucket rows when a parent enters a limited result`, async () => { + type Parent = { id: number; groupId: number } + type Child = { id: number; groupId: number } + + const parents = createCollection( + mockSyncCollectionOptions({ + id: `limited-late-route-parents`, + getKey: (parent) => parent.id, + initialData: [ + { id: 1, groupId: 1 }, + { id: 2, groupId: 2 }, + ], + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + }), + ) + const children = createCollection( + mockSyncCollectionOptions({ + id: `limited-late-route-children`, + getKey: (child) => child.id, + initialData: [ + { id: 10, groupId: 1 }, + { id: 20, groupId: 2 }, + ], + }), + ) + const collection = createLiveQueryCollection((q) => + q + .from({ parent: parents }) + .orderBy(({ parent }) => parent.id) + .limit(1) + .select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children }) + .where(({ child }) => eq(child.groupId, parent.groupId)) + .select(({ child }) => ({ id: child.id })), + })), + ) + + await collection.preload() + expect(childItems(collection.get(1)!.children)).toEqual([{ id: 10 }]) + + parents.utils.begin() + parents.utils.write({ + type: `delete`, + value: { id: 1, groupId: 1 }, + }) + parents.utils.commit() + + expect(collection.get(1)).toBeUndefined() + expect(childItems(collection.get(2)!.children)).toEqual([{ id: 20 }]) + }) + it(`does not publish facade changes when root publication fails`, async () => { type Parent = { id: number; groupId: number } type Child = { id: number; groupId: number } From f053f961cd108e9919cdffa33a79b706a6ef8b54 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 17 Aug 2026 16:30:38 +0100 Subject: [PATCH 07/21] fix(db): close materialization boundary gaps --- packages/db/src/collection/subscription.ts | 19 +- packages/db/src/query/compiler/joins.ts | 4 +- packages/db/src/query/effect.ts | 9 +- packages/db/src/query/live/ARCHITECTURE.md | 31 +- .../src/query/live/bucket-facade-adapter.ts | 52 ++- .../query/live/collection-config-builder.ts | 6 +- .../src/query/live/materialized-pipeline.ts | 14 +- .../query/live/subset-demand-controller.ts | 22 +- packages/db/tests/effect.test.ts | 3 + ...ncludes-collection-oracle.property.test.ts | 355 ++++++++---------- .../powersync-db-collection/src/powersync.ts | 18 +- 11 files changed, 323 insertions(+), 210 deletions(-) diff --git a/packages/db/src/collection/subscription.ts b/packages/db/src/collection/subscription.ts index 1a92568940..87f3bbc519 100644 --- a/packages/db/src/collection/subscription.ts +++ b/packages/db/src/collection/subscription.ts @@ -31,6 +31,8 @@ type RequestSnapshotOptions = { limit?: number /** Callback that receives the raw loadSubset result for external tracking */ onLoadSubsetResult?: (result: Promise | true) => void + /** Called when the local snapshot must fall back from an index to a scan. */ + onUnoptimized?: () => void } type RequestLimitedSnapshotOptions = { @@ -400,7 +402,22 @@ export class CollectionSubscription } // Also load data immediately from the collection - const snapshot = this.collection.currentStateAsChanges(stateOpts) + let snapshot: Array> | void + if (opts?.onUnoptimized) { + snapshot = this.collection.currentStateAsChanges({ + ...stateOpts, + optimizedOnly: true, + }) + if (snapshot === undefined) { + opts.onUnoptimized() + snapshot = this.collection.currentStateAsChanges({ + ...stateOpts, + optimizedOnly: false, + }) + } + } else { + snapshot = this.collection.currentStateAsChanges(stateOpts) + } if (snapshot === undefined) { // Couldn't load from indexes diff --git a/packages/db/src/query/compiler/joins.ts b/packages/db/src/query/compiler/joins.ts index df87cfcfaa..6d0bee863e 100644 --- a/packages/db/src/query/compiler/joins.ts +++ b/packages/db/src/query/compiler/joins.ts @@ -42,6 +42,7 @@ import type { CollectionSubscription } from '../../collection/subscription.js' export type LazyDemandPlan = { id: string path: Array + collectionId: string initialKeys: Set } @@ -55,12 +56,13 @@ let nextLazyDemandPlanId = 0 export function registerLazyDemandPlan( callbacks: Record, - target: { sourceId: string; path: Array }, + target: { sourceId: string; path: Array; collection: Collection }, initialKeys: Set = new Set(), ): LazyDemandPlan { const plan: LazyDemandPlan = { id: `lazy-demand-${++nextLazyDemandPlanId}`, path: target.path, + collectionId: target.collection.id, initialKeys: new Set(initialKeys), } const state = (callbacks[target.sourceId] ??= {}) diff --git a/packages/db/src/query/effect.ts b/packages/db/src/query/effect.ts index 101e718d61..047991cd46 100644 --- a/packages/db/src/query/effect.ts +++ b/packages/db/src/query/effect.ts @@ -662,7 +662,14 @@ class EffectPipelineRunner { plan: LazyDemandPlan, keys: Set, ): void { - this.demand.setDemand(subscription, plan, keys) + const update = this.demand.setDemand(subscription, plan, keys) + if (update.ready instanceof Promise) { + void update.ready.catch((error: unknown) => { + this.onSourceError( + error instanceof Error ? error : new Error(String(error)), + ) + }) + } } /** diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 780bb5b1aa..fbfdea182c 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -76,6 +76,26 @@ and results move in opposite conceptual directions. The graph owns the data plane. A small adapter owns asynchronous demand. The normal Collection transaction boundary owns public publication. +## Concrete implementation map + +The relation and identity names in this document describe the graph's logical +model. They are not a second set of runtime objects, nor does every name need a +matching TypeScript type. The implementation maps this model onto existing D2 +operators and a few boundary adapters: + +| Architectural role | Concrete implementation | +| ------------------------------------- | ------------------------------------------------------------------------------------------------------ | +| Compile relation IDs and demand plans | `packages/db/src/query/compiler/index.ts`, `packages/db/src/query/compiler/joins.ts` | +| Reduce public keys and build routes | `packages/db/src/query/live/materialized-pipeline.ts` | +| Run the graph and publish root rows | `packages/db/src/query/live/collection-config-builder.ts` | +| Publish Collection-valued buckets | `packages/db/src/query/live/bucket-facade-adapter.ts` | +| Start and release asynchronous demand | `packages/db/src/query/live/subset-demand-controller.ts`, `packages/db/src/collection/subscription.ts` | + +Queries without includes keep the original compiled pipeline and do not pay +for facade state. The one exception is a joined query with a custom public-key +function: its possible duplicate contributors still pass through the keyed +reduction that enforces public-key congruence and multiplicity. + ## Identity Aliases are lexical query-language names. They are not runtime identities. The @@ -432,7 +452,9 @@ Correct relation state does not prove efficient work. When an applicable index exists, irrelevant correlated rows must not cause scans of unrelated rows or activate unrelated downstream routes. Relation rows, indexed keys, active demands, materialization cells, and public facades are the relevant space -units. Inline materialization must not create recursive Collection machinery. +units. Queries without includes retain their original pipeline unless a joined +custom-key query needs contributor reduction. Inline materialization must not +create recursive Collection machinery. ## Normative laws @@ -496,6 +518,7 @@ units. Inline materialization must not create recursive Collection machinery. | Demand, cancellation, and progressive timing | `packages/db/tests/query/includes-temporal-oracle.test.ts` | | Optimistic confirmation, rollback, and later reactivity | `packages/db/tests/query/includes-optimistic-oracle.property.test.ts` | | Coherent layered publication | `packages/db/tests/query/includes-publication-oracle.test.ts` | +| Collection facades, event coherence, and route activation | `packages/db/tests/query/includes-collection-oracle.property.test.ts` | | Correlated physical work | `packages/db/tests/query/includes-work-counter-oracle.test.ts` | | Query-db ownership | `packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts` | | Reachable nested shape | `packages/query-db-collection/tests/includes-work-counter-oracle.test.ts` | @@ -518,6 +541,6 @@ not own. - Never use a public Collection, emitted event, or materialized row as internal routing or contribution state. - Add a reduced oracle trace before adding any special lifecycle branch. -- Measure retained relation rows, active demands, and public facades so the - simpler architecture remains a space improvement as well as a correctness - improvement. +- Measure retained relation rows, active demands, and public facades. Preserve + the no-includes fast path and verify any claimed space improvement with those + counters. diff --git a/packages/db/src/query/live/bucket-facade-adapter.ts b/packages/db/src/query/live/bucket-facade-adapter.ts index 39b131e65c..9da66e59f6 100644 --- a/packages/db/src/query/live/bucket-facade-adapter.ts +++ b/packages/db/src/query/live/bucket-facade-adapter.ts @@ -58,6 +58,7 @@ export class BucketFacadeAdapter { private readonly pendingActivity = new Map>() private readonly activeBuckets = new Map>() private readonly entries = new Map>() + private readonly retiredEntries = new Map>() private resolvedValues = new WeakMap() constructor( @@ -123,6 +124,9 @@ export class BucketFacadeAdapter { const sync = entry.sync if (!sync || changes.size === 0) continue + for (const change of changes.values()) { + this.prepareChange(entry, change) + } deferPublication(entry) sync.begin() for (const change of changes.values()) { @@ -130,8 +134,6 @@ export class BucketFacadeAdapter { } sync.commit() } - this.pending.delete(compilation.edgeId) - for (const entry of newBaselines) entry.sync?.markReady() for (const [bucketKey, multiplicity] of activity ?? []) { @@ -139,12 +141,15 @@ export class BucketFacadeAdapter { active.delete(bucketKey) this.retireEntry(compilation.edgeId, bucketKey, deferPublication) } - this.pendingActivity.delete(compilation.edgeId) } } catch (error) { - for (const publication of publications) publication.publish() + this.restore(snapshot, deferredEntries) + this.retiredEntries.clear() + for (const publication of publications) publication.discard() throw error } + this.pending.clear() + this.pendingActivity.clear() let closed = false return { @@ -152,11 +157,13 @@ export class BucketFacadeAdapter { if (closed) return closed = true for (const publication of publications) publication.publish() + this.cleanupRetiredEntries() }, rollback: () => { if (closed) return closed = true this.restore(snapshot, deferredEntries) + this.retiredEntries.clear() for (const publication of publications) publication.discard() }, } @@ -173,6 +180,7 @@ export class BucketFacadeAdapter { } } this.entries.clear() + this.cleanupRetiredEntries() this.pending.clear() this.pendingActivity.clear() this.activeBuckets.clear() @@ -333,6 +341,12 @@ export class BucketFacadeAdapter { } byBucket!.delete(bucketKey) if (byBucket!.size === 0) this.entries.delete(edgeId) + let retired = this.retiredEntries.get(edgeId) + if (!retired) { + retired = new Map() + this.retiredEntries.set(edgeId, retired) + } + retired.set(bucketKey, entry) } private getEntry(edgeId: string, bucketKey: string): FacadeEntry { @@ -349,7 +363,13 @@ export class BucketFacadeAdapter { let sync: FacadeSync | undefined const collection = createCollection({ id: `__bucket-facade:${this.parentId}:${edgeId}:${bucketKey}`, - getKey: (row) => keys.get(row)!, + getKey: (row) => { + const key = keys.get(row) ?? row?.$key + if (typeof key !== `string` && typeof key !== `number`) { + throw new Error(`Bucket facade row has no public key`) + } + return key + }, compare: (left, right) => { const leftOrder = order.get(left) const rightOrder = order.get(right) @@ -420,6 +440,14 @@ export class BucketFacadeAdapter { if (hasOrderBy && orderChanged) sync.collection._markLayoutChange() } + /** Resolve and validate every public key before opening a sync transaction. */ + private prepareChange(entry: FacadeEntry, change: PendingRow): void { + const key = change.value.publicKey as string | number + const row = this.resolve(change.value.value) + entry.keys.set(row, key) + entry.collection.getKeyFromItem(row) + } + private resolveValue(value: unknown): unknown { if (value !== null && typeof value === `object`) { const cached = this.resolvedValues.get(value) @@ -427,7 +455,10 @@ export class BucketFacadeAdapter { } if (isBucketFacadeRef(value)) { const { edgeId, bucketKey } = value[BUCKET_FACADE_REF] - const facade = this.getEntry(edgeId, bucketKey).collection + const facade = + this.entries.get(edgeId)?.get(bucketKey)?.collection ?? + this.retiredEntries.get(edgeId)?.get(bucketKey)?.collection ?? + this.getEntry(edgeId, bucketKey).collection this.resolvedValues.set(value, facade) return facade } @@ -447,6 +478,15 @@ export class BucketFacadeAdapter { } return result } + + private cleanupRetiredEntries(): void { + for (const byBucket of this.retiredEntries.values()) { + for (const entry of byBucket.values()) { + void entry.collection.cleanup() + } + } + this.retiredEntries.clear() + } } function isBucketFacadeRef(value: unknown): value is BucketFacadeRef { diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index f395aa787f..9b432c474e 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -744,7 +744,11 @@ export class CollectionConfigBuilder< }, ) - const materialized = materializeCompilation(compilation, this.config.getKey) + const materialized = materializeCompilation( + compilation, + this.config.getKey, + this.hasJoins(this.query), + ) this.pipelineCache = materialized.pipeline this.sourceWhereClausesCache = compilation.sourceWhereClauses this.compiledAliasToCollectionId = compilation.aliasToCollectionId diff --git a/packages/db/src/query/live/materialized-pipeline.ts b/packages/db/src/query/live/materialized-pipeline.ts index c918a0f07b..01eef49195 100644 --- a/packages/db/src/query/live/materialized-pipeline.ts +++ b/packages/db/src/query/live/materialized-pipeline.ts @@ -88,7 +88,15 @@ let nextBucketFacadeEdgeId = 0 export function materializeCompilation( compilation: CompilationResult, getRootKey?: (row: any) => unknown, + reduceJoinedPublicKeys = false, ): MaterializedCompilation { + if ( + !compilation.includes?.length && + !(getRootKey && reduceJoinedPublicKeys) + ) { + return { pipeline: compilation.pipeline, facades: [] } + } + const built: BuiltRelations = new WeakMap() const materialized = materializeRelation( compilation, @@ -259,7 +267,7 @@ function attachInlineInclude( if (rows.length === 0) return [] rows.sort(compareBucketRows) - return [[materializeRows(rows, include), 1]] + return [[{ value: materializeRows(rows, include) }, 1]] }), ) const routedParents = parentPipeline.pipe( @@ -285,7 +293,9 @@ function attachInlineInclude( ] } const materialized = - bucketValue ?? emptyMaterializedValue(include.materialization) + bucketValue === null + ? emptyMaterializedValue(include.materialization) + : bucketValue.value return [ parent!.parentKey, [ diff --git a/packages/db/src/query/live/subset-demand-controller.ts b/packages/db/src/query/live/subset-demand-controller.ts index 0dee57e068..8ccda3d1f9 100644 --- a/packages/db/src/query/live/subset-demand-controller.ts +++ b/packages/db/src/query/live/subset-demand-controller.ts @@ -31,6 +31,7 @@ export type DemandUpdate = { */ export class SubsetDemandController { private readonly states = new Map() + private readonly warnedPlans = new Set() setDemand( subscription: CollectionSubscription, @@ -70,7 +71,11 @@ export class SubsetDemandController { [...nextKeys].filter(([key]) => !coveredKeys.has(key)), ) if (added.size > 0) { - segments.push(requestSegment(subscription, plan, added)) + segments.push( + requestSegment(subscription, plan, added, () => + this.warnUnoptimized(plan), + ), + ) } if (nextKeys.size === 0) { @@ -98,6 +103,19 @@ export class SubsetDemandController { for (const segment of state.segments) segment.abortController.abort() } this.states.clear() + this.warnedPlans.clear() + } + + private warnUnoptimized(plan: LazyDemandPlan): void { + if (this.warnedPlans.has(plan.id)) return + this.warnedPlans.add(plan.id) + const path = plan.path.join(`.`) + console.warn( + `[TanStack DB]${plan.collectionId ? ` [${plan.collectionId}]` : ``} Join requires an index on "${path}" for efficient loading. ` + + `Falling back to scanning local data. ` + + `Consider creating an index on the collection with collection.createIndex((row) => row.${path}) ` + + `or enable auto-indexing with autoIndex: 'eager' and a defaultIndexType.`, + ) } } @@ -125,6 +143,7 @@ function requestSegment( subscription: CollectionSubscription, plan: LazyDemandPlan, keys: Map, + onUnoptimized: () => void, ): DemandSegment { const where = inArray(new PropRef(plan.path), [...keys.values()]) const abortController = new AbortController() @@ -133,6 +152,7 @@ function requestSegment( where, signal: abortController.signal, trackLoadSubsetPromise: false, + onUnoptimized, onLoadSubsetResult: (result) => { load.ready = result }, diff --git a/packages/db/tests/effect.test.ts b/packages/db/tests/effect.test.ts index 193ece5819..407932980d 100644 --- a/packages/db/tests/effect.test.ts +++ b/packages/db/tests/effect.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it, vi } from 'vitest' import { createCollection } from '../src/collection/index.js' +import { BTreeIndex } from '../src/indexes/btree-index.js' import { Query, createEffect, createTransaction, eq } from '../src/index.js' import { mockSyncCollectionOptions, @@ -1443,6 +1444,8 @@ describe(`createEffect`, () => { id: `effect-rejected-lazy-issues`, getKey: (issue) => issue.id, syncMode: `on-demand`, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, sync: { sync: () => ({ loadSubset: () => Promise.reject(new Error(`lazy load failed`)), diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index 4e67a4298f..833de03216 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -6,7 +6,6 @@ import { eq, materialize, } from '../../src/query/index.js' -import { expectAssertionFailure } from '../expected-failure.js' import { runTrace } from '../trace-runner.js' import { flushPromises, @@ -81,6 +80,7 @@ function createControlledCollection( id: `${name}-${nextCollectionOracleId++}`, getKey: (row) => row.id, initialData: initialData.map((row) => ({ ...row })), + autoIndex: `eager`, }) options.sync.rowUpdateMode = `full` const collection = createCollection(options) @@ -315,155 +315,131 @@ describe(`Collection-valued includes oracle`, () => { }), ) - fcTest( - `discovered trace: retiring a route cleans up its facade`, - async () => { - let retiredFacade: IncludedChildCollection | undefined - const driver = createCollectionDriver( - [{ id: 1, group: 1 }], - [{ id: 10, parentGroup: 1, value: 1 }], - ) - const lifecycleDriver: TraceDriver = - { - ...driver, - apply(action, context, checkpoint) { - retiredFacade ??= context.live.get(1)?.children - return driver.apply(action, context, checkpoint) - }, - } - const projection: TraceProjection< - CollectionContext, - { rows: Array; retiredStatus: string | undefined } - > = { - observe: (context) => ({ - rows: projectLive(context.live), - retiredStatus: retiredFacade?.status, - }), - recompute: (context) => ({ - rows: recompute(context), - retiredStatus: - context.model.parents.size === 0 - ? `cleaned-up` - : retiredFacade?.status, - }), - assertEqual(observed, expected) { - expect(observed).toEqual(expected) - return undefined - }, - } + fcTest(`retiring a route cleans up its facade`, async () => { + let retiredFacade: IncludedChildCollection | undefined + const driver = createCollectionDriver( + [{ id: 1, group: 1 }], + [{ id: 10, parentGroup: 1, value: 1 }], + ) + const lifecycleDriver: TraceDriver = { + ...driver, + apply(action, context, checkpoint) { + retiredFacade ??= context.live.get(1)?.children + return driver.apply(action, context, checkpoint) + }, + } + const projection: TraceProjection< + CollectionContext, + { rows: Array; retiredStatus: string | undefined } + > = { + observe: (context) => ({ + rows: projectLive(context.live), + retiredStatus: retiredFacade?.status, + }), + recompute: (context) => ({ + rows: recompute(context), + retiredStatus: + context.model.parents.size === 0 + ? `cleaned-up` + : retiredFacade?.status, + }), + assertEqual(observed, expected) { + expect(observed).toEqual(expected) + return undefined + }, + } - await expectAssertionFailure( - () => - runTrace({ - steps: [{ type: `deleteParent`, id: 1 }], - driver: lifecycleDriver, - projection, - }), - { - checkpoint: 1, - classify: ({ actual, expected }) => - JSON.stringify(actual) === - JSON.stringify({ rows: [], retiredStatus: `ready` }) && - JSON.stringify(expected) === - JSON.stringify({ rows: [], retiredStatus: `cleaned-up` }), - }, - )() - }, - ) + await runTrace({ + steps: [{ type: `deleteParent`, id: 1 }], + driver: lifecycleDriver, + projection, + }) + }) - fcTest( - `discovered trace: a delete event preserves the published facade identity`, - async () => { - let publishedFacade: IncludedChildCollection | undefined - let previousFacadeMatched = true - const driver = createCollectionDriver( - [{ id: 1, group: 1 }], - [{ id: 10, parentGroup: 1, value: 1 }], - ) - const eventDriver: TraceDriver = { - ...driver, - async start(context) { - await driver.start?.(context) - publishedFacade = context.live.get(1)?.children - context.subscription = context.live.subscribeChanges( - (changes: Array>) => { - for (const change of changes) { - if (change.type === `delete`) { - previousFacadeMatched = - change.previousValue?.children === publishedFacade - } + fcTest(`a delete event preserves the published facade identity`, async () => { + let publishedFacade: IncludedChildCollection | undefined + let previousFacadeMatched = true + const driver = createCollectionDriver( + [{ id: 1, group: 1 }], + [{ id: 10, parentGroup: 1, value: 1 }], + ) + const eventDriver: TraceDriver = { + ...driver, + async start(context) { + await driver.start?.(context) + publishedFacade = context.live.get(1)?.children + context.subscription = context.live.subscribeChanges( + (changes: Array>) => { + for (const change of changes) { + if (change.type === `delete`) { + previousFacadeMatched = + change.value.children === publishedFacade } - }, - { includeInitialState: false }, - ) - }, - } - const projection: TraceProjection< - CollectionContext, - { previousFacadeMatched: boolean } - > = { - observe: () => ({ previousFacadeMatched }), - recompute: () => ({ previousFacadeMatched: true }), - assertEqual(observed, expected) { - expect(observed).toEqual(expected) - return undefined - }, - } + } + }, + { includeInitialState: false }, + ) + }, + } + const projection: TraceProjection< + CollectionContext, + { previousFacadeMatched: boolean } + > = { + observe: () => ({ previousFacadeMatched }), + recompute: () => ({ previousFacadeMatched: true }), + assertEqual(observed, expected) { + expect(observed).toEqual(expected) + return undefined + }, + } - await expectAssertionFailure( - () => - runTrace({ - steps: [{ type: `deleteParent`, id: 1 }], - driver: eventDriver, - projection, - }), - { - checkpoint: 1, - classify: ({ actual, expected }) => - JSON.stringify(actual) === - JSON.stringify({ previousFacadeMatched: false }) && - JSON.stringify(expected) === - JSON.stringify({ previousFacadeMatched: true }), - }, - )() - }, - ) + await runTrace({ + steps: [{ type: `deleteParent`, id: 1 }], + driver: eventDriver, + projection, + }) + }) - fcTest( - `discovered trace: facade public keys survive row cloning`, - async () => { - const driver = createCollectionDriver( - [{ id: 1, group: 1 }], - [{ id: 10, parentGroup: 1, value: 1 }], - ) - const projection: TraceProjection< - CollectionContext, - { clonedKey: unknown }, - { clonedKey: number } - > = { - observe(context) { - const facade = context.live.get(1)!.children - const row = facade.get(10)! - return { clonedKey: facade.getKeyFromItem({ ...row }) } - }, - recompute: () => ({ clonedKey: 10 }), - assertEqual(observed, expected) { - expect(observed).toEqual(expected) - return undefined - }, - } + fcTest(`facade public keys survive row cloning`, async () => { + const driver = createCollectionDriver( + [{ id: 1, group: 1 }], + [{ id: 10, parentGroup: 1, value: 1 }], + ) + const projection: TraceProjection< + CollectionContext, + { clonedKey: unknown }, + { clonedKey: number } + > = { + observe(context) { + const facade = context.live.get(1)!.children + const row = facade.get(10)! + return { clonedKey: facade.getKeyFromItem({ ...row }) } + }, + recompute: () => ({ clonedKey: 10 }), + assertEqual(observed, expected) { + expect(observed).toEqual(expected) + return undefined + }, + } - await expectAssertionFailure( - () => runTrace({ steps: [], driver, projection }), - { - checkpoint: 0, - classify: ({ actual, expected }) => - JSON.stringify(actual) === - JSON.stringify({ clonedKey: undefined }) && - JSON.stringify(expected) === JSON.stringify({ clonedKey: 10 }), - }, - )() - }, + await runTrace({ steps: [], driver, projection }) + }) + + fcTest(`reactivating a retired route restores its current snapshot`, () => + runTrace({ + steps: [ + { type: `putParent`, row: { id: 1, group: 2 } }, + { type: `putParent`, row: { id: 1, group: 1 } }, + ], + driver: createCollectionDriver( + [{ id: 1, group: 1 }], + [ + { id: 10, parentGroup: 1, value: 1 }, + { id: 20, parentGroup: 2, value: 2 }, + ], + ), + projection: collectionProjection, + }), ) fcTest( @@ -484,13 +460,8 @@ describe(`Collection-valued includes oracle`, () => { (batch) => changes.push(...batch), { includeInitialState: false }, ) - const internalConfig = ( - facade._state as unknown as { - config: typeof facade.config - } - ).config - const originalGetKey = internalConfig.getKey - internalConfig.getKey = (row) => { + const originalGetKey = facade.config.getKey + facade.config.getKey = (row) => { if (row.id === 20) throw new Error(`facade key failed`) return originalGetKey(row) } @@ -520,8 +491,18 @@ describe(`Collection-valued includes oracle`, () => { }, ]) expect(changes).toEqual([]) + + facade.config.getKey = originalGetKey + context.parents.write(`insert`, { id: 2, group: 2 }) + await flushPromises() + expect(context.live.get(1)!.children).toBe(facade) + expect(projectLive(context.live)[0]!.children).toEqual([ + { id: 10, parentGroup: 1, value: 10 }, + { id: 20, parentGroup: 1, value: 20 }, + ]) + expect(changes).toHaveLength(2) } finally { - internalConfig.getKey = originalGetKey + facade.config.getKey = originalGetKey subscription.unsubscribe() await driver.cleanup(context) } @@ -597,45 +578,39 @@ describe(`Collection-valued includes oracle`, () => { }, ) - fcTest( - `discovered trace: a matched null singleton remains null`, - expectAssertionFailure( - async () => { - type NullableChild = { id: number; parentGroup: number; value: null } - const parents = createControlledCollection(`nullable-oracle-parents`, [ - { id: 1, group: 1 }, - ]) - const children = createControlledCollection( - `nullable-oracle-children`, - [{ id: 10, parentGroup: 1, value: null }], - ) - const live = createLiveQueryCollection((q) => - q.from({ parent: parents.collection }).select(({ parent }) => ({ - id: parent.id, - value: materialize( - q - .from({ child: children.collection }) - .where(({ child }) => eq(child.parentGroup, parent.group)) - .select(({ child }) => child.value) - .findOne(), - ), - })), - ) + fcTest(`a matched null singleton remains null`, async () => { + type NullableChild = { id: number; parentGroup: number; value: null } + const parents = createControlledCollection(`nullable-oracle-parents`, [ + { id: 1, group: 1 }, + ]) + const children = createControlledCollection( + `nullable-oracle-children`, + [{ id: 10, parentGroup: 1, value: null }], + ) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => ({ + id: parent.id, + value: materialize( + q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .select(({ child }) => child.value) + .findOne(), + ), + })), + ) - try { - await live.preload() - expect(live.get(1)!.value).toBeNull() - } finally { - await Promise.all([ - live.cleanup(), - parents.collection.cleanup(), - children.collection.cleanup(), - ]) - } - }, - { message: `expected undefined to be null` }, - ), - ) + try { + await live.preload() + expect(live.get(1)!.value).toBeNull() + } finally { + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }) fcTest.prop( [ diff --git a/packages/powersync-db-collection/src/powersync.ts b/packages/powersync-db-collection/src/powersync.ts index ed73b07016..b799575566 100644 --- a/packages/powersync-db-collection/src/powersync.ts +++ b/packages/powersync-db-collection/src/powersync.ts @@ -529,7 +529,8 @@ export function powerSyncCollectionOptions< // On-demand mode. // Registers a diff trigger for the active WHERE expressions. function runOnDemandSync() { - let onUnloadSubset: CleanupFn | void | null = null + const unloadSubsetCallbacks = new Map() + const releasedSubsets = new WeakSet() start().catch((error) => database.logger.error( @@ -547,7 +548,14 @@ export function powerSyncCollectionOptions< ): Promise => { if (options) { activeWhereExpressions.push(options.where) - onUnloadSubset = await restConfig.onLoadSubset?.(options) + const cleanup = await restConfig.onLoadSubset?.(options) + if (cleanup) { + if (releasedSubsets.has(options) || options.signal?.aborted) { + cleanup() + } else { + unloadSubsetCallbacks.set(options, cleanup) + } + } } // No predicates remain, so stop tracking entirely. Both calls are no-ops @@ -627,7 +635,9 @@ export function powerSyncCollectionOptions< } const unloadSubset = async (options: LoadSubsetOptions) => { - onUnloadSubset?.() + releasedSubsets.add(options) + unloadSubsetCallbacks.get(options)?.() + unloadSubsetCallbacks.delete(options) const idx = activeWhereExpressions.indexOf(options.where) if (idx !== -1) { @@ -680,6 +690,8 @@ export function powerSyncCollectionOptions< `Sync has been stopped for ${viewName} into ${trackedTableName}`, ) abortController.abort() + for (const cleanup of unloadSubsetCallbacks.values()) cleanup() + unloadSubsetCallbacks.clear() }, loadSubset: (options: LoadSubsetOptions) => loadSubset(options), unloadSubset: (options: LoadSubsetOptions) => unloadSubset(options), From 5f9b30e7ed773b02d88d7504d59c8989dc5aa43e Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 17 Aug 2026 16:34:11 +0100 Subject: [PATCH 08/21] test(db): expose inactive facade window --- packages/db/tests/query/includes.test.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/packages/db/tests/query/includes.test.ts b/packages/db/tests/query/includes.test.ts index 6bd0eb7c93..bbc6e992e2 100644 --- a/packages/db/tests/query/includes.test.ts +++ b/packages/db/tests/query/includes.test.ts @@ -1305,7 +1305,7 @@ describe(`includes subqueries`, () => { }) it(`replays existing bucket rows when a parent enters a limited result`, async () => { - type Parent = { id: number; groupId: number } + type Parent = { id: number; rank: number; groupId: number } type Child = { id: number; groupId: number } const parents = createCollection( @@ -1313,11 +1313,9 @@ describe(`includes subqueries`, () => { id: `limited-late-route-parents`, getKey: (parent) => parent.id, initialData: [ - { id: 1, groupId: 1 }, - { id: 2, groupId: 2 }, + { id: 1, rank: 1, groupId: 1 }, + { id: 2, rank: 2, groupId: 2 }, ], - autoIndex: `eager`, - defaultIndexType: BTreeIndex, }), ) const children = createCollection( @@ -1333,7 +1331,7 @@ describe(`includes subqueries`, () => { const collection = createLiveQueryCollection((q) => q .from({ parent: parents }) - .orderBy(({ parent }) => parent.id) + .orderBy(({ parent }) => parent.rank) .limit(1) .select(({ parent }) => ({ id: parent.id, @@ -1350,7 +1348,7 @@ describe(`includes subqueries`, () => { parents.utils.begin() parents.utils.write({ type: `delete`, - value: { id: 1, groupId: 1 }, + value: { id: 1, rank: 1, groupId: 1 }, }) parents.utils.commit() From 0c4e31173b3640e8e75130b2e9895d1071a40ac5 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 17 Aug 2026 16:35:22 +0100 Subject: [PATCH 09/21] fix(db): replay active facade buckets --- packages/db/src/query/live/ARCHITECTURE.md | 14 ++++++++------ .../db/src/query/live/materialized-pipeline.ts | 6 +++++- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index fbfdea182c..9619ec0604 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -316,18 +316,20 @@ A bare child query is a Collection-valued include. It exposes one stable public Collection facade per active bucket in that edge: ```text -BucketRow -> BucketFacade(bucket, Collection) +ActiveBucket + BucketRow -> ActiveBucketRow -> BucketFacade(bucket, Collection) Route + BucketFacade -> CellValue(cell, Collection) ``` Parents sharing a bucket share its facade. Child changes update that Collection without re-emitting every parent, and moving a route changes the parent field to the destination bucket's facade. A facade is never retargeted to another -bucket. The adapter retains a facade only while at least one parent route uses -its bucket. When the last route leaves, it retracts the facade's rows and drops -its strong reference. An external holder may keep that empty Collection alive, -but a later active interval gets a new facade. Inline modes do not create child -Collections. +bucket. The D2 join retains inactive bucket rows and emits their current +snapshot when the bucket becomes active; the facade adapter does not buffer +discarded deltas. The adapter retains a facade only while at least one parent +route uses its bucket. When the last route leaves, it retracts the facade's rows +and drops its strong reference. An external holder may keep that empty +Collection alive, but a later active interval gets a new facade. Inline modes +do not create child Collections. Composition is pure. It constructs a new result along changed paths and does not mutate a previously published row or use public routing metadata: diff --git a/packages/db/src/query/live/materialized-pipeline.ts b/packages/db/src/query/live/materialized-pipeline.ts index 01eef49195..a840c8ba8a 100644 --- a/packages/db/src/query/live/materialized-pipeline.ts +++ b/packages/db/src/query/live/materialized-pipeline.ts @@ -139,9 +139,13 @@ function materializeRelation( if (include.materialization === `collection`) { const edgeId = `bucket-facade-${++nextBucketFacadeEdgeId}` const activeBuckets = createActiveBuckets(pipeline, include) + const activeBucketRows = activeBuckets.pipe( + join(bucketRows), + map(([bucketKey, [, row]]) => [bucketKey, row]), + ) as IStreamBuilder<[string, BucketRow]> facades.push({ edgeId, - rows: bucketRows, + rows: activeBucketRows, activeBuckets, hasOrderBy: include.hasOrderBy, }) From 1d1207b5660da028823fdb9f7c129871d3d78f1c Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 17 Aug 2026 16:54:12 +0100 Subject: [PATCH 10/21] test(db): expose include key ordering mismatch --- packages/db/tests/query/includes.test.ts | 61 ++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/packages/db/tests/query/includes.test.ts b/packages/db/tests/query/includes.test.ts index bbc6e992e2..84f62930dd 100644 --- a/packages/db/tests/query/includes.test.ts +++ b/packages/db/tests/query/includes.test.ts @@ -7316,6 +7316,67 @@ describe(`includes subqueries`, () => { }) describe(`materialize`, () => { + it(`uses the same public-key tie-breaker for Collection and inline includes`, async () => { + type OrderingParent = { id: number } + type OrderingChild = { + id: number + parentId: number + label: string + } + + const orderingParents = createCollection( + mockSyncCollectionOptions({ + id: `includes-ordering-parents`, + getKey: (parent) => parent.id, + initialData: [{ id: 1 }], + }), + ) + const orderingChildren = createCollection( + mockSyncCollectionOptions({ + id: `includes-ordering-children`, + getKey: (child) => child.id, + initialData: [ + { id: 2, parentId: 1, label: `two` }, + { id: 3, parentId: 1, label: `three` }, + { id: 10, parentId: 1, label: `ten` }, + ], + }), + ) + const collection = createLiveQueryCollection((q) => { + const childRows = (parentId: number) => + q + .from({ child: orderingChildren }) + .where(({ child }) => eq(child.parentId, parentId)) + .select(({ child }) => ({ id: child.id, label: child.label })) + + return q.from({ parent: orderingParents }).select(({ parent }) => ({ + id: parent.id, + facade: childRows(parent.id), + array: toArray(childRows(parent.id)), + joined: concat( + toArray( + q + .from({ child: orderingChildren }) + .where(({ child }) => eq(child.parentId, parent.id)) + .select(({ child }) => child.label), + ), + ), + first: materialize(childRows(parent.id).findOne()), + materialized: materialize(childRows(parent.id)), + })) + }) + await collection.preload() + + const result = collection.get(1)! + const facadeIds = result.facade.toArray.map((child) => child.id) + + expect(facadeIds).toEqual([2, 3, 10]) + expect(result.array.map((child) => child.id)).toEqual(facadeIds) + expect(result.materialized.map((child) => child.id)).toEqual(facadeIds) + expect(result.first?.id).toBe(facadeIds[0]) + expect(result.joined).toBe(`twothreeten`) + }) + // For singleton behavior we look up each issue's parent project. // Each issue references exactly one project via projectId. function buildSingletonQuery() { From 2df33a158927ea9ec36a71e7d29daafbb16ee952 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 17 Aug 2026 16:58:10 +0100 Subject: [PATCH 11/21] fix(db): align include public key ordering --- .../src/query/live/materialized-pipeline.ts | 9 +++ ...ncludes-collection-oracle.property.test.ts | 69 +++++++++++++++++++ packages/db/tests/query/includes.test.ts | 41 ++++++----- 3 files changed, 100 insertions(+), 19 deletions(-) diff --git a/packages/db/src/query/live/materialized-pipeline.ts b/packages/db/src/query/live/materialized-pipeline.ts index a840c8ba8a..915113ef3e 100644 --- a/packages/db/src/query/live/materialized-pipeline.ts +++ b/packages/db/src/query/live/materialized-pipeline.ts @@ -1,4 +1,5 @@ import { + compareKeys, distinct, filter, join, @@ -421,6 +422,14 @@ function compareBucketRows(left: BucketRow, right: BucketRow): number { return left.order < right.order ? -1 : 1 } + if ( + (typeof left.publicKey === `string` || + typeof left.publicKey === `number`) && + (typeof right.publicKey === `string` || typeof right.publicKey === `number`) + ) { + return compareKeys(left.publicKey, right.publicKey) + } + const leftKey = serializeValue(left.publicKey) const rightKey = serializeValue(right.publicKey) return leftKey < rightKey ? -1 : leftKey > rightKey ? 1 : 0 diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index 833de03216..0d03d9d975 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -2,9 +2,11 @@ import { fc, test as fcTest } from '@fast-check/vitest' import { describe, expect } from 'vitest' import { createCollection } from '../../src/collection/index.js' import { + concat, createLiveQueryCollection, eq, materialize, + toArray, } from '../../src/query/index.js' import { runTrace } from '../trace-runner.js' import { @@ -612,6 +614,73 @@ describe(`Collection-valued includes oracle`, () => { } }) + fcTest.prop( + [ + fc.record({ + smallId: fc.integer({ min: 2, max: 9 }), + wideId: fc.integer({ min: 10, max: 19 }), + }), + ], + { numRuns: 20 }, + )( + `uses one raw public-key order across Collection and inline materializations`, + async ({ smallId, wideId }) => { + const parents = createControlledCollection(`ordering-oracle-parents`, [ + { id: 1, group: 1 }, + ]) + const children = createControlledCollection(`ordering-oracle-children`, [ + { id: smallId, parentGroup: 1, value: smallId }, + { id: wideId, parentGroup: 1, value: wideId }, + ]) + const live = createLiveQueryCollection((q) => { + return q.from({ parent: parents.collection }).select(({ parent }) => { + const childRows = () => + q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .select(({ child }) => ({ id: child.id, value: child.value })) + + return { + id: parent.id, + facade: childRows(), + array: toArray(childRows()), + materialized: materialize(childRows()), + first: materialize(childRows().findOne()), + joined: concat( + toArray( + q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .select(({ child }) => child.value), + ), + ), + } + }) + }) + + try { + await live.preload() + const result = live.get(1)! + const expectedIds = [smallId, wideId] + const facadeIds = result.facade.toArray.map((child) => child.id) + + expect(facadeIds).toEqual(expectedIds) + expect(result.array.map((child) => child.id)).toEqual(expectedIds) + expect(result.materialized.map((child) => child.id)).toEqual( + expectedIds, + ) + expect(result.first?.id).toBe(smallId) + expect(result.joined).toBe(`${smallId}${wideId}`) + } finally { + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }, + ) + fcTest.prop( [ fc.record({ diff --git a/packages/db/tests/query/includes.test.ts b/packages/db/tests/query/includes.test.ts index 84f62930dd..4443b92120 100644 --- a/packages/db/tests/query/includes.test.ts +++ b/packages/db/tests/query/includes.test.ts @@ -7335,6 +7335,7 @@ describe(`includes subqueries`, () => { mockSyncCollectionOptions({ id: `includes-ordering-children`, getKey: (child) => child.id, + autoIndex: `eager`, initialData: [ { id: 2, parentId: 1, label: `two` }, { id: 3, parentId: 1, label: `three` }, @@ -7343,27 +7344,29 @@ describe(`includes subqueries`, () => { }), ) const collection = createLiveQueryCollection((q) => { - const childRows = (parentId: number) => - q - .from({ child: orderingChildren }) - .where(({ child }) => eq(child.parentId, parentId)) - .select(({ child }) => ({ id: child.id, label: child.label })) + return q.from({ parent: orderingParents }).select(({ parent }) => { + const childRows = () => + q + .from({ child: orderingChildren }) + .where(({ child }) => eq(child.parentId, parent.id)) + .select(({ child }) => ({ id: child.id, label: child.label })) - return q.from({ parent: orderingParents }).select(({ parent }) => ({ - id: parent.id, - facade: childRows(parent.id), - array: toArray(childRows(parent.id)), - joined: concat( - toArray( - q - .from({ child: orderingChildren }) - .where(({ child }) => eq(child.parentId, parent.id)) - .select(({ child }) => child.label), + return { + id: parent.id, + facade: childRows(), + array: toArray(childRows()), + joined: concat( + toArray( + q + .from({ child: orderingChildren }) + .where(({ child }) => eq(child.parentId, parent.id)) + .select(({ child }) => child.label), + ), ), - ), - first: materialize(childRows(parent.id).findOne()), - materialized: materialize(childRows(parent.id)), - })) + first: materialize(childRows().findOne()), + materialized: materialize(childRows()), + } + }) }) await collection.preload() From 600ee6f134683a9032c05e571371abf7ec858473 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 17 Aug 2026 17:05:18 +0100 Subject: [PATCH 12/21] test(query-db): expose cached subset readiness gap --- .../query-db-collection/tests/query.test.ts | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/packages/query-db-collection/tests/query.test.ts b/packages/query-db-collection/tests/query.test.ts index a4e4350d05..8e2fd2cea6 100644 --- a/packages/query-db-collection/tests/query.test.ts +++ b/packages/query-db-collection/tests/query.test.ts @@ -860,6 +860,85 @@ describe(`QueryCollection`, () => { }) }) + it(`reconciles a retained cached subset before an include becomes ready`, async () => { + type LineItem = { id: string; productId: string } + type Product = { id: string; name: string } + + const queryKey = [`cached-includes-product`] + const cachedProducts: Array = [ + { id: `product-1`, name: `Cached widget` }, + ] + queryClient.setQueryData(queryKey, cachedProducts) + const queryHash = hashKey(queryKey) + const metadataHarness = createInMemorySyncMetadataApi< + string | number, + Product + >({ + collectionMetadata: new Map([ + [ + `queryCollection:gc:${queryHash}`, + { queryHash, mode: `until-revalidated` }, + ], + ]), + }) + + const lineItems = createCollection( + mockSyncCollectionOptions({ + id: `cached-includes-line-items`, + getKey: (lineItem) => lineItem.id, + initialData: [{ id: `line-1`, productId: `product-1` }], + }), + ) + const productOptions = queryCollectionOptions({ + id: `cached-includes-products`, + queryClient, + queryKey: () => queryKey, + queryFn: vi.fn().mockResolvedValue(cachedProducts), + getKey: (product) => product.id, + syncMode: `on-demand`, + startSync: true, + staleTime: Infinity, + }) + const originalSync = productOptions.sync + const products = createCollection({ + ...productOptions, + sync: { + sync: (params: Parameters[0]) => + originalSync.sync({ ...params, metadata: metadataHarness.api }), + }, + }) + const live = createLiveQueryCollection((q) => + q.from({ lineItem: lineItems }).select(({ lineItem }) => ({ + id: lineItem.id, + product: q + .from({ product: products }) + .where(({ product }) => eq(product.id, lineItem.productId)) + .select(({ product }) => ({ + id: product.id, + name: product.name, + })), + })), + ) + + try { + await new Promise((resolve) => products.onFirstReady(resolve)) + await live.preload() + + expect(live.status).toBe(`ready`) + expect( + (live.get(`line-1`) as any).product.toArray.map((product: Product) => + stripVirtualProps(product), + ), + ).toEqual(cachedProducts) + } finally { + await Promise.all([ + live.cleanup(), + products.cleanup(), + lineItems.cleanup(), + ]) + } + }) + it(`should update collection when query data changes`, async () => { const queryKey = [`testItems`] const initialItems: Array = [ From 8fe67417eff66bc0a2d579061d1b06bec4d29d56 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 17 Aug 2026 17:08:46 +0100 Subject: [PATCH 13/21] fix(query-db): await cached subset reconciliation --- .changeset/fix-includes-materialization.md | 3 ++- packages/query-db-collection/src/query.ts | 24 +++++++++++++++++----- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/.changeset/fix-includes-materialization.md b/.changeset/fix-includes-materialization.md index ce759d556d..d64904b14c 100644 --- a/.changeset/fix-includes-materialization.md +++ b/.changeset/fix-includes-materialization.md @@ -1,5 +1,6 @@ --- '@tanstack/db': patch +'@tanstack/query-db-collection': patch --- -Rebuild correlated include materialization as one D2 graph, fixing stale or missing nested results across route changes, batching, lazy loading, optimistic updates, and layered queries. Add abortable subset demand and coherent publication for Collection-valued includes. +Rebuild correlated include materialization as one D2 graph, fixing stale or missing nested results across route changes, batching, lazy loading, optimistic updates, and layered queries. Add abortable subset demand and coherent publication for Collection-valued includes. Wait for retained Query Collection cache results to reach the collection before resolving subset demand. diff --git a/packages/query-db-collection/src/query.ts b/packages/query-db-collection/src/query.ts index 82f2a8dde5..13b25ada8a 100644 --- a/packages/query-db-collection/src/query.ts +++ b/packages/query-db-collection/src/query.ts @@ -886,6 +886,7 @@ export function queryCollectionOptions( let syncStarted = false let startupRetentionSettled = false const retainedQueriesPendingRevalidation = new Set() + const pendingResultApplications = new Map>() const effectivePersistedGcTimes = new Map() const persistedRetentionTimers = new Map< string, @@ -1308,8 +1309,7 @@ export function queryCollectionOptions( const currentResult = observer.getCurrentResult() if (currentResult.isSuccess) { - // Data is already available, return true synchronously - return true + return pendingResultApplications.get(hashedQueryKey) ?? true } else if (currentResult.isError) { // Error already occurred, reject immediately return Promise.reject(currentResult.error) @@ -1320,7 +1320,9 @@ export function queryCollectionOptions( // cycles (e.g., when a live query is cleaned up and recreated). const cachedData = queryClient.getQueryData(key) if (cachedData !== undefined) { - return true + return waitForQueryReady(observer, hashedQueryKey).then(() => + pendingResultApplications.get(hashedQueryKey), + ) } // Query is still loading, wait for the first result @@ -1391,7 +1393,7 @@ export function queryCollectionOptions( if (syncStarted || collection.subscriberCount > 0) { subscribeToQuery(localObserver, hashedQueryKey) } - return true + return pendingResultApplications.get(hashedQueryKey) ?? true } // Create a promise that resolves when the query result is first available @@ -1564,12 +1566,23 @@ export function queryCollectionOptions( return } - void reconcileSuccessfulResult(queryKey, result).catch((error) => { + const application = reconcileSuccessfulResult( + queryKey, + result, + ).catch((error) => { console.error( `[QueryCollection] Error reconciling query ${String(queryKey)}:`, error, ) }) + pendingResultApplications.set(hashedQueryKey, application) + void application.finally(() => { + if ( + pendingResultApplications.get(hashedQueryKey) === application + ) { + pendingResultApplications.delete(hashedQueryKey) + } + }) } else { applySuccessfulResult(queryKey, result) } @@ -1693,6 +1706,7 @@ export function queryCollectionOptions( unsubscribePendingReadyListeners(hashedQueryKey) cancelPersistedRetentionExpiry(hashedQueryKey) retainedQueriesPendingRevalidation.delete(hashedQueryKey) + pendingResultApplications.delete(hashedQueryKey) const nextOwnersByRow = removeQueryOwnership(hashedQueryKey) const rowsToDelete: Array = [] From ddfc142dcdff17b0d92ca43e2a0aa53045478575 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 17 Aug 2026 17:14:06 +0100 Subject: [PATCH 14/21] test(db): expose retired facade cleanup --- .../query/includes-collection-oracle.property.test.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index 0d03d9d975..797e6e5748 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -317,7 +317,7 @@ describe(`Collection-valued includes oracle`, () => { }), ) - fcTest(`retiring a route cleans up its facade`, async () => { + fcTest(`retiring a route leaves a held facade empty and ready`, async () => { let retiredFacade: IncludedChildCollection | undefined const driver = createCollectionDriver( [{ id: 1, group: 1 }], @@ -342,7 +342,7 @@ describe(`Collection-valued includes oracle`, () => { rows: recompute(context), retiredStatus: context.model.parents.size === 0 - ? `cleaned-up` + ? `ready` : retiredFacade?.status, }), assertEqual(observed, expected) { @@ -356,6 +356,9 @@ describe(`Collection-valued includes oracle`, () => { driver: lifecycleDriver, projection, }) + + expect(retiredFacade?.toArray).toEqual([]) + await expect(retiredFacade?.preload()).resolves.toBeUndefined() }) fcTest(`a delete event preserves the published facade identity`, async () => { From 63194480d59120e2e94588d643ebffde41951635 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 17 Aug 2026 17:15:37 +0100 Subject: [PATCH 15/21] fix(db): keep retired facades usable --- packages/db/src/query/live/bucket-facade-adapter.ts | 4 +++- .../tests/query/includes-collection-oracle.property.test.ts | 4 +--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/db/src/query/live/bucket-facade-adapter.ts b/packages/db/src/query/live/bucket-facade-adapter.ts index 9da66e59f6..f350e88850 100644 --- a/packages/db/src/query/live/bucket-facade-adapter.ts +++ b/packages/db/src/query/live/bucket-facade-adapter.ts @@ -157,7 +157,9 @@ export class BucketFacadeAdapter { if (closed) return closed = true for (const publication of publications) publication.publish() - this.cleanupRetiredEntries() + // Drop only the adapter's strong reference. External holders keep an + // empty, ready facade; a later active interval receives a new one. + this.retiredEntries.clear() }, rollback: () => { if (closed) return diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index 797e6e5748..89a630b9d4 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -341,9 +341,7 @@ describe(`Collection-valued includes oracle`, () => { recompute: (context) => ({ rows: recompute(context), retiredStatus: - context.model.parents.size === 0 - ? `ready` - : retiredFacade?.status, + context.model.parents.size === 0 ? `ready` : retiredFacade?.status, }), assertEqual(observed, expected) { expect(observed).toEqual(expected) From 2940a002bd2fecdd9385adcfb01fc0c935c9a0bf Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 17 Aug 2026 17:19:01 +0100 Subject: [PATCH 16/21] test(db): expose signal-sensitive subset duplication --- packages/db/tests/query/subset-dedupe.test.ts | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/packages/db/tests/query/subset-dedupe.test.ts b/packages/db/tests/query/subset-dedupe.test.ts index c11c3ac188..9e4ab89d29 100644 --- a/packages/db/tests/query/subset-dedupe.test.ts +++ b/packages/db/tests/query/subset-dedupe.test.ts @@ -45,6 +45,46 @@ function not(expression: BasicExpression): Func { } describe(`createDeduplicatedLoadSubset`, () => { + it(`shares in-flight work while any cancellation owner remains active`, async () => { + let resolveLoad: (() => void) | undefined + let sharedSignal: AbortSignal | undefined + const loadSubset = vi.fn( + (options: LoadSubsetOptions) => + new Promise((resolve) => { + sharedSignal = options.signal + resolveLoad = resolve + }), + ) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const first = new AbortController() + const second = new AbortController() + const where = gt(ref(`age`), val(10)) + + const firstLoad = deduplicated.loadSubset({ where, signal: first.signal }) + const secondLoad = deduplicated.loadSubset({ + where, + signal: second.signal, + }) + + expect(loadSubset).toHaveBeenCalledTimes(1) + expect(secondLoad).toBe(firstLoad) + expect(sharedSignal).not.toBe(first.signal) + expect(sharedSignal).not.toBe(second.signal) + + first.abort() + expect(sharedSignal?.aborted).toBe(false) + + resolveLoad?.() + await Promise.all([firstLoad, secondLoad]) + + expect( + deduplicated.loadSubset({ + where, + signal: new AbortController().signal, + }), + ).toBe(true) + }) + it(`keeps independently abortable in-flight requests isolated`, async () => { const releases: Array<() => void> = [] let callCount = 0 From e347302ff638c9716dc90283197e3aeb547b0b88 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 17 Aug 2026 17:24:42 +0100 Subject: [PATCH 17/21] fix(db): share cancellable subset loads --- packages/db/src/query/live/ARCHITECTURE.md | 7 +- packages/db/src/query/subset-dedupe.ts | 103 +++++++++++++++--- packages/db/tests/query/subset-dedupe.test.ts | 101 ++++++++++++----- 3 files changed, 165 insertions(+), 46 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 9619ec0604..52e6fbff79 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -367,7 +367,12 @@ type DemandSet = readonly [ ``` One request may cover many buckets, and the adapter may coalesce or reuse -requests according to the compiled demand plan. Its semantic contract is: +requests according to the compiled demand plan. A coalesced request has one +shared abort lease. If one owner releases its lease, the source request remains +active while another owner still needs its coverage. The source signal aborts +only after every attached owner has released it. + +Its semantic contract is: > Every active, satisfiable bucket must be covered by a settled current demand > request before initial preload completes. diff --git a/packages/db/src/query/subset-dedupe.ts b/packages/db/src/query/subset-dedupe.ts index b1fcead473..c0036ee25a 100644 --- a/packages/db/src/query/subset-dedupe.ts +++ b/packages/db/src/query/subset-dedupe.ts @@ -7,6 +7,19 @@ import { import type { BasicExpression } from './ir.js' import type { LoadSubsetOptions } from '../types.js' +type SharedAbortLease = { + signal: AbortSignal | undefined + aborted: boolean + attach: (signal: AbortSignal | undefined) => void + dispose: () => void +} + +type InflightCall = { + options: LoadSubsetOptions + promise: Promise + lease: SharedAbortLease +} + /** * Deduplicated wrapper for a loadSubset function. * Tracks what data has been loaded and avoids redundant calls by applying @@ -53,11 +66,8 @@ export class DeduplicatedLoadSubset { private limitedCalls: Array = [] // Track in-flight calls to prevent concurrent duplicate requests - // We store both the options and the promise so we can apply subset logic - private inflightCalls: Array<{ - options: LoadSubsetOptions - promise: Promise - }> = [] + // Each entry also owns the shared cancellation lease for its requesters. + private inflightCalls: Array = [] // Generation counter to invalidate in-flight requests after reset() // When reset() is called, this increments, and any in-flight completion handlers @@ -114,12 +124,11 @@ export class DeduplicatedLoadSubset { // This prevents duplicate requests when concurrent calls have subset relationships const matchingInflight = this.inflightCalls.find( (inflight) => - !inflight.options.signal?.aborted && - inflight.options.signal === options.signal && - isPredicateSubset(options, inflight.options), + !inflight.lease.aborted && isPredicateSubset(options, inflight.options), ) if (matchingInflight !== undefined) { + matchingInflight.lease.attach(options.signal) // An in-flight call will load data that covers this request // Return the same promise so this caller waits for the data to load // The in-flight promise already handles tracking updates when it completes @@ -131,8 +140,9 @@ export class DeduplicatedLoadSubset { // Preserve the original request for tracking and in-flight dedupe, but allow // the backend request to be narrowed to only the missing subset. - const trackingOptions = cloneOptions(options) - const loadOptions = cloneOptions(options) + const lease = createSharedAbortLease(options.signal) + const trackingOptions = cloneOptions({ ...options, signal: lease.signal }) + const loadOptions = cloneOptions({ ...options, signal: lease.signal }) if (this.unlimitedWhere !== undefined && options.limit === undefined) { // Compute difference to get only the missing data // We can only do this for unlimited queries @@ -144,11 +154,18 @@ export class DeduplicatedLoadSubset { } // Call underlying loadSubset to load the missing data - const resultPromise = this._loadSubset(loadOptions) + let resultPromise: true | Promise + try { + resultPromise = this._loadSubset(loadOptions) + } catch (error) { + lease.dispose() + throw error + } // Handle both sync (true) and async (Promise) return values if (resultPromise === true) { - this.updateTracking(trackingOptions) + if (!lease.aborted) this.updateTracking(trackingOptions) + lease.dispose() return true } else { // Async return - track the promise and update tracking after it resolves @@ -160,15 +177,13 @@ export class DeduplicatedLoadSubset { // We need to create a reference to the in-flight entry so we can remove it later const inflightEntry = { options: trackingOptions, + lease, promise: resultPromise .then((result) => { // Only update tracking if this request is still from the current generation // If reset() was called, the generation will have incremented and we should // not repopulate the state that was just cleared - if ( - capturedGeneration === this.generation && - !trackingOptions.signal?.aborted - ) { + if (capturedGeneration === this.generation && !lease.aborted) { this.updateTracking(trackingOptions) } return result @@ -180,6 +195,7 @@ export class DeduplicatedLoadSubset { if (index !== -1) { this.inflightCalls.splice(index, 1) } + lease.dispose() }), } @@ -234,6 +250,61 @@ export class DeduplicatedLoadSubset { } } +function createSharedAbortLease( + initialSignal: AbortSignal | undefined, +): SharedAbortLease { + const controller = initialSignal ? new AbortController() : undefined + const listeners = new Map void>() + let hasUnabortableOwner = initialSignal === undefined + let activeAbortableOwners = 0 + + const abortIfUnused = (reason?: unknown) => { + if ( + !hasUnabortableOwner && + listeners.size > 0 && + activeAbortableOwners === 0 + ) { + controller?.abort(reason) + } + } + + const attach = (signal: AbortSignal | undefined) => { + if (!signal) { + hasUnabortableOwner = true + return + } + if (listeners.has(signal)) return + + const onAbort = () => { + activeAbortableOwners -= 1 + abortIfUnused(signal.reason) + } + listeners.set(signal, onAbort) + if (signal.aborted) { + abortIfUnused(signal.reason) + } else { + activeAbortableOwners += 1 + signal.addEventListener(`abort`, onAbort, { once: true }) + } + } + + attach(initialSignal) + + return { + signal: controller?.signal, + get aborted() { + return controller?.signal.aborted ?? false + }, + attach, + dispose: () => { + for (const [signal, listener] of listeners) { + signal.removeEventListener(`abort`, listener) + } + listeners.clear() + }, + } +} + /** * Clones a LoadSubsetOptions object to prevent mutation of stored predicates. * This is crucial because callers often reuse the same options object and mutate diff --git a/packages/db/tests/query/subset-dedupe.test.ts b/packages/db/tests/query/subset-dedupe.test.ts index 9e4ab89d29..4c5d8d4f96 100644 --- a/packages/db/tests/query/subset-dedupe.test.ts +++ b/packages/db/tests/query/subset-dedupe.test.ts @@ -56,26 +56,22 @@ describe(`createDeduplicatedLoadSubset`, () => { }), ) const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) - const first = new AbortController() - const second = new AbortController() + const owners = Array.from({ length: 10 }, () => new AbortController()) const where = gt(ref(`age`), val(10)) - const firstLoad = deduplicated.loadSubset({ where, signal: first.signal }) - const secondLoad = deduplicated.loadSubset({ - where, - signal: second.signal, - }) + const loads = owners.map((owner) => + deduplicated.loadSubset({ where, signal: owner.signal }), + ) expect(loadSubset).toHaveBeenCalledTimes(1) - expect(secondLoad).toBe(firstLoad) - expect(sharedSignal).not.toBe(first.signal) - expect(sharedSignal).not.toBe(second.signal) + for (const load of loads) expect(load).toBe(loads[0]) + for (const owner of owners) expect(sharedSignal).not.toBe(owner.signal) - first.abort() + for (const owner of owners.slice(0, -1)) owner.abort() expect(sharedSignal?.aborted).toBe(false) resolveLoad?.() - await Promise.all([firstLoad, secondLoad]) + await Promise.all(loads) expect( deduplicated.loadSubset({ @@ -85,12 +81,14 @@ describe(`createDeduplicatedLoadSubset`, () => { ).toBe(true) }) - it(`keeps independently abortable in-flight requests isolated`, async () => { + it(`aborts shared in-flight work after every cancellation owner leaves`, async () => { const releases: Array<() => void> = [] + let sharedSignal: AbortSignal | undefined let callCount = 0 const deduplicated = new DeduplicatedLoadSubset({ - loadSubset: () => { + loadSubset: (options) => { callCount += 1 + sharedSignal = options.signal return new Promise((resolve) => releases.push(resolve)) }, }) @@ -100,19 +98,54 @@ describe(`createDeduplicatedLoadSubset`, () => { const firstLoad = deduplicated.loadSubset({ where, signal: first.signal }) const secondLoad = deduplicated.loadSubset({ where, signal: second.signal }) - expect(callCount).toBe(2) + expect(callCount).toBe(1) + expect(secondLoad).toBe(firstLoad) first.abort() - for (const release of releases) release() + expect(sharedSignal?.aborted).toBe(false) + second.abort() + expect(sharedSignal?.aborted).toBe(true) + releases[0]?.() await Promise.all([firstLoad, secondLoad]) - expect( - deduplicated.loadSubset({ - where, - signal: new AbortController().signal, - }), - ).toBe(true) + const retry = deduplicated.loadSubset({ + where, + signal: new AbortController().signal, + }) expect(callCount).toBe(2) + expect(retry).toBeInstanceOf(Promise) + releases[1]?.() + await retry + }) + + it(`keeps shared work active for a signal-less owner`, async () => { + let resolveLoad: (() => void) | undefined + let sharedSignal: AbortSignal | undefined + const loadSubset = vi.fn( + (options: LoadSubsetOptions) => + new Promise((resolve) => { + sharedSignal = options.signal + resolveLoad = resolve + }), + ) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const controller = new AbortController() + const where = gt(ref(`age`), val(10)) + + const abortable = deduplicated.loadSubset({ + where, + signal: controller.signal, + }) + const persistent = deduplicated.loadSubset({ where }) + + expect(loadSubset).toHaveBeenCalledTimes(1) + expect(persistent).toBe(abortable) + controller.abort() + expect(sharedSignal?.aborted).toBe(false) + + resolveLoad?.() + await Promise.all([abortable, persistent]) + expect(deduplicated.loadSubset({ where })).toBe(true) }) it(`should call underlying loadSubset on first call`, async () => { @@ -1170,15 +1203,21 @@ describe(`createDeduplicatedLoadSubset`, () => { expect(onDeduplicate).toHaveBeenCalledWith(subsetOptions) }) - it(`does not share in-flight work across independent cancellation owners`, async () => { + it(`reports a signal-bearing request deduplicated after shared work completes`, async () => { const pending: Array<() => void> = [] + let sharedSignal: AbortSignal | undefined const loadSubset = vi.fn( - () => + (options: LoadSubsetOptions) => new Promise((resolve) => { + sharedSignal = options.signal pending.push(resolve) }), ) - const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const onDeduplicate = vi.fn() + const deduplicated = new DeduplicatedLoadSubset({ + loadSubset, + onDeduplicate, + }) const firstController = new AbortController() const secondController = new AbortController() @@ -1186,17 +1225,21 @@ describe(`createDeduplicatedLoadSubset`, () => { where: gt(ref(`age`), val(10)), signal: firstController.signal, }) - const second = deduplicated.loadSubset({ + const secondOptions = { where: gt(ref(`age`), val(20)), signal: secondController.signal, - }) + } + const second = deduplicated.loadSubset(secondOptions) - expect(loadSubset).toHaveBeenCalledTimes(2) - expect(second).not.toBe(first) + expect(loadSubset).toHaveBeenCalledTimes(1) + expect(second).toBe(first) firstController.abort() + expect(sharedSignal?.aborted).toBe(false) for (const resolve of pending) resolve() await Promise.all([first, second]) + expect(onDeduplicate).toHaveBeenCalledTimes(1) + expect(onDeduplicate).toHaveBeenCalledWith(secondOptions) }) }) From df4acc49d88f4899613dc6b18597b62c4532eb13 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 17 Aug 2026 18:50:55 +0100 Subject: [PATCH 18/21] fix(db): close includes review gaps --- .changeset/fix-includes-materialization.md | 4 +- packages/db-ivm/src/utils.ts | 213 +++++++++++++++++- .../db-ivm/tests/operators/groupBy.test.ts | 87 +++---- packages/db-ivm/tests/utils.test.ts | 25 +- packages/db/package.json | 3 +- packages/db/src/collection/changes.ts | 35 +-- packages/db/src/query/compiler/index.ts | 2 +- .../db/src/query/compiler/lazy-targets.ts | 3 +- packages/db/src/query/effect.ts | 8 +- packages/db/src/query/live/ARCHITECTURE.md | 7 + .../query/live/collection-config-builder.ts | 22 +- packages/db/src/scheduler.ts | 33 +++ packages/db/tests/oracle-config.ts | 13 ++ .../tests/query/compiler/lazy-targets.test.ts | 48 ++++ ...ncludes-collection-oracle.property.test.ts | 185 ++++++++++++--- ...ncludes-optimistic-oracle.property.test.ts | 17 +- .../query/includes-oracle.property.test.ts | 35 +-- .../query/includes-publication-oracle.test.ts | 37 +-- .../query/includes-query-shape-oracle.test.ts | 7 +- .../query/includes-temporal-oracle.test.ts | 84 +++++++ .../includes-work-counter-oracle.test.ts | 7 +- packages/db/tests/query/includes.test.ts | 163 ++++++++++++++ packages/db/tests/query/order-by.test.ts | 33 +++ packages/db/tests/query/scheduler.test.ts | 58 ++++- .../powersync-db-collection/src/powersync.ts | 10 + .../tests/load-hooks.test.ts | 42 ++++ packages/query-db-collection/src/query.ts | 18 +- .../query-db-collection/tests/query.test.ts | 59 +++++ 28 files changed, 1095 insertions(+), 163 deletions(-) create mode 100644 packages/db/tests/oracle-config.ts create mode 100644 packages/db/tests/query/compiler/lazy-targets.test.ts diff --git a/.changeset/fix-includes-materialization.md b/.changeset/fix-includes-materialization.md index d64904b14c..a9fb6ac71c 100644 --- a/.changeset/fix-includes-materialization.md +++ b/.changeset/fix-includes-materialization.md @@ -1,6 +1,8 @@ --- '@tanstack/db': patch +'@tanstack/db-ivm': patch +'@tanstack/powersync-db-collection': patch '@tanstack/query-db-collection': patch --- -Rebuild correlated include materialization as one D2 graph, fixing stale or missing nested results across route changes, batching, lazy loading, optimistic updates, and layered queries. Add abortable subset demand and coherent publication for Collection-valued includes. Wait for retained Query Collection cache results to reach the collection before resolving subset demand. +Rebuild correlated include materialization as one D2 graph, fixing stale or missing nested results across route changes, batching, lazy loading, optimistic updates, and layered queries. Add canonical structural relation keys, abortable subset demand, and coherent publication for Collection-valued includes. Dispose delayed PowerSync subset hooks after cleanup, and prevent released Query Collection cache results from reaching the collection. diff --git a/packages/db-ivm/src/utils.ts b/packages/db-ivm/src/utils.ts index 70cfda48c6..2c4170c897 100644 --- a/packages/db-ivm/src/utils.ts +++ b/packages/db-ivm/src/utils.ts @@ -193,19 +193,214 @@ export function compareKeys(a: string | number, b: string | number): number { return typeof a === `string` ? -1 : 1 } +type CanonicalValue = + | readonly [`undefined`] + | readonly [`null`] + | readonly [`boolean`, boolean] + | readonly [`number`, number | `NaN` | `Infinity` | `-Infinity`] + | readonly [`bigint`, string] + | readonly [`string`, string] + | readonly [`date`, number | `Invalid`] + | readonly [`regexp`, string, string] + | readonly [`bytes`, Array] + | readonly [`array`, Array] + | readonly [`map`, Array] + | readonly [`set`, Array] + | readonly [`object`, Array] + /** - * Serializes a value for use as a key, handling BigInt and Date values that JSON.stringify cannot handle. - * Uses JSON.stringify with a replacer function to convert BigInt values to strings and Date values to ISO strings. - * This is used for creating string keys in groupBy operations. + * Serializes a supported query value into one canonical key. + * + * JSON's native encoding is not suitable for relation keys: it merges BigInt + * with strings when a replacer is used, merges NaN with null, drops undefined, + * and depends on object insertion order. Ordinary JSON values keep their + * established wire form. Values that need richer types use a reserved prefix + * plus a structural, type-tagged encoding. */ export function serializeValue(value: unknown): string { - return JSON.stringify(value, (_, val) => { - if (typeof val === 'bigint') { - return val.toString() + if (isJsonSafeStructuralValue(value, new Set())) { + return JSON.stringify(toStableJsonValue(value)) + } + + return `~${JSON.stringify(toCanonicalValue(value, new Set()))}` +} + +type JsonValue = + | null + | boolean + | number + | string + | Array + | { [key: string]: JsonValue } + +function isJsonSafeStructuralValue( + value: unknown, + ancestors: Set, +): boolean { + if (value === null) return true + + switch (typeof value) { + case `boolean`: + case `string`: + return true + case `number`: + return Number.isFinite(value) + case `undefined`: + case `bigint`: + case `symbol`: + case `function`: + return false + } + + return withAcyclicValue(value, ancestors, () => { + if ( + value instanceof Date || + value instanceof RegExp || + value instanceof Uint8Array || + value instanceof Map || + value instanceof Set + ) { + return false } - if (val instanceof Date) { - return val.toISOString() + + return Array.isArray(value) + ? value.every((item) => isJsonSafeStructuralValue(item, ancestors)) + : Object.keys(value).every((key) => + isJsonSafeStructuralValue( + (value as Record)[key], + ancestors, + ), + ) + }) +} + +function toStableJsonValue(value: unknown): JsonValue { + if ( + value === null || + typeof value === `boolean` || + typeof value === `number` || + typeof value === `string` + ) { + return value + } + + if (Array.isArray(value)) { + return value.map(toStableJsonValue) + } + + return Object.fromEntries( + Object.keys(value as object) + .sort() + .map((key) => [ + key, + toStableJsonValue((value as Record)[key]), + ]), + ) +} + +function toCanonicalValue( + value: unknown, + ancestors: Set, +): CanonicalValue { + if (value === undefined) return [`undefined`] + if (value === null) return [`null`] + + switch (typeof value) { + case `boolean`: + return [`boolean`, value] + case `number`: + if (Number.isNaN(value)) return [`number`, `NaN`] + if (value === Infinity) return [`number`, `Infinity`] + if (value === -Infinity) return [`number`, `-Infinity`] + return [`number`, value === 0 ? 0 : value] + case `bigint`: + return [`bigint`, value.toString()] + case `string`: + return [`string`, value] + case `symbol`: + case `function`: + throw new TypeError( + `Cannot serialize ${typeof value} as a structural relation key`, + ) + } + + return withAcyclicValue(value, ancestors, () => { + if (value instanceof Date) { + const timestamp = value.getTime() + return Number.isNaN(timestamp) + ? ([`date`, `Invalid`] as const) + : ([`date`, timestamp] as const) + } + + if (value instanceof RegExp) { + return [`regexp`, value.source, value.flags] + } + + if (value instanceof Uint8Array) { + return [`bytes`, Array.from(value)] + } + + if (Array.isArray(value)) { + return [`array`, value.map((item) => toCanonicalValue(item, ancestors))] + } + + if (value instanceof Map) { + const entries = [...value.entries()].map( + ([key, entryValue]) => + [ + toCanonicalValue(key, ancestors), + toCanonicalValue(entryValue, ancestors), + ] as const, + ) + entries.sort((left, right) => + compareSerializedValues(JSON.stringify(left), JSON.stringify(right)), + ) + return [`map`, entries] } - return val + + if (value instanceof Set) { + const entries = [...value].map((entry) => + toCanonicalValue(entry, ancestors), + ) + entries.sort((left, right) => + compareSerializedValues(JSON.stringify(left), JSON.stringify(right)), + ) + return [`set`, entries] + } + + const entries = Object.keys(value) + .sort() + .map( + (key) => + [ + key, + toCanonicalValue( + (value as Record)[key], + ancestors, + ), + ] as const, + ) + return [`object`, entries] }) } + +function compareSerializedValues(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0 +} + +function withAcyclicValue( + value: object, + ancestors: Set, + encode: () => T, +): T { + if (ancestors.has(value)) { + throw new TypeError(`Cannot serialize a cyclic structural relation key`) + } + + ancestors.add(value) + try { + return encode() + } finally { + ancestors.delete(value) + } +} diff --git a/packages/db-ivm/tests/operators/groupBy.test.ts b/packages/db-ivm/tests/operators/groupBy.test.ts index fbe50fb33a..615e31fdb3 100644 --- a/packages/db-ivm/tests/operators/groupBy.test.ts +++ b/packages/db-ivm/tests/operators/groupBy.test.ts @@ -12,6 +12,7 @@ import { sum, } from '../../src/operators/groupBy.js' import { output } from '../../src/operators/index.js' +import { serializeValue } from '../../src/utils.js' describe(`Operators`, () => { describe(`GroupBy operation`, () => { @@ -50,7 +51,7 @@ describe(`Operators`, () => { const expectedResult = [ [ [ - `{"category":"A"}`, + serializeValue({ category: `A` }), { category: `A`, }, @@ -59,7 +60,7 @@ describe(`Operators`, () => { ], [ [ - `{"category":"B"}`, + serializeValue({ category: `B` }), { category: `B`, }, @@ -108,7 +109,7 @@ describe(`Operators`, () => { const expectedResult = [ [ [ - `{"category":"A"}`, + serializeValue({ category: `A` }), { total: 30, category: `A`, @@ -118,7 +119,7 @@ describe(`Operators`, () => { ], [ [ - `{"category":"B"}`, + serializeValue({ category: `B` }), { total: 30, category: `B`, @@ -177,7 +178,7 @@ describe(`Operators`, () => { const expectedResult = [ [ [ - `{"category":"A","region":"East"}`, + serializeValue({ category: `A`, region: `East` }), { total: 30, count: 2, @@ -189,7 +190,7 @@ describe(`Operators`, () => { ], [ [ - `{"category":"A","region":"West"}`, + serializeValue({ category: `A`, region: `West` }), { total: 30, count: 1, @@ -201,7 +202,7 @@ describe(`Operators`, () => { ], [ [ - `{"category":"B","region":"East"}`, + serializeValue({ category: `B`, region: `East` }), { total: 40, count: 1, @@ -228,7 +229,7 @@ describe(`Operators`, () => { const expectedAddResult = [ [ [ - `{"category":"A","region":"East"}`, + serializeValue({ category: `A`, region: `East` }), { category: `A`, region: `East`, @@ -240,7 +241,7 @@ describe(`Operators`, () => { ], [ [ - `{"category":"A","region":"East"}`, + serializeValue({ category: `A`, region: `East` }), { category: `A`, region: `East`, @@ -252,7 +253,7 @@ describe(`Operators`, () => { ], [ [ - `{"category":"B","region":"West"}`, + serializeValue({ category: `B`, region: `West` }), { category: `B`, region: `West`, @@ -277,7 +278,7 @@ describe(`Operators`, () => { const expectedDeleteResult = [ [ [ - `{"category":"A","region":"East"}`, + serializeValue({ category: `A`, region: `East` }), { category: `A`, region: `East`, @@ -289,7 +290,7 @@ describe(`Operators`, () => { ], [ [ - `{"category":"A","region":"East"}`, + serializeValue({ category: `A`, region: `East` }), { category: `A`, region: `East`, @@ -349,7 +350,7 @@ describe(`Operators`, () => { const expectedResult = [ [ [ - `{"category":"A"}`, + serializeValue({ category: `A` }), { category: `A`, countNotNull: 1, @@ -360,7 +361,7 @@ describe(`Operators`, () => { ], [ [ - `{"category":"B"}`, + serializeValue({ category: `B` }), { category: `B`, countNotNull: 1, @@ -412,7 +413,7 @@ describe(`Operators`, () => { const expectedResult = [ [ [ - `{"category":"A"}`, + serializeValue({ category: `A` }), { category: `A`, average: 15, @@ -423,7 +424,7 @@ describe(`Operators`, () => { ], [ [ - `{"category":"B"}`, + serializeValue({ category: `B` }), { category: `B`, average: 30, @@ -448,7 +449,7 @@ describe(`Operators`, () => { const expectedAddResult = [ [ [ - `{"category":"A"}`, + serializeValue({ category: `A` }), { category: `A`, average: 15, @@ -459,7 +460,7 @@ describe(`Operators`, () => { ], [ [ - `{"category":"A"}`, + serializeValue({ category: `A` }), { category: `A`, average: 20, @@ -470,7 +471,7 @@ describe(`Operators`, () => { ], [ [ - `{"category":"C"}`, + serializeValue({ category: `C` }), { category: `C`, average: 50, @@ -494,7 +495,7 @@ describe(`Operators`, () => { const expectedDeleteResult = [ [ [ - `{"category":"A"}`, + serializeValue({ category: `A` }), { category: `A`, average: 20, @@ -505,7 +506,7 @@ describe(`Operators`, () => { ], [ [ - `{"category":"A"}`, + serializeValue({ category: `A` }), { category: `A`, average: 25, @@ -561,7 +562,7 @@ describe(`Operators`, () => { const expectedResult = [ [ [ - `{"category":"A"}`, + serializeValue({ category: `A` }), { category: `A`, minimum: 5, @@ -574,7 +575,7 @@ describe(`Operators`, () => { ], [ [ - `{"category":"B"}`, + serializeValue({ category: `B` }), { category: `B`, minimum: 15, @@ -637,7 +638,7 @@ describe(`Operators`, () => { const expectedResult = [ [ [ - `{"category":"A"}`, + serializeValue({ category: `A` }), { category: `A`, middle: 20, @@ -648,7 +649,7 @@ describe(`Operators`, () => { ], [ [ - `{"category":"B"}`, + serializeValue({ category: `B` }), { category: `B`, middle: 12.5, @@ -699,7 +700,7 @@ describe(`Operators`, () => { // Find the group for category A const categoryAGroup = result.find( - ([key]: any) => key[0] === `{"category":"A"}`, + ([key]: any) => key[0] === serializeValue({ category: `A` }), ) expect(categoryAGroup).toBeDefined() expect(categoryAGroup[0][1].total).toBe(30) // Sum of 10 + 20 @@ -722,7 +723,7 @@ describe(`Operators`, () => { const expectedResult = [ [ [ - `{"category":"A"}`, + serializeValue({ category: `A` }), { category: `A`, total: 30, @@ -737,7 +738,8 @@ describe(`Operators`, () => { // Verify no new group with total: 0 was created by checking that // we don't have any positive weight entries for category A const positiveCategoryAEntries = result.filter( - ([key, , weight]: any) => key[0] === `{"category":"A"}` && weight > 0, + ([key, , weight]: any) => + key[0] === serializeValue({ category: `A` }) && weight > 0, ) expect(positiveCategoryAEntries).toHaveLength(0) }) @@ -788,7 +790,8 @@ describe(`Operators`, () => { // Find the group for category A, region East const categoryAEastGroup = result.find( - ([key]: any) => key[0] === `{"category":"A","region":"East"}`, + ([key]: any) => + key[0] === serializeValue({ category: `A`, region: `East` }), ) expect(categoryAEastGroup).toBeDefined() expect(categoryAEastGroup[0][1]).toEqual({ @@ -816,7 +819,7 @@ describe(`Operators`, () => { const expectedResult = [ [ [ - `{"category":"A","region":"East"}`, + serializeValue({ category: `A`, region: `East` }), { category: `A`, region: `East`, @@ -834,7 +837,8 @@ describe(`Operators`, () => { // Verify no new group with zero/empty values was created const positiveCategoryAEastEntries = result.filter( ([key, , weight]: any) => - key[0] === `{"category":"A","region":"East"}` && weight > 0, + key[0] === serializeValue({ category: `A`, region: `East` }) && + weight > 0, ) expect(positiveCategoryAEastEntries).toHaveLength(0) }) @@ -875,7 +879,7 @@ describe(`Operators`, () => { // Find the group for category A const categoryAGroup = result.find( - ([key]: any) => key[0] === `{"category":"A"}`, + ([key]: any) => key[0] === serializeValue({ category: `A` }), ) expect(categoryAGroup).toBeDefined() expect(categoryAGroup[0][1].total).toBe(30) // Sum of 10 + 20 @@ -894,7 +898,7 @@ describe(`Operators`, () => { const expectedRemovalResult = [ [ [ - `{"category":"A"}`, + serializeValue({ category: `A` }), { category: `A`, total: 30, @@ -919,7 +923,7 @@ describe(`Operators`, () => { const expectedReAdditionResult = [ [ [ - `{"category":"A"}`, + serializeValue({ category: `A` }), { category: `A`, total: 75, // 50 + 25 (new values, not the old 30) @@ -939,7 +943,7 @@ describe(`Operators`, () => { const expectedUpdateResult = [ [ [ - `{"category":"A"}`, + serializeValue({ category: `A` }), { category: `A`, total: 75, // Previous total @@ -949,7 +953,7 @@ describe(`Operators`, () => { ], [ [ - `{"category":"A"}`, + serializeValue({ category: `A` }), { category: `A`, total: 90, // 75 + 15 @@ -1009,7 +1013,8 @@ describe(`Operators`, () => { // Find the group for category A, region East const categoryAEastGroup = result.find( - ([key]: any) => key[0] === `{"category":"A","region":"East"}`, + ([key]: any) => + key[0] === serializeValue({ category: `A`, region: `East` }), ) expect(categoryAEastGroup).toBeDefined() expect(categoryAEastGroup[0][1]).toEqual({ @@ -1037,7 +1042,7 @@ describe(`Operators`, () => { const expectedRemovalResult = [ [ [ - `{"category":"A","region":"East"}`, + serializeValue({ category: `A`, region: `East` }), { category: `A`, region: `East`, @@ -1069,7 +1074,7 @@ describe(`Operators`, () => { const expectedReAdditionResult = [ [ [ - `{"category":"A","region":"East"}`, + serializeValue({ category: `A`, region: `East` }), { category: `A`, region: `East`, @@ -1098,7 +1103,7 @@ describe(`Operators`, () => { const expectedPartialRemovalResult = [ [ [ - `{"category":"A","region":"East"}`, + serializeValue({ category: `A`, region: `East` }), { category: `A`, region: `East`, @@ -1113,7 +1118,7 @@ describe(`Operators`, () => { ], [ [ - `{"category":"A","region":"East"}`, + serializeValue({ category: `A`, region: `East` }), { category: `A`, region: `East`, diff --git a/packages/db-ivm/tests/utils.test.ts b/packages/db-ivm/tests/utils.test.ts index a3eb0685bb..3e6b17f4c1 100644 --- a/packages/db-ivm/tests/utils.test.ts +++ b/packages/db-ivm/tests/utils.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest' import { Temporal } from 'temporal-polyfill' -import { DefaultMap } from '../src/utils.js' +import { DefaultMap, serializeValue } from '../src/utils.js' import { hash } from '../src/hashing/index.js' describe(`DefaultMap`, () => { @@ -30,6 +30,29 @@ describe(`DefaultMap`, () => { }) }) +describe(`serializeValue`, () => { + it(`preserves the established JSON form for ordinary keys`, () => { + expect(serializeValue(`user1`)).toBe(`"user1"`) + expect(serializeValue([1, `completed`])).toBe(`[1,"completed"]`) + }) + + it(`keeps distinct primitive types and special numbers distinct`, () => { + expect(serializeValue(1n)).not.toBe(serializeValue(`1`)) + expect(serializeValue(Number.NaN)).not.toBe(serializeValue(null)) + expect(serializeValue(undefined)).not.toBe(serializeValue(null)) + expect(serializeValue(new Date(0))).not.toBe( + serializeValue(`1970-01-01T00:00:00.000Z`), + ) + expect(serializeValue(new Date(Number.NaN))).not.toBe( + serializeValue(Number.NaN), + ) + }) + + it(`canonicalizes plain-object property order`, () => { + expect(serializeValue({ a: 1, b: 2 })).toBe(serializeValue({ b: 2, a: 1 })) + }) +}) + const hashType = `number` describe(`hash`, () => { describe(`primitive types`, () => { diff --git a/packages/db/package.json b/packages/db/package.json index 70f9ea2bb7..df40e4b36a 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -20,7 +20,8 @@ "build:minified": "vite build --minify", "dev": "vite build --watch", "lint": "eslint . --fix", - "test": "vitest --run" + "test": "vitest --run", + "test:oracles": "vitest --run tests/query/includes-oracle.property.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts" }, "type": "module", "main": "dist/cjs/index.cjs", diff --git a/packages/db/src/collection/changes.ts b/packages/db/src/collection/changes.ts index f7802c0e98..d51a0e799c 100644 --- a/packages/db/src/collection/changes.ts +++ b/packages/db/src/collection/changes.ts @@ -1,4 +1,5 @@ import { NegativeActiveSubscribersError } from '../errors' +import { withPublicationContext } from '../scheduler.js' import { createSingleRowRefProxy, toExpression, @@ -81,10 +82,11 @@ export class CollectionChangesManager< * This bypasses the normal empty array check in emitEvents */ public emitEmptyReadyEvent(): void { - // Emit empty array directly to all subscribers - for (const subscription of this.changeSubscriptions) { - subscription.emitEvents([]) - } + withPublicationContext(() => { + for (const subscription of this.changeSubscriptions) { + subscription.emitEvents([]) + } + }) } /** @@ -183,23 +185,26 @@ export class CollectionChangesManager< return } - // Notify both internal layout consumers and the public subscription API. - // Public subscribers historically receive an empty batch for order-only - // moves because there is no row-value ChangeMessage to publish. - if (rawEvents.length === 0) { - for (const listener of this.layoutChangeListeners) listener() - } - // Enrich all change messages with virtual properties // This uses the "add-if-missing" pattern to preserve pass-through semantics const enrichedEvents: Array< ChangeMessage, TKey> > = rawEvents.map((change) => this.enrichChangeWithVirtualProps(change)) - // Emit to all listeners - for (const subscription of this.changeSubscriptions) { - subscription.emitEvents(enrichedEvents) - } + // Every subscriber sees one committed source batch before dependent query + // graphs run. This keeps repeated aliases and sibling subqueries coherent. + withPublicationContext(() => { + // Notify both internal layout consumers and the public subscription API. + // Public subscribers historically receive an empty batch for order-only + // moves because there is no row-value ChangeMessage to publish. + if (rawEvents.length === 0) { + for (const listener of this.layoutChangeListeners) listener() + } + + for (const subscription of this.changeSubscriptions) { + subscription.emitEvents(enrichedEvents) + } + }) } /** Subscribe to layout-only publications. Internal observer channel. */ diff --git a/packages/db/src/query/compiler/index.ts b/packages/db/src/query/compiler/index.ts index 7331786142..0b07fc6a59 100644 --- a/packages/db/src/query/compiler/index.ts +++ b/packages/db/src/query/compiler/index.ts @@ -960,7 +960,7 @@ export function compileQuery( (row as any)?.__correlationKey const parentContext = (row as any)?.__parentContext if (parentContext != null) { - return JSON.stringify([correlationKey, parentContext]) + return serializeValue([correlationKey, parentContext]) } return correlationKey } diff --git a/packages/db/src/query/compiler/lazy-targets.ts b/packages/db/src/query/compiler/lazy-targets.ts index 963f62ebff..cf4f5cfc4c 100644 --- a/packages/db/src/query/compiler/lazy-targets.ts +++ b/packages/db/src/query/compiler/lazy-targets.ts @@ -212,7 +212,8 @@ function resolveLazySource( if ( lazyFrom.type === `collectionRef` && - lazyFrom.collection === target.collection + lazyFrom.collection === target.collection && + lazyFrom.alias === target.alias ) { return lazyFrom } diff --git a/packages/db/src/query/effect.ts b/packages/db/src/query/effect.ts index 047991cd46..3976c2fde8 100644 --- a/packages/db/src/query/effect.ts +++ b/packages/db/src/query/effect.ts @@ -1,5 +1,8 @@ import { D2, output } from '@tanstack/db-ivm' -import { transactionScopedScheduler } from '../scheduler.js' +import { + getActivePublicationContext, + transactionScopedScheduler, +} from '../scheduler.js' import { getActiveTransaction } from '../transactions.js' import { compileQuery } from './compiler/index.js' import { @@ -683,7 +686,8 @@ class EffectPipelineRunner { * live query collections, ensuring parent queries run before effects. */ private scheduleGraphRun(sourceId?: string): void { - const contextId = getActiveTransaction()?.id + const contextId = + getActiveTransaction()?.id ?? getActivePublicationContext() // Collect dependencies for this schedule call const deps = new Set(this.builderDependencies) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 52e6fbff79..8c42930c20 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -536,6 +536,13 @@ scenarios use direct assertions. A boundary suite may retain an exact expected-failure guard for a planner or ownership defect that this graph does not own. +Run the DB oracle set with `pnpm test:oracles` from `packages/db`. Broad +properties use FastCheck's random seed, while structural matrices keep fixed +seeds so each run covers the same named cells. Increase both corpora with +`TANSTACK_DB_ORACLE_RUNS_MULTIPLIER=10 pnpm test:oracles`. Preserve FastCheck's +reported seed and shrink path while reducing a failure, then add the smallest +case as a deterministic regression trace. + ## Implementation discipline - Express relation state with existing D2 inputs, joins, reductions, grouping, diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index 9b432c474e..fadcaa39f0 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -4,7 +4,10 @@ import { MissingAliasInputsError, SetWindowRequiresOrderByError, } from '../../errors.js' -import { transactionScopedScheduler } from '../../scheduler.js' +import { + getActivePublicationContext, + transactionScopedScheduler, +} from '../../scheduler.js' import { getActiveTransaction } from '../../transactions.js' import { deepEquals } from '../../utils.js' import { CollectionSubscriber } from './collection-subscriber.js' @@ -398,14 +401,15 @@ export class CollectionConfigBuilder< let callbackCalled = false while (syncState.graph.pendingWork()) { syncState.graph.run() - // Flush accumulated changes after each graph step to commit them as one transaction. - // This ensures intermediate join states (like null on one side) don't cause - // duplicate key errors when the full join result arrives in the same step. - syncState.flushPendingChanges?.() callback?.() callbackCalled = true } + // Publish only after every operator has reached quiescence. A source + // change can reach sibling materializations in different graph steps; + // flushing between those steps would expose a mixed root snapshot. + syncState.flushPendingChanges?.() + // Ensure the callback runs at least once even when the graph has no pending work. // This handles lazy loading scenarios where setWindow() increases the limit or // an async loadSubset completes and we need to re-check if more data is needed. @@ -459,7 +463,10 @@ export class CollectionConfigBuilder< dependencies?: Array> }, ) { - const contextId = options?.contextId ?? getActiveTransaction()?.id + const contextId = + options?.contextId ?? + getActiveTransaction()?.id ?? + getActivePublicationContext() // Use the builder instance as the job ID for deduplication. This is memory-safe // because the scheduler's context Map is deleted after flushing (no long-term retention). const jobId = options?.jobId ?? this @@ -981,7 +988,8 @@ export class CollectionConfigBuilder< // Mark ready when: // 1. All subscriptions are set up (subscribedToAllCollections) // 2. All source collections are ready - // 3. The live query collection is not loading subset data + // 3. Every active route demand has settled + // 4. The live query collection is not loading subset data // This prevents marking the live query ready before its data is processed // (fixes issue where useLiveQuery returns isReady=true with empty data) if (subscribedToAll && allReady && allDemandsSettled && !isLoading) { diff --git a/packages/db/src/scheduler.ts b/packages/db/src/scheduler.ts index 8ffe8b768c..d87ac05319 100644 --- a/packages/db/src/scheduler.ts +++ b/packages/db/src/scheduler.ts @@ -220,3 +220,36 @@ export class Scheduler { } export const transactionScopedScheduler = new Scheduler() + +let activePublicationContext: SchedulerContextId | undefined + +/** + * Returns the Collection publication that currently owns synchronous change + * delivery. Live-query jobs use it to coalesce all source subscriptions that + * observe one committed batch. + */ +export function getActivePublicationContext(): SchedulerContextId | undefined { + return activePublicationContext +} + +/** + * Runs one synchronous Collection publication inside a scheduler context. + * Nested publications share the outer context, so downstream live queries run + * only after every subscriber to the original committed batch has observed it. + */ +export function withPublicationContext(publish: () => T): T { + if (activePublicationContext !== undefined) return publish() + + const contextId = Symbol(`collection-publication`) + activePublicationContext = contextId + try { + const result = publish() + transactionScopedScheduler.flush(contextId) + return result + } catch (error) { + transactionScopedScheduler.clear(contextId) + throw error + } finally { + activePublicationContext = undefined + } +} diff --git a/packages/db/tests/oracle-config.ts b/packages/db/tests/oracle-config.ts new file mode 100644 index 0000000000..8f30fbda41 --- /dev/null +++ b/packages/db/tests/oracle-config.ts @@ -0,0 +1,13 @@ +const multiplierText = process.env.TANSTACK_DB_ORACLE_RUNS_MULTIPLIER +const multiplier = multiplierText === undefined ? 1 : Number(multiplierText) + +if (!Number.isSafeInteger(multiplier) || multiplier < 1) { + throw new Error( + `TANSTACK_DB_ORACLE_RUNS_MULTIPLIER must be a positive integer`, + ) +} + +/** Keeps ordinary CI bounded while allowing long randomized oracle campaigns. */ +export function oracleRuns(baseRuns: number): number { + return baseRuns * multiplier +} diff --git a/packages/db/tests/query/compiler/lazy-targets.test.ts b/packages/db/tests/query/compiler/lazy-targets.test.ts new file mode 100644 index 0000000000..280b0572e1 --- /dev/null +++ b/packages/db/tests/query/compiler/lazy-targets.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from 'vitest' +import { getLazyLoadTargets } from '../../../src/query/compiler/lazy-targets.js' +import { CollectionRef, PropRef, QueryRef } from '../../../src/query/ir.js' +import type { QueryIR } from '../../../src/query/ir.js' +import type { CollectionImpl } from '../../../src/collection/index.js' + +describe(`lazy load target identity`, () => { + const collection = { id: `items` } as CollectionImpl + + function targetsForAlias(alias: string) { + const inner: QueryIR = { + from: new CollectionRef(collection, `inner`), + } + const query: QueryIR = { + from: new QueryRef(inner, `selected`), + } + const optimizedSource = new CollectionRef(collection, `optimized`) + + return { + optimizedSource, + targets: getLazyLoadTargets( + query, + optimizedSource, + `selected`, + new PropRef([`selected`, `id`]), + collection, + { selected: alias }, + ), + } + } + + it(`uses a fallback source when its lexical alias matches`, () => { + const { optimizedSource, targets } = targetsForAlias(`optimized`) + + expect(targets).toEqual([ + { + sourceId: optimizedSource.sourceId, + alias: `optimized`, + collection, + path: [`id`], + }, + ]) + }) + + it(`does not route demand through a fallback with another alias`, () => { + expect(targetsForAlias(`other`).targets).toEqual([]) + }) +}) diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index 89a630b9d4..937e8d1916 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -9,6 +9,7 @@ import { toArray, } from '../../src/query/index.js' import { runTrace } from '../trace-runner.js' +import { oracleRuns } from '../oracle-config.js' import { flushPromises, mockSyncCollectionOptions, @@ -40,6 +41,8 @@ type ProjectedParent = { group: number childrenReady: boolean children: Array + arrayChildren: Array + materializedChildren: Array } type CollectionObservation = { @@ -111,24 +114,37 @@ function createControlledCollection( function createCollectionQuery( parents: Collection, children: Collection, + reuseChildRelation = true, ) { return createLiveQueryCollection((q) => q .from({ parent: parents }) .orderBy(({ parent }) => parent.id) - .select(({ parent }) => ({ - id: parent.id, - group: parent.group, - children: q - .from({ child: children }) - .where(({ child }) => eq(child.parentGroup, parent.group)) - .orderBy(({ child }) => child.id) - .select(({ child }) => ({ - id: child.id, - parentGroup: child.parentGroup, - value: child.value, - })), - })), + .select(({ parent }) => { + const createChildRows = () => + q + .from({ child: children }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ child }) => child.id) + .select(({ child }) => ({ + id: child.id, + parentGroup: child.parentGroup, + value: child.value, + })) + const childRows = createChildRows() + + return { + id: parent.id, + group: parent.group, + children: childRows, + arrayChildren: toArray( + reuseChildRelation ? childRows : createChildRows(), + ), + materializedChildren: materialize( + reuseChildRelation ? childRows : createChildRows(), + ), + } + }), ) } @@ -139,32 +155,48 @@ type IncludedChildCollection = ReturnType< function projectLive( live: ReturnType, ): Array { - return [...live.values()].map((parent) => ({ - id: parent.id, - group: parent.group, - childrenReady: parent.children.isReady(), - children: [...parent.children.values()] - .map(({ id, parentGroup, value }) => ({ id, parentGroup, value })) - .sort((left, right) => left.id - right.id), - })) + return [...live.values()].map((parent) => { + const projectChildren = (rows: Iterable) => + [...rows].map(({ id, parentGroup, value }) => ({ + id, + parentGroup, + value, + })) + + return { + id: parent.id, + group: parent.group, + childrenReady: parent.children.isReady(), + children: projectChildren(parent.children.values()), + arrayChildren: projectChildren(parent.arrayChildren), + materializedChildren: projectChildren(parent.materializedChildren), + } + }) } function recompute(context: CollectionContext): Array { return [...context.model.parents.values()] .sort((left, right) => left.id - right.id) - .map((parent) => ({ - ...parent, - childrenReady: true, - children: [...context.model.children.values()] + .map((parent) => { + const children = [...context.model.children.values()] .filter((child) => child.parentGroup === parent.group) .sort((left, right) => left.id - right.id) - .map((child) => ({ ...child })), - })) + .map((child) => ({ ...child })) + + return { + ...parent, + childrenReady: true, + children, + arrayChildren: children, + materializedChildren: children, + } + }) } function createCollectionDriver( initialParents: ReadonlyArray, initialChildren: ReadonlyArray, + reuseChildRelation = true, ): TraceDriver { return { setup() { @@ -179,7 +211,11 @@ function createCollectionDriver( return { parents, children, - live: createCollectionQuery(parents.collection, children.collection), + live: createCollectionQuery( + parents.collection, + children.collection, + reuseChildRelation, + ), model: { parents: new Map(initialParents.map((row) => [row.id, { ...row }])), children: new Map(initialChildren.map((row) => [row.id, { ...row }])), @@ -292,9 +328,33 @@ const collectionScenarioArbitrary = fc.record({ actions: fc.array(actionArbitrary, { minLength: 1, maxLength: 16 }), }) +function enumerateActionSequences( + actions: ReadonlyArray, + maxLength: number, +): Array> { + const sequences: Array> = [[]] + let frontier: Array> = [[]] + for (let length = 1; length <= maxLength; length++) { + frontier = frontier.flatMap((prefix) => + actions.map((action) => [...prefix, action]), + ) + sequences.push(...frontier) + } + return sequences +} + +const exhaustiveActions: ReadonlyArray = [ + { type: `putParent`, row: { id: 0, group: 0 } }, + { type: `putParent`, row: { id: 0, group: 1 } }, + { type: `deleteParent`, id: 0 }, + { type: `putChild`, row: { id: 10, parentGroup: 0, value: 0 } }, + { type: `putChild`, row: { id: 10, parentGroup: 1, value: 1 } }, + { type: `deleteChild`, id: 10 }, +] + describe(`Collection-valued includes oracle`, () => { - fcTest.prop([collectionScenarioArbitrary], { numRuns: 30 })( - `matches recomputation and publishes coherent snapshots across generated relationship histories`, + fcTest.prop([collectionScenarioArbitrary], { numRuns: oracleRuns(30) })( + `keeps Collection, toArray, and materialize equivalent across generated relationship histories`, ({ parentGroup, childValue, actions }) => runTrace({ steps: actions, @@ -306,6 +366,56 @@ describe(`Collection-valued includes oracle`, () => { }), ) + fcTest( + `exhaustively matches every two-step history in the smallest relationship domain`, + async () => { + const initialStates = [ + { parents: [] as Array, children: [] as Array }, + { parents: [{ id: 0, group: 0 }], children: [] as Array }, + { + parents: [] as Array, + children: [{ id: 10, parentGroup: 0, value: 0 }], + }, + { + parents: [{ id: 0, group: 0 }], + children: [{ id: 10, parentGroup: 0, value: 0 }], + }, + ] + const histories = enumerateActionSequences(exhaustiveActions, 2) + + for (const initial of initialStates) { + for (const steps of histories) { + try { + await runTrace({ + steps, + driver: createCollectionDriver(initial.parents, initial.children), + projection: collectionProjection, + }) + } catch (cause) { + throw new Error( + `Exhaustive Collection include history failed: ${JSON.stringify({ initial, steps })}`, + { cause }, + ) + } + } + } + }, + ) + + fcTest( + `publishes independently compiled equivalent child relations coherently`, + () => + runTrace({ + steps: [{ type: `deleteChild`, id: 10 }], + driver: createCollectionDriver( + [{ id: 0, group: 0 }], + [{ id: 10, parentGroup: 0, value: 0 }], + false, + ), + projection: collectionProjection, + }), + ) + fcTest(`replays a dormant bucket when its first parent route activates`, () => runTrace({ steps: [{ type: `putParent`, row: { id: 1, group: 7 } }], @@ -491,6 +601,14 @@ describe(`Collection-valued includes oracle`, () => { { id: 10, parentGroup: 1, value: 1 }, { id: 20, parentGroup: 1, value: 2 }, ], + arrayChildren: [ + { id: 10, parentGroup: 1, value: 1 }, + { id: 20, parentGroup: 1, value: 2 }, + ], + materializedChildren: [ + { id: 10, parentGroup: 1, value: 1 }, + { id: 20, parentGroup: 1, value: 2 }, + ], }, ]) expect(changes).toEqual([]) @@ -622,7 +740,7 @@ describe(`Collection-valued includes oracle`, () => { wideId: fc.integer({ min: 10, max: 19 }), }), ], - { numRuns: 20 }, + { numRuns: oracleRuns(20) }, )( `uses one raw public-key order across Collection and inline materializations`, async ({ smallId, wideId }) => { @@ -691,7 +809,7 @@ describe(`Collection-valued includes oracle`, () => { value: fc.integer({ min: -10, max: 10 }), }), ], - { numRuns: 20 }, + { numRuns: oracleRuns(20) }, )( `matches recomputation through optimistic child insert and delete confirmation and rollback`, async ({ group, insertedId, confirmedId, value }) => { @@ -702,12 +820,14 @@ describe(`Collection-valued includes oracle`, () => { const driver: TraceDriver = { ...base, async apply(action, context, checkpoint) { + context.publications = [] if (action.type === `insert`) { const transaction = context.children.collection.insert({ ...action.row, }) context.model.children.set(action.row.id, { ...action.row }) checkpoint() + context.publications = [] if (action.settlement === `confirm`) { context.children.write(`insert`, action.row) context.children.resolveSync() @@ -732,6 +852,7 @@ describe(`Collection-valued includes oracle`, () => { const transaction = context.children.collection.delete(action.id) context.model.children.delete(action.id) checkpoint() + context.publications = [] if (action.settlement === `confirm`) { context.children.write(`delete`, previous) context.children.resolveSync() diff --git a/packages/db/tests/query/includes-optimistic-oracle.property.test.ts b/packages/db/tests/query/includes-optimistic-oracle.property.test.ts index 24e599018f..1b010abb6f 100644 --- a/packages/db/tests/query/includes-optimistic-oracle.property.test.ts +++ b/packages/db/tests/query/includes-optimistic-oracle.property.test.ts @@ -12,6 +12,7 @@ import { withExpectedRejection, } from '../utils.js' import { runTrace } from '../trace-runner.js' +import { oracleRuns } from '../oracle-config.js' import type { TraceDriver, TraceProjection } from '../trace-runner.js' type RootRow = { @@ -530,7 +531,7 @@ describe(`optimistic relationship-transition oracle`, () => { }, ) - fcTest.prop([routeValuesArbitrary], { numRuns: 12 })( + fcTest.prop([routeValuesArbitrary], { numRuns: oracleRuns(12) })( `an optimistic rekey detaches its old descendants immediately`, async (routes) => { await expectHistoryMatches(routes, [ @@ -545,7 +546,7 @@ describe(`optimistic relationship-transition oracle`, () => { }, ) - fcTest.prop([routeValuesArbitrary], { numRuns: 12 })( + fcTest.prop([routeValuesArbitrary], { numRuns: oracleRuns(12) })( `restores the authoritative relationship after an optimistic rekey rolls back`, async (routes) => { await expectHistoryMatches(routes, [ @@ -559,7 +560,7 @@ describe(`optimistic relationship-transition oracle`, () => { }, ) - fcTest.prop([routeValuesArbitrary], { numRuns: 12 })( + fcTest.prop([routeValuesArbitrary], { numRuns: oracleRuns(12) })( `rolls back a descendant update made while its ancestor is reparented`, async (routes) => { await expectHistoryMatches(routes, [ @@ -583,7 +584,7 @@ describe(`optimistic relationship-transition oracle`, () => { }, ) - fcTest.prop([routeValuesArbitrary], { numRuns: 12 })( + fcTest.prop([routeValuesArbitrary], { numRuns: oracleRuns(12) })( `rolls back a reparented ancestor while its descendant update remains pending`, async (routes) => { await expectHistoryMatches(routes, [ @@ -607,7 +608,7 @@ describe(`optimistic relationship-transition oracle`, () => { }, ) - fcTest.prop([routeValuesArbitrary], { numRuns: 12 })( + fcTest.prop([routeValuesArbitrary], { numRuns: oracleRuns(12) })( `settles a confirmed optimistic reparent on the same authoritative route`, async (routes) => { await expectHistoryMatches(routes, [ @@ -645,7 +646,7 @@ describe(`optimistic relationship-transition oracle`, () => { }, ) - fcTest.prop([routeValuesArbitrary], { numRuns: 12 })( + fcTest.prop([routeValuesArbitrary], { numRuns: oracleRuns(12) })( `settles a confirmed optimistic reparent on a different authoritative route`, async (routes) => { await expectHistoryMatches(routes, [ @@ -685,7 +686,7 @@ describe(`optimistic relationship-transition oracle`, () => { }, ) - fcTest.prop([routeValuesArbitrary], { numRuns: 12 })( + fcTest.prop([routeValuesArbitrary], { numRuns: oracleRuns(12) })( `restores a rekey after a sibling enters its old route`, async (routes) => { await expectHistoryMatches(routes, [ @@ -730,7 +731,7 @@ describe(`optimistic relationship-transition oracle`, () => { }, ) - fcTest.prop([routeValuesArbitrary], { numRuns: 12 })( + fcTest.prop([routeValuesArbitrary], { numRuns: oracleRuns(12) })( `supports repeated rollback and confirmation histories`, async (routes) => { await expectHistoryMatches(routes, [ diff --git a/packages/db/tests/query/includes-oracle.property.test.ts b/packages/db/tests/query/includes-oracle.property.test.ts index c0c0f71109..c044d74b8c 100644 --- a/packages/db/tests/query/includes-oracle.property.test.ts +++ b/packages/db/tests/query/includes-oracle.property.test.ts @@ -14,6 +14,7 @@ import { mockSyncCollectionOptions, withExpectedRejection, } from '../utils.js' +import { oracleRuns } from '../oracle-config.js' import { runTrace } from '../trace-runner.js' import type { TraceCheckpoint, @@ -3698,7 +3699,7 @@ describe(`includes recompute oracle`, () => { for (const [shapeIndex, shape] of independentTransitionShapes.entries()) { fcTest.prop([independentTransitionScenarioArbitrary(shape)], { - numRuns: 4, + numRuns: oracleRuns(4), seed: 1734 + shapeIndex, })( `matches recomputation for independent ${shape} relationship targets`, @@ -3711,7 +3712,7 @@ describe(`includes recompute oracle`, () => { for (const [historyIndex, history] of destinationHistories.entries()) { fcTest.prop([destinationHistoryScenarioArbitrary(history)], { - numRuns: 4, + numRuns: oracleRuns(4), seed: 1740 + historyIndex, })( `matches recomputation for the ${history} route destination history`, @@ -3990,7 +3991,7 @@ describe(`includes recompute oracle`, () => { for (const depth of [2, 3, 4] as const) { for (const sourceBranch of [0, 1] as const) { fcTest.prop([rekeyRouteReuseScenarioArbitrary(depth, sourceBranch)], { - numRuns: 4, + numRuns: oracleRuns(4), seed: 1726 + depth * 10 + sourceBranch, })( `reuses a retired route at depth ${depth}, branch ${sourceBranch}`, @@ -4006,7 +4007,7 @@ describe(`includes recompute oracle`, () => { ), ], { - numRuns: 4, + numRuns: oracleRuns(4), seed: 1733 + depth * 10 + sourceBranch, }, )( @@ -4038,7 +4039,7 @@ describe(`includes recompute oracle`, () => { ), ], { - numRuns: 4, + numRuns: oracleRuns(4), seed: 1727 + depth * 100 + targetLevel * 10 + sourceBranch, }, )( @@ -4063,7 +4064,7 @@ describe(`includes recompute oracle`, () => { [`route-only`, `route-and-position`] as const ).entries()) { fcTest.prop([relationshipBatchFixtureArbitrary], { - numRuns: 4, + numRuns: oracleRuns(4), seed: 1738 + publicIdIndex * 100 + routeIndex * 10 + updateIndex, })( `matches the split/atomic replacement matrix for ${publicId} public id, ${route} route, ${ancestorUpdate}`, @@ -4103,7 +4104,7 @@ describe(`includes recompute oracle`, () => { flatMaterializationScenarioArbitrary, ], { - numRuns: 30, + numRuns: oracleRuns(30), seed: 1721, }, )(`matches recomputation for flat materializations`, (kind, scenario) => @@ -4112,7 +4113,7 @@ describe(`includes recompute oracle`, () => { for (const depth of [1, 2, 3, 4] as const) { fcTest.prop([fullRowBatchScenarioAtDepthArbitrary(depth)], { - numRuns: 10, + numRuns: oracleRuns(10), seed: 1719 + depth, })( `matches recomputation for visible multi-row batches at depth ${depth}`, @@ -4134,7 +4135,7 @@ describe(`includes recompute oracle`, () => { ), ], { - numRuns: 4, + numRuns: oracleRuns(4), seed: 1721 + depth + targetLevel, }, )( @@ -4187,7 +4188,7 @@ describe(`includes recompute oracle`, () => { ), ], { - numRuns: 3, + numRuns: oracleRuns(3), seed: 1725 + depth * 100 + @@ -4279,7 +4280,7 @@ describe(`includes recompute oracle`, () => { }) }) - fcTest.prop([scenarioArbitrary], { numRuns: 40 })( + fcTest.prop([scenarioArbitrary], { numRuns: oracleRuns(40) })( `matches naive recomputation after every incremental change`, expectScenarioMatches, ) @@ -4290,7 +4291,7 @@ describe(`includes recompute oracle`, () => { ({ sharedIntermediate }) => !sharedIntermediate, ), ], - { numRuns: 30 }, + { numRuns: oracleRuns(30) }, )( `matches recomputation for nested scalar materialization`, expectMaterializeScenarioMatches, @@ -4318,7 +4319,7 @@ describe(`includes recompute oracle`, () => { { selector: (row) => row.id, maxLength: 7 }, ), ], - { numRuns: 25 }, + { numRuns: oracleRuns(25) }, )( `is unchanged by alpha-renaming, sibling declaration order, or an unrelated sibling`, async (rootRows, childRows) => { @@ -4430,7 +4431,7 @@ describe(`includes recompute oracle`, () => { fcTest.prop( [fc.integer({ min: -5, max: 5 }).filter((value) => value !== 0)], - { numRuns: 15 }, + { numRuns: oracleRuns(15) }, )( `optimistic updates converge to confirmed-only state`, async (confirmedValue) => { @@ -4643,9 +4644,9 @@ describe(`includes recompute oracle`, () => { })), ) - expect(stripVirtualProperties(duplicateAliases)).toEqual( - stripVirtualProperties(uniqueAliases), - ) + const expected = [{ id: 1, issues: [{ id: 10 }], tags: [{ id: 20 }] }] + expect(stripVirtualProperties(uniqueAliases)).toEqual(expected) + expect(stripVirtualProperties(duplicateAliases)).toEqual(expected) } finally { await Promise.all([ roots.collection.cleanup(), diff --git a/packages/db/tests/query/includes-publication-oracle.test.ts b/packages/db/tests/query/includes-publication-oracle.test.ts index 584b117bee..037a7aeddd 100644 --- a/packages/db/tests/query/includes-publication-oracle.test.ts +++ b/packages/db/tests/query/includes-publication-oracle.test.ts @@ -8,6 +8,7 @@ import { materialize, } from '../../src/query/index.js' import { runTrace } from '../trace-runner.js' +import { oracleRuns } from '../oracle-config.js' import { flushPromises, mockSyncCollectionOptions, @@ -449,7 +450,7 @@ describe(`layered-query publication oracle`, () => { for (const q1Shape of q1Shapes) { for (const q2Shape of q2Shapes) { - fcTest.prop([changedValueArbitrary], { numRuns: 12 })( + fcTest.prop([changedValueArbitrary], { numRuns: oracleRuns(12) })( `publishes #1713 updates through a ${q1Shape} Q1 and ${q2Shape} Q2`, async (value) => { await expectPublicationMatches( @@ -462,7 +463,7 @@ describe(`layered-query publication oracle`, () => { ) fcTest.prop([changedValueArbitrary, changedChildValueArbitrary], { - numRuns: 12, + numRuns: oracleRuns(12), })( `recovers a ${q1Shape} Q1 and ${q2Shape} Q2 after a child update`, async (parentValue, childValue) => { @@ -475,7 +476,7 @@ describe(`layered-query publication oracle`, () => { }, ) - fcTest.prop([changedValueArbitrary], { numRuns: 8 })( + fcTest.prop([changedValueArbitrary], { numRuns: oracleRuns(8) })( `publishes optimistic state before confirmation through a ${q1Shape} Q1 and ${q2Shape} Q2`, async (value) => { await expectPublicationMatches( @@ -487,7 +488,7 @@ describe(`layered-query publication oracle`, () => { }, ) - fcTest.prop([changedValueArbitrary], { numRuns: 8 })( + fcTest.prop([changedValueArbitrary], { numRuns: oracleRuns(8) })( `publishes state after optimistic confirmation through a ${q1Shape} Q1 and ${q2Shape} Q2`, async (value) => { await expectPublicationMatches( @@ -501,33 +502,33 @@ describe(`layered-query publication oracle`, () => { } } - fcTest.prop([changedChildValueArbitrary])( + fcTest.prop([changedChildValueArbitrary], { numRuns: oracleRuns(100) })( `publishes child-only scalar updates through both layers`, async (value) => { await expectPublicationMatches({ type: `childScalar`, value }) }, ) - fcTest.prop([fc.constantFrom(20, 30)])( + fcTest.prop([fc.constantFrom(20, 30)], { numRuns: oracleRuns(100) })( `compares route transitions at both query layers`, async (group) => { await expectPublicationMatches({ type: `parentRoute`, group }) }, ) - fcTest.prop([ - fc.record({ - group: fc.constantFrom(10, 20, 30), - value: changedValueArbitrary, - }), - ])( - `compares atomic parent replacements at both query layers`, - async (row) => { - await expectPublicationMatches({ type: `atomicReplace`, ...row }) - }, - ) + fcTest.prop( + [ + fc.record({ + group: fc.constantFrom(10, 20, 30), + value: changedValueArbitrary, + }), + ], + { numRuns: oracleRuns(100) }, + )(`compares atomic parent replacements at both query layers`, async (row) => { + await expectPublicationMatches({ type: `atomicReplace`, ...row }) + }) - fcTest.prop([changedValueArbitrary])( + fcTest.prop([changedValueArbitrary], { numRuns: oracleRuns(100) })( `publishes restored state after optimistic rollback`, async (value) => { await expectPublicationMatches({ diff --git a/packages/db/tests/query/includes-query-shape-oracle.test.ts b/packages/db/tests/query/includes-query-shape-oracle.test.ts index 4a6f4b2bec..6e11994420 100644 --- a/packages/db/tests/query/includes-query-shape-oracle.test.ts +++ b/packages/db/tests/query/includes-query-shape-oracle.test.ts @@ -8,6 +8,7 @@ import { materialize, } from '../../src/query/index.js' import { runTrace } from '../trace-runner.js' +import { oracleRuns } from '../oracle-config.js' import { mockSyncCollectionOptions } from '../utils.js' import type { TraceDriver, TraceProjection } from '../trace-runner.js' @@ -385,7 +386,7 @@ const nullableProjection: TraceProjection< describe(`includes query-shape recompute oracle`, () => { fcTest.prop([fc.integer({ min: 2, max: 5 })], { - numRuns: 12, + numRuns: oracleRuns(12), seed: 1703, })( `deleting one joined contributor preserves remaining multiplicity (#1703)`, @@ -415,7 +416,7 @@ describe(`includes query-shape recompute oracle`, () => { productionId: fc.integer({ min: 101, max: 200 }), }), ], - { numRuns: 12, seed: 1704 }, + { numRuns: oracleRuns(12), seed: 1704 }, )( `materialization follows correlation through a joined alias (#1704)`, async ({ correlationId, productionId }) => { @@ -438,7 +439,7 @@ describe(`includes query-shape recompute oracle`, () => { ) fcTest.prop([fc.integer({ min: 1, max: 100 })], { - numRuns: 12, + numRuns: oracleRuns(12), seed: 1706, })( `findOne maps a null correlation key to undefined (#1706)`, diff --git a/packages/db/tests/query/includes-temporal-oracle.test.ts b/packages/db/tests/query/includes-temporal-oracle.test.ts index 97b1fa8edc..e1abca79ac 100644 --- a/packages/db/tests/query/includes-temporal-oracle.test.ts +++ b/packages/db/tests/query/includes-temporal-oracle.test.ts @@ -1,6 +1,8 @@ +import { fc, test as fcTest } from '@fast-check/vitest' import { describe, expect, it, vi } from 'vitest' import { createCollection } from '../../src/collection/index.js' import { createDeferred } from '../../src/deferred.js' +import { BasicIndex } from '../../src/indexes/basic-index.js' import { extractSimpleComparisons } from '../../src/query/expression-helpers.js' import { DeduplicatedLoadSubset } from '../../src/query/subset-dedupe.js' import { @@ -9,11 +11,13 @@ import { toArray, } from '../../src/query/index.js' import { runTrace } from '../trace-runner.js' +import { oracleRuns } from '../oracle-config.js' import { flushPromises } from '../utils.js' import type { Collection } from '../../src/collection/index.js' import type { Deferred } from '../../src/deferred.js' import type { LoadSubsetOptions } from '../../src/types.js' import type { TraceDriver, TraceProjection } from '../trace-runner.js' +import type { Scheduler } from 'fast-check' type Post = { id: number @@ -492,6 +496,77 @@ async function expectObsoleteDemandCannotPublishAfterReactivation(): Promise { + const { collection: posts, remove, add } = createRemovablePost() + const requests: Array<{ + outcome: Promise + signal: AbortSignal | undefined + }> = [] + const comments = createCollection({ + id: nextCollectionId(`temporal-scheduled-generation-comments`), + getKey: (comment) => comment.id, + autoIndex: `eager`, + defaultIndexType: BasicIndex, + syncMode: `on-demand`, + sync: { + sync: ({ begin, write, commit, markReady }) => ({ + loadSubset: (options) => { + const requestIndex = requests.length + const signal = options.signal + const outcome = scheduler + .schedule(Promise.resolve(), `demand-${requestIndex}`) + .then(() => { + if (signal?.aborted) return + begin() + write({ + type: `insert`, + value: { + id: requestIndex === 0 ? 100 : 200, + postId: 1, + body: requestIndex === 0 ? `obsolete` : `current`, + }, + }) + commit() + markReady() + }) + requests.push({ outcome, signal }) + return outcome + }, + }), + }, + }) + const live = createPostsWithCommentsLive(posts, comments) + const preload = live.preload() + + try { + await flushPromises() + expect(requests).toHaveLength(1) + + remove() + await preload + add() + await flushPromises() + expect(requests).toHaveLength(2) + expect(requests[0]!.signal?.aborted).toBe(true) + + await scheduler.waitAll() + await Promise.all(requests.map(({ outcome }) => outcome)) + await flushPromises() + + expect(live.isReady()).toBe(true) + expect(live.get(1)?.comments.map(({ id, body }) => ({ id, body }))).toEqual( + [{ id: 200, body: `current` }], + ) + } finally { + if (scheduler.count() > 0) await scheduler.waitAll() + await Promise.allSettled(requests.map(({ outcome }) => outcome)) + await live.cleanup() + await Promise.all([posts.cleanup(), comments.cleanup()]) + } +} + function createMutablePosts( initial: ReadonlyArray, options: { markReadyInitially?: boolean } = {}, @@ -800,6 +875,10 @@ async function expectPartialShrinkRetainsCoverage(): Promise { await flushPromises() expect(live.get(1)?.comments).toHaveLength(1) expect(unloads).toEqual([]) + + posts.write(`delete`, firstPost) + await flushPromises() + expect(unloads).toEqual([[1, 2]]) } finally { initialLoad.resolve() await live.cleanup() @@ -1059,6 +1138,11 @@ describe(`includes temporal oracle`, () => { expectObsoleteDemandCannotPublishAfterReactivation, ) + fcTest.prop([fc.scheduler()], { numRuns: oracleRuns(20) })( + `obsolete and current demand completions are generation-safe in either order`, + expectScheduledDemandCompletionsStayGenerationSafe, + ) + it( `retained pending demand blocks readiness after demand expands`, expectRetainedDemandBlocksReadiness, diff --git a/packages/db/tests/query/includes-work-counter-oracle.test.ts b/packages/db/tests/query/includes-work-counter-oracle.test.ts index 46649e3830..78beb5ca3d 100644 --- a/packages/db/tests/query/includes-work-counter-oracle.test.ts +++ b/packages/db/tests/query/includes-work-counter-oracle.test.ts @@ -1,5 +1,6 @@ import { fc, test as fcTest } from '@fast-check/vitest' import { beforeAll, describe, expect, it } from 'vitest' +import { oracleRuns } from '../oracle-config.js' import { createCollection } from '../../src/collection/index.js' import { BTreeIndex } from '../../src/indexes/btree-index.js' import { localOnlyCollectionOptions } from '../../src/local-only.js' @@ -397,7 +398,7 @@ describe(`includes deterministic work-counter oracle`, () => { ) fcTest.prop([fc.integer({ min: 1, max: 24 })], { - numRuns: 6, + numRuns: oracleRuns(6), seed: 1709, })( `a join preserves correlated source pushdown (#1709)`, @@ -405,7 +406,7 @@ describe(`includes deterministic work-counter oracle`, () => { ) fcTest.prop([fc.integer({ min: 1, max: 24 })], { - numRuns: 6, + numRuns: oracleRuns(6), seed: 170_900, })( `indexed join-target growth keeps source work flat (#1709 direction control)`, @@ -429,7 +430,7 @@ describe(`includes deterministic work-counter oracle`, () => { ) fcTest.prop([fc.integer({ min: 1, max: 24 })], { - numRuns: 6, + numRuns: oracleRuns(6), seed: 17_090, })( `join-free correlated includes keep source work flat (#1709 control)`, diff --git a/packages/db/tests/query/includes.test.ts b/packages/db/tests/query/includes.test.ts index 4443b92120..b58bcf71a1 100644 --- a/packages/db/tests/query/includes.test.ts +++ b/packages/db/tests/query/includes.test.ts @@ -329,6 +329,31 @@ describe(`includes subqueries`, () => { ]) }) + it(`publishes a non-empty child Collection as ready with its rows`, async () => { + const collection = buildIncludesQuery() + const publications: Array<{ ready: boolean; issueIds: Array }> = + [] + const subscription = collection.subscribeChanges( + () => { + const project = collection.get(1) + if (!project) return + publications.push({ + ready: project.issues.isReady(), + issueIds: [...project.issues.values()].map((issue) => issue.id), + }) + }, + { includeInitialState: true }, + ) + + try { + await collection.preload() + expect(publications[0]).toEqual({ ready: true, issueIds: [10, 11] }) + } finally { + subscription.unsubscribe() + await collection.cleanup() + } + }) + it(`gives an active null correlation an empty child Collection`, async () => { const parents = createCollection( mockSyncCollectionOptions<{ id: number; groupId: number | null }>({ @@ -1356,6 +1381,144 @@ describe(`includes subqueries`, () => { expect(childItems(collection.get(2)!.children)).toEqual([{ id: 20 }]) }) + it(`keeps a limited child facade complete as the window widens and receives later changes`, async () => { + type Parent = { id: number; rank: number; groupId: number } + type Child = { id: number; groupId: number; label: string } + + const parents = createCollection( + mockSyncCollectionOptions({ + id: `widened-limited-facade-parents`, + getKey: (parent) => parent.id, + initialData: [ + { id: 1, rank: 1, groupId: 1 }, + { id: 2, rank: 2, groupId: 2 }, + ], + }), + ) + const children = createCollection( + mockSyncCollectionOptions({ + id: `widened-limited-facade-children`, + getKey: (child) => child.id, + initialData: [ + { id: 10, groupId: 1, label: `first` }, + { id: 20, groupId: 2, label: `preloaded` }, + ], + }), + ) + const buildQuery = () => + createLiveQueryCollection((q) => + q + .from({ parent: parents }) + // Keep this as orderBy + limit: parent keys branch before top-K, + // so the second bucket's initial rows arrive while it is inactive. + .orderBy(({ parent }) => parent.rank) + .limit(1) + .select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children }) + .where(({ child }) => eq(child.groupId, parent.groupId)) + .select(({ child }) => ({ + id: child.id, + label: child.label, + })), + })), + ) + const collection = buildQuery() + + await collection.preload() + const windowResult = collection.utils.setWindow({ offset: 0, limit: 2 }) + if (windowResult instanceof Promise) { + await windowResult + } + + const secondFacade = collection.get(2)!.children + expect(secondFacade.status).toBe(`ready`) + expect(secondFacade.isReady()).toBe(true) + expect(plainRows(secondFacade)).toEqual([{ id: 20, label: `preloaded` }]) + + children.utils.begin() + children.utils.write({ + type: `insert`, + value: { id: 21, groupId: 2, label: `fresh` }, + }) + children.utils.commit() + children.utils.begin() + children.utils.write({ + type: `update`, + value: { id: 20, groupId: 2, label: `updated` }, + previousValue: { id: 20, groupId: 2, label: `preloaded` }, + }) + children.utils.commit() + + expect(plainRows(secondFacade)).toEqual([ + { id: 20, label: `updated` }, + { id: 21, label: `fresh` }, + ]) + + parents.utils.begin() + parents.utils.write({ + type: `delete`, + value: { id: 1, rank: 1, groupId: 1 }, + }) + parents.utils.commit() + await collection.cleanup() + + const replayed = buildQuery() + await replayed.preload() + expect(plainRows(replayed.get(2)!.children)).toEqual([ + { id: 20, label: `updated` }, + { id: 21, label: `fresh` }, + ]) + await replayed.cleanup() + }) + + it(`replays existing child rows when a parent where predicate becomes true`, async () => { + type Parent = { id: number; active: boolean; groupId: number } + type Child = { id: number; groupId: number } + + const parents = createCollection( + mockSyncCollectionOptions({ + id: `where-activated-facade-parents`, + getKey: (parent) => parent.id, + initialData: [{ id: 1, active: false, groupId: 1 }], + }), + ) + const children = createCollection( + mockSyncCollectionOptions({ + id: `where-activated-facade-children`, + getKey: (child) => child.id, + initialData: [{ id: 10, groupId: 1 }], + }), + ) + const collection = createLiveQueryCollection((q) => + q + .from({ parent: parents }) + .where(({ parent }) => eq(parent.active, true)) + .select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children }) + .where(({ child }) => eq(child.groupId, parent.groupId)), + })), + ) + + await collection.preload() + expect(collection.size).toBe(0) + + parents.utils.begin() + parents.utils.write({ + type: `update`, + value: { id: 1, active: true, groupId: 1 }, + previousValue: { id: 1, active: false, groupId: 1 }, + }) + parents.utils.commit() + + expect(plainRows(collection.get(1)!.children)).toEqual([ + { id: 10, groupId: 1 }, + ]) + }) + it(`does not publish facade changes when root publication fails`, async () => { type Parent = { id: number; groupId: number } type Child = { id: number; groupId: number } diff --git a/packages/db/tests/query/order-by.test.ts b/packages/db/tests/query/order-by.test.ts index 0b45fb351e..b9441b420d 100644 --- a/packages/db/tests/query/order-by.test.ts +++ b/packages/db/tests/query/order-by.test.ts @@ -1946,6 +1946,39 @@ function createOrderByTests(autoIndex: `off` | `eager`): void { }, ) + itWhenAutoIndex( + `loads an ordered self-join through the ordered alias`, + async () => { + const collection = createLiveQueryCollection((q) => + q + .from({ employee: employeesCollection }) + .join({ manager: employeesCollection }, ({ employee, manager }) => + eq(employee.id, manager.id), + ) + .orderBy(({ manager }) => manager.name, `asc`) + .limit(3) + .select(({ employee, manager }) => ({ + id: employee.id, + employeeName: employee.name, + managerName: manager.name, + })), + ) + + await collection.preload() + + expect( + Array.from(collection.values()).map((row) => [ + row.employeeName, + row.managerName, + ]), + ).toEqual([ + [`Alice`, `Alice`], + [`Bob`, `Bob`], + [`Charlie`, `Charlie`], + ]) + }, + ) + itWhenAutoIndex( `optimizes single-column orderBy when passed as array with single element`, async () => { diff --git a/packages/db/tests/query/scheduler.test.ts b/packages/db/tests/query/scheduler.test.ts index 1361231e7a..3faf361648 100644 --- a/packages/db/tests/query/scheduler.test.ts +++ b/packages/db/tests/query/scheduler.test.ts @@ -3,7 +3,11 @@ import { createCollection } from '../../src/collection/index.js' import { createLiveQueryCollection, eq, isNull } from '../../src/query/index.js' import { createTransaction } from '../../src/transactions.js' import { createOptimisticAction } from '../../src/optimistic-action.js' -import { transactionScopedScheduler } from '../../src/scheduler.js' +import { + getActivePublicationContext, + transactionScopedScheduler, + withPublicationContext, +} from '../../src/scheduler.js' import { CollectionConfigBuilder } from '../../src/query/live/collection-config-builder.js' import { mockSyncCollectionOptions, stripVirtualProps } from '../utils.js' import type { OutputWithVirtual } from '../utils.js' @@ -87,6 +91,58 @@ afterEach(() => { transactionScopedScheduler.flushAll() }) +describe(`Collection publication scheduler context`, () => { + it(`shares one context and flushes after the outer publication`, () => { + const calls: Array = [] + let contextId: ReturnType + + withPublicationContext(() => { + contextId = getActivePublicationContext() + expect(contextId).toBeDefined() + + transactionScopedScheduler.schedule({ + contextId, + jobId: `outer`, + run: () => calls.push(`outer`), + }) + withPublicationContext(() => { + expect(getActivePublicationContext()).toBe(contextId) + transactionScopedScheduler.schedule({ + contextId, + jobId: `inner`, + run: () => calls.push(`inner`), + }) + }) + + expect(calls).toEqual([]) + }) + + expect(calls).toEqual([`outer`, `inner`]) + expect(getActivePublicationContext()).toBeUndefined() + }) + + it(`clears queued work when publication throws`, () => { + const run = vi.fn() + let contextId: ReturnType + + expect(() => + withPublicationContext(() => { + contextId = getActivePublicationContext() + transactionScopedScheduler.schedule({ + contextId, + jobId: `discarded`, + run, + }) + throw new Error(`publication failed`) + }), + ).toThrow(`publication failed`) + + expect(run).not.toHaveBeenCalled() + expect(getActivePublicationContext()).toBeUndefined() + expect(transactionScopedScheduler.hasPendingJobs(contextId!)).toBe(false) + }) +}) + describe(`live query scheduler`, () => { it(`runs the live query graph once per transaction that touches multiple collections`, async () => { const { users, tasks, assignments } = diff --git a/packages/powersync-db-collection/src/powersync.ts b/packages/powersync-db-collection/src/powersync.ts index b799575566..51075af426 100644 --- a/packages/powersync-db-collection/src/powersync.ts +++ b/packages/powersync-db-collection/src/powersync.ts @@ -531,6 +531,8 @@ export function powerSyncCollectionOptions< function runOnDemandSync() { const unloadSubsetCallbacks = new Map() const releasedSubsets = new WeakSet() + let stopped = false + const hasStopped = () => stopped start().catch((error) => database.logger.error( @@ -546,9 +548,15 @@ export function powerSyncCollectionOptions< const loadSubset = async ( options?: LoadSubsetOptions, ): Promise => { + if (hasStopped()) return + if (options) { activeWhereExpressions.push(options.where) const cleanup = await restConfig.onLoadSubset?.(options) + if (hasStopped()) { + cleanup?.() + return + } if (cleanup) { if (releasedSubsets.has(options) || options.signal?.aborted) { cleanup() @@ -686,12 +694,14 @@ export function powerSyncCollectionOptions< return { cleanup: () => { + stopped = true database.logger.info( `Sync has been stopped for ${viewName} into ${trackedTableName}`, ) abortController.abort() for (const cleanup of unloadSubsetCallbacks.values()) cleanup() unloadSubsetCallbacks.clear() + activeWhereExpressions.length = 0 }, loadSubset: (options: LoadSubsetOptions) => loadSubset(options), unloadSubset: (options: LoadSubsetOptions) => unloadSubset(options), diff --git a/packages/powersync-db-collection/tests/load-hooks.test.ts b/packages/powersync-db-collection/tests/load-hooks.test.ts index 9c7a8b4f7d..d6ead38733 100644 --- a/packages/powersync-db-collection/tests/load-hooks.test.ts +++ b/packages/powersync-db-collection/tests/load-hooks.test.ts @@ -179,4 +179,46 @@ describe(`Sync Streams`, () => { { timeout: 2000 }, ) }) + + it(`disposes a subset hook that resolves after collection cleanup`, async () => { + const db = await createDatabase() + await createTestProducts(db) + let resolveHook!: (cleanup: () => void) => void + const hook = new Promise<() => void>((resolve) => { + resolveHook = resolve + }) + const cleanupHook = vi.fn() + + const collection = createCollection( + powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + syncMode: `on-demand`, + onLoadSubset: () => hook, + }), + ) + await collection.stateWhenReady() + const query = createLiveQueryCollection({ + query: (q) => + q + .from({ product: collection }) + .where(({ product }) => eq(product.category, `electronics`)) + .select(({ product }) => ({ + id: product.id, + name: product.name, + price: product.price, + category: product.category, + })), + }) + const preload = query.preload() + void preload.catch(() => {}) + + await vi.waitFor(() => expect(resolveHook).toBeTypeOf(`function`)) + const collectionCleanup = collection.cleanup() + resolveHook(cleanupHook) + await collectionCleanup + + await vi.waitFor(() => expect(cleanupHook).toHaveBeenCalledOnce()) + await query.cleanup() + }) }) diff --git a/packages/query-db-collection/src/query.ts b/packages/query-db-collection/src/query.ts index 13b25ada8a..ed618e5e6b 100644 --- a/packages/query-db-collection/src/query.ts +++ b/packages/query-db-collection/src/query.ts @@ -887,6 +887,7 @@ export function queryCollectionOptions( let startupRetentionSettled = false const retainedQueriesPendingRevalidation = new Set() const pendingResultApplications = new Map>() + const resultApplicationTokens = new Map() const effectivePersistedGcTimes = new Map() const persistedRetentionTimers = new Map< string, @@ -894,6 +895,11 @@ export function queryCollectionOptions( >() let persistedRetentionMaintenance = Promise.resolve() + const invalidatePendingResultApplication = (hashedQueryKey: string) => { + pendingResultApplications.delete(hashedQueryKey) + resultApplicationTokens.delete(hashedQueryKey) + } + const getRowMetadata = (rowKey: string | number) => { return (metadata?.row.get(rowKey) ?? collection._state.syncedMetadata.get(rowKey)) as @@ -1517,11 +1523,15 @@ export function queryCollectionOptions( const reconcileSuccessfulResult = async ( queryKey: QueryKey, result: QueryObserverResult, + applicationToken: object, ) => { const hashedQueryKey = hashKey(queryKey) const persistedBaseline = await loadPersistedBaselineForQuery(hashedQueryKey) - if (collection.status === `cleaned-up`) { + if ( + collection.status === `cleaned-up` || + resultApplicationTokens.get(hashedQueryKey) !== applicationToken + ) { return } applySuccessfulResult(queryKey, result, persistedBaseline) @@ -1566,9 +1576,12 @@ export function queryCollectionOptions( return } + const applicationToken = {} + resultApplicationTokens.set(hashedQueryKey, applicationToken) const application = reconcileSuccessfulResult( queryKey, result, + applicationToken, ).catch((error) => { console.error( `[QueryCollection] Error reconciling query ${String(queryKey)}:`, @@ -1706,7 +1719,7 @@ export function queryCollectionOptions( unsubscribePendingReadyListeners(hashedQueryKey) cancelPersistedRetentionExpiry(hashedQueryKey) retainedQueriesPendingRevalidation.delete(hashedQueryKey) - pendingResultApplications.delete(hashedQueryKey) + invalidatePendingResultApplication(hashedQueryKey) const nextOwnersByRow = removeQueryOwnership(hashedQueryKey) const rowsToDelete: Array = [] @@ -1794,6 +1807,7 @@ export function queryCollectionOptions( metadata && persistedMetadata?.row.scanPersisted ) { + invalidatePendingResultApplication(hashedQueryKey) begin() metadata.collection.set( `${QUERY_COLLECTION_GC_PREFIX}${hashedQueryKey}`, diff --git a/packages/query-db-collection/tests/query.test.ts b/packages/query-db-collection/tests/query.test.ts index 8e2fd2cea6..08f3912a85 100644 --- a/packages/query-db-collection/tests/query.test.ts +++ b/packages/query-db-collection/tests/query.test.ts @@ -6255,6 +6255,65 @@ describe(`QueryCollection`, () => { ).toBe(false) }) + it(`does not apply a retained-query result after its subset is released`, async () => { + const queryKey = [`stale-retained-reconciliation`] + const queryHash = hashKey(queryKey) + const item = { id: `1`, name: `Stale result`, category: `A` } + const persistedScan = + createDeferred< + Array<{ key: string; value: CategorisedItem; metadata?: unknown }> + >() + const metadataHarness = createInMemorySyncMetadataApi({ + collectionMetadata: new Map([ + [ + `queryCollection:gc:${queryHash}`, + { queryHash, mode: `until-revalidated` }, + ], + ]), + }) + const scanPersisted = vi.fn().mockReturnValue(persistedScan.promise) + const metadataApi = { + ...metadataHarness.api, + row: { + ...metadataHarness.api.row, + scanPersisted, + }, + } as SyncMetadataApi + + const baseOptions = queryCollectionOptions({ + id: `stale-retained-reconciliation`, + queryClient, + queryKey: () => queryKey, + queryFn: async () => [item], + getKey: (value) => value.id, + syncMode: `on-demand`, + startSync: true, + }) + const originalSync = baseOptions.sync + const collection = createCollection({ + ...baseOptions, + sync: { + sync: (params: Parameters[0]) => + originalSync.sync({ + ...params, + metadata: metadataApi, + }), + }, + }) + const load = collection._sync.loadSubset({}) + await vi.waitFor(() => { + expect(scanPersisted).toHaveBeenCalledOnce() + }) + + collection._sync.unloadSubset({}) + persistedScan.resolve([]) + await load + await flushPromises() + + expect(collection.has(item.id)).toBe(false) + await collection.cleanup() + }) + it(`should clean up expired persisted ttl placeholders on startup`, async () => { const baseQueryKey = [`persisted-ttl-cleanup-test`] const queryFn = vi.fn().mockResolvedValue([]) From bf9c03a2270ea11cf25907a4c5e884e7aedf6fff Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 17 Aug 2026 19:20:51 +0100 Subject: [PATCH 19/21] test(db): broaden includes oracle strategies --- packages/db/package.json | 2 +- packages/db/src/query/live/ARCHITECTURE.md | 13 +- packages/db/tests/oracle-config.ts | 17 + ...ncludes-collection-oracle.property.test.ts | 8 +- ...-cross-formulation-oracle.property.test.ts | 427 ++++++++++++++++++ ...ncludes-optimistic-oracle.property.test.ts | 18 +- .../query/includes-oracle.property.test.ts | 72 ++- .../query/includes-publication-oracle.test.ts | 23 +- .../query/includes-temporal-oracle.test.ts | 4 +- 9 files changed, 543 insertions(+), 41 deletions(-) create mode 100644 packages/db/tests/query/includes-cross-formulation-oracle.property.test.ts diff --git a/packages/db/package.json b/packages/db/package.json index df40e4b36a..391283d968 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -21,7 +21,7 @@ "dev": "vite build --watch", "lint": "eslint . --fix", "test": "vitest --run", - "test:oracles": "vitest --run tests/query/includes-oracle.property.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts" + "test:oracles": "vitest --run tests/query/includes-oracle.property.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts" }, "type": "module", "main": "dist/cjs/index.cjs", diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 8c42930c20..41bdda884e 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -540,8 +540,17 @@ Run the DB oracle set with `pnpm test:oracles` from `packages/db`. Broad properties use FastCheck's random seed, while structural matrices keep fixed seeds so each run covers the same named cells. Increase both corpora with `TANSTACK_DB_ORACLE_RUNS_MULTIPLIER=10 pnpm test:oracles`. Preserve FastCheck's -reported seed and shrink path while reducing a failure, then add the smallest -case as a deterministic regression trace. +reported seed and shrink path while reducing a failure. Replay a broad +campaign with `TANSTACK_DB_ORACLE_SEED= pnpm test:oracles`, then add the +smallest case as a deterministic regression trace. + +The broad relationship history changes correlation keys rather than freezing +them. Set `TANSTACK_DB_ORACLE_STATISTICS=1` to print its generated depth, +relationship-change, optimistic, and delete distribution. Collection-valued, +array, and materialized includes are checked together for every Collection +scenario instead of relying on a random mode sample. A separate metamorphic +oracle compares nested includes with a flat join, fresh per-parent queries, and +three-valued predicate partitioning. ## Implementation discipline diff --git a/packages/db/tests/oracle-config.ts b/packages/db/tests/oracle-config.ts index 8f30fbda41..8ec28f414b 100644 --- a/packages/db/tests/oracle-config.ts +++ b/packages/db/tests/oracle-config.ts @@ -1,5 +1,7 @@ const multiplierText = process.env.TANSTACK_DB_ORACLE_RUNS_MULTIPLIER const multiplier = multiplierText === undefined ? 1 : Number(multiplierText) +const seedText = process.env.TANSTACK_DB_ORACLE_SEED +const seed = seedText === undefined ? undefined : Number(seedText) if (!Number.isSafeInteger(multiplier) || multiplier < 1) { throw new Error( @@ -7,7 +9,22 @@ if (!Number.isSafeInteger(multiplier) || multiplier < 1) { ) } +if (seed !== undefined && !Number.isSafeInteger(seed)) { + throw new Error(`TANSTACK_DB_ORACLE_SEED must be an integer`) +} + /** Keeps ordinary CI bounded while allowing long randomized oracle campaigns. */ export function oracleRuns(baseRuns: number): number { return baseRuns * multiplier } + +/** Replays broad randomized properties when a campaign seed is supplied. */ +export function oraclePropertyOptions(baseRuns: number): { + numRuns: number + seed?: number +} { + return { + numRuns: oracleRuns(baseRuns), + ...(seed === undefined ? {} : { seed }), + } +} diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index 937e8d1916..dacd01ed08 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -9,7 +9,7 @@ import { toArray, } from '../../src/query/index.js' import { runTrace } from '../trace-runner.js' -import { oracleRuns } from '../oracle-config.js' +import { oraclePropertyOptions } from '../oracle-config.js' import { flushPromises, mockSyncCollectionOptions, @@ -353,7 +353,7 @@ const exhaustiveActions: ReadonlyArray = [ ] describe(`Collection-valued includes oracle`, () => { - fcTest.prop([collectionScenarioArbitrary], { numRuns: oracleRuns(30) })( + fcTest.prop([collectionScenarioArbitrary], oraclePropertyOptions(30))( `keeps Collection, toArray, and materialize equivalent across generated relationship histories`, ({ parentGroup, childValue, actions }) => runTrace({ @@ -740,7 +740,7 @@ describe(`Collection-valued includes oracle`, () => { wideId: fc.integer({ min: 10, max: 19 }), }), ], - { numRuns: oracleRuns(20) }, + oraclePropertyOptions(20), )( `uses one raw public-key order across Collection and inline materializations`, async ({ smallId, wideId }) => { @@ -809,7 +809,7 @@ describe(`Collection-valued includes oracle`, () => { value: fc.integer({ min: -10, max: 10 }), }), ], - { numRuns: oracleRuns(20) }, + oraclePropertyOptions(20), )( `matches recomputation through optimistic child insert and delete confirmation and rollback`, async ({ group, insertedId, confirmedId, value }) => { diff --git a/packages/db/tests/query/includes-cross-formulation-oracle.property.test.ts b/packages/db/tests/query/includes-cross-formulation-oracle.property.test.ts new file mode 100644 index 0000000000..59920e5ea1 --- /dev/null +++ b/packages/db/tests/query/includes-cross-formulation-oracle.property.test.ts @@ -0,0 +1,427 @@ +import { fc, test as fcTest } from '@fast-check/vitest' +import { describe, expect } from 'vitest' +import { createCollection } from '../../src/collection/index.js' +import { + createLiveQueryCollection, + eq, + isNull, + lt, + not, + queryOnce, + toArray, +} from '../../src/query/index.js' +import { oraclePropertyOptions } from '../oracle-config.js' +import { flushPromises, mockSyncCollectionOptions } from '../utils.js' +import type { Collection } from '../../src/collection/index.js' + +type ParentRow = { + id: number + group: number + position: number +} + +type ChildRow = { + id: number + parentGroup: number + score: number | null + position: number +} + +type CrossFormulationAction = + | { type: `putParent`; row: ParentRow } + | { type: `deleteParent`; id: number } + | { type: `putChild`; row: ChildRow } + | { type: `deleteChild`; id: number } + +type CrossFormulationScenario = { + parents: Array + children: Array + pivot: number + actions: Array +} + +type NormalizedParent = ParentRow & { + children: Array +} + +type ControlledCollection = { + collection: Collection + write: (type: `insert` | `update` | `delete`, value: T) => void +} + +type FlatRow = { + parentId: number + parentGroup: number + parentPosition: number + child: ChildRow | undefined +} + +let nextCrossFormulationId = 0 + +function createControlledCollection( + name: string, + initialData: ReadonlyArray, +): ControlledCollection { + const options = mockSyncCollectionOptions({ + id: `${name}-${nextCrossFormulationId++}`, + getKey: (row) => row.id, + initialData: initialData.map((row) => ({ ...row })), + autoIndex: `eager`, + }) + options.sync.rowUpdateMode = `full` + return { + collection: createCollection(options), + write(type, value) { + options.utils.begin() + options.utils.write({ type, value: { ...value } }) + options.utils.commit() + }, + } +} + +function compareParents(left: ParentRow, right: ParentRow): number { + return left.position - right.position || left.id - right.id +} + +function compareChildren(left: ChildRow, right: ChildRow): number { + return left.position - right.position || left.id - right.id +} + +function normalizeChild(child: ChildRow): ChildRow { + return { + id: child.id, + parentGroup: child.parentGroup, + score: child.score, + position: child.position, + } +} + +function normalizeNested( + rows: ReadonlyArray, +): Array { + return rows + .map((parent) => ({ + id: parent.id, + group: parent.group, + position: parent.position, + children: parent.children.map(normalizeChild).sort(compareChildren), + })) + .sort(compareParents) +} + +function normalizeFlat(rows: ReadonlyArray): Array { + const parents = new Map() + for (const row of rows) { + const parent = parents.get(row.parentId) ?? { + id: row.parentId, + group: row.parentGroup, + position: row.parentPosition, + children: [], + } + if (row.child) parent.children.push(normalizeChild(row.child)) + parents.set(row.parentId, parent) + } + return normalizeNested([...parents.values()]) +} + +function recompute( + parents: Map, + children: Map, +): Array { + return normalizeNested( + [...parents.values()].map((parent) => ({ + ...parent, + children: [...children.values()].filter( + (child) => child.parentGroup === parent.group, + ), + })), + ) +} + +function createNestedQuery( + parents: Collection, + children: Collection, +) { + return createLiveQueryCollection({ + getKey: (row) => row.id, + query: (q) => + q + .from({ parent: parents }) + .orderBy(({ parent }) => parent.position) + .orderBy(({ parent }) => parent.id) + .select(({ parent }) => ({ + id: parent.id, + group: parent.group, + position: parent.position, + children: toArray( + q + .from({ child: children }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ child }) => child.position) + .orderBy(({ child }) => child.id) + .select(({ child }) => ({ + id: child.id, + parentGroup: child.parentGroup, + score: child.score, + position: child.position, + })), + ), + })), + }) +} + +function createFlatQuery( + parents: Collection, + children: Collection, +) { + return createLiveQueryCollection({ + getKey: (row) => `${row.parentId}:${row.child?.id ?? `empty`}`, + query: (q) => + q + .from({ parent: parents }) + .leftJoin({ child: children }, ({ parent, child }) => + eq(parent.group, child.parentGroup), + ) + .select(({ parent, child }) => ({ + parentId: parent.id, + parentGroup: parent.group, + parentPosition: parent.position, + child, + })), + }) +} + +type ChildPartition = `all` | `predicate` | `complement` | `unknown` + +async function queryChildren( + children: Collection, + parentGroup: number, + pivot: number, + partition: ChildPartition, +): Promise> { + return queryOnce((q) => { + const correlated = q + .from({ child: children }) + .where(({ child }) => eq(child.parentGroup, parentGroup)) + const partitioned = (() => { + switch (partition) { + case `all`: + return correlated + case `predicate`: + return correlated.where(({ child }) => lt(child.score, pivot)) + case `complement`: + return correlated.where(({ child }) => not(lt(child.score, pivot))) + case `unknown`: + return correlated.where(({ child }) => isNull(lt(child.score, pivot))) + } + })() + + return partitioned + .orderBy(({ child }) => child.position) + .orderBy(({ child }) => child.id) + .select(({ child }) => ({ + id: child.id, + parentGroup: child.parentGroup, + score: child.score, + position: child.position, + })) + }) +} + +async function queryPerParent( + parents: ReadonlyArray, + children: Collection, + pivot: number, + useTlp: boolean, +): Promise> { + return normalizeNested( + await Promise.all( + parents.map(async (parent) => { + const childRows = useTlp + ? ( + await Promise.all( + ([`predicate`, `complement`, `unknown`] as const).map( + (partition) => + queryChildren(children, parent.group, pivot, partition), + ), + ) + ).flat() + : await queryChildren(children, parent.group, pivot, `all`) + + return { ...parent, children: childRows } + }), + ), + ) +} + +function applyAction( + action: CrossFormulationAction, + parentSource: ControlledCollection, + childSource: ControlledCollection, + parents: Map, + children: Map, +): void { + switch (action.type) { + case `putParent`: { + const type = parents.has(action.row.id) ? `update` : `insert` + parents.set(action.row.id, { ...action.row }) + parentSource.write(type, action.row) + return + } + case `deleteParent`: { + const previous = parents.get(action.id) + if (!previous) return + parents.delete(action.id) + parentSource.write(`delete`, previous) + return + } + case `putChild`: { + const type = children.has(action.row.id) ? `update` : `insert` + children.set(action.row.id, { ...action.row }) + childSource.write(type, action.row) + return + } + case `deleteChild`: { + const previous = children.get(action.id) + if (!previous) return + children.delete(action.id) + childSource.write(`delete`, previous) + } + } +} + +async function expectFormulationsEquivalent( + scenario: CrossFormulationScenario, +): Promise { + const parentSource = createControlledCollection( + `cross-form-parents`, + scenario.parents, + ) + const childSource = createControlledCollection( + `cross-form-children`, + scenario.children, + ) + const parents = new Map(scenario.parents.map((row) => [row.id, { ...row }])) + const children = new Map(scenario.children.map((row) => [row.id, { ...row }])) + const nested = createNestedQuery( + parentSource.collection, + childSource.collection, + ) + const flat = createFlatQuery(parentSource.collection, childSource.collection) + + const assertEquivalent = async () => { + const expected = recompute(parents, children) + const nestedResult = normalizeNested(nested.toArray) + const flatResult = normalizeFlat(flat.toArray) + const parentRows = [...parents.values()] + const standaloneResult = await queryPerParent( + parentRows, + childSource.collection, + scenario.pivot, + false, + ) + const tlpResult = await queryPerParent( + parentRows, + childSource.collection, + scenario.pivot, + true, + ) + + expect({ + nested: nestedResult, + flat: flatResult, + standalone: standaloneResult, + tlp: tlpResult, + }).toEqual({ + nested: expected, + flat: expected, + standalone: expected, + tlp: expected, + }) + } + + try { + await Promise.all([nested.preload(), flat.preload()]) + await assertEquivalent() + for (const action of scenario.actions) { + applyAction(action, parentSource, childSource, parents, children) + await flushPromises() + await assertEquivalent() + } + } finally { + await Promise.allSettled([ + nested.cleanup(), + flat.cleanup(), + parentSource.collection.cleanup(), + childSource.collection.cleanup(), + ]) + } +} + +const parentRowArbitrary = (id: number) => + fc.record({ + id: fc.constant(id), + group: fc.integer({ min: -1, max: 1 }), + position: fc.integer({ min: -2, max: 2 }), + }) + +const childRowArbitrary = (id: number) => + fc.record({ + id: fc.constant(id), + parentGroup: fc.integer({ min: -1, max: 1 }), + score: fc.option(fc.integer({ min: -2, max: 2 }), { nil: null }), + position: fc.integer({ min: -2, max: 2 }), + }) + +const actionArbitrary: fc.Arbitrary = fc.oneof( + fc.record({ + type: fc.constant(`putParent` as const), + row: fc.integer({ min: 0, max: 2 }).chain(parentRowArbitrary), + }), + fc.record({ + type: fc.constant(`deleteParent` as const), + id: fc.integer({ min: 0, max: 2 }), + }), + fc.record({ + type: fc.constant(`putChild` as const), + row: fc.integer({ min: 10, max: 14 }).chain(childRowArbitrary), + }), + fc.record({ + type: fc.constant(`deleteChild` as const), + id: fc.integer({ min: 10, max: 14 }), + }), +) + +const scenarioArbitrary: fc.Arbitrary = fc.record({ + parents: fc.tuple(parentRowArbitrary(0), parentRowArbitrary(1)), + children: fc.tuple( + childRowArbitrary(10), + childRowArbitrary(11), + childRowArbitrary(12), + ), + pivot: fc.integer({ min: -2, max: 2 }), + actions: fc.array(actionArbitrary, { minLength: 1, maxLength: 5 }), +}) + +describe(`includes cross-formulation oracle`, () => { + fcTest(`shared-route child deletion agrees across formulations`, () => + expectFormulationsEquivalent({ + parents: [ + { id: 0, group: 0, position: 0 }, + { id: 1, group: 0, position: 0 }, + ], + children: [ + { id: 10, parentGroup: 0, score: null, position: 0 }, + { id: 11, parentGroup: 0, score: null, position: 0 }, + { id: 12, parentGroup: 0, score: null, position: 0 }, + ], + pivot: 0, + actions: [{ type: `deleteChild`, id: 10 }], + }), + ) + + fcTest.prop([scenarioArbitrary], oraclePropertyOptions(8))( + `agrees across nested includes, flat joins, per-parent queries, and TLP partitions`, + expectFormulationsEquivalent, + ) +}) diff --git a/packages/db/tests/query/includes-optimistic-oracle.property.test.ts b/packages/db/tests/query/includes-optimistic-oracle.property.test.ts index 1b010abb6f..1a190a911b 100644 --- a/packages/db/tests/query/includes-optimistic-oracle.property.test.ts +++ b/packages/db/tests/query/includes-optimistic-oracle.property.test.ts @@ -12,7 +12,7 @@ import { withExpectedRejection, } from '../utils.js' import { runTrace } from '../trace-runner.js' -import { oracleRuns } from '../oracle-config.js' +import { oraclePropertyOptions } from '../oracle-config.js' import type { TraceDriver, TraceProjection } from '../trace-runner.js' type RootRow = { @@ -531,7 +531,7 @@ describe(`optimistic relationship-transition oracle`, () => { }, ) - fcTest.prop([routeValuesArbitrary], { numRuns: oracleRuns(12) })( + fcTest.prop([routeValuesArbitrary], oraclePropertyOptions(12))( `an optimistic rekey detaches its old descendants immediately`, async (routes) => { await expectHistoryMatches(routes, [ @@ -546,7 +546,7 @@ describe(`optimistic relationship-transition oracle`, () => { }, ) - fcTest.prop([routeValuesArbitrary], { numRuns: oracleRuns(12) })( + fcTest.prop([routeValuesArbitrary], oraclePropertyOptions(12))( `restores the authoritative relationship after an optimistic rekey rolls back`, async (routes) => { await expectHistoryMatches(routes, [ @@ -560,7 +560,7 @@ describe(`optimistic relationship-transition oracle`, () => { }, ) - fcTest.prop([routeValuesArbitrary], { numRuns: oracleRuns(12) })( + fcTest.prop([routeValuesArbitrary], oraclePropertyOptions(12))( `rolls back a descendant update made while its ancestor is reparented`, async (routes) => { await expectHistoryMatches(routes, [ @@ -584,7 +584,7 @@ describe(`optimistic relationship-transition oracle`, () => { }, ) - fcTest.prop([routeValuesArbitrary], { numRuns: oracleRuns(12) })( + fcTest.prop([routeValuesArbitrary], oraclePropertyOptions(12))( `rolls back a reparented ancestor while its descendant update remains pending`, async (routes) => { await expectHistoryMatches(routes, [ @@ -608,7 +608,7 @@ describe(`optimistic relationship-transition oracle`, () => { }, ) - fcTest.prop([routeValuesArbitrary], { numRuns: oracleRuns(12) })( + fcTest.prop([routeValuesArbitrary], oraclePropertyOptions(12))( `settles a confirmed optimistic reparent on the same authoritative route`, async (routes) => { await expectHistoryMatches(routes, [ @@ -646,7 +646,7 @@ describe(`optimistic relationship-transition oracle`, () => { }, ) - fcTest.prop([routeValuesArbitrary], { numRuns: oracleRuns(12) })( + fcTest.prop([routeValuesArbitrary], oraclePropertyOptions(12))( `settles a confirmed optimistic reparent on a different authoritative route`, async (routes) => { await expectHistoryMatches(routes, [ @@ -686,7 +686,7 @@ describe(`optimistic relationship-transition oracle`, () => { }, ) - fcTest.prop([routeValuesArbitrary], { numRuns: oracleRuns(12) })( + fcTest.prop([routeValuesArbitrary], oraclePropertyOptions(12))( `restores a rekey after a sibling enters its old route`, async (routes) => { await expectHistoryMatches(routes, [ @@ -731,7 +731,7 @@ describe(`optimistic relationship-transition oracle`, () => { }, ) - fcTest.prop([routeValuesArbitrary], { numRuns: oracleRuns(12) })( + fcTest.prop([routeValuesArbitrary], oraclePropertyOptions(12))( `supports repeated rollback and confirmation histories`, async (routes) => { await expectHistoryMatches(routes, [ diff --git a/packages/db/tests/query/includes-oracle.property.test.ts b/packages/db/tests/query/includes-oracle.property.test.ts index c044d74b8c..c487f4897c 100644 --- a/packages/db/tests/query/includes-oracle.property.test.ts +++ b/packages/db/tests/query/includes-oracle.property.test.ts @@ -14,7 +14,7 @@ import { mockSyncCollectionOptions, withExpectedRejection, } from '../utils.js' -import { oracleRuns } from '../oracle-config.js' +import { oraclePropertyOptions, oracleRuns } from '../oracle-config.js' import { runTrace } from '../trace-runner.js' import type { TraceCheckpoint, @@ -241,6 +241,58 @@ const scenarioArbitrary: fc.Arbitrary = depthArbitrary.chain( })), ) +function classifyScenarioCoverage({ depth, history }: Scenario): Array { + const routesByLevel = Array.from( + { length: depth + 1 }, + () => new Map(), + ) + let relationshipChanges = 0 + + for (const action of history) { + const routes = routesByLevel[action.level]! + if (action.type === `delete`) { + routes.delete(action.id) + continue + } + + const previous = routes.get(action.id) + if ( + previous && + (previous.group !== action.group || + (action.level > 0 && previous.parentGroup !== action.parentGroup)) + ) { + relationshipChanges += 1 + } + routes.set(action.id, { + parentGroup: action.parentGroup, + group: action.group, + }) + } + + return [ + `depth=${depth}`, + `relationship-changes=${ + relationshipChanges === 0 + ? `none` + : relationshipChanges === 1 + ? `one` + : `many` + }`, + `optimistic=${history.some((action) => + action.type.startsWith(`optimistic`), + )}`, + `delete=${history.some((action) => action.type === `delete`)}`, + ] +} + +if (process.env.TANSTACK_DB_ORACLE_STATISTICS === `1`) { + fc.statistics( + scenarioArbitrary, + classifyScenarioCoverage, + oraclePropertyOptions(1_000), + ) +} + const materializeScenarioArbitrary: fc.Arbitrary = fc .boolean() .chain((sharedIntermediate) => { @@ -694,11 +746,9 @@ async function applyAction( return } - // Keep correlation keys stable in green fuzz histories. The known - // correlation-key update failure has its own deterministic seed below. const next: RootRow = { id: action.id, - group: current?.group ?? action.group, + group: action.group, value: action.value, position: action.type === `put` ? action.position : (current?.position ?? 0), @@ -754,12 +804,10 @@ async function applyAction( return } - // Keep correlation keys stable in green fuzz histories. The known - // correlation-key update failure has its own deterministic seed below. const next: ChildRow = { id: action.id, - parentGroup: current?.parentGroup ?? action.parentGroup, - group: current?.group ?? action.group, + parentGroup: action.parentGroup, + group: action.group, value: action.value, position: action.type === `put` ? action.position : (current?.position ?? 0), @@ -4280,7 +4328,7 @@ describe(`includes recompute oracle`, () => { }) }) - fcTest.prop([scenarioArbitrary], { numRuns: oracleRuns(40) })( + fcTest.prop([scenarioArbitrary], oraclePropertyOptions(40))( `matches naive recomputation after every incremental change`, expectScenarioMatches, ) @@ -4291,7 +4339,7 @@ describe(`includes recompute oracle`, () => { ({ sharedIntermediate }) => !sharedIntermediate, ), ], - { numRuns: oracleRuns(30) }, + oraclePropertyOptions(30), )( `matches recomputation for nested scalar materialization`, expectMaterializeScenarioMatches, @@ -4319,7 +4367,7 @@ describe(`includes recompute oracle`, () => { { selector: (row) => row.id, maxLength: 7 }, ), ], - { numRuns: oracleRuns(25) }, + oraclePropertyOptions(25), )( `is unchanged by alpha-renaming, sibling declaration order, or an unrelated sibling`, async (rootRows, childRows) => { @@ -4431,7 +4479,7 @@ describe(`includes recompute oracle`, () => { fcTest.prop( [fc.integer({ min: -5, max: 5 }).filter((value) => value !== 0)], - { numRuns: oracleRuns(15) }, + oraclePropertyOptions(15), )( `optimistic updates converge to confirmed-only state`, async (confirmedValue) => { diff --git a/packages/db/tests/query/includes-publication-oracle.test.ts b/packages/db/tests/query/includes-publication-oracle.test.ts index 037a7aeddd..0201d57de0 100644 --- a/packages/db/tests/query/includes-publication-oracle.test.ts +++ b/packages/db/tests/query/includes-publication-oracle.test.ts @@ -8,7 +8,7 @@ import { materialize, } from '../../src/query/index.js' import { runTrace } from '../trace-runner.js' -import { oracleRuns } from '../oracle-config.js' +import { oraclePropertyOptions } from '../oracle-config.js' import { flushPromises, mockSyncCollectionOptions, @@ -450,7 +450,7 @@ describe(`layered-query publication oracle`, () => { for (const q1Shape of q1Shapes) { for (const q2Shape of q2Shapes) { - fcTest.prop([changedValueArbitrary], { numRuns: oracleRuns(12) })( + fcTest.prop([changedValueArbitrary], oraclePropertyOptions(12))( `publishes #1713 updates through a ${q1Shape} Q1 and ${q2Shape} Q2`, async (value) => { await expectPublicationMatches( @@ -462,9 +462,10 @@ describe(`layered-query publication oracle`, () => { }, ) - fcTest.prop([changedValueArbitrary, changedChildValueArbitrary], { - numRuns: oracleRuns(12), - })( + fcTest.prop( + [changedValueArbitrary, changedChildValueArbitrary], + oraclePropertyOptions(12), + )( `recovers a ${q1Shape} Q1 and ${q2Shape} Q2 after a child update`, async (parentValue, childValue) => { await expectPublicationMatches( @@ -476,7 +477,7 @@ describe(`layered-query publication oracle`, () => { }, ) - fcTest.prop([changedValueArbitrary], { numRuns: oracleRuns(8) })( + fcTest.prop([changedValueArbitrary], oraclePropertyOptions(8))( `publishes optimistic state before confirmation through a ${q1Shape} Q1 and ${q2Shape} Q2`, async (value) => { await expectPublicationMatches( @@ -488,7 +489,7 @@ describe(`layered-query publication oracle`, () => { }, ) - fcTest.prop([changedValueArbitrary], { numRuns: oracleRuns(8) })( + fcTest.prop([changedValueArbitrary], oraclePropertyOptions(8))( `publishes state after optimistic confirmation through a ${q1Shape} Q1 and ${q2Shape} Q2`, async (value) => { await expectPublicationMatches( @@ -502,14 +503,14 @@ describe(`layered-query publication oracle`, () => { } } - fcTest.prop([changedChildValueArbitrary], { numRuns: oracleRuns(100) })( + fcTest.prop([changedChildValueArbitrary], oraclePropertyOptions(100))( `publishes child-only scalar updates through both layers`, async (value) => { await expectPublicationMatches({ type: `childScalar`, value }) }, ) - fcTest.prop([fc.constantFrom(20, 30)], { numRuns: oracleRuns(100) })( + fcTest.prop([fc.constantFrom(20, 30)], oraclePropertyOptions(100))( `compares route transitions at both query layers`, async (group) => { await expectPublicationMatches({ type: `parentRoute`, group }) @@ -523,12 +524,12 @@ describe(`layered-query publication oracle`, () => { value: changedValueArbitrary, }), ], - { numRuns: oracleRuns(100) }, + oraclePropertyOptions(100), )(`compares atomic parent replacements at both query layers`, async (row) => { await expectPublicationMatches({ type: `atomicReplace`, ...row }) }) - fcTest.prop([changedValueArbitrary], { numRuns: oracleRuns(100) })( + fcTest.prop([changedValueArbitrary], oraclePropertyOptions(100))( `publishes restored state after optimistic rollback`, async (value) => { await expectPublicationMatches({ diff --git a/packages/db/tests/query/includes-temporal-oracle.test.ts b/packages/db/tests/query/includes-temporal-oracle.test.ts index e1abca79ac..8f4ab2e1f1 100644 --- a/packages/db/tests/query/includes-temporal-oracle.test.ts +++ b/packages/db/tests/query/includes-temporal-oracle.test.ts @@ -11,7 +11,7 @@ import { toArray, } from '../../src/query/index.js' import { runTrace } from '../trace-runner.js' -import { oracleRuns } from '../oracle-config.js' +import { oraclePropertyOptions } from '../oracle-config.js' import { flushPromises } from '../utils.js' import type { Collection } from '../../src/collection/index.js' import type { Deferred } from '../../src/deferred.js' @@ -1138,7 +1138,7 @@ describe(`includes temporal oracle`, () => { expectObsoleteDemandCannotPublishAfterReactivation, ) - fcTest.prop([fc.scheduler()], { numRuns: oracleRuns(20) })( + fcTest.prop([fc.scheduler()], oraclePropertyOptions(20))( `obsolete and current demand completions are generation-safe in either order`, expectScheduledDemandCompletionsStayGenerationSafe, ) From 06a61b17a566d5a727a7397af834dc352ec7f73c Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 18 Aug 2026 14:32:18 +0100 Subject: [PATCH 20/21] test(db): harden includes oracle coverage --- ...ncludes-collection-oracle.property.test.ts | 323 ++++++++++++++++++ .../query/includes-temporal-oracle.test.ts | 61 ++++ 2 files changed, 384 insertions(+) diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index dacd01ed08..6f001b586d 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -630,6 +630,329 @@ describe(`Collection-valued includes oracle`, () => { }, ) + fcTest( + `root application failure rolls back a prepared facade publication`, + async () => { + type NodeRow = { + id: number + kind: `parent` | `child` + group: number + value: number + } + const initialParent: NodeRow = { + id: 1, + kind: `parent`, + group: 1, + value: 1, + } + const initialChild: NodeRow = { + id: 10, + kind: `child`, + group: 1, + value: 1, + } + const nodes = createControlledCollection(`rollback-nodes`, [ + initialParent, + initialChild, + ]) + const live = createLiveQueryCollection((q) => + q + .from({ parent: nodes.collection }) + .where(({ parent }) => eq(parent.kind, `parent`)) + .select(({ parent }) => ({ + id: parent.id, + value: parent.value, + children: q + .from({ child: nodes.collection }) + .where(({ child }) => eq(child.kind, `child`)) + .where(({ child }) => eq(child.group, parent.group)), + })), + ) + + await live.preload() + const facade = live.get(1)!.children + const rootPublications: Array = [] + const childPublications: Array = [] + const rootSubscription = live.subscribeChanges( + (batch) => rootPublications.push(...batch), + { includeInitialState: false }, + ) + const childSubscription = facade.subscribeChanges( + (batch) => childPublications.push(...batch), + { includeInitialState: false }, + ) + const originalGetKey = live.config.getKey + live.config.getKey = (row) => { + if (row.value === 2) throw new Error(`root key failed`) + return originalGetKey(row) + } + + try { + expect(() => + nodes.writeBatch([ + { + type: `update`, + value: { ...initialParent, value: 2 }, + }, + { + type: `update`, + value: { ...initialChild, value: 2 }, + }, + ]), + ).toThrow(`root key failed`) + expect(live.get(1)!.value).toBe(1) + expect(facade.get(10)!.value).toBe(1) + expect(rootPublications).toEqual([]) + expect(childPublications).toEqual([]) + + live.config.getKey = originalGetKey + nodes.writeBatch([ + { + type: `update`, + value: { ...initialParent, value: 3 }, + }, + { + type: `update`, + value: { ...initialChild, value: 3 }, + }, + ]) + expect(live.get(1)!.value).toBe(3) + expect(facade.get(10)!.value).toBe(3) + expect(rootPublications).toHaveLength(1) + expect(childPublications).toHaveLength(1) + } finally { + live.config.getKey = originalGetKey + rootSubscription.unsubscribe() + childSubscription.unsubscribe() + await Promise.all([live.cleanup(), nodes.collection.cleanup()]) + } + }, + ) + + fcTest( + `child-only changes flush the facade without republishing the parent`, + async () => { + const parents = createControlledCollection(`facade-only-parents`, [ + { id: 1, group: 1 }, + ]) + const children = createControlledCollection(`facade-only-children`, [ + { id: 10, parentGroup: 1, value: 1 }, + ]) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)), + })), + ) + + await live.preload() + const facade = live.get(1)!.children + const rootPublications: Array = [] + const childPublications: Array = [] + const rootSubscription = live.subscribeChanges( + (batch) => rootPublications.push(...batch), + { includeInitialState: false }, + ) + const childSubscription = facade.subscribeChanges( + (batch) => childPublications.push(...batch), + { includeInitialState: false }, + ) + + try { + children.write(`update`, { + id: 10, + parentGroup: 1, + value: 2, + }) + + expect(rootPublications).toEqual([]) + expect(childPublications).toHaveLength(1) + expect(live.get(1)!.children).toBe(facade) + expect( + [...facade.values()].map(({ id, parentGroup, value }) => ({ + id, + parentGroup, + value, + })), + ).toEqual([{ id: 10, parentGroup: 1, value: 2 }]) + } finally { + rootSubscription.unsubscribe() + childSubscription.unsubscribe() + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }, + ) + + fcTest( + `outer fn.select recomputes nested values after a union branch include changes`, + async () => { + const messages = createControlledCollection(`fn-select-messages`, [ + { id: 1, group: 1 }, + ]) + const tools = createControlledCollection(`fn-select-tools`, [ + { id: 2, group: 2 }, + ]) + const children = createControlledCollection(`fn-select-children`, [ + { id: 10, parentGroup: 1, value: 1 }, + ]) + const live = createLiveQueryCollection((q) => { + const messageRows = q + .from({ message: messages.collection }) + .select(({ message }) => ({ + kind: `message` as const, + id: message.id, + children: toArray( + q + .from({ messageChild: children.collection }) + .where(({ messageChild }) => + eq(messageChild.parentGroup, message.group), + ) + .select(({ messageChild }) => ({ + id: messageChild.id, + value: messageChild.value, + })), + ), + })) + const toolRows = q + .from({ tool: tools.collection }) + .select(({ tool }) => ({ + kind: `tool` as const, + id: tool.id, + })) + + return q.unionAll(messageRows, toolRows).fn.select((row) => ({ + kind: row.kind, + id: row.id, + payload: { children: row.children }, + })) + }) + + try { + await live.preload() + const message = live.toArray.find((row) => row.kind === `message`)! + expect(message.payload.children).toEqual([{ id: 10, value: 1 }]) + + children.write(`update`, { + id: 10, + parentGroup: 1, + value: 2, + }) + expect( + live.toArray.find((row) => row.kind === `message`)!.payload.children, + ).toEqual([{ id: 10, value: 2 }]) + } finally { + await Promise.all([ + live.cleanup(), + messages.collection.cleanup(), + tools.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }, + ) + + fcTest( + `rejects collapsed contributors that disagree by value, order, or outgoing route`, + async () => { + type CommentRow = { + id: number + userId: number + text: string + } + const users = createControlledCollection(`congruence-users`, [ + { id: 1, group: 1 }, + ]) + + const expectIncrementalRejection = async ( + mode: `value` | `order`, + ): Promise => { + const comments = createControlledCollection( + `congruence-${mode}-comments`, + [{ id: 1, userId: 1, text: `first` }], + ) + const live = createLiveQueryCollection({ + query: (q) => { + const joined = q + .from({ comment: comments.collection }) + .join({ user: users.collection }, ({ comment, user }) => + eq(comment.userId, user.id), + ) + return mode === `value` + ? joined.select(({ comment }) => ({ + publicId: comment.userId, + visible: comment.text, + })) + : joined + .orderBy(({ comment }) => comment.id) + .select(({ comment }) => ({ + publicId: comment.userId, + visible: `same`, + })) + }, + getKey: (row) => row.publicId, + }) + + try { + await live.preload() + expect(() => + comments.write(`insert`, { + id: 2, + userId: 1, + text: mode === `value` ? `second` : `ignored`, + }), + ).toThrow(`not congruent`) + } finally { + await live.cleanup() + await comments.collection.cleanup() + } + } + + await expectIncrementalRejection(`value`) + await expectIncrementalRejection(`order`) + + const parents = createControlledCollection(`congruence-parents`, [ + { id: 1, group: 1 }, + ]) + const children = createControlledCollection( + `congruence-children`, + [], + ) + const routed = createLiveQueryCollection({ + query: (q) => + q.from({ parent: parents.collection }).select(({ parent }) => ({ + publicId: 1, + visible: `same`, + children: toArray( + q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)), + ), + })), + getKey: (row) => row.publicId, + }) + + try { + await routed.preload() + expect(() => parents.write(`insert`, { id: 2, group: 2 })).toThrow( + `not congruent`, + ) + } finally { + await routed.cleanup() + await Promise.all([ + parents.collection.cleanup(), + children.collection.cleanup(), + users.collection.cleanup(), + ]) + } + }, + ) + fcTest(`facade events observe the matching root publication`, async () => { const driver = createCollectionDriver( [{ id: 1, group: 1 }], diff --git a/packages/db/tests/query/includes-temporal-oracle.test.ts b/packages/db/tests/query/includes-temporal-oracle.test.ts index 8f4ab2e1f1..5c89ff3f66 100644 --- a/packages/db/tests/query/includes-temporal-oracle.test.ts +++ b/packages/db/tests/query/includes-temporal-oracle.test.ts @@ -4,6 +4,7 @@ import { createCollection } from '../../src/collection/index.js' import { createDeferred } from '../../src/deferred.js' import { BasicIndex } from '../../src/indexes/basic-index.js' import { extractSimpleComparisons } from '../../src/query/expression-helpers.js' +import { SubsetDemandController } from '../../src/query/live/subset-demand-controller.js' import { DeduplicatedLoadSubset } from '../../src/query/subset-dedupe.js' import { createLiveQueryCollection, @@ -16,6 +17,7 @@ import { flushPromises } from '../utils.js' import type { Collection } from '../../src/collection/index.js' import type { Deferred } from '../../src/deferred.js' import type { LoadSubsetOptions } from '../../src/types.js' +import type { LazyDemandPlan } from '../../src/query/compiler/joins.js' import type { TraceDriver, TraceProjection } from '../trace-runner.js' import type { Scheduler } from 'fast-check' @@ -781,6 +783,60 @@ async function expectRejectedDemandEntersError(): Promise { } } +async function expectFailedDemandRetriesSameCoverage(): Promise { + let loadCount = 0 + let shouldReject = true + const comments = createCollection({ + id: nextCollectionId(`temporal-demand-retry-comments`), + getKey: (comment) => comment.id, + syncMode: `on-demand`, + autoIndex: `eager`, + defaultIndexType: BasicIndex, + sync: { + sync: ({ markReady }) => ({ + loadSubset: () => { + loadCount += 1 + if (shouldReject) { + return Promise.reject(new Error(`child load failed`)) + } + markReady() + return true + }, + }), + }, + }) + comments.createIndex((comment) => comment.postId) + const subscription = comments.subscribeChanges(() => {}, { + includeInitialState: false, + }) + const controller = new SubsetDemandController() + const plan: LazyDemandPlan = { + id: `same-coverage-retry`, + path: [`postId`], + collectionId: comments.id, + initialKeys: new Set(), + } + + try { + const first = controller.setDemand(subscription, plan, new Set([1])) + expect(first.ready).toBeInstanceOf(Promise) + if (!(first.ready instanceof Promise)) { + throw new Error(`Expected failed demand to be asynchronous`) + } + await expect(first.ready).rejects.toThrow(`child load failed`) + + shouldReject = false + const retry = controller.setDemand(subscription, plan, new Set([1])) + expect(retry.changed).toBe(true) + expect(loadCount).toBe(2) + if (retry.ready instanceof Promise) await retry.ready + } finally { + controller.clear() + subscription.unsubscribe() + await comments.cleanup() + } +} + async function expectSynchronousEmptyDemandIsReady(): Promise { const posts = createMutablePosts([ { id: 1, authorId: `selected`, title: `one` }, @@ -1155,6 +1211,11 @@ describe(`includes temporal oracle`, () => { it(`rejected demand enters error`, expectRejectedDemandEntersError) + it( + `failed demand retries the same coverage`, + expectFailedDemandRetriesSameCoverage, + ) + it( `a synchronous empty demand can establish ready coverage`, expectSynchronousEmptyDemandIsReady, From 33982457ca5f12a53eccdd0f757b4be317ff1a1d Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 18 Aug 2026 14:50:56 +0100 Subject: [PATCH 21/21] test(react-db): bound include rerenders --- packages/react-db/tests/useLiveQuery.test.tsx | 50 +++++++++++++------ 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/packages/react-db/tests/useLiveQuery.test.tsx b/packages/react-db/tests/useLiveQuery.test.tsx index fbb48d882c..981c4b20e2 100644 --- a/packages/react-db/tests/useLiveQuery.test.tsx +++ b/packages/react-db/tests/useLiveQuery.test.tsx @@ -2520,7 +2520,7 @@ describe(`Query Collections`, () => { { id: `i3`, title: `Bug in Beta`, projectId: `p2` }, ] - it(`should render includes results and reactively update child collections`, async () => { + it(`renders only the affected child hook once when an include changes`, async () => { const projectsCollection = createCollection( mockSyncCollectionOptions({ id: `includes-react-projects`, @@ -2537,9 +2537,14 @@ describe(`Query Collections`, () => { }), ) + let parentRenderCount = 0 + let alphaRenderCount = 0 + let betaRenderCount = 0 + // Parent hook: runs includes query that produces child Collections - const { result: parentResult } = renderHook(() => - useLiveQuery((q) => + const { result: parentResult } = renderHook(() => { + parentRenderCount += 1 + return useLiveQuery((q) => q.from({ p: projectsCollection }).select(({ p }) => ({ id: p.id, name: p.name, @@ -2551,8 +2556,8 @@ describe(`Query Collections`, () => { title: i.title, })), })), - ), - ) + ) + }) // Wait for parent to be ready await waitFor(() => { @@ -2562,24 +2567,38 @@ describe(`Query Collections`, () => { const alphaProject = parentResult.current.data.find( (p: any) => p.id === `p1`, )! + const betaProject = parentResult.current.data.find( + (p: any) => p.id === `p2`, + )! expect(alphaProject.name).toBe(`Alpha`) - // Child hook: subscribes to the child Collection from the parent row, - // simulating a subcomponent using useLiveQuery(project.issues) - const { result: childResult } = renderHook(() => - useLiveQuery((alphaProject as any).issues), - ) + // Child hooks simulate sibling subcomponents subscribing to the child + // Collections from their parent rows. + const { result: alphaResult } = renderHook(() => { + alphaRenderCount += 1 + return useLiveQuery((alphaProject as any).issues) + }) + const { result: betaResult } = renderHook(() => { + betaRenderCount += 1 + return useLiveQuery((betaProject as any).issues) + }) await waitFor(() => { - expect(childResult.current.data).toHaveLength(2) + expect(alphaResult.current.data).toHaveLength(2) + expect(alphaResult.current.isReady).toBe(true) + expect(betaResult.current.data).toHaveLength(1) + expect(betaResult.current.isReady).toBe(true) }) - expect(childResult.current.data).toEqual( + expect(alphaResult.current.data).toEqual( expect.arrayContaining([ expect.objectContaining({ id: `i1`, title: `Bug in Alpha` }), expect.objectContaining({ id: `i2`, title: `Feature for Alpha` }), ]), ) + const settledParentRenders = parentRenderCount + const settledAlphaRenders = alphaRenderCount + const settledBetaRenders = betaRenderCount // Add a new issue to Alpha — the child hook should reactively update act(() => { @@ -2592,16 +2611,19 @@ describe(`Query Collections`, () => { }) await waitFor(() => { - expect(childResult.current.data).toHaveLength(3) + expect(alphaResult.current.data).toHaveLength(3) }) - expect(childResult.current.data).toEqual( + expect(alphaResult.current.data).toEqual( expect.arrayContaining([ expect.objectContaining({ id: `i1`, title: `Bug in Alpha` }), expect.objectContaining({ id: `i2`, title: `Feature for Alpha` }), expect.objectContaining({ id: `i4`, title: `New Alpha issue` }), ]), ) + expect(parentRenderCount).toBe(settledParentRenders) + expect(alphaRenderCount).toBe(settledAlphaRenders + 1) + expect(betaRenderCount).toBe(settledBetaRenders) }) }) })