From 6de8915749aa5fea9d01bb68f2daf64832a07c2e Mon Sep 17 00:00:00 2001 From: David de Boer Date: Mon, 16 Nov 2026 20:18:04 +0100 Subject: [PATCH] =?UTF-8?q?feat(search)!:=20filter=20and=20facet=20a=20roo?= =?UTF-8?q?t=20document=20on=20whether=20an=20entry=E2=80=99s=20leaf=20has?= =?UTF-8?q?=20a=20value?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An inline reference may list presenceLeaves: the projection writes one ${reference}_${leaf}_exists boolean per root document, true when any entry holds a value for the leaf. A weld of one exists condition selects it, the facet ‹reference›.‹leaf›.exists counts it, and GraphQL serves it under the leaf in the reference’s where and facets. The presence ADR is renumbered to 40, after the other ADR that took number 38. BREAKING CHANGE: PhysicalFields gains a required leafExists member, so code constructing a PhysicalFields must supply it. An engine adapter must declare each leaf boolean and compile a one-condition exists weld against it. --- ...t-a-root-document-on-its-entries-leaves.md | 2 +- ...acet-a-field-on-whether-it-has-a-value.md} | 8 +- ...t-on-whether-an-entrys-leaf-has-a-value.md | 75 ++ docs/reference/search-api-graphql.md | 18 +- docs/reference/search-typesense.md | 6 +- docs/reference/search.md | 58 +- .../search-api-graphql/src/build-schema.ts | 217 ++++-- .../test/leaf-presence.test.ts | 329 +++++++++ packages/search-api-graphql/vite.config.ts | 2 +- .../src/collection-definition.ts | 11 +- .../search-typesense/src/query-compiler.ts | 13 + packages/search-typesense/src/search.ts | 5 +- .../test/leaf-presence.integration.test.ts | 245 +++++++ .../test/query-compiler.test.ts | 116 ++++ packages/search-typesense/vite.config.ts | 4 +- packages/search/src/engine.ts | 12 + packages/search/src/project.ts | 36 +- packages/search/src/query.ts | 62 +- packages/search/src/schema.ts | 286 ++++++-- packages/search/test/leaf-presence.test.ts | 640 ++++++++++++++++++ packages/search/test/presence.test.ts | 2 +- packages/search/test/schema.test.ts | 5 + 22 files changed, 2034 insertions(+), 118 deletions(-) rename docs/decisions/{0038-filter-and-facet-a-field-on-whether-it-has-a-value.md => 0040-filter-and-facet-a-field-on-whether-it-has-a-value.md} (95%) create mode 100644 docs/decisions/0041-filter-and-facet-a-root-document-on-whether-an-entrys-leaf-has-a-value.md create mode 100644 packages/search-api-graphql/test/leaf-presence.test.ts create mode 100644 packages/search-typesense/test/leaf-presence.integration.test.ts create mode 100644 packages/search/test/leaf-presence.test.ts diff --git a/docs/decisions/0036-facet-a-root-document-on-its-entries-leaves.md b/docs/decisions/0036-facet-a-root-document-on-its-entries-leaves.md index a1f3187a..1d15f959 100644 --- a/docs/decisions/0036-facet-a-root-document-on-its-entries-leaves.md +++ b/docs/decisions/0036-facet-a-root-document-on-its-entries-leaves.md @@ -71,5 +71,5 @@ An inline reference on a Root Type may list leaves of its Reference Type in - A derived judgement over entries, such as LOL’s `hasMedia` (“some reproduction has a thumbnail”), stays a `derive`. Typesense has no native presence filter, so a generic one would be the same index-time boolean. - (Superseded 2026-11-16 for a Root Type’s own field: [ADR 38](0038-filter-and-facet-a-field-on-whether-it-has-a-value.md) + (Superseded 2026-11-16 for a Root Type’s own field: [ADR 40](0040-filter-and-facet-a-field-on-whether-it-has-a-value.md) writes that boolean once, in LDE.) diff --git a/docs/decisions/0038-filter-and-facet-a-field-on-whether-it-has-a-value.md b/docs/decisions/0040-filter-and-facet-a-field-on-whether-it-has-a-value.md similarity index 95% rename from docs/decisions/0038-filter-and-facet-a-field-on-whether-it-has-a-value.md rename to docs/decisions/0040-filter-and-facet-a-field-on-whether-it-has-a-value.md index ffed9f79..f254cba2 100644 --- a/docs/decisions/0038-filter-and-facet-a-field-on-whether-it-has-a-value.md +++ b/docs/decisions/0040-filter-and-facet-a-field-on-whether-it-has-a-value.md @@ -1,4 +1,4 @@ -# 38. Filter and facet a field on whether it has a value +# 40. Filter and facet a field on whether it has a value Date: 2026-11-16 @@ -8,7 +8,8 @@ Accepted Supersedes the last consequence of [ADR 36](0036-facet-a-root-document-on-its-entries-leaves.md) for a Root -Type’s own field. +Type’s own field. Extended to an inline entry’s leaf by +[ADR 41](0041-filter-and-facet-a-root-document-on-whether-an-entrys-leaf-has-a-value.md). ## Context @@ -78,7 +79,8 @@ count } } }`, as `‹Type›‹Field›Facets`, the per-field grouping ADR 36 ga refuses it on: - a field of a Reference Type and an inline reference: presence on an entry’s leaf, counted per root document as ADR 36’s faceted leaves are, - is [#1091](https://github.com/ldelements/lde/issues/1091)’s second part; + is [#1091](https://github.com/ldelements/lde/issues/1091)’s second part + (done since in ADR 41, declared on the reference); - a field that is already `facetable`: its facet would have to become an object to hold both kinds of bucket, the shape change [#1090](https://github.com/ldelements/lde/issues/1090) makes once for diff --git a/docs/decisions/0041-filter-and-facet-a-root-document-on-whether-an-entrys-leaf-has-a-value.md b/docs/decisions/0041-filter-and-facet-a-root-document-on-whether-an-entrys-leaf-has-a-value.md new file mode 100644 index 00000000..401e5da4 --- /dev/null +++ b/docs/decisions/0041-filter-and-facet-a-root-document-on-whether-an-entrys-leaf-has-a-value.md @@ -0,0 +1,75 @@ +# 41. Filter and facet a root document on whether an entry’s leaf has a value + +Date: 2026-11-16 + +## Status + +Accepted + +Extends [ADR 40](0040-filter-and-facet-a-field-on-whether-it-has-a-value.md) +to a leaf of an inline reference. + +## Context + +ADR 40 lets a Root Type’s own field declare presence and refuses it on an +inline reference and on a Reference Type’s field. Linked Open Limburg still +derives `hasMedia`, “an `associatedMedia` entry has a `thumbnailUrl`”, beside +the reference it describes. + +A boolean inside each entry does not answer that question. Typesense flattens +a leaf under an `object[]` into one list per document +([ADR 36](0036-facet-a-root-document-on-its-entries-leaves.md)), so a work +with one entry that has a thumbnail and one that lacks it would count in both +buckets, and a work with no entries in neither. + +## Decision + +An inline reference on a Root Type may list leaves of its Reference Type in +`ref.presenceLeaves`. For each, the root document is filtered and faceted on +whether any entry holds a value for the leaf. + +- **One boolean per root document.** The projection writes + `${reference}_${leaf}_exists`, true when any entry holds a value for the + leaf in ADR 40’s sense and false otherwise, a document with no entries + included. It reads the entries before pruning, so a leaf with no Role of its + own still counts. `physicalFields(reference).leafExists` names it, and the reserved-name + check covers it. +- **Declared on the reference**, as ADR 36 declares faceted leaves: the count + is a fact about the root, and a Reference Type nested from several places + may be counted from one and not another. +- **A weld of one condition selects it.** + `{ field: 'associatedMedia', entry: [{ field: 'thumbnailUrl', exists: true }] }` + compiles to the root’s boolean. `exists` welded to other conditions is + refused (`exists-in-weld`): a root-level boolean cannot hold of the same + entry as another condition. Through a join it stays refused, as in ADR 40. +- **The facet is `‹reference›.‹leaf›.exists`**, on the axis of the same name. + It owns the one-condition weld on `exists`, never a weld on the leaf’s value, + and a faceted leaf’s facet never owns it. +- **GraphQL nests it under the leaf.** The leaf’s key in `‹Edge›Where` takes + `exists`, typed as an own field’s is (`ExistsFilter`, `‹Filter›OrExists`). + Setting it beside another key of the same weld is an error naming the + rewrite. The facet is served as `facets { associatedMedia { thumbnailUrl { +exists } } }`, inside the reference’s facet object. +- **One set of leaves per Reference Type.** `‹Edge›Where` is shared by every + reference nesting that type, and its keys follow the presence leaves. So + `searchSchema` requires every reference nesting one type to declare the same + leaves, rather than emit a filter input per reference. Through a join, a + reference with presence leaves takes `‹Edge›JoinedFilter`, without `exists`. + +## Consequences + +- LOL drops its derived `hasMedia`, as it drops `hasCoordinates` once + [#1090](https://github.com/ldelements/lde/issues/1090) lands. +- The reference must be `filterable`, since a bucket is selected through a + condition on one entry. A leaf is held to what an own field is: neither + `text` nor `boolean`. A leaf that is itself an inline reference is refused, + since an entry’s entries are not counted. +- Until [#1090](https://github.com/ldelements/lde/issues/1090) gives each + field one facet object, a leaf cannot be both a faceted leaf and a presence + leaf of one reference, and a `facetable` reference cannot declare presence + leaves: both facets would claim one GraphQL key. +- A lookup reference may not declare presence leaves until #1090 brings a + target’s values onto the root document. +- Two `filterable` references on Root Types nesting one Reference Type with + different leaves are refused, until #1090’s per-reference filter inputs. A + deployment that needs both before then declares a second Reference Type. diff --git a/docs/reference/search-api-graphql.md b/docs/reference/search-api-graphql.md index 47352905..9409e9ee 100644 --- a/docs/reference/search-api-graphql.md +++ b/docs/reference/search-api-graphql.md @@ -280,6 +280,18 @@ role: { in: ["etser"] } } }` asks for _this person in this role_, where the without the keys of fields declaring presence alone. A type declaring none keeps one `‹Type›Where` for both. + An inline reference’s + [presence leaves](./search#presence-on-an-entry-s-leaf) take `exists` in its + reference type’s `‹Edge›Where`, typed as a field of the root’s own is + (`ExistsFilter`, or `‹Filter›OrExists` for a `filterable` leaf): + `where: { associatedMedia: { where: { thumbnailUrl: { exists: true } } } }`. + Whether any entry holds a value is recorded per document, so `exists` is + welded to nothing else: setting it beside another key of the same + `‹Edge›Where` is an error naming the rewrite into two conditions. Every + reference nesting one reference type declares the same presence leaves, so + `‹Edge›Where` stays shared; through a join, where no companion is read, a + reference with presence leaves takes `‹Edge›JoinedFilter`, without `exists`. + - **`orderBy`**: `RELEVANCE` plus every `sortable` field, as an enum – field names SCREAMING_SNAKE_CASEd (`datePosted` → `DATE_POSTED`); `direction` defaults to `DESC`. @@ -343,7 +355,11 @@ license { buckets { value count } } } }`, typed `‹Type›‹Reference›Facets dateCreated { exists { value count } } }`, typed `‹Type›‹Field›Facets` (`WorkDateCreatedFacets`) with `exists: [BooleanBucket!]!`. The object leaves room for a field’s value buckets beside it, once a field may be both - `facetable` and declare presence. + `facetable` and declare presence. A presence leaf’s facet sits under the + reference, one level further down: `facets { associatedMedia { thumbnailUrl { +exists { value count } } } }`, typed `‹Type›‹Reference›‹Leaf›Facets` + (`WorkAssociatedMediaThumbnailUrlFacets`), beside the reference’s faceted + leaves. Selecting facet fields IS the request, and the whole selection is **batched per request**: one diff --git a/docs/reference/search-typesense.md b/docs/reference/search-typesense.md index d50986d0..2d7ffc9e 100644 --- a/docs/reference/search-typesense.md +++ b/docs/reference/search-typesense.md @@ -139,7 +139,11 @@ value, so `exists` compiles to `${name}_exists:=true` or `:=false`, and the facet `‹field›.exists` counts the companion. The projection writes it on every document, `false` included, so the companion is not optional. A field declaring presence alone is internal, so the companion is all the collection -declares for it. +declares for it. An inline reference’s +[presence leaves](./search#presence-on-an-entry-s-leaf) each get a top-level +`${reference}_${leaf}_exists` of the same kind, true when any entry holds a +value: an `exists` welded alone to an entry compiles to it, never to a +condition inside the nested object. An `InPlaceRebuild` additionally refuses a type whose declared [dataset field](#provenance) inherits a policy: it enumerates the indexed diff --git a/docs/reference/search.md b/docs/reference/search.md index df2e4e44..b11d8bdf 100644 --- a/docs/reference/search.md +++ b/docs/reference/search.md @@ -1258,12 +1258,64 @@ declaring no presence (`no-presence`), which is also what a weld meets. `validateSearchType` accepts `presence` on a Root Type’s own field only, and `searchSchema` says why in full. It refuses it on a field of a Reference Type and on an inline reference, whose entries -hold no companion yet ([#1091](https://github.com/ldelements/lde/issues/1091)); -on a `text` field, which stores its values per language; on a `boolean`, which +are counted through the reference’s +[presence leaves](#presence-on-an-entry-s-leaf) instead; on a `text` field, which stores its values per language; on a `boolean`, which states its own absence as `false`; and on a field that is already `facetable`, since its facet would have to become an object to hold both kinds of bucket ([#1090](https://github.com/ldelements/lde/issues/1090)) -([ADR 38](../decisions/0038-filter-and-facet-a-field-on-whether-it-has-a-value.md)). +([ADR 40](../decisions/0040-filter-and-facet-a-field-on-whether-it-has-a-value.md)). + +### Presence on an entry’s leaf + +A root document can be filtered and faceted on whether **any entry** of an +inline reference holds a value for a leaf – “a work with a reproduction that +has a thumbnail”. List the leaves in `ref.presenceLeaves`, beside +[`facetedLeaves`](#faceting-on-an-entry-s-leaf): + +```ts +{ + name: 'associatedMedia', + kind: 'reference', + array: true, + output: true, + filterable: true, + ref: { + strategy: 'inline', + typeName: 'MediaObject', + presenceLeaves: ['thumbnailUrl'], + }, +} +``` + +The projection writes one boolean per root document, +`associatedMedia_thumbnailUrl_exists` (`physicalFields(field).leafExists`): `true` when +any entry holds a value for the leaf, in the sense above, and `false` +otherwise – a document with no entries included. It is read before pruning, so +a leaf declaring no role of its own still counts. A boolean inside each entry +would not answer the question: an engine flattens it into one list per +document, so a work with one entry that has a thumbnail and one that lacks it +would count in both buckets. + +The facet is `associatedMedia.thumbnailUrl.exists`, and a bucket is selected +by `exists` welded to one entry alone: +`{ field: 'associatedMedia', entry: [{ field: 'thumbnailUrl', exists: true }] }` +(GraphQL: `where: { associatedMedia: { where: { thumbnailUrl: { exists: true } } } }`). +That weld is the facet’s own clause; a condition on the leaf’s value is not, +nor is `exists` the selection of a facet on the leaf’s values. `exists` +welded to other conditions is refused (`exists-in-weld`): whether any entry +holds a value is recorded per document, so it cannot hold of the same entry +as another condition. Through a join it is refused like any `exists` +(`joined-presence`). + +The reference is `filterable`, sits on a Root Type and – for now – is not +`facetable` itself; each leaf is declared by the reference type, named once, +and neither `text`, `boolean`, an inline reference nor – for now – one of the +reference’s faceted leaves. Every `filterable` inline reference on a Root +Type nesting the same reference type declares the same presence leaves, +because they share the type’s filter input. The two “for now” refusals and the +shared leaves are lifted by per-field facet objects and per-reference filter +inputs ([#1090](https://github.com/ldelements/lde/issues/1090)) +([ADR 41](../decisions/0041-filter-and-facet-a-root-document-on-whether-an-entrys-leaf-has-a-value.md)). ### Facet policy diff --git a/packages/search-api-graphql/src/build-schema.ts b/packages/search-api-graphql/src/build-schema.ts index e7ee1709..bf5fa0a4 100644 --- a/packages/search-api-graphql/src/build-schema.ts +++ b/packages/search-api-graphql/src/build-schema.ts @@ -41,7 +41,6 @@ import { AND_KEY, assertValidQuery, facetsOf, - filterableFields, fieldNamed, labelSourceNamesOf, labelTargetNamesOf, @@ -60,6 +59,7 @@ import { nestedReferenceType, outputFields, pageForOffset, + physicalFields, sortableFields, unixSecondsToIso, type FacetDeclaration, @@ -689,7 +689,7 @@ export function buildGraphQLSchema( */ function presenceFilterFor( field: SearchField, - owner: RootType, + owner: SearchType, ): GraphQLInputObjectType { if (field.filterable !== true) { return existsFilter; @@ -839,6 +839,21 @@ export function buildGraphQLSchema( floatRange.name, dateRange.name, ]); + /** + * Claim a generated type name, throwing a clear error when a declaration + * already took it – graphql-js would reject the duplicate too, but only once + * the whole schema is assembled and without saying what caused it. `what` + * says what the name is for. + */ + function claimTypeName(name: string, what: string): string { + if (takenTypeNames.has(name)) { + throw new Error( + `“${name}”, the type name of ${what}, is already the name of another type; rename that type.`, + ); + } + takenTypeNames.add(name); + return name; + } // The declared joins, and the input types they make shareable. Each map is // keyed by the SearchType `name`, so a type reached as a join target and the @@ -1309,6 +1324,7 @@ export function buildGraphQLSchema( function whereFieldType( field: SearchField, owner: SearchType, + joined = false, ): GraphQLInputType { // An inline reference is filterable in two ways at once, exactly as a // joinable one is: by the ids its entries hold (its identity), @@ -1316,7 +1332,7 @@ export function buildGraphQLSchema( // question one hop out – only what the hop costs differs. const nested = nestedReferenceType(schema, field); if (nested !== undefined) { - return nestedFilterFor(nested, field); + return nestedFilterFor(nested, field, joined); } // A joinable reference is filterable in two ways at once – by the ids it // holds, and by a condition on the referent – so it earns a richer input @@ -1343,15 +1359,40 @@ export function buildGraphQLSchema( function nestedFilterFor( referenceType: ReferenceType, field: SearchField, + joined: boolean, ): GraphQLInputObjectType { - const existing = nestedFilters.get(referenceType.name); + // Every reference nesting the type declares the same presence leaves + // (`searchSchema`), so the filter is shared – except through a join, which + // reads no presence companion yet and so gets a variant without `exists`. + const presenceLeaves = joined ? [] : presenceLeavesOf(field); + const variant = + joined && presenceLeavesOf(field).length > 0 + ? `${referenceType.name}Joined` + : referenceType.name; + const existing = nestedFilters.get(variant); if (existing !== undefined) { return existing; } + // The join variants are names of their own, so they are claimed like + // every other name a declaration could clash with. + if (variant !== referenceType.name) { + claimTypeName( + `${variant}Filter`, + `the filter on “${referenceType.name}” through a join`, + ); + claimTypeName( + `${variant}Where`, + `the condition on one “${referenceType.name}” entry through a join`, + ); + } const identityTargets = labelTargetNamesOf(field, schema); - const nestedWhere = nestedWhereInputFor(referenceType); + const nestedWhere = nestedWhereInputFor( + referenceType, + presenceLeaves, + variant, + ); const created = new GraphQLInputObjectType({ - name: `${referenceType.name}Filter`, + name: `${variant}Filter`, description: identityTargets.length > 0 ? `A condition on ${referenceType.name}: the ids its entries reference, or a condition on one entry.` @@ -1364,7 +1405,7 @@ export function buildGraphQLSchema( where: { type: nestedWhere }, }), }); - nestedFilters.set(referenceType.name, created); + nestedFilters.set(variant, created); return created; } @@ -1373,22 +1414,33 @@ export function buildGraphQLSchema( * else. Deliberately without the `id` key every Root Type's `where` carries: * an entry is read, not addressed, so it has no document key to filter on * ({@link validateQuery} refuses one). + * + * A presence leaf of the references nesting the type takes `exists` too, + * as a field of the root’s own does: `ExistsFilter`, or `‹Filter›OrExists` + * for a `filterable` leaf. Whether any entry holds a value is the root + * document’s, so `exists` is welded to nothing else – a `where` setting it + * beside another key is refused with the rewrite ({@link + * weldedCriterionOf}). */ function nestedWhereInputFor( referenceType: ReferenceType, + presenceLeaves: readonly string[], + variant: string, ): GraphQLInputObjectType { // Memoized by its only caller, which is memoized itself – so this is built - // once per Reference Type however many fields nest it. + // once per Reference Type (and variant) however many fields nest it. return new GraphQLInputObjectType({ - name: `${referenceType.name}Where`, + name: `${variant}Where`, description: 'Sibling keys are combined with AND, and all of them must hold of the SAME entry.', fields: () => Object.fromEntries( - filterableFields(referenceType).map((nestedField) => [ + whereFieldsOf(referenceType, presenceLeaves).map((nestedField) => [ nestedField.name, { - type: whereFieldType(nestedField, referenceType), + type: presenceLeaves.includes(nestedField.name) + ? presenceFilterFor(nestedField, referenceType) + : whereFieldType(nestedField, referenceType), description: nestedField.description, }, ]), @@ -1498,7 +1550,7 @@ export function buildGraphQLSchema( type: field.presence === true && !joined ? presenceFilterFor(field, searchType) - : whereFieldType(field, searchType), + : whereFieldType(field, searchType, joined), description: field.description, }; } @@ -1514,14 +1566,21 @@ export function buildGraphQLSchema( * `creativeWorks(where: { dataset: { where: … } })`. * * The one exception is presence, which no join reads yet: a type declaring - * it is reached through a join by `‹Type›JoinedWhere`, the same input without - * `exists` ({@link fieldKeys}). A type declaring none shares one input. + * it – on a field or on an inline reference’s leaves – is reached through a + * join by `‹Type›JoinedWhere`, the same input without `exists` + * ({@link fieldKeys}). A type declaring none shares one input. */ function whereInputFor( searchType: RootType, joined = false, ): GraphQLInputObjectType { - if (joined && !searchType.fields.some((field) => field.presence === true)) { + if ( + joined && + !searchType.fields.some( + (field) => + field.presence === true || presenceLeavesOf(field).length > 0, + ) + ) { return whereInputFor(searchType); } const prefix = joined ? `${searchType.name}Joined` : searchType.name; @@ -1756,8 +1815,11 @@ export function buildGraphQLSchema( const own: FacetDeclaration[] = []; const byGroup = new Map(); for (const facet of facets) { + // A leaf’s facets, a presence leaf’s included, go under the reference; + // an own field’s presence facet under the field. const group = - facet.presence === true ? facet.field.name : facet.reference?.name; + facet.reference?.name ?? + (facet.presence === true ? facet.field.name : undefined); if (group === undefined) { own.push(facet); continue; @@ -1770,31 +1832,40 @@ export function buildGraphQLSchema( return new GraphQLObjectType({ name: `${typeName}Facets`, fields: () => ({ - ...facetFieldsFor(own), + ...facetFieldsFor(own, typeName), ...Object.fromEntries( - [...byGroup].map(([group, grouped]) => [ - group, - { - type: new GraphQLNonNull( - new GraphQLObjectType({ - name: `${typeName}${group.charAt(0).toUpperCase()}${group.slice(1)}Facets`, - fields: () => facetFieldsFor(grouped), - }), - ), - // The source carries the facet loader; the grouped facets read - // it. - resolve: (source: Source) => source, - }, - ]), + [...byGroup].map(([group, grouped]) => { + const groupBaseName = `${typeName}${capitalized(group)}`; + return [ + group, + { + type: new GraphQLNonNull( + new GraphQLObjectType({ + name: `${groupBaseName}Facets`, + fields: () => facetFieldsFor(grouped, groupBaseName), + }), + ), + // The source carries the facet loader; the grouped facets + // read it. + resolve: (source: Source) => source, + }, + ]; + }), ), }), }); } - /** One GraphQL field per facet, keyed by its field’s name – `exists` for a - * presence facet – and loaded by the facet’s logical name. */ + /** + * One GraphQL field per facet, keyed by its field’s name – `exists` for an + * own field’s presence facet, served under the field – and loaded by the + * facet’s logical name. A presence leaf’s facet is one level further down, + * `‹leaf› { exists }`, in an object typed `‹Base›‹Leaf›Facets` after the + * base name of the object it sits in (`WorkAssociatedMedia`). + */ function facetFieldsFor( facets: readonly FacetDeclaration[], + baseName: string, ): Record> { const fields: Record< string, @@ -1803,17 +1874,35 @@ export function buildGraphQLSchema( // The keys share one engine round-trip, but each is its own facet // calculation in it. const facetCost = costExtensions(operationCost); - for (const { name, field, presence } of facets) { + for (const { name, field, presence, reference: reachedThrough } of facets) { // The batching into one engine dispatch and the degrade-to-empty // error handling live in the loader (facet-batch.ts). if (presence === true) { - fields.exists = { + const exists: GraphQLFieldConfig = { type: nonNullListOf(booleanBucket), extensions: facetCost, - description: `Documents that hold a value for “${field.name}”, and documents that hold none.`, + description: reachedThrough + ? `Documents with an entry that holds a value for “${field.name}”, and documents with none.` + : `Documents that hold a value for “${field.name}”, and documents that hold none.`, resolve: async (source: Source) => (await (source.loadFacet as FacetLoader)(name, 'matching')).buckets, }; + if (reachedThrough === undefined) { + fields.exists = exists; + continue; + } + fields[field.name] = { + type: new GraphQLNonNull( + new GraphQLObjectType({ + name: claimTypeName( + `${baseName}${capitalized(field.name)}Facets`, + `the presence facet of “${name}”`, + ), + fields: { exists }, + }), + ), + resolve: (source: Source) => source, + }; continue; } if (isRangeFacet(field)) { @@ -2062,12 +2151,17 @@ function whereToFilters( * a resolver runs: a disjunction inside a weld is not a weld, and the flat IR * has nowhere to put one. `undefined` when the `where` states nothing, which * constrains nothing and so contributes no clause. + * + * `exists` on a presence leaf is the root document’s, not one entry’s, so it + * is welded to nothing else: beside another key it is refused, naming the + * rewrite into two conditions. */ function weldedCriterionOf( entry: { readonly name: string; readonly nested: Record; readonly target: SearchType; + readonly presenceLeaves: readonly string[]; }, joins: JoinGraph, schema: SearchSchema, @@ -2082,7 +2176,16 @@ function weldedCriterionOf( entry.target, joins, schema, + entry.presenceLeaves, ).map((nested) => (nested as { readonly criterion: Criterion }).criterion); + const presence = conditions.find((condition) => 'exists' in condition); + if (presence !== undefined && conditions.length > 1) { + const other = conditions.find((condition) => condition !== presence) + ?.field as string; + throw userError( + `“${entry.name}” welds “exists” on “${presence.field}” to “${other}” on one entry, but whether any entry holds a value for “${presence.field}” is recorded per document, not per entry. Write each as its own condition – and: [{ ${entry.name}: { where: { ${presence.field}: { exists: ${String((presence as { exists: boolean }).exists)} } } } }, { ${entry.name}: { where: { ${other}: … } } }].`, + ); + } return conditions.length === 0 ? undefined : withPath({ field: entry.name, entry: conditions }, on); @@ -2167,6 +2270,9 @@ type KeyEntry = // A Root Type for a JOIN hop, a Reference Type for a NESTING hop. Both // extend the `on` path the same way; only the compiler tells them apart. readonly target: SearchType; + // The presence leaves of a NESTING hop’s reference, whose keys in the + // entry’s `where` take `exists`; none for a join. + readonly presenceLeaves: readonly string[]; }; /** @@ -2184,18 +2290,19 @@ function keyEntriesOf( searchType: SearchType, joins: JoinGraph, schema: SearchSchema, + presenceLeaves: readonly string[] = [], ): KeyEntry[] { const entries: KeyEntry[] = []; const id = keyed[ID_FIELD] as { in?: string[] } | undefined | null; if (id !== undefined && id !== null) { entries.push({ criterion: { field: ID_FIELD, in: id.in ?? [] } }); } - for (const field of whereFieldsOf(searchType)) { + for (const field of whereFieldsOf(searchType, presenceLeaves)) { const value = keyed[field.name]; if (value === undefined || value === null) { continue; } - if (field.presence === true) { + if (field.presence === true || presenceLeaves.includes(field.name)) { const presence = presenceCriterionOf(field.name, value); if (presence !== undefined) { entries.push({ criterion: presence }); @@ -2216,7 +2323,12 @@ function keyEntriesOf( const nested = (value as { where?: Record | null }) .where; if (nested !== undefined && nested !== null) { - entries.push({ name: field.name, nested, target }); + entries.push({ + name: field.name, + nested, + target, + presenceLeaves: presenceLeavesOf(field), + }); continue; } } @@ -2249,11 +2361,18 @@ function keyEntriesOf( } /** The fields a `where` input carries a key for, in declaration order: the - * `filterable` ones, and those declaring presence, which take `exists` - * whether or not they are filterable. */ -function whereFieldsOf(searchType: SearchType): readonly SearchField[] { + * `filterable` ones, and those declaring presence – or, for a reference + * type’s `where`, named among the presence leaves of the references nesting + * it – which take `exists` whether or not they are filterable. */ +function whereFieldsOf( + searchType: SearchType, + presenceLeaves: readonly string[] = [], +): readonly SearchField[] { return searchType.fields.filter( - (field) => field.filterable === true || field.presence === true, + (field) => + field.filterable === true || + field.presence === true || + presenceLeaves.includes(field.name), ); } @@ -2328,6 +2447,18 @@ function rangeInput( }); } +/** The presence leaves an inline reference declares, read off the physical + * companions that record them – none for any other field. */ +function presenceLeavesOf(field: SearchField): readonly string[] { + return physicalFields(field).leafExists.map(({ leaf }) => leaf); +} + +/** A name with its first letter upper-cased, as it is spelled inside a type + * name: `associatedMedia` → `AssociatedMedia`. */ +function capitalized(name: string): string { + return `${name.charAt(0).toUpperCase()}${name.slice(1)}`; +} + /** Type names as prose, with no article to get wrong: `Person`, * `Person or Organization`, `Person, Organization or Group`. */ function disjunction(names: readonly string[]): string { diff --git a/packages/search-api-graphql/test/leaf-presence.test.ts b/packages/search-api-graphql/test/leaf-presence.test.ts new file mode 100644 index 00000000..5ce90ef4 --- /dev/null +++ b/packages/search-api-graphql/test/leaf-presence.test.ts @@ -0,0 +1,329 @@ +import { describe, expect, it } from 'vitest'; +import { graphql, printSchema } from 'graphql'; +import { + defineSearchType, + searchSchema, + type SearchEngine, + type SearchQuery, + type SearchResult, + type SearchType, +} from '@lde/search'; +import { buildGraphQLSchema, type SearchContext } from '../src/build-schema.js'; + +const SCHEMA_ORG = 'https://schema.org/'; +const CC_BY = 'https://creativecommons.org/licenses/by/4.0/'; + +const mediaObject = defineSearchType({ + name: 'MediaObject', + fields: [ + { + name: 'license', + kind: 'reference', + output: true, + filterable: true, + ref: { strategy: 'idOnly' }, + }, + { name: 'thumbnailUrl', kind: 'keyword', output: true }, + { + name: 'encodingFormat', + kind: 'keyword', + output: true, + filterable: true, + }, + ], +}); + +const work = defineSearchType({ + name: 'Work', + class: `${SCHEMA_ORG}CreativeWork`, + fields: [ + { name: 'title', kind: 'text', locales: ['nl'], output: true }, + { + name: 'associatedMedia', + kind: 'reference', + array: true, + output: true, + filterable: true, + ref: { + strategy: 'inline', + typeName: 'MediaObject', + facetedLeaves: ['license'], + presenceLeaves: ['thumbnailUrl', 'encodingFormat'], + }, + }, + ], +}); + +const schema = searchSchema(work, mediaObject); +const gqlSchema = buildGraphQLSchema(schema, {}); +const printed = printSchema(gqlSchema); + +function recordingEngine() { + const searched: SearchQuery[] = []; + const faceted: SearchQuery[] = []; + const engine: SearchEngine = { + schema, + async search(_searchType: SearchType, query: SearchQuery) { + searched.push(query); + return { total: 0, hits: [], facets: {} } as SearchResult; + }, + async searchFacets(_searchType, queries) { + faceted.push(...queries); + return queries.map(() => ({ + facets: { + 'associatedMedia.license': { + buckets: [{ value: CC_BY, count: 2 }], + }, + 'associatedMedia.thumbnailUrl.exists': { + buckets: [ + { value: 'false', count: 2, is: false }, + { value: 'true', count: 1, is: true }, + ], + }, + }, + })) as never; + }, + }; + return { engine, searched, faceted }; +} + +async function run(source: string) { + const recording = recordingEngine(); + const response = await graphql({ + schema: gqlSchema, + source, + contextValue: { + engine: recording.engine, + acceptLanguage: ['nl'], + } as SearchContext, + }); + return { response, ...recording }; +} + +describe('a root document filtered on whether an entry’s leaf has a value', () => { + it('gives a presence leaf `exists` in the reference type’s where', () => { + expect(printed).toMatch( + /input MediaObjectWhere \{[^}]*license: IRIFilter\n[^}]*thumbnailUrl: ExistsFilter\n[^}]*encodingFormat: KeywordFilterOrExists\n/, + ); + }); + + it('compiles `exists` into a weld of that one condition', async () => { + const { response, searched } = await run( + `{ works(where: { associatedMedia: { where: { thumbnailUrl: { exists: true } } } }) { pagination { total } } }`, + ); + + expect(response.errors).toBeUndefined(); + expect(searched[0]?.where).toEqual([ + { + or: [ + { + field: 'associatedMedia', + entry: [{ field: 'thumbnailUrl', exists: true }], + }, + ], + }, + ]); + }); + + it('reads a filterable presence leaf’s value when `exists` is unset', async () => { + const { response, searched } = await run( + `{ works(where: { associatedMedia: { where: { encodingFormat: { in: ["image/png"] } } } }) { pagination { total } } }`, + ); + + expect(response.errors).toBeUndefined(); + expect(searched[0]?.where).toEqual([ + { + or: [ + { + field: 'associatedMedia', + entry: [{ field: 'encodingFormat', in: ['image/png'] }], + }, + ], + }, + ]); + }); + + it('refuses `exists` welded to another condition, naming the rewrite', async () => { + const { response, searched } = await run( + `{ works(where: { associatedMedia: { where: { thumbnailUrl: { exists: true }, license: { in: ["${CC_BY}"] } } } }) { pagination { total } } }`, + ); + + expect(searched).toEqual([]); + expect(response.errors?.[0]?.message).toMatch( + /“associatedMedia” welds “exists” on “thumbnailUrl” to “license” on one entry.*and: \[\{ associatedMedia: \{ where: \{ thumbnailUrl: \{ exists: true \} \} \} \}, \{ associatedMedia: \{ where: \{ license: … \} \} \}\]/, + ); + expect(response.errors?.[0]?.extensions?.code).toBe('BAD_USER_INPUT'); + }); +}); + +describe('a root document faceted on whether an entry’s leaf has a value', () => { + it('serves the facet under the leaf, beside the reference’s faceted leaves', () => { + expect(printed).toMatch( + /type WorkAssociatedMediaFacets \{[^}]*\): IRIFacet!\s*thumbnailUrl: WorkAssociatedMediaThumbnailUrlFacets!\s*encodingFormat: WorkAssociatedMediaEncodingFormatFacets!\s*\}/, + ); + expect(printed).toMatch( + /type WorkAssociatedMediaThumbnailUrlFacets \{[^}]*exists: \[BooleanBucket!\]!\s*\}/, + ); + }); + + it('asks the engine for the facet by its path, and serves boolean buckets', async () => { + const { response, faceted } = await run( + `{ works { facets { associatedMedia { license { buckets { value count } } thumbnailUrl { exists { value count } } } } } }`, + ); + + expect(response.errors).toBeUndefined(); + expect(faceted.map((query) => query.facets)).toEqual([ + ['associatedMedia.license', 'associatedMedia.thumbnailUrl.exists'], + ]); + expect(response.data).toEqual({ + works: { + facets: { + associatedMedia: { + license: { buckets: [{ value: CC_BY, count: 2 }] }, + thumbnailUrl: { + exists: [ + { value: false, count: 2 }, + { value: true, count: 1 }, + ], + }, + }, + }, + }, + }); + }); +}); + +describe('a reference with presence leaves on a joined type', () => { + const joined = defineSearchType({ + name: 'Collection', + class: `${SCHEMA_ORG}Collection`, + fields: [ + { + name: 'label', + kind: 'text', + locales: ['nl'], + output: true, + searchable: { weight: 1 }, + }, + { + name: 'associatedMedia', + kind: 'reference', + array: true, + filterable: true, + ref: { + strategy: 'inline', + typeName: 'MediaObject', + presenceLeaves: ['thumbnailUrl', 'encodingFormat'], + }, + }, + ], + }); + const member = defineSearchType({ + name: 'Item', + class: `${SCHEMA_ORG}Thing`, + fields: [ + { + name: 'collection', + kind: 'reference', + filterable: true, + labelSource: 'Collection', + joinable: true, + }, + ], + }); + const joinedPrinted = printSchema( + buildGraphQLSchema(searchSchema(joined, member, mediaObject), {}), + ); + + it('reaches the entries through a join without `exists`, which no join reads', () => { + expect(joinedPrinted).toMatch( + /input CollectionJoinedWhere \{[^}]*associatedMedia: MediaObjectJoinedFilter\n/, + ); + const where = /input MediaObjectJoinedWhere \{([^}]*)\}/.exec( + joinedPrinted, + ); + expect(where?.[1]).toMatch(/encodingFormat: KeywordFilter\n/); + expect(where?.[1]).not.toMatch(/thumbnailUrl|exists|OrExists/); + expect(joinedPrinted).toMatch( + /input CollectionWhere \{[^}]*associatedMedia: MediaObjectFilter\n/, + ); + }); +}); + +describe('the names a presence leaf adds', () => { + const clashing = (name: string) => + defineSearchType({ + name, + class: `${SCHEMA_ORG}Thing`, + fields: [ + { + name: 'label', + kind: 'text', + locales: ['nl'], + output: true, + searchable: { weight: 1 }, + }, + ], + }); + + it('refuses a type that takes the name of a leaf’s facet object', () => { + expect(() => + printSchema( + buildGraphQLSchema( + searchSchema( + work, + mediaObject, + clashing('WorkAssociatedMediaThumbnailUrlFacets'), + ), + {}, + ), + ), + ).toThrow( + /“WorkAssociatedMediaThumbnailUrlFacets”, the type name of the presence facet of “associatedMedia.thumbnailUrl.exists”, is already the name of another type/, + ); + }); + + it('refuses a type that takes the name of a join variant', () => { + const item = defineSearchType({ + name: 'Item', + class: `${SCHEMA_ORG}Product`, + fields: [ + { + name: 'work', + kind: 'reference', + filterable: true, + labelSource: 'Work', + joinable: true, + }, + ], + }); + const labelled = defineSearchType({ + ...work, + fields: [ + ...work.fields, + { + name: 'label', + kind: 'text', + locales: ['nl'], + output: true, + searchable: { weight: 1 }, + }, + ], + }); + expect(() => + printSchema( + buildGraphQLSchema( + searchSchema( + labelled, + item, + mediaObject, + clashing('MediaObjectJoinedFilter'), + ), + {}, + ), + ), + ).toThrow( + /“MediaObjectJoinedFilter”, the type name of the filter on “MediaObject” through a join, is already the name of another type/, + ); + }); +}); diff --git a/packages/search-api-graphql/vite.config.ts b/packages/search-api-graphql/vite.config.ts index 6797460d..242dc709 100644 --- a/packages/search-api-graphql/vite.config.ts +++ b/packages/search-api-graphql/vite.config.ts @@ -31,7 +31,7 @@ export default mergeConfig( // from a direct caller, since GraphQL validates the query first. // Re-anchored when facet grouping moved to `@lde/search`, taking its // covered branches with it. - branches: 98.99, + branches: 99.05, statements: 100, }, }, diff --git a/packages/search-typesense/src/collection-definition.ts b/packages/search-typesense/src/collection-definition.ts index 8a703e5c..e0c1f17a 100644 --- a/packages/search-typesense/src/collection-definition.ts +++ b/packages/search-typesense/src/collection-definition.ts @@ -443,10 +443,13 @@ function periodFields( * the field stores – a value, a period, a nested object – and for a field * declaring presence alone, which is internal and stores nothing else. */ function presenceFields(field: SearchField): CollectionFieldSchema[] { - const exists = physicalFields(field).exists; - return exists === undefined - ? [] - : [{ name: exists, type: 'bool', facet: true }]; + const { exists, leafExists } = physicalFields(field); + return [ + ...(exists === undefined ? [] : [exists]), + // One per presence leaf of an inline reference: whether any entry holds + // a value, a top-level boolean of the root document. + ...leafExists.map((companion) => companion.exists), + ].map((name) => ({ name, type: 'bool', facet: true })); } /** A `date` field’s precision companion ({@link physicalFields}): read back diff --git a/packages/search-typesense/src/query-compiler.ts b/packages/search-typesense/src/query-compiler.ts index c89e0bc9..7730bba2 100644 --- a/packages/search-typesense/src/query-compiler.ts +++ b/packages/search-typesense/src/query-compiler.ts @@ -636,6 +636,19 @@ function compileWelded( if (entryType === undefined) { return UNUSABLE; } + // Presence on a leaf is recorded once per root document, not inside an + // entry: a weld of that one condition reads the root’s companion. Welded to + // anything else, or under a nesting prefix, it has no entry to hold of – + // `assertValidQuery` rejects both first. + const [only, ...more] = criterion.entry; + if (only !== undefined && 'exists' in only) { + const companion = physicalFields( + fieldNamed(searchType, criterion.field) as SearchField, + ).leafExists.find(({ leaf }) => leaf === only.field); + return more.length === 0 && prefix === '' && companion !== undefined + ? `${companion.exists}:=${only.exists}` + : UNUSABLE; + } const conditions: string[] = []; for (const condition of criterion.entry) { const compiled = compileLeaf(condition, entryType, schema); diff --git a/packages/search-typesense/src/search.ts b/packages/search-typesense/src/search.ts index d38ada3f..3bf67932 100644 --- a/packages/search-typesense/src/search.ts +++ b/packages/search-typesense/src/search.ts @@ -220,9 +220,10 @@ export function createTypesenseSearchEngine< name: field.name, field, })), - // A faceted leaf labels its buckets from the leaf’s own targets. + // A faceted leaf labels its buckets from the leaf’s own targets; a + // presence leaf’s buckets are booleans, and labelled by nothing. ...facetsOf(searchType, schema).filter( - (facet) => facet.reference !== undefined, + (facet) => facet.reference !== undefined && facet.presence !== true, ), ] // An INLINE reference names its label target one level in, through diff --git a/packages/search-typesense/test/leaf-presence.integration.test.ts b/packages/search-typesense/test/leaf-presence.integration.test.ts new file mode 100644 index 00000000..070510c6 --- /dev/null +++ b/packages/search-typesense/test/leaf-presence.integration.test.ts @@ -0,0 +1,245 @@ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import type { Client } from 'typesense'; +import { + searchSchema, + type Filter, + type SearchEngine, + type SearchQuery, + type SearchType, +} from '@lde/search'; +import { describeSearchEngineContract } from '@lde/search/testing'; +import { buildCollectionDefinition } from '../src/collection-definition.js'; +import { createTypesenseSearchEngine } from '../src/search.js'; +import { TypesenseContainer } from './typesense-container.js'; + +/** + * Presence on an inline entry’s leaf – “a work with a reproduction that has a + * thumbnail” – pinned against a real Typesense. A per-entry boolean would not + * answer it: Typesense flattens a leaf under an `object[]` into one list per + * document, so a work with entries `[true, false]` would count in both + * buckets, and a work with no entries in neither. The projection therefore + * writes one boolean per root document, `associatedMedia_thumbnailUrl_exists`, + * true when any entry holds a value. + */ +describe('a root document filtered and faceted on whether an entry’s leaf has a value', () => { + const container = new TypesenseContainer(); + let client: Client; + let engine: SearchEngine; + + const mediaObject: SearchType = { + name: 'MediaObject', + fields: [ + { + name: 'thumbnailUrl', + kind: 'keyword', + path: 'https://schema.org/thumbnailUrl', + output: true, + }, + { + name: 'encodingFormat', + kind: 'keyword', + path: 'https://schema.org/encodingFormat', + output: true, + filterable: true, + }, + ], + }; + const work = { + name: 'Work', + class: 'https://schema.org/CreativeWork', + fields: [ + { + name: 'title', + kind: 'keyword', + path: 'https://schema.org/name', + output: true, + filterable: true, + sortable: true, + }, + { + name: 'associatedMedia', + kind: 'reference', + path: 'https://schema.org/associatedMedia', + array: true, + output: true, + filterable: true, + ref: { + strategy: 'inline', + typeName: 'MediaObject', + presenceLeaves: ['thumbnailUrl'], + }, + }, + ], + } as const satisfies SearchType; + const schema = searchSchema(work, mediaObject); + const facet = 'associatedMedia.thumbnailUrl.exists'; + const base: SearchQuery = { + where: [], + orderBy: [{ field: 'title', direction: 'asc' }], + limit: 10, + offset: 0, + facets: [facet], + locale: 'nl', + }; + const thumbnailed = (exists: boolean): Filter => ({ + or: [ + { + field: 'associatedMedia', + entry: [{ field: 'thumbnailUrl', exists }], + }, + ], + }); + const counts = (result: { facets: Record }) => + Object.fromEntries( + ( + result.facets[facet] as { + buckets: { count: number; is?: boolean }[]; + } + ).buckets.map((bucket) => [String(bucket.is), bucket.count]), + ); + const titles = (result: { hits: readonly { id: string }[] }) => + result.hits.map((hit) => hit.id.replace('https://ex/', '')); + + beforeAll(async () => { + client = await container.start(); + await client.collections().create( + buildCollectionDefinition(work, { + collectionNameFor: () => 'works', + schema, + }), + ); + const results = await client + .collections('works') + .documents() + .import( + [ + { + // One entry has a thumbnail, the other lacks one: the work has + // one, and counts once. + id: 'https://ex/mixed', + title: 'mixed', + associatedMedia: [ + { + thumbnailUrl: 'https://ex/t.jpg', + encodingFormat: 'image/jpeg', + }, + { encodingFormat: 'image/tiff' }, + ], + associatedMedia_thumbnailUrl_exists: true, + }, + { + id: 'https://ex/lacking', + title: 'lacking', + associatedMedia: [{ encodingFormat: 'image/png' }], + associatedMedia_thumbnailUrl_exists: false, + }, + // No entries at all: no thumbnail either. + { + id: 'https://ex/bare', + title: 'bare', + associatedMedia_thumbnailUrl_exists: false, + }, + ], + { action: 'create' }, + ); + expect(results.every((result) => result.success)).toBe(true); + engine = createTypesenseSearchEngine(client, schema, { + collections: { Work: 'works' }, + }); + }, 120_000); + + afterAll(async () => { + await container.stop(); + }); + + // The port contract, with the leaf’s presence facet among those it checks. + describeSearchEngineContract( + 'a Typesense engine serving an entry leaf’s presence', + () => engine, + ); + + it('declares one boolean per root document, faceted', async () => { + const { fields } = await client.collections('works').retrieve(); + + expect(fields).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + name: 'associatedMedia_thumbnailUrl_exists', + type: 'bool', + facet: true, + }), + ]), + ); + }); + + it('counts a work whose entries differ once, under `true`', async () => { + expect(counts(await engine.search(work, base))).toEqual({ + true: 1, + false: 2, + }); + }); + + it('filters on any entry holding a value, and on none holding one', async () => { + const some = await engine.search(work, { + ...base, + where: [thumbnailed(true)], + }); + const none = await engine.search(work, { + ...base, + where: [thumbnailed(false)], + }); + + expect(titles(some)).toEqual(['mixed']); + expect(titles(none)).toEqual(['bare', 'lacking']); + }); + + it('counts without its own selection, and filters the hits by it', async () => { + const result = await engine.search(work, { + ...base, + where: [thumbnailed(true)], + }); + + expect(titles(result)).toEqual(['mixed']); + expect(counts(result)).toEqual({ true: 1, false: 2 }); + }); + + it('counts under a condition on the same entries, which it does not own', async () => { + const result = await engine.search(work, { + ...base, + where: [ + { + or: [ + { + field: 'associatedMedia', + entry: [{ field: 'encodingFormat', in: ['image/png'] }], + }, + ], + }, + ], + }); + + expect(titles(result)).toEqual(['lacking']); + expect(counts(result)).toEqual({ false: 1 }); + }); + + it('refuses `exists` welded to another condition on one entry', async () => { + await expect( + engine.search(work, { + ...base, + where: [ + { + or: [ + { + field: 'associatedMedia', + entry: [ + { field: 'thumbnailUrl', exists: true }, + { field: 'encodingFormat', in: ['image/jpeg'] }, + ], + }, + ], + }, + ], + }), + ).rejects.toThrow(/associatedMedia\.thumbnailUrl.*exists-in-weld/); + }); +}); diff --git a/packages/search-typesense/test/query-compiler.test.ts b/packages/search-typesense/test/query-compiler.test.ts index 0e486fbe..42aca64b 100644 --- a/packages/search-typesense/test/query-compiler.test.ts +++ b/packages/search-typesense/test/query-compiler.test.ts @@ -885,3 +885,119 @@ describe('a field filtered and faceted on whether it has a value', () => { expect(params.filter_by).toBe('id:=[]'); }); }); + +describe('`exists` welded to an inline entry', () => { + const media: SearchType = { + name: 'Media', + fields: [ + { name: 'thumbnailUrl', kind: 'keyword' }, + { name: 'format', kind: 'keyword', filterable: true }, + ], + }; + const work: SearchType = { + name: 'Work', + class: 'https://schema.org/CreativeWork', + fields: [ + { + name: 'media', + kind: 'reference', + filterable: true, + ref: { + strategy: 'inline', + typeName: 'Media', + presenceLeaves: ['thumbnailUrl'], + }, + }, + ], + }; + const mediaSchema = searchSchema(work, media); + const filterBy = (entry: Record[]) => + buildSearchParams( + { + ...base, + where: [{ or: [{ field: 'media', entry }] } as never], + }, + work, + { schema: mediaSchema }, + ).filter_by; + + it('reads the root document’s boolean when welded alone', () => { + expect(filterBy([{ field: 'thumbnailUrl', exists: false }])).toBe( + 'media_thumbnailUrl_exists:=false', + ); + }); + + it('compiles to a term matching nothing beside another condition, or on a leaf named no presence', () => { + // `assertValidQuery` rejects both first; this is the guard for a direct + // caller. + expect( + filterBy([ + { field: 'thumbnailUrl', exists: true }, + { field: 'format', in: ['image/png'] }, + ]), + ).toBe('id:=[]'); + expect( + filterBy([ + { field: 'format', in: ['image/png'] }, + { field: 'thumbnailUrl', exists: true }, + ]), + ).toBe('id:=[]'); + expect(filterBy([{ field: 'format', exists: true }])).toBe('id:=[]'); + }); + + it('compiles to a term matching nothing under a nesting, where no boolean is written', () => { + // `searchSchema` refuses presence leaves, and a filterable reference, on + // a Reference Type, so the declaration is changed after the fact, as a + // direct caller could. + const nestedRef = { strategy: 'inline', typeName: 'Media' } as const; + const nestedMedia = { + name: 'media', + kind: 'reference', + output: true, + ref: nestedRef, + } as const; + const edge: SearchType = { + name: 'Edge', + fields: [ + nestedMedia, + { name: 'role', kind: 'keyword', filterable: true }, + ], + }; + const root: SearchType = { + name: 'Work', + class: 'https://schema.org/CreativeWork', + fields: [ + { + name: 'edges', + kind: 'reference', + filterable: true, + ref: { strategy: 'inline', typeName: 'Edge' }, + }, + ], + }; + const nestedSchema = searchSchema(root, edge, media); + Object.assign(nestedRef, { presenceLeaves: ['thumbnailUrl'] }); + Object.assign(nestedMedia, { filterable: true }); + + expect( + buildSearchParams( + { + ...base, + where: [ + { + or: [ + { + on: ['edges'], + field: 'media', + entry: [{ field: 'thumbnailUrl', exists: true }], + }, + ], + }, + ], + }, + root, + { schema: nestedSchema }, + ).filter_by, + ).toBe('id:=[]'); + }); +}); diff --git a/packages/search-typesense/vite.config.ts b/packages/search-typesense/vite.config.ts index 6bba127b..06da98e0 100644 --- a/packages/search-typesense/vite.config.ts +++ b/packages/search-typesense/vite.config.ts @@ -22,7 +22,7 @@ export default mergeConfig( // by-name map for the shared `rootTypeNamed`: one COVERED arrow fewer // over one function fewer, which lowers the ratio without uncovering // anything. - functions: 99.37, + functions: 99.38, // Lines, branches and statements moved the same way when the range // bound reading went to `@lde/search` as `storedRangeBound`. lines: 99.59, @@ -34,7 +34,7 @@ export default mergeConfig( // companion: covered branches deleted, nothing uncovered. And when // a field declaring presence alone went back to being internal, // deleting the covered branches that kept its value unindexed. - branches: 97.15, + branches: 97.3, statements: 99.6, }, }, diff --git a/packages/search/src/engine.ts b/packages/search/src/engine.ts index c14b8c63..e5f7ff1d 100644 --- a/packages/search/src/engine.ts +++ b/packages/search/src/engine.ts @@ -161,8 +161,20 @@ export type FacetFieldsOf = RootType extends Type ? string : | Extract['name'] | FacetedLeavesOf + | PresenceLeavesOf | `${Extract['name']}.exists`; +/** The `‹reference›.‹leaf›.exists` facet names an inline reference declares + * through its presence leaves (`ref.presenceLeaves`). */ +type PresenceLeavesOf = Field extends { + readonly name: infer Name extends string; + readonly ref: { + readonly presenceLeaves: readonly (infer Leaf extends string)[]; + }; +} + ? `${Name}.${Leaf}.exists` + : never; + /** The `‹reference›.‹leaf›` facet names an inline reference declares through * its faceted leaves (`ref.facetedLeaves`). */ type FacetedLeavesOf = Field extends { diff --git a/packages/search/src/project.ts b/packages/search/src/project.ts index b379800e..bd437512 100644 --- a/packages/search/src/project.ts +++ b/packages/search/src/project.ts @@ -150,7 +150,8 @@ export function projectDocument( unreadableSink(context, key), ) as SearchDocument; // After every field, so a presence reads what a later derive left too; on - // the root alone, since presence is declared only on a Root Type’s own field. + // the root alone, since presence is declared only on a Root Type’s own field + // and on its inline references’ leaves. for (const field of searchType.fields) { if (field.presence === true) { document[physicalFields(field).exists as string] = holdsPresence( @@ -158,6 +159,7 @@ export function projectDocument( field, ); } + writeLeafPresence(document, field, schema); } pruneInternalFields(document, searchType, schema); return document; @@ -490,6 +492,38 @@ function holdsPresence(document: ProjectedNode, field: SearchField): boolean { ); } +/** + * Write whether any entry of an inline reference holds a value for each of + * its {@link ReferenceStrategy.presenceLeaves presence leaves}: one boolean + * per root document, `false` with no entries at all. Read before pruning, so + * a leaf declaring no Role of its own still counts. Without a schema the + * entries are not projected, so there is nothing to count – and nothing is + * written. + */ +function writeLeafPresence( + document: ProjectedNode, + field: SearchField, + schema: SearchSchema | undefined, +): void { + const leaves = physicalFields(field).leafExists; + const referenceType = + schema === undefined || leaves.length === 0 || !isInlineReference(field) + ? undefined + : referenceTypeNamed(schema, field.ref.typeName); + if (referenceType === undefined) { + return; + } + const stored = document[field.name]; + const entries = ( + stored === undefined ? [] : Array.isArray(stored) ? stored : [stored] + ) as ProjectedNode[]; + for (const { leaf: name, exists } of leaves) { + // `searchSchema` has resolved every presence leaf against the type. + const leaf = fieldNamed(referenceType, name) as SearchField; + document[exists] = entries.some((entry) => holdsPresence(entry, leaf)); + } +} + /** Whether a stored value is one: neither `undefined` nor `null`, nor an * empty list. */ function holdsValue(value: unknown): boolean { diff --git a/packages/search/src/query.ts b/packages/search/src/query.ts index 17cfab93..8d22275b 100644 --- a/packages/search/src/query.ts +++ b/packages/search/src/query.ts @@ -5,6 +5,7 @@ import { ID_FIELD, holdsNoIds, nestedReferenceType, + physicalFields, presenceFacetName, referencedTargetsOf, storedRangeBound, @@ -310,7 +311,9 @@ export interface FacetSearch { * * An `exists` criterion is on the axis `‹field›.exists`, which the field’s * presence facet owns. It owns nothing else: a range on the same field is - * another axis, and so a presence facet counts under it. + * another axis, and so a presence facet counts under it. Welded alone to an + * entry, `exists` is on the axis `‹reference›.‹leaf›.exists`, owned by that + * presence leaf’s facet and never by a facet on the leaf’s values. * * A default a surface adds to `where` (a valid-only `status`) is a clause * like any other: its facet counts across every value and reports the @@ -418,16 +421,24 @@ export interface QueryIssue { | 'inverted-range' /** * An `exists` criterion on a field that declares no - * {@link SearchFieldBase.presence}, so no document records whether it - * holds a value. A field of a Reference Type never declares one, which - * makes this the reason for an `exists` welded to an entry, too. + * {@link SearchFieldBase.presence} – or, welded to one entry, on a leaf + * its inline reference does not name among its + * {@link ReferenceStrategy.presenceLeaves presence leaves} – so no + * document records whether it holds a value. */ | 'no-presence' /** * An `exists` criterion reached through a join path. The joined type’s * own field may declare presence, but no join reads its companion yet. */ - | 'joined-presence'; + | 'joined-presence' + /** + * An `exists` welded to one entry together with other conditions. Whether + * any entry holds a value for a leaf is recorded once per root document, + * not per entry, so it cannot hold of the same entry as another + * condition. State it as a weld of its own. + */ + | 'exists-in-weld'; } /** @@ -649,7 +660,12 @@ export function validateQuery( } constrained = resolved.leafType; nested = resolved.nestedPath.length > 0; - if ('exists' in criterion && resolved.joinPath.length > 0) { + if ( + ('exists' in criterion || + (isWelded(criterion) && + criterion.entry.some((welded) => 'exists' in welded))) && + resolved.joinPath.length > 0 + ) { issues.push({ part: 'where', field, reason: 'joined-presence' }); continue; } @@ -688,6 +704,30 @@ export function validateQuery( continue; } for (const welded of criterion.entry) { + // Presence on an entry’s leaf is the root document’s, so it is + // answered by a weld of that one condition alone, on a leaf the + // reference names among its presence leaves. + if ('exists' in welded) { + const path = [...on, name, welded.field].join('.'); + if (criterion.entry.length > 1) { + issues.push({ + part: 'where', + field: path, + reason: 'exists-in-weld', + }); + } else if ( + !physicalFields(declared as SearchField).leafExists.some( + ({ leaf }) => leaf === welded.field, + ) + ) { + issues.push({ + part: 'where', + field: path, + reason: 'no-presence', + }); + } + continue; + } collectLeafIssues( welded, entryType, @@ -879,10 +919,14 @@ function axisOf(criterion: Criterion): string | undefined { // One condition on one leaf selects on that leaf’s axis, the same documents // whether read per entry or per document. Several are welded to one entry: // like a clause spanning two fields, that constrains no single axis. + // An `exists` welded alone is on its leaf’s presence axis, which no facet + // on the leaf’s values owns. const [only, ...more] = criterion.entry; - return only !== undefined && more.length === 0 - ? [...path, only.field].join('.') - : undefined; + if (only === undefined || more.length > 0) { + return undefined; + } + const leafPath = [...path, only.field].join('.'); + return 'exists' in only ? presenceFacetName(leafPath) : leafPath; } /** diff --git a/packages/search/src/schema.ts b/packages/search/src/schema.ts index 6601ab4d..c6c6b8f2 100644 --- a/packages/search/src/schema.ts +++ b/packages/search/src/schema.ts @@ -282,6 +282,7 @@ export type ReferenceStrategy = readonly target?: never; readonly identity?: never; readonly facetedLeaves?: never; + readonly presenceLeaves?: never; } | { readonly strategy: 'lookup'; @@ -349,6 +350,7 @@ export type ReferenceStrategy = readonly literal?: LiteralKind; readonly identity?: never; readonly facetedLeaves?: never; + readonly presenceLeaves?: never; } | { readonly strategy: 'inline'; @@ -401,6 +403,29 @@ export type ReferenceStrategy = * ({@link searchSchema} enforces it), and the reference must be too. */ readonly facetedLeaves?: readonly string[]; + /** + * The `name`s of **leaf fields** of this reference type the root + * document is filtered and faceted on by whether **any entry** holds a + * value: `where: { associatedMedia: { where: { thumbnailUrl: { exists: + * true } } } }` and the facet `‹reference›.‹leaf›.exists`. + * + * One boolean per root document, `${reference}_${leaf}_exists` + * ({@link PhysicalFields.leafExists}), true when any entry holds a value for + * the leaf – in the sense of {@link SearchFieldBase.presence} – and false + * otherwise, a document with no entries included. A boolean inside each + * entry would not answer the question: an engine flattens it into one + * list per document, so a work with one entry that has the leaf and one + * that lacks it would count in both buckets. + * + * Declared on the reference, like {@link facetedLeaves}, because the + * count is a fact about the root. The reference must be `filterable` and + * sit on a Root Type; each leaf is neither `text`, `boolean`, an inline + * reference nor one of the reference’s faceted leaves; and every inline + * reference nesting the same reference type declares the same leaves, + * since they share one filter input ({@link searchSchema} enforces all + * of it). + */ + readonly presenceLeaves?: readonly string[]; }; /** The `literal` kinds a declaration may name, checked at runtime for one @@ -910,7 +935,7 @@ export function searchSchema( const names = new Set(); // First, so a refused presence gets its full sentence rather than the // reason `validateSearchType` files it under. - assertPresenceOnOwnFields(types); + assertServiceablePresence(types); for (const searchType of types) { assertValidSearchType(searchType); if (isRootType(searchType)) { @@ -1271,6 +1296,7 @@ function assertResolvableInlineReferences( assertFacetedLeaves(searchType, field, referenceType); } } + assertSharedPresenceLeaves(types); for (const searchType of types) { for (const field of searchType.fields) { assertServiceableLocalLookup(searchType, field); @@ -1377,9 +1403,13 @@ function assertIdentity( `${where} declares “filterable” without an “identity”, but “${searchType.name}” is a reference type: a condition on one entry cannot be welded inside another. Declare it on a Root Type’s reference, or drop “filterable”.`, ); } - if (filterableFields(referenceType).length === 0) { + // A presence leaf is a key a condition on one entry can name too. + if ( + filterableFields(referenceType).length === 0 && + physicalFields(field).leafExists.length === 0 + ) { throw new Error( - `${where} declares “filterable” without an “identity”, but “${field.ref.typeName}” declares no “filterable” field: a condition on one entry is the only filter it can take, and there is nothing for one to name. Declare “filterable” on a field of “${field.ref.typeName}”, or name an “identity”.`, + `${where} declares “filterable” without an “identity”, but “${field.ref.typeName}” declares no “filterable” field: a condition on one entry is the only filter it can take, and there is nothing for one to name. Declare “filterable” on a field of “${field.ref.typeName}”, name one in “presenceLeaves”, or name an “identity”.`, ); } return; @@ -1512,34 +1542,90 @@ function assertFacetedLeaves( /** * {@link SearchFieldBase.presence Presence} counts what one field of a root - * document holds, written to a companion beside it. Refused wherever that - * reading does not hold yet: on a Reference Type’s field, whose entries no - * companion is written inside; on an inline reference, which stores entries - * rather than a value; on a `text` field, which stores its values per - * language; on a `boolean`, which states its own absence as `false`; and on a - * field that is already `facetable`, whose facet would have to become an - * object to hold both. The full-sentence form of {@link presenceIssues}, - * linking the issue that lifts each refusal. - */ -function assertPresenceOnOwnFields(types: readonly SearchType[]): void { + * document holds, and an inline reference’s + * {@link ReferenceStrategy.presenceLeaves presence leaves} what any of its + * entries holds, each written to a companion. Throws the first refusal + * {@link presenceIssues} files – resolving each presence leaf against the + * reference type it names – as a sentence linking the issue that lifts it, + * where there is one. + */ +function assertServiceablePresence(types: readonly SearchType[]): void { + const referenceTypes = new Map( + types + .filter((searchType) => searchType.class === undefined) + .map((searchType) => [searchType.name, searchType]), + ); for (const searchType of types) { - const [issue] = presenceIssues(searchType); + const [issue] = presenceIssues(searchType, referenceTypes); if (issue === undefined) { continue; } - const where = `Field “${searchType.name}.${issue.field}” declares “presence”`; + const [name, leaf] = issue.field.split('.'); + const at = `“${searchType.name}.${name}”`; + const field = `Field ${at} declares “presence”`; + const leaves = `Inline reference ${at} declares “presenceLeaves”`; + const named = `Inline reference ${at} names presence leaf “${leaf}”`; + const objectFacet = + 'would have to become an object to hold the presence buckets beside the value buckets, a change https://github.com/ldelements/lde/issues/1090 makes. Declare one of the two until then.'; throw new Error( { - 'presence-on-reference-type': `${where}, but “${searchType.name}” is a reference type: presence is counted on a Root Type’s own field, and a leaf of an entry waits for https://github.com/ldelements/lde/issues/1091. Declare it on a field of the Root Type instead.`, - 'presence-with-inline-ref': `${where} on an inline reference, which stores entries rather than a value; presence on an entry’s leaf waits for https://github.com/ldelements/lde/issues/1091.`, - 'presence-on-text': `${where} on a “text” field, which stores its values per language for display and search, so no one stored value says whether it has one. Declare presence on a “keyword” field over the same path.`, - 'presence-on-boolean': `${where} on a “boolean”, which states its own absence as “false”: filter it with “is” instead.`, - 'presence-with-facetable': `${where} and is “facetable”: its facet would have to become an object to hold the presence buckets beside the value buckets, a change https://github.com/ldelements/lde/issues/1090 makes. Declare one of the two for now.`, + 'presence-on-reference-type': `${field}, but “${searchType.name}” is a reference type: presence is counted on a root document. To count whether any entry holds a value for this leaf, name it in the “presenceLeaves” of the Root Type’s inline reference instead.`, + 'presence-with-inline-ref': `${field} on an inline reference, which stores entries rather than a value. To count whether any entry holds a value for one of their leaves, name the leaf in the reference’s “presenceLeaves”.`, + 'presence-on-text': `${field} on a “text” field, which stores its values per language for display and search, so no one stored value says whether it has one. Declare presence on a “keyword” field over the same path.`, + 'presence-on-boolean': `${field} on a “boolean”, which states its own absence as “false”: filter it with “is” instead.`, + 'presence-with-facetable': `${field} and is “facetable”: its facet ${objectFacet}`, + 'presence-leaves-on-reference-type': `${leaves}, but “${searchType.name}” is a reference type: presence is counted on a root document, and an entry is not one. Declare it on the Root Type’s reference instead.`, + 'presence-leaves-not-inline': `${at} declares “presenceLeaves” but is no inline reference, so it has no entries to count; a lookup target’s leaf waits for https://github.com/ldelements/lde/issues/1090.`, + 'presence-leaves-without-filterable': `${leaves} without “filterable”: a bucket is selected by a condition on one entry, which the reference must accept. Declare “filterable” on it.`, + 'presence-leaves-with-facetable': `${leaves} and is “facetable”: its facet ${objectFacet}`, + 'presence-leaves-duplicate': `${leaves} and names one leaf twice.`, + 'presence-leaf-unknown': `${named}, which “${(fieldNamed(searchType, name) as { readonly ref?: { readonly typeName?: string } } | undefined)?.ref?.typeName}” does not declare.`, + 'presence-leaf-on-text': `${named}, a “text”: it stores its values per language, so no one stored value says whether it has one.`, + 'presence-leaf-on-boolean': `${named}, a “boolean”: it states its own absence as “false”.`, + 'presence-leaf-inline-ref': `${named}, which is itself an inline reference: a leaf of an entry’s entries is not counted.`, + 'presence-leaf-faceted': `${named}, which is also one of its faceted leaves: the leaf’s facet ${objectFacet}`, }[issue.reason as PresenceIssueReason], ); } } +/** + * Every inline reference that gets a filter input – a `filterable` one on a + * Root Type – and nests one reference type must declare the same + * {@link ReferenceStrategy.presenceLeaves presence leaves}: the reference + * type’s `where` input is shared by all of them, and which of its keys take + * `exists` is decided by those leaves. Two references disagreeing would offer + * `exists` where one of them cannot answer it. A temporary rule: per-reference + * filter inputs ([#1090](https://github.com/ldelements/lde/issues/1090)) lift + * it. + */ +function assertSharedPresenceLeaves(types: readonly SearchType[]): void { + const declared = new Map(); + for (const searchType of types) { + if (searchType.class === undefined) { + continue; + } + for (const field of searchType.fields) { + if (!isInlineReference(field) || field.filterable !== true) { + continue; + } + const at = `${searchType.name}.${field.name}`; + const leaves = physicalFields(field) + .leafExists.map(({ leaf }) => leaf) + .sort() + .join(', '); + const first = declared.get(field.ref.typeName); + if (first === undefined) { + declared.set(field.ref.typeName, { at, leaves }); + } else if (first.leaves !== leaves) { + throw new Error( + `Inline references “${first.at}” and “${at}” both nest “${field.ref.typeName}” but declare different “presenceLeaves” (${first.leaves || 'none'}; ${leaves || 'none'}): they share one filter input on “${field.ref.typeName}”, so declare the same leaves on both.`, + ); + } + } + } +} + /** The reasons {@link presenceIssues} files a refused presence under. */ type PresenceIssueReason = Extract< SearchTypeIssue['reason'], @@ -1547,29 +1633,93 @@ type PresenceIssueReason = Extract< >; /** - * The issues a type’s {@link SearchFieldBase.presence presence} declarations - * carry: presence counts what one field of a root document holds, written to - * a companion beside it, and is refused wherever that reading does not hold - * yet – one issue per field, the first rule it breaks. + * The issues a type’s presence declarations carry – a field’s + * {@link SearchFieldBase.presence presence}, refused wherever a root + * document’s companion cannot answer it yet, and an inline reference’s + * {@link ReferenceStrategy.presenceLeaves presence leaves}. One issue per + * field, the first rule it breaks; a presence leaf’s is filed under + * `‹reference›.‹leaf›`. The leaves are checked against the reference type + * only where `referenceTypes` resolves it – one declaration cannot see it. */ -function presenceIssues(searchType: SearchType): readonly SearchTypeIssue[] { +function presenceIssues( + searchType: SearchType, + referenceTypes?: ReadonlyMap, +): readonly SearchTypeIssue[] { return searchType.fields.flatMap((field): SearchTypeIssue[] => { - if (field.presence !== true) { - return []; - } const reason: PresenceIssueReason | undefined = - searchType.class === undefined - ? 'presence-on-reference-type' - : field.kind === 'reference' && field.ref?.strategy === 'inline' - ? 'presence-with-inline-ref' - : field.kind === 'text' - ? 'presence-on-text' - : field.kind === 'boolean' - ? 'presence-on-boolean' - : field.facetable === true - ? 'presence-with-facetable' + field.presence !== true + ? undefined + : searchType.class === undefined + ? 'presence-on-reference-type' + : field.kind === 'reference' && field.ref?.strategy === 'inline' + ? 'presence-with-inline-ref' + : field.kind === 'text' + ? 'presence-on-text' + : field.kind === 'boolean' + ? 'presence-on-boolean' + : field.facetable === true + ? 'presence-with-facetable' + : undefined; + const issues: SearchTypeIssue[] = + reason === undefined ? [] : [{ field: field.name, reason }]; + return [ + ...issues, + ...presenceLeavesIssues(searchType, field, referenceTypes), + ]; + }); +} + +/** The issues an inline reference’s + * {@link ReferenceStrategy.presenceLeaves presence leaves} carry: the + * reference’s own first, then – where the reference type resolves – each + * leaf’s. */ +function presenceLeavesIssues( + searchType: SearchType, + field: SearchField, + referenceTypes: ReadonlyMap | undefined, +): readonly SearchTypeIssue[] { + const leaves = (field as FlatField).ref?.presenceLeaves; + if (leaves === undefined || leaves.length === 0) { + return []; + } + const reason: PresenceIssueReason | undefined = + searchType.class === undefined + ? 'presence-leaves-on-reference-type' + : !isInlineReference(field) + ? 'presence-leaves-not-inline' + : field.filterable !== true + ? 'presence-leaves-without-filterable' + : field.facetable === true + ? 'presence-leaves-with-facetable' + : new Set(leaves).size !== leaves.length + ? 'presence-leaves-duplicate' + : undefined; + if (reason !== undefined) { + return [{ field: field.name, reason }]; + } + const referenceType = referenceTypes?.get( + (field as { readonly ref: { readonly typeName: string } }).ref.typeName, + ); + if (referenceType === undefined) { + return []; + } + return leaves.flatMap((name): SearchTypeIssue[] => { + const leaf = fieldNamed(referenceType, name); + const leafReason: PresenceIssueReason | undefined = + leaf === undefined + ? 'presence-leaf-unknown' + : leaf.kind === 'text' + ? 'presence-leaf-on-text' + : leaf.kind === 'boolean' + ? 'presence-leaf-on-boolean' + : isInlineReference(leaf) + ? 'presence-leaf-inline-ref' + : facetedLeavesOf(field).includes(name) + ? 'presence-leaf-faceted' : undefined; - return reason === undefined ? [] : [{ field: field.name, reason }]; + return leafReason === undefined + ? [] + : [{ field: `${field.name}.${name}`, reason: leafReason }]; }); } @@ -2127,7 +2277,17 @@ export interface SearchTypeIssue { | 'presence-with-inline-ref' | 'presence-on-text' | 'presence-on-boolean' - | 'presence-with-facetable'; + | 'presence-with-facetable' + | 'presence-leaves-on-reference-type' + | 'presence-leaves-not-inline' + | 'presence-leaves-without-filterable' + | 'presence-leaves-with-facetable' + | 'presence-leaves-duplicate' + | 'presence-leaf-unknown' + | 'presence-leaf-on-text' + | 'presence-leaf-on-boolean' + | 'presence-leaf-inline-ref' + | 'presence-leaf-faceted'; } /** @@ -2277,7 +2437,9 @@ const LOCALE_PATTERN = /^[A-Za-z0-9]+(-[A-Za-z0-9]+)*$/; * ({@link facetKeysIssues}); a {@link ReferenceType} declares no policy at * all. * - {@link SearchFieldBase.presence presence} sits on a Root Type’s own field - * that is neither an inline reference, `text`, `boolean` nor `facetable` + * that is neither an inline reference, `text`, `boolean` nor `facetable`, + * and {@link ReferenceStrategy.presenceLeaves presence leaves} on a Root + * Type’s `filterable` inline reference, each named once * ({@link presenceIssues}). * * Pure and total: returns every issue rather than throwing; @@ -2299,6 +2461,9 @@ export function validateSearchType( ...(field.kind === 'text' || physicalFields(field).exists === undefined ? [] : [physicalFields(field).exists as string]), + ...(field.kind === 'text' + ? [] + : physicalFields(field).leafExists.map(({ exists }) => exists)), ]), ); // An `output` text field stores its display under every `${name}_${lang}` @@ -2737,6 +2902,18 @@ export interface PhysicalFields { * every other field. */ readonly exists: string | undefined; + /** + * The **presence leaf companions** of an inline reference declaring + * {@link ReferenceStrategy.presenceLeaves presence leaves}: per leaf, + * `${name}_${leaf}_exists`, a top-level boolean of the root document stating + * whether any entry holds a value for the leaf, written on every document – + * `false` with no entries. Indexed and faceted; never served. Empty for + * every other field. + */ + readonly leafExists: readonly { + readonly leaf: string; + readonly exists: string; + }[]; } /** The physical name of a {@link ReferenceStrategy.local local} lookup’s @@ -2923,7 +3100,8 @@ export function facetableFields( */ export interface FacetDeclaration { /** The logical name a query asks for and a result files buckets under: the - * field’s own name, `‹reference›.‹leaf›`, or `‹field›.exists`. */ + * field’s own name, `‹reference›.‹leaf›`, `‹field›.exists`, or + * `‹reference›.‹leaf›.exists`. */ readonly name: string; /** The field whose values the buckets are – the facetable field, or the * faceted leaf. Its kind types the buckets, and its label source labels @@ -2936,9 +3114,11 @@ export interface FacetDeclaration { /** The physical field the engine facets ({@link physicalFields}): the field * itself or one of its companions, or the nested path of a leaf. */ readonly engineField: string; - /** Set on the facet counting whether {@link field} holds a value: its two - * buckets are `true` and `false`, selected by an `exists` criterion, and - * carry no label or bin. Absent on every other facet. */ + /** Set on the facet counting whether {@link field} holds a value – for a + * leaf reached through {@link reference}, whether any entry holds one: its + * two buckets are `true` and `false`, selected by an `exists` criterion + * (welded to one entry, for a leaf), and carry no label or bin. Absent on + * every other facet. */ readonly presence?: true; } @@ -3022,7 +3202,7 @@ function declaredFacets( return own; } // Only an inline reference nests a reference type, and `searchSchema` - // has resolved every faceted leaf against it. + // has resolved every faceted and presence leaf against it. const leaves = facetedLeavesOf(field); return [ ...own, @@ -3035,6 +3215,15 @@ function declaredFacets( engineField: path, }; }), + ...physicalFields(field, schema).leafExists.map( + ({ leaf, exists }): FacetDeclaration => ({ + name: presenceFacetName(nestedFieldName(field.name, leaf)), + field: fieldNamed(referenceType, leaf) as SearchField, + reference: field, + engineField: exists, + presence: true, + }), + ), ]; }); } @@ -3662,6 +3851,7 @@ export function physicalFields( period: undefined, // Nor does it declare presence (`searchSchema`). exists: undefined, + leafExists: [], }; } // A field storing a nested OBJECT cannot be filtered or faceted as itself, so @@ -3706,6 +3896,10 @@ export function physicalFields( } : undefined, exists: field.presence === true ? `${field.name}_exists` : undefined, + leafExists: (isInlineReference(field) + ? (field.ref.presenceLeaves ?? []) + : [] + ).map((leaf) => ({ leaf, exists: `${field.name}_${leaf}_exists` })), }; } diff --git a/packages/search/test/leaf-presence.test.ts b/packages/search/test/leaf-presence.test.ts new file mode 100644 index 00000000..a8f3463f --- /dev/null +++ b/packages/search/test/leaf-presence.test.ts @@ -0,0 +1,640 @@ +import { describe, expect, it } from 'vitest'; +import type { FacetFieldsOf } from '../src/engine.js'; +import { joinGraph } from '../src/join-graph.js'; +import { projectDocument } from '../src/project.js'; +import { + defineSearchType, + facetNamed, + facetsOf, + physicalFields, + searchSchema, + validateSearchType, + type FacetDeclaration, + type SearchField, + type SearchType, +} from '../src/schema.js'; +import { + facetSearches, + selectionOf, + validateQuery, + type Criterion, + type Filter, + type SearchQuery, +} from '../src/query.js'; + +const SCHEMA_ORG = 'https://schema.org/'; + +/** A reproduction: a thumbnail only counted, a format filtered by, a licence + * faceted on, a caption only shown and a flag. */ +const mediaObject = defineSearchType({ + name: 'MediaObject', + fields: [ + { + name: 'thumbnailUrl', + kind: 'keyword', + path: `${SCHEMA_ORG}thumbnailUrl`, + }, + { + name: 'encodingFormat', + kind: 'keyword', + path: `${SCHEMA_ORG}encodingFormat`, + output: true, + filterable: true, + }, + { + name: 'license', + kind: 'reference', + path: `${SCHEMA_ORG}license`, + output: true, + filterable: true, + ref: { strategy: 'idOnly' }, + }, + { + name: 'caption', + kind: 'text', + path: `${SCHEMA_ORG}caption`, + locales: ['nl'], + output: true, + }, + { + name: 'isFree', + kind: 'boolean', + path: `${SCHEMA_ORG}isAccessibleForFree`, + output: true, + }, + ], +}); + +const workWith = ( + ref: Record, + extra: Partial = {}, +): SearchType => + ({ + name: 'Work', + class: `${SCHEMA_ORG}CreativeWork`, + fields: [ + { + name: 'associatedMedia', + kind: 'reference', + path: `${SCHEMA_ORG}associatedMedia`, + array: true, + output: true, + filterable: true, + ...extra, + ref: { strategy: 'inline', typeName: 'MediaObject', ...ref }, + }, + ], + }) as never; + +const work = workWith({ + presenceLeaves: ['thumbnailUrl'], + facetedLeaves: ['license'], +}); +const schema = searchSchema(work, mediaObject); +const facet = 'associatedMedia.thumbnailUrl.exists'; +const browse: SearchQuery = { + where: [], + orderBy: [], + limit: 10, + offset: 0, + facets: [], + locale: 'nl', +}; +const welded = (...entry: Criterion[]): Filter => ({ + or: [{ field: 'associatedMedia', entry }], +}); + +describe('declaring presence leaves', () => { + it('names one boolean per leaf on the root document', () => { + expect(physicalFields(work.fields[0]).leafExists).toEqual([ + { leaf: 'thumbnailUrl', exists: 'associatedMedia_thumbnailUrl_exists' }, + ]); + expect(physicalFields(mediaObject.fields[1]).leafExists).toEqual([]); + expect(physicalFields(mediaObject.fields[3]).leafExists).toEqual([]); + }); + + it('reserves the boolean’s name', () => { + expect( + validateSearchType({ + ...work, + fields: [ + ...work.fields, + { name: 'associatedMedia_thumbnailUrl_exists', kind: 'keyword' }, + ], + }), + ).toContainEqual({ + field: 'associatedMedia_thumbnailUrl_exists', + reason: 'reserved-field-name', + }); + }); + + it('files what one declaration can tell under a reason', () => { + const issues = (field: Record, type: object = {}) => + validateSearchType({ + name: 'Work', + class: `${SCHEMA_ORG}CreativeWork`, + ...type, + fields: [ + { + name: 'associatedMedia', + kind: 'reference', + filterable: true, + ref: { + strategy: 'inline', + typeName: 'MediaObject', + presenceLeaves: ['thumbnailUrl'], + }, + ...field, + }, + ], + } as SearchType); + + expect(issues({}, { class: undefined })).toEqual([ + { field: 'associatedMedia', reason: 'presence-leaves-on-reference-type' }, + ]); + expect( + issues({ + ref: { + strategy: 'lookup', + target: 'MediaObject', + presenceLeaves: ['thumbnailUrl'], + }, + }), + ).toEqual([ + { field: 'associatedMedia', reason: 'presence-leaves-not-inline' }, + ]); + expect(issues({ filterable: undefined })).toEqual([ + { + field: 'associatedMedia', + reason: 'presence-leaves-without-filterable', + }, + ]); + expect( + issues({ + ref: { + strategy: 'inline', + typeName: 'MediaObject', + presenceLeaves: ['thumbnailUrl', 'thumbnailUrl'], + }, + }), + ).toEqual([ + { field: 'associatedMedia', reason: 'presence-leaves-duplicate' }, + ]); + expect(issues({ facetable: true })).toEqual([ + { field: 'associatedMedia', reason: 'presence-leaves-with-facetable' }, + ]); + // A leaf is resolved against the reference type, which one declaration + // cannot see: `searchSchema` checks it. + expect( + issues({ + ref: { + strategy: 'inline', + typeName: 'MediaObject', + presenceLeaves: ['nope'], + }, + }), + ).toEqual([]); + expect(issues({})).toEqual([]); + }); + + it('refuses a facetable reference, whose facet would collide with its leaves’ until #1090', () => { + expect(() => + searchSchema( + workWith( + { presenceLeaves: ['thumbnailUrl'], identity: 'license' }, + { facetable: true }, + ), + mediaObject, + ), + ).toThrow( + /declares “presenceLeaves” and is “facetable”: its facet would have to become an object.*issues\/1090/, + ); + }); + + it('accepts a reference type whose only key a weld can name is a presence leaf', () => { + const counted = defineSearchType({ + name: 'MediaObject', + fields: [ + { + name: 'thumbnailUrl', + kind: 'keyword', + path: `${SCHEMA_ORG}thumbnailUrl`, + output: true, + }, + ], + }); + expect(() => + searchSchema(workWith({ presenceLeaves: ['thumbnailUrl'] }), counted), + ).not.toThrow(); + }); + + it('says why in full when the schema is built', () => { + expect(() => + searchSchema( + workWith( + { presenceLeaves: ['thumbnailUrl'] }, + { filterable: undefined }, + ), + mediaObject, + ), + ).toThrow(/declares “presenceLeaves” without “filterable”/); + expect(() => + searchSchema( + workWith({ presenceLeaves: ['thumbnailUrl', 'thumbnailUrl'] }), + mediaObject, + ), + ).toThrow(/names one leaf twice/); + expect(() => + searchSchema( + { + ...work, + fields: [ + { + name: 'creator', + kind: 'reference', + filterable: true, + ref: { strategy: 'idOnly', presenceLeaves: ['x'] }, + } as never, + ], + }, + mediaObject, + ), + ).toThrow(/but is no inline reference.*issues\/1090/); + }); + + it('refuses a leaf the reference type does not declare', () => { + expect(() => + searchSchema(workWith({ presenceLeaves: ['nope'] }), mediaObject), + ).toThrow( + /names presence leaf “nope”, which “MediaObject” does not declare/, + ); + }); + + it('refuses a text or boolean leaf, as on a field of the root’s own', () => { + expect(() => + searchSchema(workWith({ presenceLeaves: ['caption'] }), mediaObject), + ).toThrow(/presence leaf “caption”, a “text”/); + expect(() => + searchSchema(workWith({ presenceLeaves: ['isFree'] }), mediaObject), + ).toThrow(/presence leaf “isFree”, a “boolean”/); + }); + + it('refuses a leaf that is itself an inline reference', () => { + const thumbnail = defineSearchType({ + name: 'Thumbnail', + fields: [ + { + name: 'url', + kind: 'keyword', + path: `${SCHEMA_ORG}url`, + output: true, + }, + ], + }); + const media = defineSearchType({ + ...mediaObject, + fields: [ + ...mediaObject.fields, + { + name: 'thumbnail', + kind: 'reference', + path: `${SCHEMA_ORG}thumbnail`, + output: true, + ref: { strategy: 'inline', typeName: 'Thumbnail' }, + }, + ], + }); + expect(() => + searchSchema( + workWith({ presenceLeaves: ['thumbnail'] }), + media, + thumbnail, + ), + ).toThrow(/presence leaf “thumbnail”, which is itself an inline reference/); + }); + + it('refuses a leaf that is also faceted on, until its facet can hold both', () => { + expect(() => + searchSchema( + workWith({ presenceLeaves: ['license'], facetedLeaves: ['license'] }), + mediaObject, + ), + ).toThrow( + /presence leaf “license”, which is also one of its faceted leaves/, + ); + }); + + it('refuses one on a reference type, which has no documents to count', () => { + const edge = defineSearchType({ + name: 'Edge', + fields: [ + { + name: 'media', + kind: 'reference', + path: `${SCHEMA_ORG}associatedMedia`, + output: true, + filterable: true, + ref: { + strategy: 'inline', + typeName: 'MediaObject', + presenceLeaves: ['thumbnailUrl'], + }, + }, + ], + }); + expect(() => searchSchema(edge, mediaObject)).toThrow( + /“Edge” is a reference type: presence is counted on a root document/, + ); + }); + + it('refuses two references that nest one type with different leaves', () => { + const collection = { + ...workWith({}), + name: 'Collection', + class: `${SCHEMA_ORG}Collection`, + } as SearchType; + expect(() => searchSchema(work, collection, mediaObject)).toThrow( + /“Work.associatedMedia” and “Collection.associatedMedia” both nest “MediaObject” but declare different “presenceLeaves” \(thumbnailUrl; none\)/, + ); + expect(() => searchSchema(collection, work, mediaObject)).toThrow( + /declare different “presenceLeaves” \(none; thumbnailUrl\)/, + ); + }); + + it('compares only the references that get a filter input', () => { + // Not filterable: no filter input, so nothing to share. + const unfiltered = { + ...workWith({}, { filterable: undefined }), + name: 'Collection', + class: `${SCHEMA_ORG}Collection`, + } as SearchType; + expect(() => searchSchema(work, unfiltered, mediaObject)).not.toThrow(); + // On a Reference Type: its references are never filtered through. + const edge = defineSearchType({ + name: 'Edge', + fields: [ + { + name: 'media', + kind: 'reference', + path: `${SCHEMA_ORG}associatedMedia`, + output: true, + ref: { strategy: 'inline', typeName: 'MediaObject' }, + }, + ], + }); + const withEdge = defineSearchType({ + name: 'Agent', + class: `${SCHEMA_ORG}Person`, + fields: [ + { + name: 'edge', + kind: 'reference', + path: `${SCHEMA_ORG}subjectOf`, + output: true, + ref: { strategy: 'inline', typeName: 'Edge' }, + }, + ], + }); + expect(() => searchSchema(work, withEdge, edge, mediaObject)).not.toThrow(); + }); +}); + +describe('projecting presence leaves', () => { + const alias = (type: string, field: string) => `urn:lde:${type}/${field}`; + const project = (entries?: Record[]) => + projectDocument( + { + '@id': 'https://ex/w', + ...(entries !== undefined && { + [alias('Work', 'associatedMedia')]: entries.map((entry, index) => ({ + '@id': `https://ex/m/${index}`, + ...Object.fromEntries( + Object.entries(entry).map(([name, value]) => [ + alias('MediaObject', name), + value, + ]), + ), + })), + }), + }, + work, + schema, + ); + + it('records a value when any entry holds one, even if another lacks it', () => { + const document = project([ + { thumbnailUrl: 'https://ex/t.jpg', encodingFormat: 'image/jpeg' }, + { encodingFormat: 'image/tiff' }, + ]); + + expect(document.associatedMedia_thumbnailUrl_exists).toBe(true); + // The leaf declares no Role, so it is pruned from the entries once counted. + expect(document.associatedMedia).toEqual([ + { encodingFormat: 'image/jpeg' }, + { encodingFormat: 'image/tiff' }, + ]); + }); + + it('records no value when no entry holds one, or there are no entries', () => { + expect( + project([{ encodingFormat: 'image/png' }]) + .associatedMedia_thumbnailUrl_exists, + ).toBe(false); + expect(project().associatedMedia_thumbnailUrl_exists).toBe(false); + }); + + it('counts the one entry of a single-valued reference', () => { + const single = { + ...work, + fields: [{ ...work.fields[0], array: undefined }], + } as SearchType; + const document = projectDocument( + { + '@id': 'https://ex/w', + [alias('Work', 'associatedMedia')]: { + '@id': 'https://ex/m', + [alias('MediaObject', 'thumbnailUrl')]: 'https://ex/t.jpg', + }, + }, + single, + searchSchema(single, mediaObject), + ); + + expect(document.associatedMedia_thumbnailUrl_exists).toBe(true); + }); + + it('writes nothing without a schema, which projects no entries', () => { + expect(projectDocument({ '@id': 'https://ex/w' }, work)).not.toHaveProperty( + 'associatedMedia_thumbnailUrl_exists', + ); + }); +}); + +describe('the presence leaf’s facet', () => { + it('is named by the leaf’s path and reads the root’s boolean', () => { + expect(facetNamed(work, facet, schema)).toEqual({ + name: facet, + field: mediaObject.fields[0], + reference: work.fields[0], + engineField: 'associatedMedia_thumbnailUrl_exists', + presence: true, + }); + expect(facetsOf(work, schema).map((declared) => declared.name)).toEqual([ + 'associatedMedia.license', + facet, + ]); + }); + + it('selects a bucket through `exists` welded to one entry alone', () => { + const clause = selectionOf( + facetNamed(work, facet, schema) as FacetDeclaration, + 'false', + ); + + expect(clause).toEqual(welded({ field: 'thumbnailUrl', exists: false })); + expect( + facetSearches({ ...browse, where: [clause], facets: [facet] })[0]?.own, + ).toBe(facet); + }); +}); + +describe('`exists` welded to an entry', () => { + const where = (filter: Filter) => + validateQuery({ ...browse, where: [filter] }, work, schema); + + it('is accepted alone, on a presence leaf', () => { + expect(where(welded({ field: 'thumbnailUrl', exists: true }))).toEqual([]); + }); + + it('is refused on a leaf the reference names no presence for', () => { + expect(where(welded({ field: 'encodingFormat', exists: true }))).toEqual([ + { + part: 'where', + field: 'associatedMedia.encodingFormat', + reason: 'no-presence', + }, + ]); + }); + + it('is refused beside another condition on the same entry', () => { + expect( + where( + welded( + { field: 'encodingFormat', in: ['image/jpeg'] }, + { field: 'thumbnailUrl', exists: true }, + ), + ), + ).toEqual([ + { + part: 'where', + field: 'associatedMedia.thumbnailUrl', + reason: 'exists-in-weld', + }, + ]); + }); + + it('is refused through a join, which reads no companion yet', () => { + const item = defineSearchType({ + name: 'Item', + class: `${SCHEMA_ORG}Thing`, + fields: [ + { + name: 'work', + kind: 'reference', + filterable: true, + labelSource: 'Work', + joinable: true, + }, + ], + }); + const labelled = { + ...work, + fields: [ + ...work.fields, + { + name: 'label', + kind: 'text', + locales: ['nl'], + output: true, + searchable: { weight: 1 }, + }, + ], + } as SearchType; + const joinedSchema = searchSchema(labelled, item, mediaObject); + + expect( + validateQuery( + { + ...browse, + where: [ + { + or: [ + { + on: ['work'], + field: 'associatedMedia', + entry: [{ field: 'thumbnailUrl', exists: true }], + }, + ], + }, + ], + }, + item, + joinedSchema, + joinGraph(joinedSchema), + ), + ).toEqual([ + { + part: 'where', + field: 'work.associatedMedia', + reason: 'joined-presence', + }, + ]); + }); +}); + +describe('the clause a presence leaf’s facet owns', () => { + it('owns `exists` welded alone, never a condition on the leaf’s value', () => { + const presence = welded({ field: 'thumbnailUrl', exists: true }); + const value = welded({ field: 'license', in: ['https://ex/l'] }); + const searches = facetSearches({ + ...browse, + where: [presence, value], + facets: [facet, 'associatedMedia.license'], + }); + + expect(searches.map((search) => [search.own, search.query.where])).toEqual([ + [facet, [value]], + ['associatedMedia.license', [presence]], + ]); + }); +}); + +describe('the facet names a typed declaration admits', () => { + it('includes each presence leaf’s facet by its path', () => { + const typed = defineSearchType({ + name: 'Work', + class: `${SCHEMA_ORG}CreativeWork`, + fields: [ + { + name: 'associatedMedia', + kind: 'reference', + path: `${SCHEMA_ORG}associatedMedia`, + array: true, + filterable: true, + ref: { + strategy: 'inline', + typeName: 'MediaObject', + presenceLeaves: ['thumbnailUrl'], + }, + }, + ], + }); + const admitted: FacetFieldsOf = + 'associatedMedia.thumbnailUrl.exists'; + // @ts-expect-error – not a presence leaf + const refused: FacetFieldsOf = + 'associatedMedia.encodingFormat.exists'; + + expect([typed.name, admitted, refused]).toHaveLength(3); + }); +}); diff --git a/packages/search/test/presence.test.ts b/packages/search/test/presence.test.ts index cedf800b..8b0ddc4f 100644 --- a/packages/search/test/presence.test.ts +++ b/packages/search/test/presence.test.ts @@ -144,7 +144,7 @@ describe('declaring presence', () => { ], }); expect(() => searchSchema(media)).toThrow( - /“MediaObject” is a reference type: presence is counted on a Root Type’s own field/, + /“MediaObject” is a reference type: presence is counted on a root document.*“presenceLeaves”/, ); }); diff --git a/packages/search/test/schema.test.ts b/packages/search/test/schema.test.ts index e008e34c..f31066e6 100644 --- a/packages/search/test/schema.test.ts +++ b/packages/search/test/schema.test.ts @@ -111,6 +111,7 @@ describe('physicalFields', () => { search: ['title_search_nl', 'title_search_en'], sort: ['title_sort_nl', 'title_sort_en'], filter: 'title', + leafExists: [], }); }); @@ -129,6 +130,7 @@ describe('physicalFields', () => { sort: [], facet: 'keyword', filter: 'keyword', + leafExists: [], }); }); @@ -144,6 +146,7 @@ describe('physicalFields', () => { search: ['creator_search_nl', 'creator_search_en'], sort: [], filter: 'creator', + leafExists: [], }); }); @@ -161,6 +164,7 @@ describe('physicalFields', () => { search: [], sort: [], filter: 'title', + leafExists: [], }); }); @@ -179,6 +183,7 @@ describe('physicalFields', () => { sort: [], facet: 'publisher', filter: 'publisher', + leafExists: [], }); });