From ffcc5b2a64b9f1055c6c76952981102ee4d6eb71 Mon Sep 17 00:00:00 2001 From: David de Boer Date: Tue, 17 Nov 2026 11:13:29 +0100 Subject: [PATCH] =?UTF-8?q?feat(search)!:=20filter=20and=20facet=20a=20roo?= =?UTF-8?q?t=20document=20on=20whether=20a=20referent=E2=80=99s=20leaf=20h?= =?UTF-8?q?as=20a=20value?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A lookup naming one target may declare presenceLeaves: the extraction reads each leaf one hop from the root, the target type’s transforms run on the referents in the batch – a referent named by IRI alone seeded with its type, through the type’s own transforms for a lookup into its own type – and the projection writes one boolean per root document, counting referents with an IRI only. A plain criterion on the root, { field, leaf, exists }, selects it, and each key of the lookup’s GraphQL where is one such criterion, ANDed like sibling keys. Everything counted on one field is served in one facet object, each part charged as a facet calculation, which lifts the refusals of presence on a facetable field, leaves on a facetable reference, a leaf counted by value and by presence, and one set of presence leaves per reference type. BREAKING CHANGE: GraphQL facet objects are renamed and reshaped. The ‹Type›‹Reference›Facets and ‹Type›‹Field›Facets objects become ‹Type›‹Field›Facet, and a presence leaf’s ‹…›‹Leaf›Facets becomes ‹Type›‹Field›‹Leaf›Facet. A field whose facet also counts its presence or a leaf is served as ‹Type›‹Field›Facet { buckets totalValues exists ‹leaf› } instead of IRIFacet or ValueFacet; select its buckets as facetField(limit) { buckets { … } }. A reference counting faceted or presence leaves takes ‹Type›‹Field›Filter, with ‹Type›‹Field›Where, instead of ‹Edge›Filter or ‹Target›Filter, and ‹Edge›JoinedFilter becomes ‹Type›‹Field›JoinedFilter; variables typed with the old names stop validating, and adding leaves to a reference later changes its filter type the same way. In @lde/search, PhysicalFields.leafExists covers a lookup’s presence leaves too, PresenceCriterion takes an optional leaf, and the SearchTypeIssue reasons presence-with-facetable, presence-leaves-not-inline, presence-leaves-with-facetable and presence-leaf-faceted are gone, replaced by presence-leaves-on-id-only, presence-leaves-with-several-targets, presence-leaves-with-joinable, presence-leaves-with-literal, presence-leaf-reserved-name and presence-leaf-without-path. A target type’s transform also receives referents seeded as rdf:type . --- AGENTS.md | 2 +- ...facet-a-field-on-whether-it-has-a-value.md | 4 +- ...t-on-whether-an-entrys-leaf-has-a-value.md | 4 +- ...-leaf-and-a-fields-facets-in-one-object.md | 124 ++++ docs/reference/search-api-graphql.md | 88 ++- docs/reference/search-pipeline.md | 32 + docs/reference/search-typesense.md | 12 +- docs/reference/search.md | 84 ++- package-lock.json | 3 +- .../search-api-graphql/src/build-schema.ts | 639 ++++++++++++------ .../test/faceted-leaves.test.ts | 4 +- .../test/leaf-presence.test.ts | 20 +- .../test/lookup-presence.test.ts | 390 +++++++++++ .../search-api-graphql/test/presence.test.ts | 4 +- .../test/query-cost.test.ts | 63 +- packages/search-api-graphql/vite.config.ts | 2 +- packages/search-pipeline/package.json | 2 +- packages/search-pipeline/src/extraction.ts | 14 +- packages/search-pipeline/src/search-stages.ts | 128 +++- .../search-pipeline/test/extraction.test.ts | 26 +- .../test/fixtures/lookup-presence-sample.ttl | 43 ++ .../test/lookup-presence.integration.test.ts | 288 ++++++++ .../search-typesense/src/query-compiler.ts | 13 +- .../test/lookup-presence.integration.test.ts | 248 +++++++ .../test/query-compiler.test.ts | 75 ++ packages/search-typesense/vite.config.ts | 8 +- packages/search/src/project.ts | 82 ++- packages/search/src/query.ts | 63 +- packages/search/src/schema.ts | 315 ++++----- packages/search/test/faceted-leaves.test.ts | 9 +- packages/search/test/leaf-presence.test.ts | 160 +++-- packages/search/test/lookup-presence.test.ts | 255 +++++++ packages/search/test/presence.test.ts | 16 +- 33 files changed, 2639 insertions(+), 581 deletions(-) create mode 100644 docs/decisions/0042-count-a-referents-leaf-and-a-fields-facets-in-one-object.md create mode 100644 packages/search-api-graphql/test/lookup-presence.test.ts create mode 100644 packages/search-pipeline/test/fixtures/lookup-presence-sample.ttl create mode 100644 packages/search-pipeline/test/lookup-presence.integration.test.ts create mode 100644 packages/search-typesense/test/lookup-presence.integration.test.ts create mode 100644 packages/search/test/lookup-presence.test.ts diff --git a/AGENTS.md b/AGENTS.md index 40433b54..c791b36a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -86,7 +86,7 @@ Each package uses conditional exports with a `development` condition for local d - Test files use `.test.ts` suffix in `test/` directory - Fixtures in `test/fixtures/` - HTTP mocking with Nock -- Tests that start a local SPARQL endpoint (`@lde/local-sparql-endpoint`) must use unique ports across packages to avoid conflicts when Nx runs tests in parallel. Current port allocations: `dataset-registry-client` (3002), `pipeline` sparqlQuery (3001), `pipeline` executor (3003), `pipeline` provenance store (3004), `pipeline-void` namespace-normalization (3005), `search-pipeline` extraction round-trip (3007), `search-pipeline` searchIndexerPipeline end-to-end (3008), `search-pipeline` registry-sourced extraction (3009), `search-pipeline` joins end-to-end (3010), `search-pipeline` keyed roots (3011), `search-pipeline` nested fan-out (3012), `pipeline-void` per-property partitions (3013), `pipeline-void` subject URI spaces (3014), `pipeline-void` distinct counts (3016), `change-detection` content hash reader (3017), `sparql-qlever` remote mode (3015 for QLever, 7070 for the task agent) +- Tests that start a local SPARQL endpoint (`@lde/local-sparql-endpoint`) must use unique ports across packages to avoid conflicts when Nx runs tests in parallel. Current port allocations: `dataset-registry-client` (3002), `pipeline` sparqlQuery (3001), `pipeline` executor (3003), `pipeline` provenance store (3004), `pipeline-void` namespace-normalization (3005), `search-pipeline` extraction round-trip (3007), `search-pipeline` searchIndexerPipeline end-to-end (3008), `search-pipeline` registry-sourced extraction (3009), `search-pipeline` joins end-to-end (3010), `search-pipeline` keyed roots (3011), `search-pipeline` nested fan-out (3012), `pipeline-void` per-property partitions (3013), `pipeline-void` subject URI spaces (3014), `pipeline-void` distinct counts (3016), `change-detection` content hash reader (3017), `search-pipeline` referents’ label transforms (3019), `search-pipeline` lookup presence (3020), `sparql-qlever` remote mode (3015 for QLever, 7070 for the task agent) ### Key Dependencies diff --git a/docs/decisions/0040-filter-and-facet-a-field-on-whether-it-has-a-value.md b/docs/decisions/0040-filter-and-facet-a-field-on-whether-it-has-a-value.md index f254cba2..898ab066 100644 --- a/docs/decisions/0040-filter-and-facet-a-field-on-whether-it-has-a-value.md +++ b/docs/decisions/0040-filter-and-facet-a-field-on-whether-it-has-a-value.md @@ -9,7 +9,9 @@ Accepted Supersedes the last consequence of [ADR 36](0036-facet-a-root-document-on-its-entries-leaves.md) for a Root Type’s own field. Extended to an inline entry’s leaf by -[ADR 41](0041-filter-and-facet-a-root-document-on-whether-an-entrys-leaf-has-a-value.md). +[ADR 41](0041-filter-and-facet-a-root-document-on-whether-an-entrys-leaf-has-a-value.md); +its refusal of a facetable field lifted by +[ADR 42](0042-count-a-referents-leaf-and-a-fields-facets-in-one-object.md). ## Context diff --git a/docs/decisions/0041-filter-and-facet-a-root-document-on-whether-an-entrys-leaf-has-a-value.md b/docs/decisions/0041-filter-and-facet-a-root-document-on-whether-an-entrys-leaf-has-a-value.md index 401e5da4..806eee25 100644 --- a/docs/decisions/0041-filter-and-facet-a-root-document-on-whether-an-entrys-leaf-has-a-value.md +++ b/docs/decisions/0041-filter-and-facet-a-root-document-on-whether-an-entrys-leaf-has-a-value.md @@ -7,7 +7,9 @@ Date: 2026-11-16 Accepted Extends [ADR 40](0040-filter-and-facet-a-field-on-whether-it-has-a-value.md) -to a leaf of an inline reference. +to a leaf of an inline reference. Extended to a lookup’s referents, and its +refusals for the facet shape lifted, by +[ADR 42](0042-count-a-referents-leaf-and-a-fields-facets-in-one-object.md). ## Context diff --git a/docs/decisions/0042-count-a-referents-leaf-and-a-fields-facets-in-one-object.md b/docs/decisions/0042-count-a-referents-leaf-and-a-fields-facets-in-one-object.md new file mode 100644 index 00000000..914cd619 --- /dev/null +++ b/docs/decisions/0042-count-a-referents-leaf-and-a-fields-facets-in-one-object.md @@ -0,0 +1,124 @@ +# 42. Count a referent’s leaf, and a field’s facets in one object + +Date: 2026-11-16 + +## Status + +Accepted + +Extends [ADR 39](0039-apply-a-label-sources-transforms-to-its-referents.md) +and [ADR 41](0041-filter-and-facet-a-root-document-on-whether-an-entrys-leaf-has-a-value.md); +lifts the refusals ADR 36, ADR 40 and ADR 41 made for the facet shape. + +## Context + +Linked Open Limburg’s map view filters works on whether a place they name has +coordinates. `contentLocation` is a lookup into `Place`, whose coordinates +come from GeoNames enrichment, not from the work’s graph. The root document +stores only the places’ ids, so LOL guesses from each place’s key: a GeoNames +IRI counts as located. When a GeoNames lookup fails, the place is indexed +without coordinates and the work still says it has them. + +ADR 41 counts a leaf of an inline reference’s entries, which the root +document stores. A lookup’s referents are other documents, possibly from +other datasets, so reading the leaf off the target’s collection would need a +pass after every dataset, or an order across them. + +Separately, the facets object had one key per facetable field, typed as a +bucket list. ADR 36, ADR 40 and ADR 41 therefore refused anything that would +give one key two meanings: presence on a facetable field, leaves on a +facetable reference, a leaf faceted both by value and by presence. + +## Decision + +- **A lookup declares `presenceLeaves`** naming leaves of its target, as an + inline reference does since ADR 41: + `ref: { strategy: 'lookup', target: 'Place', presenceLeaves: ['latitude'] }`. + One boolean per root document, `contentLocation_latitude_exists`, in + `PhysicalFields.leafExists`. Only on a lookup naming one target: a leaf is + read through one declaration. +- **Counted at index time, in the referring batch.** The extraction reads each + presence leaf one hop from the root, beside the target’s key and label. The + stage passes the referents through the target type’s transforms, as ADR 39 + does for labels, and the projection reads the leaf off each referent’s node + as the target declares it. No pass after all datasets: what the root’s graph + states, or a transform can say from the referent’s key, is counted; + anything else is not, and the work counts as holding none – so a failed + lookup gives `false`, not the guess’s `true`. Only a referent named by IRI + counts: a blank node is no id the lookup stores. +- **A referent named by IRI alone is seeded.** ADR 39 hands a transform only + the quads the batch has, so such a referent reached no transform. A referent + a presence leaf is counted for enters its target’s transforms as + ` rdf:type `, and the seed is dropped from what comes + back. A transform describing a node from its key, as LOL’s place transform + does, then supplies the leaf. A lookup into its own type seeds through that + type’s transforms too, though they saw the batch: they never saw a referent + the batch states nothing of. +- **The target declares the leaf, with a `path`.** A leaf is read off the + referent’s node through the target’s declaration, not derived. +- **A plain criterion on the root selects it**, naming the reference and the + leaf: `{ field: 'contentLocation', leaf: 'latitude', exists: true }`. The + value is the root document’s, so it combines with other criteria like any, + and each key of a lookup’s GraphQL `where` is one such criterion, ANDed like + sibling keys – two presence leaves in one `where` are two conditions. A + lookup takes no weld: it has no entries. +- **One facet object per field.** A field counting more than its own values + is served as `‹Type›‹Field›Facet`: its own values in `buckets` and + `totalValues` (a numeric field’s range facet parts instead), capped by the + field’s `limit`; `exists` for its presence; one key per leaf, a plain facet + type for a leaf counted by value only and `‹Type›‹Field›‹Leaf›Facet` for + one whose presence is counted. A field counting only its own values keeps + `IRIFacet`, `ValueFacet` or its range facet type. A leaf is not named + `buckets`, `totalValues` or `exists`. +- **One filter input per reference counting leaves.** `‹Type›‹Field›Filter` + holds the reference’s `in` and a `where`, `‹Type›‹Field›Where`, naming its + filterable entry leaves and its presence leaves. Two references nesting one + Reference Type may then declare different leaves, which ADR 41 refused. + Adding leaves to a reference therefore changes its filter type, from the + shared `‹Edge›Filter` or `‹Target›Filter` to its own – a breaking change + for a client whose variables name the old type. + +## Consequences + +- LOL declares `presenceLeaves: ['latitude']` on `contentLocation` and + `locationCreated`, and drops its key-based guess. +- Lifted: presence on a facetable field (ADR 40), leaves on a facetable + reference (ADR 36, ADR 41), a leaf faceted by value and by presence (ADR + 41), and one set of presence leaves per Reference Type (ADR 41). +- Breaking at the GraphQL surface: ADR 36’s `‹Type›‹Reference›Facets` and ADR + 40 and 41’s `…Facets` objects are renamed `…Facet`; a field that gains a + leaf or `exists` changes from `IRIFacet`/`ValueFacet` to its own object; a + reference counting leaves takes `‹Type›‹Field›Filter` instead of + `‹Edge›Filter` or `‹Target›Filter`, so the refined discovery strategy no + longer reaches it – and a reference that gains its first leaf later breaks + the variables typed with its old filter. +- Each part of a field’s facet object is charged as a facet key, so selecting + `buckets` with `totalValues` costs two where a plain facet costs one – an + overestimate, never a part that costs nothing. +- A target’s transform now also receives nodes carrying a type alone, once per + referring batch, and pays for any I/O it does there unless it caches. +- A value is a copy, changed only when the referring document is reindexed. +- Refused for now: presence leaves on a lookup naming several targets, on a + `joinable` lookup, whose `where` already joins, and on a lookup reading its + literals as periods. A lookup’s leaves faceted by value have no consumer + and are not offered. + +## Alternatives considered + +- **A pass after all datasets, from the collection being built**, as the + issue first proposed. Complete for referents described only in another + dataset’s graph, but a pass over every referring document per run, skipped + datasets included, outside the batch. +- **A Typesense join facet** (`facet_by=$places(latitude)`). On 30.2 it counted + the joined documents, not the roots, and returned nothing on a text query + without a filter. +- **A weld of one `exists` condition**, as for an inline entry: + `{ field: 'contentLocation', entry: [{ field: 'latitude', exists }] }`. + It pretends a lookup has entries, needs a special case wherever a weld is + read, and cannot hold two leaves, which a weld puts on one entry. +- **Counting through any of several targets.** A referent of either kind + would be read through its own target’s declaration, at the cost of a leaf + every target must declare alike; no consumer names several targets yet. +- **The reference’s own buckets under `id`**, beside its leaves. Every + facetable reference would change shape, where a field counting only its own + values needs none. diff --git a/docs/reference/search-api-graphql.md b/docs/reference/search-api-graphql.md index 9409e9ee..05386c98 100644 --- a/docs/reference/search-api-graphql.md +++ b/docs/reference/search-api-graphql.md @@ -124,8 +124,14 @@ fails this – accepts. The endpoint path defaults to `/graphql` | A joined reference (`lookup`), at any depth | 1 per target type | | A facet key | 1 | | A range facet’s `all` extent | 1 more | + | Each part of a `‹Type›‹Field›Facet` object | 1 | | A plain stored field or an inline reference | 0, at any depth | + In a field’s facet object every part selected is charged on its own – + `buckets`, `totalValues`, `exists`, a range facet’s `all`, `matching` and + `buckets`, and each part of a leaf’s – so `buckets` with `totalValues` + costs 2 where the same pair on a plain `IRIFacet` costs 1. + A lookup fetches the referents of a whole page in one request, and the facet keys of a search share one request, so `perPage` does not enter the cost. Fragments cost what their contents cost, a selection under `@skip` or @@ -280,22 +286,41 @@ role: { in: ["etser"] } } }` asks for _this person in this role_, where the without the keys of fields declaring presence alone. A type declaring none keeps one `‹Type›Where` for both. - An inline reference’s - [presence leaves](./search#presence-on-an-entry-s-leaf) take `exists` in its - reference type’s `‹Edge›Where`, typed as a field of the root’s own is - (`ExistsFilter`, or `‹Filter›OrExists` for a `filterable` leaf): - `where: { associatedMedia: { where: { thumbnailUrl: { exists: true } } } }`. - Whether any entry holds a value is recorded per document, so `exists` is - welded to nothing else: setting it beside another key of the same - `‹Edge›Where` is an error naming the rewrite into two conditions. Every - reference nesting one reference type declares the same presence leaves, so - `‹Edge›Where` stays shared; through a join, where no companion is read, a - reference with presence leaves takes `‹Edge›JoinedFilter`, without `exists`. + A reference counting leaves of its referents – an inline reference’s + [faceted](./search#faceting-on-an-entry-s-leaf) or + [presence leaves](./search#presence-on-an-entry-s-leaf), a lookup’s + [presence leaves](./search#on-a-lookup-s-referents) – takes a filter of its + own, **`‹Type›‹Field›Filter`**: `@oneOf` over `in` (the ids it holds, for a + lookup or an inline reference naming an identity) and `where`, typed + **`‹Type›‹Field›Where`**. For an inline reference that `where` names the + entries’ filterable leaves, as the shared `‹Edge›Where` does; for both + kinds it names each presence leaf, taking `exists` – `ExistsFilter`, or + `‹Filter›OrExists` for an inline entry’s filterable leaf: + `where: { contentLocation: { where: { latitude: { exists: true } } } }`. + A lookup’s `where` names its presence leaves alone, and each key is a plain + criterion on the root document, ANDed like sibling keys: + `{ latitude: { exists: true }, longitude: { exists: true } }` is two + conditions, each on whether any referent holds that leaf; inside an `or` + alternative it sets one key, as a joined condition does. In an inline + reference’s `where`, whether any entry holds a value is recorded per + document, so `exists` is welded to nothing else: setting it beside another + key of the same `where` is an error naming the rewrite into two conditions. + Through a join, where no companion is read, an inline reference with + presence leaves takes `‹Type›‹Field›JoinedFilter`, without `exists`, and a + lookup its plain `‹Target›Filter`. + + **Adding leaves changes the reference’s filter type.** A reference that + gains faceted or presence leaves moves from the shared `‹Edge›Filter` or + `‹Target›Filter` to its own `‹Type›‹Field›Filter`, so a client’s variables + typed with the old name stop validating – a breaking change to the API, + even where the reference’s `in` arm is unchanged. Like `‹Filter›OrExists`, + the per-reference filter is a type of its own, so the refined discovery + strategy does not reach it. - **`orderBy`**: `RELEVANCE` plus every `sortable` field, as an enum – field names SCREAMING_SNAKE_CASEd (`datePosted` → `DATE_POSTED`); `direction` defaults to `DESC`. -- **Facets**: a keyed object with one field per `facetable` field, typed by +- **Facets**: a keyed object with one key per field anything is counted on, typed by the field’s declaration: - a **reference** facet returns an `IRIFacet!` – `buckets: [IRIBucket!]!`, each a `value` (an `IRI`) + `count` + the resolved data `label`, and @@ -348,18 +373,33 @@ role: { in: ["etser"] } } }` asks for _this person in this role_, where the the engine counted documents for it, so a uniform result set yields one bucket, not two. - An inline reference’s [faceted leaves](./search#faceting-on-an-entry-s-leaf) - are served one level down, under the reference: `facets { associatedMedia { -license { buckets { value count } } } }`, typed `‹Type›‹Reference›Facets`. A presence - facet is served the same way, under the field it counts: `facets { -dateCreated { exists { value count } } }`, typed `‹Type›‹Field›Facets` - (`WorkDateCreatedFacets`) with `exists: [BooleanBucket!]!`. The object - leaves room for a field’s value buckets beside it, once a field may be both - `facetable` and declare presence. A presence leaf’s facet sits under the - reference, one level further down: `facets { associatedMedia { thumbnailUrl { -exists { value count } } } }`, typed `‹Type›‹Reference›‹Leaf›Facets` - (`WorkAssociatedMediaThumbnailUrlFacets`), beside the reference’s faceted - leaves. + **One object per field.** A field counting more than its own values – + its presence, or a leaf of its referents – is served as an object of its + own, **`‹Type›‹Field›Facet`**, holding everything counted on it: + + ```graphql + facets { + material(limit: 20) { buckets { value count } totalValues } + dateCreated { exists { value count } } + contentLocation(limit: 20) { + buckets { value count } + totalValues + latitude { exists { value count } } + } + associatedMedia { license(limit: 5) { buckets { value count } } } + } + ``` + + Its own values are `buckets` and `totalValues` (a numeric field’s `all`, + `matching` and `buckets` instead), capped by the field’s `limit`, which caps + nothing else; `exists` counts its presence. Each leaf is a key of its own: a + leaf counted only by value keeps its plain facet type (`license(limit): +IRIFacet!`), and a leaf whose presence is counted is an object again, + **`‹Type›‹Field›‹Leaf›Facet`** (`WorkContentLocationLatitudeFacet`), with + `exists` and – when it is faceted by value too – its `buckets`, + `totalValues` and `limit`. A field counting only its own values keeps + `IRIFacet`, `ValueFacet`, its range facet type or `[BooleanBucket!]!`. A + name a declaration took already is refused with a clear error. Selecting facet fields IS the request, and the whole selection is **batched per request**: one diff --git a/docs/reference/search-pipeline.md b/docs/reference/search-pipeline.md index 03441d24..fae24da5 100644 --- a/docs/reference/search-pipeline.md +++ b/docs/reference/search-pipeline.md @@ -398,6 +398,38 @@ readers: the stage’s projected documents into the write. A projected document is far heavier than a quad, so lower it where documents are large. +## Referents through their type’s transforms + +A stage passes the referents of its references through each target type’s own +`transforms`, inside the batch, so memory stays bounded by `batchSize`: the +quads the extraction read off each referent go in, under the target’s IR +Aliases, and what the transforms return replaces them. Two kinds of reference +are described this way: + +- a `searchable` reference, whose referents’ key and label are read, so it is + searched by the label the target’s transform leaves + ([ADR 39](../decisions/0039-apply-a-label-sources-transforms-to-its-referents.md)); +- a `lookup` declaring [presence leaves](./search#on-a-lookup-s-referents), + whose referents’ key and leaves are read, so the projection can write whether + any referent holds a leaf. + +A referent a presence leaf is counted for, named by IRI, and that the batch +states nothing about – the root’s graph names it by IRI alone – enters its +target’s transforms all the same, as ` rdf:type `, so a +transform that describes a node from its key (an Authority’s coordinates for a +GeoNames place) supplies the leaf. The seed is not kept. A transform on such a +target therefore also receives nodes carrying a type alone, or only a key and +a few leaves, once per referring batch; a transform doing I/O pays for that +unless it caches. A blank-node referent is not seeded: the lookup stores no id +for it, so it is not counted. + +A reference to the stage’s own type passes no referent’s quads through again, +since its transforms already saw the batch – but a lookup into its own type +declaring presence leaves still has the referents the batch states nothing +about seeded through them. A target wired with its own `readers` has no +transform to apply +([ADR 42](../decisions/0042-count-a-referents-leaf-and-a-fields-facets-in-one-object.md)). + ## Per-component isolation Each root type is an independent engine run – its own collection, alias and diff --git a/docs/reference/search-typesense.md b/docs/reference/search-typesense.md index 2d7ffc9e..6ba53139 100644 --- a/docs/reference/search-typesense.md +++ b/docs/reference/search-typesense.md @@ -139,11 +139,13 @@ value, so `exists` compiles to `${name}_exists:=true` or `:=false`, and the facet `‹field›.exists` counts the companion. The projection writes it on every document, `false` included, so the companion is not optional. A field declaring presence alone is internal, so the companion is all the collection -declares for it. An inline reference’s -[presence leaves](./search#presence-on-an-entry-s-leaf) each get a top-level -`${reference}_${leaf}_exists` of the same kind, true when any entry holds a -value: an `exists` welded alone to an entry compiles to it, never to a -condition inside the nested object. +declares for it. A reference’s presence leaves – an +[inline entry’s](./search#presence-on-an-entry-s-leaf) or a +[lookup’s referents’](./search#on-a-lookup-s-referents) – each get a top-level +`${reference}_${leaf}_exists` of the same kind, true when any entry or +referent holds a value: an `exists` welded alone to an inline reference, and a +lookup’s `{ field, leaf, exists }`, compile to it, never to a condition inside +the nested object. An `InPlaceRebuild` additionally refuses a type whose declared [dataset field](#provenance) inherits a policy: it enumerates the indexed diff --git a/docs/reference/search.md b/docs/reference/search.md index b11d8bdf..992a2ba0 100644 --- a/docs/reference/search.md +++ b/docs/reference/search.md @@ -601,7 +601,9 @@ label with an Authority’s, the referring document is searched by the Authority’s. A transform on a label source therefore also receives nodes that carry only those two fields, once per referring batch. A referent stated by IRI alone, with no triple of its own, gives a transform nothing to act on and adds -nothing to search. A copied label changes when the document is reindexed. +nothing to search – unless the reference also counts a +[presence leaf](#on-a-lookup-s-referents), which seeds it. A copied label +changes when the document is reindexed. A reference naming no label source still folds its IRIs, and a `derive`d or `from` reference reads no referent, so it is never searched by labels @@ -1205,8 +1207,10 @@ nobody’s own, as a clause spanning two fields is not `searchSchema` holds each leaf to what that round trip needs: it is declared on the reference type, `filterable`, and a `keyword`, `boolean` or `reference`; -the reference is `filterable`, not `facetable` itself (both facets would be -named after it), and sits on a Root Type. A `lookup` leaf whose target declares +the reference is `filterable` and sits on a Root Type, and a leaf is not named +`buckets`, `totalValues` or `exists`, which name parts of the reference’s +facet object. A reference that is `facetable` itself counts its own values +beside its leaves, in one facet object. A `lookup` leaf whose target declares a [facet policy](#facet-policy) is refused for now. A condition welded to one entry is owned by a faceted leaf, never by an @@ -1259,11 +1263,12 @@ declaring no presence (`no-presence`), which is also what a weld meets. `searchSchema` says why in full. It refuses it on a field of a Reference Type and on an inline reference, whose entries are counted through the reference’s -[presence leaves](#presence-on-an-entry-s-leaf) instead; on a `text` field, which stores its values per language; on a `boolean`, which -states its own absence as `false`; and on a field that is already `facetable`, -since its facet would have to become an object to hold both kinds of bucket -([#1090](https://github.com/ldelements/lde/issues/1090)) +[presence leaves](#presence-on-an-entry-s-leaf) instead; on a `text` field, +which stores its values per language; and on a `boolean`, which states its own +absence as `false` ([ADR 40](../decisions/0040-filter-and-facet-a-field-on-whether-it-has-a-value.md)). +A field that is also `facetable` counts both, its values and its presence +([ADR 42](../decisions/0042-count-a-referents-leaf-and-a-fields-facets-in-one-object.md)). ### Presence on an entry’s leaf @@ -1307,15 +1312,62 @@ holds a value is recorded per document, so it cannot hold of the same entry as another condition. Through a join it is refused like any `exists` (`joined-presence`). -The reference is `filterable`, sits on a Root Type and – for now – is not -`facetable` itself; each leaf is declared by the reference type, named once, -and neither `text`, `boolean`, an inline reference nor – for now – one of the -reference’s faceted leaves. Every `filterable` inline reference on a Root -Type nesting the same reference type declares the same presence leaves, -because they share the type’s filter input. The two “for now” refusals and the -shared leaves are lifted by per-field facet objects and per-reference filter -inputs ([#1090](https://github.com/ldelements/lde/issues/1090)) -([ADR 41](../decisions/0041-filter-and-facet-a-root-document-on-whether-an-entrys-leaf-has-a-value.md)). +The reference is `filterable` and sits on a Root Type; each leaf is declared +by the reference type, named once, neither `text`, `boolean` nor an inline +reference, and not named `buckets`, `totalValues` or `exists`. A leaf may be +faceted on by value and by presence at once, and the reference may be +`facetable` itself: everything counted on one field is served in one facet +object +([ADR 41](../decisions/0041-filter-and-facet-a-root-document-on-whether-an-entrys-leaf-has-a-value.md), +[ADR 42](../decisions/0042-count-a-referents-leaf-and-a-fields-facets-in-one-object.md)). + +#### On a lookup’s referents + +A `lookup` stores only its referents’ ids, so it declares `presenceLeaves` to +count a leaf of the documents it names – “a work with a place that has +coordinates”: + +```ts +{ + name: 'contentLocation', + kind: 'reference', + path: 'https://schema.org/contentLocation', + array: true, + filterable: true, + ref: { strategy: 'lookup', target: 'Place', presenceLeaves: ['latitude'] }, +} +``` + +It is counted at index time, in the referring batch: the extraction reads +each presence leaf one hop from the root, beside the target’s key and label, +the target type’s transforms run on the referents +([`@lde/search-pipeline`](./search-pipeline#referents-through-their-type-s-transforms)), +and the projection writes `contentLocation_latitude_exists` from what they +return. A referent the root’s graph states the leaf for counts; so does one a +transform describes from its IRI alone – the type’s own transforms, for a +lookup into its own type; one that neither states nor any transform describes +– a failed lookup, say – counts as holding none. Only a referent with an IRI +counts: a blank node is no id the lookup stores, so what the graph states of +it does not count either. + +The facet is `contentLocation.latitude.exists`. Since the value is recorded on +the root document, its criterion is a plain one on the root, naming the +reference and the leaf: +`{ field: 'contentLocation', leaf: 'latitude', exists: true }` +(GraphQL: `where: { contentLocation: { where: { latitude: { exists: true } } } }`). +It combines with other criteria like any: the keys of a lookup’s `where` are +ANDed as sibling keys are, so `{ latitude: { exists: true }, longitude: { +exists: true } }` asks for a work with a place that has a latitude and one +that has a longitude, each counted on its own. A weld (`entry`) on a lookup is +refused (`not-weldable`): a lookup has no entries. + +The lookup names one `target`, which declares each leaf, with a `path`: a leaf +is read off the referent’s own node, never derived, through that target’s +declaration. Refused on a lookup naming several targets +(`presence-leaves-with-several-targets`), on a `joinable` lookup, whose `where` +already joins, and on one reading its literals as periods. A reference’s +values faceted by a leaf of a lookup’s referents (`facetedLeaves` on a lookup) +are not offered. ### Facet policy diff --git a/package-lock.json b/package-lock.json index 1555ae54..58e92989 100644 --- a/package-lock.json +++ b/package-lock.json @@ -42129,11 +42129,11 @@ "@traqula/generator-sparql-1-1": "^1.3.1", "@traqula/parser-sparql-1-1": "^1.3.1", "@traqula/rules-sparql-1-1": "^1.2.0", + "n3": "^2.13.12", "tslib": "^2.3.0" }, "devDependencies": { "@lde/search-typesense": "^0.53.0", - "n3": "^2.13.12", "testcontainers": "^12.1.0", "typesense": "^3.2.0" }, @@ -42146,7 +42146,6 @@ "version": "2.13.12", "resolved": "https://registry.npmjs.org/n3/-/n3-2.13.12.tgz", "integrity": "sha512-z944NsyJi58WXcYevtNkUL2JhVr0KP6nvED9AgymQq5V35TDaNn1N8kO+OWJk/kj4PR0L/ilWB3d7/bWceg1CQ==", - "dev": true, "license": "MIT", "dependencies": { "buffer": "^6.0.3", diff --git a/packages/search-api-graphql/src/build-schema.ts b/packages/search-api-graphql/src/build-schema.ts index bf5fa0a4..e6ccd92c 100644 --- a/packages/search-api-graphql/src/build-schema.ts +++ b/packages/search-api-graphql/src/build-schema.ts @@ -16,6 +16,7 @@ import { responsePathAsArray, type GraphQLEnumValueConfigMap, type GraphQLFieldConfig, + type GraphQLArgumentConfig, type GraphQLInputFieldConfig, type GraphQLInputType, type GraphQLOutputType, @@ -24,6 +25,7 @@ import { import { NESTED_DOCUMENT_TYPE, type Criterion, + type FacetBucket, type Filter, type LocalizedValue, type NestedDocument, @@ -41,6 +43,7 @@ import { AND_KEY, assertValidQuery, facetsOf, + filterableFields, fieldNamed, labelSourceNamesOf, labelTargetNamesOf, @@ -59,6 +62,7 @@ import { nestedReferenceType, outputFields, pageForOffset, + isInlineReference, physicalFields, sortableFields, unixSecondsToIso, @@ -465,6 +469,13 @@ export function buildGraphQLSchema( }); const valueFacet = facetTypeOf('ValueFacet', valueBucket); const iriFacet = facetTypeOf('IRIFacet', iriBucket); + // The keys share one engine round-trip, but each is its own facet + // calculation in it. + const facetCost = costExtensions(operationCost); + const limitArgument: GraphQLArgumentConfig = { + type: GraphQLInt, + description: `At most this many buckets, between 1 and ${maxFacetLimit}: the values with the highest counts. A larger limit returns a superset of a smaller one; there is no next page. Omit it for the deployment’s default.`, + }; // A range facet’s extent, in the type its field’s `range` filter takes // (IntRange, FloatRange, DateRange), so a slider’s bounds feed straight back // into the filter that selects them. A date is stored as Unix seconds and @@ -861,6 +872,32 @@ export function buildGraphQLSchema( const joins = joinGraph(schema); const whereInputs = new Map(); const nestedFilters = new Map(); + const leafFilters = new Map(); + const leafCountingReferences = new Map< + SearchType, + ReadonlySet + >(); + + /** The references of a Root Type counting leaves of their referents – + * faceted or presence leaves – computed once per type. None for a + * Reference Type. */ + function leafCountingReferencesOf( + owner: SearchType, + ): ReadonlySet { + const existing = leafCountingReferences.get(owner); + if (existing !== undefined) { + return existing; + } + const counting = new Set( + owner.class === undefined + ? [] + : facetsOf(owner, schema).flatMap((facet) => + facet.reference === undefined ? [] : [facet.reference], + ), + ); + leafCountingReferences.set(owner, counting); + return counting; + } const referenceFilters = new Map(); /** The name a reference’s emitted type is keyed under: a `lookup`’s target @@ -1326,13 +1363,18 @@ export function buildGraphQLSchema( owner: SearchType, joined = false, ): GraphQLInputType { + // A reference counting leaves of its referents takes a filter of its own, + // whose `where` names exactly the keys a condition on it can take. + if (leafCountingReferencesOf(owner).has(field)) { + return leafFilterFor(field, owner as RootType, joined); + } // An inline reference is filterable in two ways at once, exactly as a // joinable one is: by the ids its entries hold (its identity), // and by a condition on an entry. Same input shape, because it is the same // question one hop out – only what the hop costs differs. const nested = nestedReferenceType(schema, field); if (nested !== undefined) { - return nestedFilterFor(nested, field, joined); + return nestedFilterFor(nested, field); } // A joinable reference is filterable in two ways at once – by the ids it // holds, and by a condition on the referent – so it earns a richer input @@ -1346,6 +1388,83 @@ export function buildGraphQLSchema( : plainWhereFieldType(field); } + /** + * The filter input of a reference counting leaves of its referents – + * faceted or presence leaves – `‹Type›‹Field›Filter`: `@oneOf` over its `in` + * (the ids it holds, for a lookup or an inline reference naming an + * identity) and a `where` of its own, `‹Type›‹Field›Where`. For an inline + * reference that `where` names the entries’ filterable leaves, as the + * shared `‹Edge›Where` does; for both kinds it names each presence leaf, + * taking `exists` – `ExistsFilter`, or `‹Filter›OrExists` for an inline + * entry’s filterable leaf. A lookup’s `where` holds its presence leaves + * alone, each recorded on the root document, so its keys combine with AND + * like any sibling keys rather than holding of one referent. + * + * Through a join no presence companion is read yet, so a reference with + * presence leaves takes `‹Type›‹Field›JoinedFilter` there, without `exists` + * – or, for a lookup, which then has nothing left to name, its plain + * filter. + */ + function leafFilterFor( + field: SearchField, + owner: RootType, + joined: boolean, + ): GraphQLInputType { + const nested = nestedReferenceType(schema, field); + const presenceLeaves = joined ? [] : presenceLeavesOf(field); + if (nested === undefined && presenceLeaves.length === 0) { + return plainWhereFieldType(field); + } + const base = `${owner.name}${capitalized(field.name)}`; + const variant = + joined && presenceLeavesOf(field).length > 0 ? `${base}Joined` : base; + const existing = leafFilters.get(variant); + if (existing !== undefined) { + return existing; + } + const at = `“${owner.name}.${field.name}”`; + const where = new GraphQLInputObjectType({ + name: claimTypeName(`${variant}Where`, `the condition on ${at}`), + description: + nested === undefined + ? 'Whether any referent holds a value for a leaf. Sibling keys are combined with AND.' + : 'Sibling keys are combined with AND, and all of them must hold of the SAME entry.', + fields: () => + nested === undefined + ? Object.fromEntries( + presenceLeaves.map((leaf) => [leaf, { type: existsFilter }]), + ) + : Object.fromEntries( + whereFieldsOf(nested, presenceLeaves).map((leaf) => [ + leaf.name, + { + type: presenceLeaves.includes(leaf.name) + ? presenceFilterFor(leaf, nested) + : whereFieldType(leaf, nested), + description: leaf.description, + }, + ]), + ), + }); + const ids = + nested === undefined || labelTargetNamesOf(field, schema).length > 0; + const created = new GraphQLInputObjectType({ + name: claimTypeName(`${variant}Filter`, `the filter on ${at}`), + description: ids + ? `A condition on ${at}: the ids it holds, or a condition on its referents.` + : `A condition on one ${at} entry.`, + isOneOf: true, + fields: () => ({ + ...(ids && { + in: { type: new GraphQLList(new GraphQLNonNull(iriScalar)) }, + }), + where: { type: where }, + }), + }); + leafFilters.set(variant, created); + return created; + } + /** * The filter input an inline reference takes: the ids its entries hold, or a * condition on one entry. `@oneOf`, so a criterion stays an atom. @@ -1354,45 +1473,22 @@ export function buildGraphQLSchema( * {@link ReferenceStrategy.identity identity} names, so a bucket a * consumer clicks feeds straight back in – the same round trip an ordinary * facetable reference already guarantees. A reference declaring no identity - * has no ids to filter, so it offers the `where` arm alone. + * has no ids to filter, so it offers the `where` arm alone. Shared by every + * reference nesting the type that counts no leaf of it + * ({@link leafFilterFor}). */ function nestedFilterFor( referenceType: ReferenceType, field: SearchField, - joined: boolean, ): GraphQLInputObjectType { - // Every reference nesting the type declares the same presence leaves - // (`searchSchema`), so the filter is shared – except through a join, which - // reads no presence companion yet and so gets a variant without `exists`. - const presenceLeaves = joined ? [] : presenceLeavesOf(field); - const variant = - joined && presenceLeavesOf(field).length > 0 - ? `${referenceType.name}Joined` - : referenceType.name; - const existing = nestedFilters.get(variant); + const existing = nestedFilters.get(referenceType.name); if (existing !== undefined) { return existing; } - // The join variants are names of their own, so they are claimed like - // every other name a declaration could clash with. - if (variant !== referenceType.name) { - claimTypeName( - `${variant}Filter`, - `the filter on “${referenceType.name}” through a join`, - ); - claimTypeName( - `${variant}Where`, - `the condition on one “${referenceType.name}” entry through a join`, - ); - } const identityTargets = labelTargetNamesOf(field, schema); - const nestedWhere = nestedWhereInputFor( - referenceType, - presenceLeaves, - variant, - ); + const nestedWhere = nestedWhereInputFor(referenceType); const created = new GraphQLInputObjectType({ - name: `${variant}Filter`, + name: `${referenceType.name}Filter`, description: identityTargets.length > 0 ? `A condition on ${referenceType.name}: the ids its entries reference, or a condition on one entry.` @@ -1405,7 +1501,7 @@ export function buildGraphQLSchema( where: { type: nestedWhere }, }), }); - nestedFilters.set(variant, created); + nestedFilters.set(referenceType.name, created); return created; } @@ -1414,33 +1510,22 @@ export function buildGraphQLSchema( * else. Deliberately without the `id` key every Root Type's `where` carries: * an entry is read, not addressed, so it has no document key to filter on * ({@link validateQuery} refuses one). - * - * A presence leaf of the references nesting the type takes `exists` too, - * as a field of the root’s own does: `ExistsFilter`, or `‹Filter›OrExists` - * for a `filterable` leaf. Whether any entry holds a value is the root - * document’s, so `exists` is welded to nothing else – a `where` setting it - * beside another key is refused with the rewrite ({@link - * weldedCriterionOf}). */ function nestedWhereInputFor( referenceType: ReferenceType, - presenceLeaves: readonly string[], - variant: string, ): GraphQLInputObjectType { // Memoized by its only caller, which is memoized itself – so this is built - // once per Reference Type (and variant) however many fields nest it. + // once per Reference Type however many fields nest it. return new GraphQLInputObjectType({ - name: `${variant}Where`, + name: `${referenceType.name}Where`, description: 'Sibling keys are combined with AND, and all of them must hold of the SAME entry.', fields: () => Object.fromEntries( - whereFieldsOf(referenceType, presenceLeaves).map((nestedField) => [ + filterableFields(referenceType).map((nestedField) => [ nestedField.name, { - type: presenceLeaves.includes(nestedField.name) - ? presenceFilterFor(nestedField, referenceType) - : whereFieldType(nestedField, referenceType), + type: whereFieldType(nestedField, referenceType), description: nestedField.description, }, ]), @@ -1800,179 +1885,250 @@ export function buildGraphQLSchema( } /** - * The keyed facets object for one type (only called with ≥ 1 facet). A - * faceted leaf is served one level down, under the reference it is reached - * through – `facets { associatedMedia { license } }` – so a facet is named - * by the same path as the value it counts. A presence facet is served one - * level down too, under the field it counts – `facets { dateCreated { - * exists } }` – in an object that has room for the field’s value buckets - * once a field may have both. + * The keyed facets object for one type (only called with ≥ 1 facet): one + * key per field anything is counted on. A field counting only its own values + * keeps its plain facet type (`IRIFacet`, `ValueFacet`, a range facet, a + * boolean bucket list). A field counting more – whether it holds a value, + * or a leaf of its referents – gets one object of its own, + * `‹Type›‹Field›Facet`, holding everything counted on it + * ({@link fieldFacetFor}). */ function facetsTypeFor( typeName: string, facets: readonly FacetDeclaration[], ): GraphQLObjectType { - const own: FacetDeclaration[] = []; - const byGroup = new Map(); + const groups = new Map(); for (const facet of facets) { - // A leaf’s facets, a presence leaf’s included, go under the reference; - // an own field’s presence facet under the field. - const group = - facet.reference?.name ?? - (facet.presence === true ? facet.field.name : undefined); - if (group === undefined) { - own.push(facet); - continue; + const name = facet.reference?.name ?? facet.field.name; + const group: FacetGroup = groups.get(name) ?? { leaves: new Map() }; + groups.set(name, group); + if (facet.reference !== undefined) { + const leaf = group.leaves.get(facet.field.name) ?? {}; + group.leaves.set(facet.field.name, leaf); + if (facet.presence === true) { + leaf.presence = facet; + } else { + leaf.values = facet; + } + } else if (facet.presence === true) { + group.presence = facet; + } else { + group.values = facet; } - const grouped = byGroup.get(group) ?? []; - grouped.push(facet); - byGroup.set(group, grouped); } - return new GraphQLObjectType({ name: `${typeName}Facets`, - fields: () => ({ - ...facetFieldsFor(own, typeName), - ...Object.fromEntries( - [...byGroup].map(([group, grouped]) => { - const groupBaseName = `${typeName}${capitalized(group)}`; - return [ - group, - { - type: new GraphQLNonNull( - new GraphQLObjectType({ - name: `${groupBaseName}Facets`, - fields: () => facetFieldsFor(grouped, groupBaseName), - }), - ), - // The source carries the facet loader; the grouped facets - // read it. - resolve: (source: Source) => source, - }, - ]; - }), + fields: () => + Object.fromEntries( + [...groups].map(([name, group]) => [ + name, + group.presence === undefined && group.leaves.size === 0 + ? valueFacetFieldFor(group.values as FacetDeclaration) + : fieldFacetFor(`${typeName}${capitalized(name)}`, group), + ]), ), - }), }); } /** - * One GraphQL field per facet, keyed by its field’s name – `exists` for an - * own field’s presence facet, served under the field – and loaded by the - * facet’s logical name. A presence leaf’s facet is one level further down, - * `‹leaf› { exists }`, in an object typed `‹Base›‹Leaf›Facets` after the - * base name of the object it sits in (`WorkAssociatedMedia`). + * The facet object of a field counting more than its own values, typed + * `‹Base›Facet` (`WorkContentLocationFacet`, `WorkDateCreatedFacet`): its + * own values in `buckets` and `totalValues` – or, for a numeric field, its + * range facet’s `all`, `matching` and `buckets` – capped by the field’s + * `limit`; `exists` for its presence; and one key per leaf of its + * referents, which is a plain facet type again for a leaf counting only its + * values, and `‹Base›‹Leaf›Facet` for one whose presence is counted. */ - function facetFieldsFor( - facets: readonly FacetDeclaration[], + function fieldFacetFor( baseName: string, - ): Record> { + group: FacetGroup, + ): GraphQLFieldConfig { + const values = group.values; + const range = values !== undefined && isRangeFacet(values.field); + const limited = values !== undefined && !range; const fields: Record< string, GraphQLFieldConfig > = {}; - // The keys share one engine round-trip, but each is its own facet - // calculation in it. - const facetCost = costExtensions(operationCost); - for (const { name, field, presence, reference: reachedThrough } of facets) { - // The batching into one engine dispatch and the degrade-to-empty - // error handling live in the loader (facet-batch.ts). - if (presence === true) { - const exists: GraphQLFieldConfig = { - type: nonNullListOf(booleanBucket), - extensions: facetCost, - description: reachedThrough - ? `Documents with an entry that holds a value for “${field.name}”, and documents with none.` - : `Documents that hold a value for “${field.name}”, and documents that hold none.`, - resolve: async (source: Source) => - (await (source.loadFacet as FacetLoader)(name, 'matching')).buckets, - }; - if (reachedThrough === undefined) { - fields.exists = exists; - continue; - } - fields[field.name] = { - type: new GraphQLNonNull( - new GraphQLObjectType({ - name: claimTypeName( - `${baseName}${capitalized(field.name)}Facets`, - `the presence facet of “${name}”`, - ), - fields: { exists }, - }), - ), - resolve: (source: Source) => source, - }; - continue; - } - if (isRangeFacet(field)) { - fields[field.name] = { - type: new GraphQLNonNull(rangeFacetTypeFor(field)), - extensions: facetCost, - // Resolved synchronously, so the scopes its fields load land in - // the same batch as the sibling facets. - resolve: (source: Source) => ({ - load: (scope: FacetScope) => - (source.loadFacet as FacetLoader)(name, scope), - }), - }; - continue; - } - if (field.kind === 'boolean') { - // Two values at most: nothing to cap and nothing to total. - fields[field.name] = { - type: nonNullListOf(booleanBucket), - extensions: facetCost, - resolve: async (source: Source) => - (await (source.loadFacet as FacetLoader)(name, 'matching')).buckets, + if (values !== undefined && range) { + // Each part is charged as the facet key it stands in for is – `all` + // already is, a facet query of its own. + const rangeFields = rangeFacetTypeFor( + values.field as NumericField, + ).toConfig().fields; + for (const [name, rangeField] of Object.entries(rangeFields)) { + fields[name] = { + ...rangeField, + extensions: { ...facetCost, ...rangeField.extensions }, }; - continue; } - // `IRI`-keyed buckets for a reference, literal keys otherwise: the - // reference test {@link whereFieldType} applies, so a bucket and the - // filter it feeds cannot disagree about what the field keys on. - const reference = field.kind === 'reference'; - fields[field.name] = { - type: new GraphQLNonNull(reference ? iriFacet : valueFacet), + } else if (values !== undefined) { + Object.assign(fields, valueFieldsFor(values)); + } + if (group.presence !== undefined) { + fields.exists = existsFieldFor(group.presence); + } + for (const [leafName, leaf] of group.leaves) { + fields[leafName] = + leaf.presence === undefined + ? valueFacetFieldFor(leaf.values as FacetDeclaration) + : fieldFacetFor(`${baseName}${capitalized(leafName)}`, { + values: leaf.values, + presence: leaf.presence, + leaves: new Map(), + }); + } + const name = `${baseName}Facet`; + return { + type: new GraphQLNonNull( + new GraphQLObjectType({ + name: claimTypeName(name, `the facet object of “${baseName}”`), + fields, + }), + ), + ...(limited && { args: { limit: limitArgument } }), + // Resolved synchronously, so what its keys load lands in the same batch + // as the sibling facets. A range facet’s keys read `load`. + resolve: (source: Source, args: { limit?: number | null }) => ({ + loadFacet: source.loadFacet, + limit: limited ? checkedLimit(args.limit) : undefined, + ...(range && { + load: (scope: FacetScope) => + (source.loadFacet as FacetLoader)( + (values as FacetDeclaration).name, + scope, + ), + }), + }), + }; + } + + /** A field counting only its own values, typed by its kind, as the facets + * object has always served it. */ + function valueFacetFieldFor( + facet: FacetDeclaration, + ): GraphQLFieldConfig { + const { name, field } = facet; + // The batching into one engine dispatch and the degrade-to-empty error + // handling live in the loader (facet-batch.ts). + if (isRangeFacet(field)) { + return { + type: new GraphQLNonNull(rangeFacetTypeFor(field)), extensions: facetCost, - args: { - limit: { - type: GraphQLInt, - description: `At most this many buckets, between 1 and ${maxFacetLimit}: the values with the highest counts. A larger limit returns a superset of a smaller one; there is no next page. Omit it for the deployment’s default.`, - }, - }, - resolve: async (source: Source, args: { limit?: number | null }) => { - const limit = args.limit ?? undefined; - if (limit !== undefined && (limit < 1 || limit > maxFacetLimit)) { - throw userError( - `limit must be between 1 and ${maxFacetLimit}; got ${limit}.`, - ); - } - const { buckets, totalValues } = await ( - source.loadFacet as FacetLoader - )(name, 'matching', limit); - return { - // A bucket keyed on something that is not an IRI is not a - // selection key – it cannot be sent back as the filter that - // selects it – so it is dropped rather than served under `IRI`. - // Dropping, not throwing: `value: IRI!` sits inside a chain of - // non-nulls up to the root field, so a raising coercion here - // would null the WHOLE response and take `items` with it, which - // is exactly what facet-batch.ts’s degradation contract forbids - // a supplementary sidebar count from doing. - buckets: reference - ? buckets.filter( - (bucket) => - typeof bucket.value === 'string' && - isAbsoluteIri(bucket.value), - ) - : buckets, - totalValues: totalValues ?? null, - }; - }, + // Resolved synchronously, so the scopes its fields load land in the + // same batch as the sibling facets. + resolve: (source: Source) => ({ + load: (scope: FacetScope) => + (source.loadFacet as FacetLoader)(name, scope), + }), }; } - return fields; + if (field.kind === 'boolean') { + // Two values at most: nothing to cap and nothing to total. + return { + type: nonNullListOf(booleanBucket), + extensions: facetCost, + resolve: async (source: Source) => + (await (source.loadFacet as FacetLoader)(name, 'matching')).buckets, + }; + } + return { + type: new GraphQLNonNull( + field.kind === 'reference' ? iriFacet : valueFacet, + ), + extensions: facetCost, + args: { limit: limitArgument }, + resolve: (source: Source, args: { limit?: number | null }) => + valueFacetOf(source, facet, checkedLimit(args.limit)), + }; + } + + /** + * A value facet’s `buckets` and `totalValues`, read through the loader. + * `IRI`-keyed buckets for a reference, literal keys otherwise: the reference + * test {@link whereFieldType} applies, so a bucket and the filter it feeds + * cannot disagree about what the field keys on. + */ + async function valueFacetOf( + source: Source, + facet: FacetDeclaration, + limit: number | undefined, + ): Promise<{ buckets: readonly FacetBucket[]; totalValues: number | null }> { + const { buckets, totalValues } = await (source.loadFacet as FacetLoader)( + facet.name, + 'matching', + limit, + ); + return { + // A bucket keyed on something that is not an IRI is not a selection key + // – it cannot be sent back as the filter that selects it – so it is + // dropped rather than served under `IRI`. Dropping, not throwing: + // `value: IRI!` sits inside a chain of non-nulls up to the root field, + // so a raising coercion here would null the WHOLE response and take + // `items` with it, which is exactly what facet-batch.ts’s degradation + // contract forbids a supplementary sidebar count from doing. + buckets: + facet.field.kind === 'reference' + ? buckets.filter( + (bucket) => + typeof bucket.value === 'string' && isAbsoluteIri(bucket.value), + ) + : buckets, + totalValues: totalValues ?? null, + }; + } + + /** `buckets` and `totalValues` of a field’s own values, inside its facet + * object, capped by the `limit` the field took ({@link fieldFacetFor}). */ + function valueFieldsFor( + facet: FacetDeclaration, + ): Record> { + const shape = ( + facet.field.kind === 'reference' ? iriFacet : valueFacet + ).toConfig().fields; + const read = (source: Source) => + valueFacetOf(source, facet, source.limit as number | undefined); + return { + buckets: { + ...shape.buckets, + extensions: facetCost, + resolve: async (source: Source) => (await read(source)).buckets, + }, + totalValues: { + ...shape.totalValues, + extensions: facetCost, + resolve: async (source: Source) => (await read(source)).totalValues, + }, + }; + } + + /** `exists`: the documents that hold a value – for a leaf, with a referent + * that holds one – and the documents that hold none. */ + function existsFieldFor( + facet: FacetDeclaration, + ): GraphQLFieldConfig { + return { + type: nonNullListOf(booleanBucket), + extensions: facetCost, + description: + facet.reference === undefined + ? `Documents that hold a value for “${facet.field.name}”, and documents that hold none.` + : `Documents with a referent that holds a value for “${facet.field.name}”, and documents with none.`, + resolve: async (source: Source) => + (await (source.loadFacet as FacetLoader)(facet.name, 'matching')) + .buckets, + }; + } + + /** A value facet’s `limit` argument, between 1 and `maxFacetLimit`. */ + function checkedLimit(limit: number | null | undefined): number | undefined { + const checked = limit ?? undefined; + if (checked !== undefined && (checked < 1 || checked > maxFacetLimit)) { + throw userError( + `limit must be between 1 and ${maxFacetLimit}; got ${checked}.`, + ); + } + return checked; } const queryFields: Record< @@ -2099,6 +2255,16 @@ function whereToFilters( // whole AND/OR mapping: which bucket a criterion lands in, never a combinator // inferred from how deeply it is nested. for (const entry of keyEntriesOf(where, searchType, joins, schema)) { + // A lookup’s presence leaves are recorded on this document, so each key + // of its `where` is a criterion of its own, ANDed like sibling keys. + if ('leafPresence' in entry) { + filters.push( + ...entry.leafPresence.map((criterion) => + filterOn(withPath(criterion, on)), + ), + ); + continue; + } if ('nested' in entry) { // A NESTING hop welds: every condition inside one edge's `where` must // hold of the SAME entry, which is the whole point of indexing an edge. @@ -2217,10 +2383,27 @@ function criteriaOf( ): Criterion[] { const criteria: Criterion[] = []; for (const entry of keyEntriesOf(keyed, searchType, joins, schema)) { - if (!('nested' in entry)) { + if ('criterion' in entry) { criteria.push(withPath(entry.criterion, on)); continue; } + // A lookup’s `where` ANDs its keys, so in an `or` alternative it must set + // exactly one, as a joined condition must. + if ('leafPresence' in entry) { + const [only, ...more] = entry.leafPresence; + if (only === undefined) { + throw userError( + `The condition on “${entry.name}” states no criterion, so it constrains nothing – and an “or” alternative that constrains nothing would make the whole disjunction match everything. Give it a condition, or leave the alternative out.`, + ); + } + if (more.length > 0) { + throw userError( + `The condition on “${entry.name}” states more than one criterion, so it is a conjunction – which cannot sit inside an “or”. Write one “or” alternative per criterion, or move the conjunction into “and”.`, + ); + } + criteria.push(withPath(only, on)); + continue; + } // A NESTING hop welds, in an `or` alternative exactly as anywhere else – // and it is the one place a weld belongs most naturally, since a welded // criterion IS an atom. Compiled as separate joined criteria it would be @@ -2260,10 +2443,17 @@ function criteriaOf( return criteria; } -/** One key of a `Where`/`Criterion` input, compiled: either a leaf criterion or - * a joined `where` to recurse into. */ +/** One key of a `Where`/`Criterion` input, compiled: a leaf criterion, a + * lookup’s presence-leaf criteria, or a joined or nested `where` to recurse + * into. */ type KeyEntry = | { readonly criterion: Criterion } + | { + readonly name: string; + // One criterion per presence leaf the lookup’s `where` sets, each on + // the root document’s companion. + readonly leafPresence: readonly Criterion[]; + } | { readonly name: string; readonly nested: Record; @@ -2312,6 +2502,27 @@ function keyEntriesOf( continue; } } + const nested = (value as { where?: Record | null }).where; + // A lookup’s `where` names its presence leaves, each a plain criterion on + // the root document’s companion – no entry, and no join. + if ( + !isInlineReference(field) && + presenceLeavesOf(field).length > 0 && + nested !== undefined && + nested !== null + ) { + entries.push({ + name: field.name, + leafPresence: presenceLeavesOf(field).flatMap((leaf): Criterion[] => { + const exists = (nested[leaf] as { exists?: boolean | null } | null) + ?.exists; + return exists === undefined || exists === null + ? [] + : [{ field: field.name, leaf, exists }]; + }), + }); + continue; + } // Both hop kinds take the same `@oneOf` input – `in` (the ids this field // holds) or `where` (a condition one hop out) – so both are read here, and // only what they resolve to differs: another collection, or this @@ -2319,18 +2530,14 @@ function keyEntriesOf( const target = nestedReferenceType(schema, field) ?? joins.resolve(searchType, [field.name]); - if (target !== undefined) { - const nested = (value as { where?: Record | null }) - .where; - if (nested !== undefined && nested !== null) { - entries.push({ - name: field.name, - nested, - target, - presenceLeaves: presenceLeavesOf(field), - }); - continue; - } + if (target !== undefined && nested !== undefined && nested !== null) { + entries.push({ + name: field.name, + nested, + target, + presenceLeaves: presenceLeavesOf(field), + }); + continue; } switch (filterOperatorFor(field)) { case 'in': @@ -2447,12 +2654,24 @@ function rangeInput( }); } -/** The presence leaves an inline reference declares, read off the physical - * companions that record them – none for any other field. */ +/** The presence leaves a reference declares, inline or `lookup`, read off + * the physical companions that record them – none for any other field. */ function presenceLeavesOf(field: SearchField): readonly string[] { return physicalFields(field).leafExists.map(({ leaf }) => leaf); } +/** What one field of a root type counts, gathered for its facet object: its + * own values, its presence, and – per leaf of its referents – the leaf’s + * values and presence. */ +interface FacetGroup { + values?: FacetDeclaration; + presence?: FacetDeclaration; + readonly leaves: Map< + string, + { values?: FacetDeclaration; presence?: FacetDeclaration } + >; +} + /** A name with its first letter upper-cased, as it is spelled inside a type * name: `associatedMedia` → `AssociatedMedia`. */ function capitalized(name: string): string { diff --git a/packages/search-api-graphql/test/faceted-leaves.test.ts b/packages/search-api-graphql/test/faceted-leaves.test.ts index d1807bf6..27ddb51c 100644 --- a/packages/search-api-graphql/test/faceted-leaves.test.ts +++ b/packages/search-api-graphql/test/faceted-leaves.test.ts @@ -78,10 +78,10 @@ describe('a root type faceted on its entries’ leaves', () => { const printed = printSchema(gqlSchema); expect(printed).toMatch( - /type WorkFacets \{\s*associatedMedia: WorkAssociatedMediaFacets!\s*\}/, + /type WorkFacets \{\s*associatedMedia: WorkAssociatedMediaFacet!\s*\}/, ); expect(printed).toMatch( - /type WorkAssociatedMediaFacets \{\s*license\([\s\S]*?\): IRIFacet!\s*\}/, + /type WorkAssociatedMediaFacet \{\s*license\([\s\S]*?\): IRIFacet!\s*\}/, ); }); diff --git a/packages/search-api-graphql/test/leaf-presence.test.ts b/packages/search-api-graphql/test/leaf-presence.test.ts index 5ce90ef4..a61d15d3 100644 --- a/packages/search-api-graphql/test/leaf-presence.test.ts +++ b/packages/search-api-graphql/test/leaf-presence.test.ts @@ -103,7 +103,7 @@ async function run(source: string) { describe('a root document filtered on whether an entry’s leaf has a value', () => { it('gives a presence leaf `exists` in the reference type’s where', () => { expect(printed).toMatch( - /input MediaObjectWhere \{[^}]*license: IRIFilter\n[^}]*thumbnailUrl: ExistsFilter\n[^}]*encodingFormat: KeywordFilterOrExists\n/, + /input WorkAssociatedMediaWhere \{[^}]*license: IRIFilter\n[^}]*thumbnailUrl: ExistsFilter\n[^}]*encodingFormat: KeywordFilterOrExists\n/, ); }); @@ -159,10 +159,10 @@ describe('a root document filtered on whether an entry’s leaf has a value', () describe('a root document faceted on whether an entry’s leaf has a value', () => { it('serves the facet under the leaf, beside the reference’s faceted leaves', () => { expect(printed).toMatch( - /type WorkAssociatedMediaFacets \{[^}]*\): IRIFacet!\s*thumbnailUrl: WorkAssociatedMediaThumbnailUrlFacets!\s*encodingFormat: WorkAssociatedMediaEncodingFormatFacets!\s*\}/, + /type WorkAssociatedMediaFacet \{[^}]*\): IRIFacet!\s*thumbnailUrl: WorkAssociatedMediaThumbnailUrlFacet!\s*encodingFormat: WorkAssociatedMediaEncodingFormatFacet!\s*\}/, ); expect(printed).toMatch( - /type WorkAssociatedMediaThumbnailUrlFacets \{[^}]*exists: \[BooleanBucket!\]!\s*\}/, + /type WorkAssociatedMediaThumbnailUrlFacet \{[^}]*exists: \[BooleanBucket!\]!\s*\}/, ); }); @@ -237,15 +237,15 @@ describe('a reference with presence leaves on a joined type', () => { it('reaches the entries through a join without `exists`, which no join reads', () => { expect(joinedPrinted).toMatch( - /input CollectionJoinedWhere \{[^}]*associatedMedia: MediaObjectJoinedFilter\n/, + /input CollectionJoinedWhere \{[^}]*associatedMedia: CollectionAssociatedMediaJoinedFilter\n/, ); - const where = /input MediaObjectJoinedWhere \{([^}]*)\}/.exec( + const where = /input CollectionAssociatedMediaJoinedWhere \{([^}]*)\}/.exec( joinedPrinted, ); expect(where?.[1]).toMatch(/encodingFormat: KeywordFilter\n/); expect(where?.[1]).not.toMatch(/thumbnailUrl|exists|OrExists/); expect(joinedPrinted).toMatch( - /input CollectionWhere \{[^}]*associatedMedia: MediaObjectFilter\n/, + /input CollectionWhere \{[^}]*associatedMedia: CollectionAssociatedMediaFilter\n/, ); }); }); @@ -273,13 +273,13 @@ describe('the names a presence leaf adds', () => { searchSchema( work, mediaObject, - clashing('WorkAssociatedMediaThumbnailUrlFacets'), + clashing('WorkAssociatedMediaThumbnailUrlFacet'), ), {}, ), ), ).toThrow( - /“WorkAssociatedMediaThumbnailUrlFacets”, the type name of the presence facet of “associatedMedia.thumbnailUrl.exists”, is already the name of another type/, + /“WorkAssociatedMediaThumbnailUrlFacet”, the type name of the facet object of “WorkAssociatedMediaThumbnailUrl”, is already the name of another type/, ); }); @@ -317,13 +317,13 @@ describe('the names a presence leaf adds', () => { labelled, item, mediaObject, - clashing('MediaObjectJoinedFilter'), + clashing('WorkAssociatedMediaJoinedFilter'), ), {}, ), ), ).toThrow( - /“MediaObjectJoinedFilter”, the type name of the filter on “MediaObject” through a join, is already the name of another type/, + /“WorkAssociatedMediaJoinedFilter”, the type name of the filter on “Work.associatedMedia”, is already the name of another type/, ); }); }); diff --git a/packages/search-api-graphql/test/lookup-presence.test.ts b/packages/search-api-graphql/test/lookup-presence.test.ts new file mode 100644 index 00000000..d1efdb42 --- /dev/null +++ b/packages/search-api-graphql/test/lookup-presence.test.ts @@ -0,0 +1,390 @@ +import { describe, expect, it } from 'vitest'; +import { graphql, printSchema } from 'graphql'; +import { + defineSearchType, + searchSchema, + type SearchEngine, + type SearchQuery, + type SearchResult, + type SearchType, +} from '@lde/search'; +import { buildGraphQLSchema, type SearchContext } from '../src/build-schema.js'; + +const SCHEMA_ORG = 'https://schema.org/'; +const VENLO = 'https://sws.geonames.org/2745707'; +const KESSEL = 'https://ex/place/kessel'; + +const place = defineSearchType({ + name: 'Place', + class: `${SCHEMA_ORG}Place`, + fields: [ + { + name: 'label', + kind: 'text', + path: `${SCHEMA_ORG}name`, + locales: ['nl'], + output: true, + searchable: { weight: 1 }, + }, + { + name: 'latitude', + kind: 'number', + path: `${SCHEMA_ORG}latitude`, + output: true, + }, + { + name: 'longitude', + kind: 'number', + path: `${SCHEMA_ORG}longitude`, + output: true, + }, + ], +}); + +const work = defineSearchType({ + name: 'Work', + class: `${SCHEMA_ORG}CreativeWork`, + fields: [ + { + name: 'label', + kind: 'text', + locales: ['nl'], + output: true, + searchable: { weight: 1 }, + }, + { name: 'material', kind: 'keyword', facetable: true, filterable: true }, + { + name: 'contentLocation', + kind: 'reference', + path: `${SCHEMA_ORG}contentLocation`, + array: true, + output: true, + filterable: true, + facetable: true, + ref: { + strategy: 'lookup', + target: 'Place', + presenceLeaves: ['latitude', 'longitude'], + }, + }, + { + name: 'locationCreated', + kind: 'reference', + path: `${SCHEMA_ORG}locationCreated`, + filterable: true, + ref: { strategy: 'lookup', target: 'Place' }, + }, + ], +}); + +const schema = searchSchema(work, place); +const gqlSchema = buildGraphQLSchema(schema, {}); +const printed = printSchema(gqlSchema); + +function recordingEngine() { + const searched: SearchQuery[] = []; + const faceted: SearchQuery[] = []; + const engine: SearchEngine = { + schema, + async search(_searchType: SearchType, query: SearchQuery) { + searched.push(query); + return { total: 0, hits: [], facets: {} } as SearchResult; + }, + async searchFacets(_searchType, queries) { + faceted.push(...queries); + return queries.map(() => ({ + facets: { + material: { buckets: [{ value: 'hout', count: 3 }], totalValues: 1 }, + contentLocation: { + buckets: [ + { value: VENLO, count: 2 }, + { value: KESSEL, count: 1 }, + ], + totalValues: 2, + }, + 'contentLocation.latitude.exists': { + buckets: [ + { value: 'true', count: 2, is: true }, + { value: 'false', count: 1, is: false }, + ], + }, + }, + })) as never; + }, + }; + return { engine, searched, faceted }; +} + +async function run(source: string) { + const recording = recordingEngine(); + const response = await graphql({ + schema: gqlSchema, + source, + contextValue: { + engine: recording.engine, + acceptLanguage: ['nl'], + } as SearchContext, + }); + return { response, ...recording }; +} + +describe('one facet object per field', () => { + it('keeps the plain facet type for a field counting only its own values', () => { + expect(printed).toMatch(/material\([^)]*\): ValueFacet!/); + }); + + it('gives a field counting more an object holding everything counted on it', () => { + expect(printed).toMatch( + /contentLocation\(\s*"""[^"]*"""\s*limit: Int\s*\): WorkContentLocationFacet!/, + ); + expect(printed).toMatch( + /type WorkContentLocationFacet \{[^}]*buckets: \[IRIBucket!\]![^}]*totalValues: Int\s*latitude: WorkContentLocationLatitudeFacet!\s*longitude: WorkContentLocationLongitudeFacet!\s*\}/, + ); + expect(printed).toMatch( + /type WorkContentLocationLatitudeFacet \{[^}]*exists: \[BooleanBucket!\]!\s*\}/, + ); + }); + + it('caps the field’s own buckets with its limit, and counts its leaf beside them', async () => { + const { response, faceted } = await run( + `{ works { facets { contentLocation(limit: 1) { buckets { value count } totalValues latitude { exists { value count } } } } } }`, + ); + + expect(response.errors).toBeUndefined(); + expect(faceted.map((query) => query.facets)).toEqual([ + ['contentLocation', 'contentLocation.latitude.exists'], + ]); + expect(faceted[0]?.facetLimits).toEqual({ contentLocation: 1 }); + expect(response.data).toEqual({ + works: { + facets: { + contentLocation: { + buckets: [ + { value: VENLO, count: 2 }, + { value: KESSEL, count: 1 }, + ], + totalValues: 2, + latitude: { + exists: [ + { value: true, count: 2 }, + { value: false, count: 1 }, + ], + }, + }, + }, + }, + }); + }); + + it('refuses a limit out of range', async () => { + const { response } = await run( + `{ works { facets { contentLocation(limit: 0) { totalValues } } } }`, + ); + + expect(response.errors?.[0]?.message).toMatch( + /limit must be between 1 and 100; got 0/, + ); + }); +}); + +describe('a lookup filtered on whether a referent’s leaf has a value', () => { + it('takes a filter of its own, beside its ids', () => { + expect(printed).toMatch(/contentLocation: WorkContentLocationFilter\n/); + expect(printed).toMatch( + /input WorkContentLocationFilter @oneOf \{[^}]*in: \[IRI!\][^}]*where: WorkContentLocationWhere\s*\}/, + ); + expect(printed).toMatch( + /input WorkContentLocationWhere \{[^}]*latitude: ExistsFilter\s*longitude: ExistsFilter\s*\}/, + ); + // A lookup counting no leaf keeps its target’s filter. + expect(printed).toMatch(/locationCreated: PlaceFilter\n/); + }); + + it('compiles `exists` on the leaf into a plain criterion on the root', async () => { + const { response, searched } = await run( + `{ works(where: { contentLocation: { where: { latitude: { exists: false } } } }) { pagination { total } } }`, + ); + + expect(response.errors).toBeUndefined(); + expect(searched[0]?.where).toEqual([ + { + or: [{ field: 'contentLocation', leaf: 'latitude', exists: false }], + }, + ]); + }); + + it('ANDs two leaves in one `where`, as sibling keys', async () => { + const { response, searched } = await run( + `{ works(where: { contentLocation: { where: { latitude: { exists: true }, longitude: { exists: false } } } }) { pagination { total } } }`, + ); + + expect(response.errors).toBeUndefined(); + expect(searched[0]?.where).toEqual([ + { or: [{ field: 'contentLocation', leaf: 'latitude', exists: true }] }, + { or: [{ field: 'contentLocation', leaf: 'longitude', exists: false }] }, + ]); + }); + + it('takes one leaf in an `or` alternative, and refuses two', async () => { + const one = await run( + `{ works(where: { or: [{ contentLocation: { where: { latitude: { exists: true } } } }, { material: { in: ["hout"] } }] }) { pagination { total } } }`, + ); + expect(one.response.errors).toBeUndefined(); + expect(one.searched[0]?.where).toEqual([ + { + or: [ + { field: 'contentLocation', leaf: 'latitude', exists: true }, + { field: 'material', in: ['hout'] }, + ], + }, + ]); + + const two = await run( + `{ works(where: { or: [{ contentLocation: { where: { latitude: { exists: true }, longitude: { exists: true } } } }] }) { pagination { total } } }`, + ); + expect(two.response.errors?.[0]?.message).toMatch( + /The condition on “contentLocation” states more than one criterion, so it is a conjunction/, + ); + + const none = await run( + `{ works(where: { or: [{ contentLocation: { where: {} } }] }) { pagination { total } } }`, + ); + expect(none.response.errors?.[0]?.message).toMatch( + /The condition on “contentLocation” states no criterion, so it constrains nothing/, + ); + }); + + it('still filters by the ids it holds', async () => { + const { response, searched } = await run( + `{ works(where: { contentLocation: { in: ["${VENLO}"] } }) { pagination { total } } }`, + ); + + expect(response.errors).toBeUndefined(); + expect(searched[0]?.where).toEqual([ + { or: [{ field: 'contentLocation', in: [VENLO] }] }, + ]); + }); +}); + +describe('a numeric field counted by its range and its presence', () => { + const dated = defineSearchType({ + name: 'Dated', + class: `${SCHEMA_ORG}Event`, + fields: [ + { + name: 'label', + kind: 'text', + locales: ['nl'], + output: true, + searchable: { weight: 1 }, + }, + { + name: 'startDate', + kind: 'date', + facetable: true, + filterable: true, + presence: true, + }, + ], + }); + const datedSchema = searchSchema(dated); + const datedPrinted = printSchema(buildGraphQLSchema(datedSchema, {})); + + it('holds the range facet’s parts and `exists` in one object', async () => { + expect(datedPrinted).toMatch( + /type DatedStartDateFacet \{[^}]*all: DateExtent[^}]*matching: DateExtent[^}]*buckets: \[DateBucket!\]![^}]*exists: \[BooleanBucket!\]!\s*\}/, + ); + const loaded: string[] = []; + const engine: SearchEngine = { + schema: datedSchema, + async search() { + return { total: 0, hits: [], facets: {} } as SearchResult; + }, + async searchFacets(_searchType, queries) { + loaded.push(...queries.flatMap((query) => query.facets)); + return queries.map(() => ({ + facets: { + startDate: { buckets: [], extent: { min: 0, max: 86400 } }, + 'startDate.exists': { + buckets: [{ value: 'true', count: 1, is: true }], + }, + }, + })) as never; + }, + }; + const response = await graphql({ + schema: buildGraphQLSchema(datedSchema, {}), + source: `{ dateds { facets { startDate { matching { min max } exists { value count } } } } }`, + contextValue: { engine, acceptLanguage: ['nl'] } as SearchContext, + }); + + expect(response.errors).toBeUndefined(); + expect(loaded).toEqual(['startDate', 'startDate.exists']); + expect(response.data).toEqual({ + dateds: { + facets: { + startDate: { + matching: { + min: '1970-01-01T00:00:00Z', + max: '1970-01-02T00:00:00Z', + }, + exists: [{ value: true, count: 1 }], + }, + }, + }, + }); + }); +}); + +describe('a lookup with presence leaves on a joined type', () => { + it('keeps its ids’ filter through the join, which reads no companion', () => { + const item = defineSearchType({ + name: 'Item', + class: `${SCHEMA_ORG}Product`, + fields: [ + { + name: 'work', + kind: 'reference', + filterable: true, + labelSource: 'Work', + joinable: true, + }, + ], + }); + const joinedPrinted = printSchema( + buildGraphQLSchema(searchSchema(work, place, item), {}), + ); + + expect(joinedPrinted).toMatch( + /input WorkJoinedWhere \{[^}]*contentLocation: PlaceFilter\n/, + ); + }); +}); + +describe('a keyword field counted by its values and its presence', () => { + it('serves its literal buckets beside `exists`', () => { + const tagged = defineSearchType({ + name: 'Tagged', + class: `${SCHEMA_ORG}Thing`, + fields: [ + { + name: 'label', + kind: 'text', + locales: ['nl'], + output: true, + searchable: { weight: 1 }, + }, + { + name: 'material', + kind: 'keyword', + facetable: true, + filterable: true, + presence: true, + }, + ], + }); + expect(printSchema(buildGraphQLSchema(searchSchema(tagged), {}))).toMatch( + /type TaggedMaterialFacet \{[^}]*buckets: \[ValueBucket!\]![^}]*totalValues: Int[^}]*exists: \[BooleanBucket!\]!\s*\}/, + ); + }); +}); diff --git a/packages/search-api-graphql/test/presence.test.ts b/packages/search-api-graphql/test/presence.test.ts index 5a7af6bc..53cd0675 100644 --- a/packages/search-api-graphql/test/presence.test.ts +++ b/packages/search-api-graphql/test/presence.test.ts @@ -165,9 +165,9 @@ describe('a field faceted on whether it has a value', () => { it('serves the facet under the field it counts', () => { const printed = printSchema(gqlSchema); - expect(printed).toMatch(/dateCreated: WorkDateCreatedFacets!/); + expect(printed).toMatch(/dateCreated: WorkDateCreatedFacet!/); expect(printed).toMatch( - /type WorkDateCreatedFacets \{[^}]*exists: \[BooleanBucket!\]!\s*\}/, + /type WorkDateCreatedFacet \{[^}]*exists: \[BooleanBucket!\]!\s*\}/, ); }); diff --git a/packages/search-api-graphql/test/query-cost.test.ts b/packages/search-api-graphql/test/query-cost.test.ts index 6ce4cbdb..8807e082 100644 --- a/packages/search-api-graphql/test/query-cost.test.ts +++ b/packages/search-api-graphql/test/query-cost.test.ts @@ -54,7 +54,12 @@ const place: SearchType = { output: true, searchable: { weight: 1 }, }, - { name: 'postalCode', kind: 'keyword', output: true }, + { + name: 'postalCode', + kind: 'keyword', + path: 'https://schema.org/postalCode', + output: true, + }, { name: 'containedIn', kind: 'reference', @@ -105,6 +110,36 @@ const work: SearchType = { output: true, ref: { strategy: 'lookup', target: ['Person', 'Organization'] }, }, + // Fields whose facet counts more than their own values, served as one + // object each. + { + name: 'material', + kind: 'keyword', + output: true, + filterable: true, + facetable: true, + presence: true, + }, + { + name: 'pages', + kind: 'integer', + output: true, + filterable: true, + facetable: true, + presence: true, + }, + { + name: 'location', + kind: 'reference', + output: true, + filterable: true, + facetable: true, + ref: { + strategy: 'lookup', + target: 'Place', + presenceLeaves: ['postalCode'], + }, + }, ], }; const schema = searchSchema(work, person, organization, place); @@ -201,6 +236,32 @@ describe('the cost of a query', () => { ).toBe(1 + 1); }); + it('charges each part of a field’s facet object it selects', async () => { + const facets = (selection: string) => + costOf(`{ creativeWorks(perPage: 0) { facets { ${selection} } } }`); + + expect(await facets('material { buckets { value } }')).toBe(1 + 1); + expect(await facets('material { totalValues }')).toBe(1 + 1); + expect(await facets('material { exists { value } }')).toBe(1 + 1); + // Each part is charged on its own, so the values’ two parts cost two. + expect( + await facets( + 'material { buckets { value } totalValues exists { value } }', + ), + ).toBe(1 + 3); + expect(await facets('pages { matching { min } }')).toBe(1 + 1); + expect(await facets('pages { buckets { value } }')).toBe(1 + 1); + expect(await facets('pages { all { min } exists { value } }')).toBe(1 + 2); + expect(await facets('location { postalCode { exists { value } } }')).toBe( + 1 + 1, + ); + expect( + await facets( + 'location { buckets { value } postalCode { exists { value } } }', + ), + ).toBe(1 + 2); + }); + it('charges a lookup inside a fragment like an inlined one', async () => { expect( await costOf( diff --git a/packages/search-api-graphql/vite.config.ts b/packages/search-api-graphql/vite.config.ts index 242dc709..453e092b 100644 --- a/packages/search-api-graphql/vite.config.ts +++ b/packages/search-api-graphql/vite.config.ts @@ -31,7 +31,7 @@ export default mergeConfig( // from a direct caller, since GraphQL validates the query first. // Re-anchored when facet grouping moved to `@lde/search`, taking its // covered branches with it. - branches: 99.05, + branches: 99.15, statements: 100, }, }, diff --git a/packages/search-pipeline/package.json b/packages/search-pipeline/package.json index 4451b4c1..0c050ce9 100644 --- a/packages/search-pipeline/package.json +++ b/packages/search-pipeline/package.json @@ -30,11 +30,11 @@ "@traqula/generator-sparql-1-1": "^1.3.1", "@traqula/parser-sparql-1-1": "^1.3.1", "@traqula/rules-sparql-1-1": "^1.2.0", + "n3": "^2.13.12", "tslib": "^2.3.0" }, "devDependencies": { "@lde/search-typesense": "^0.64.0", - "n3": "^2.13.12", "testcontainers": "^12.1.0", "typesense": "^3.2.0" }, diff --git a/packages/search-pipeline/src/extraction.ts b/packages/search-pipeline/src/extraction.ts index 3030aa50..ee40a98c 100644 --- a/packages/search-pipeline/src/extraction.ts +++ b/packages/search-pipeline/src/extraction.ts @@ -18,6 +18,7 @@ import { isInlineReference, labelFieldOf, localLookupTargetsOf, + physicalFields, referencedTargetsOf, referenceTypeNamed, } from '@lde/search/adapter'; @@ -143,20 +144,27 @@ export function extractionQueryString( /** * The fields a reference reads off a referent of a target it does not expand - * `local`ly: the target’s key field, where it is keyed, and – for a - * `searchable` reference – its label field, by which the reference is searched. + * `local`ly: the target’s key field, where it is keyed; – for a `searchable` + * reference – its label field, by which the reference is searched; and each of + * the reference’s presence leaves, whose companions say whether any referent + * holds one. Each field once, though it plays several of these parts. */ export function referentHopFields( field: SearchField, target: RootType, ): PathField[] { - return [ + const hopFields = [ keyedFieldOf(target), // `searchSchema` guarantees a searchable reference’s label a path. field.searchable === undefined ? undefined : (labelFieldOf(target) as PathField), + // …and a lookup’s target each presence leaf, with a path. + ...physicalFields(field).leafExists.map( + ({ leaf }) => fieldNamed(target, leaf) as PathField, + ), ].filter((hopField) => hopField !== undefined); + return [...new Set(hopFields)]; } /** A per-query counter minting distinct value/referent variable names. */ diff --git a/packages/search-pipeline/src/search-stages.ts b/packages/search-pipeline/src/search-stages.ts index 4361c82d..4fba1348 100644 --- a/packages/search-pipeline/src/search-stages.ts +++ b/packages/search-pipeline/src/search-stages.ts @@ -1,4 +1,5 @@ -import type { Quad, Term } from '@rdfjs/types'; +import type { NamedNode, Quad, Term } from '@rdfjs/types'; +import { DataFactory } from 'n3'; import { Distribution, type Dataset } from '@lde/dataset'; import { SparqlConstructReader, @@ -23,6 +24,7 @@ import { import { endpointFieldOf, irAlias, + physicalFields, isInlineReference, referencedTargetsOf, referenceTypeNamed, @@ -397,10 +399,13 @@ export function registrySource( export const defaultBatchSize = { endpoint: 10, imported: 100 } as const; /** - * A transform passing each referent of the type’s `searchable` references, - * its inline referents’ included, through its target type’s transforms: the + * A transform passing each referent of the type’s `searchable` references – + * its inline referents’ included – and of its lookups declaring presence + * leaves, through its target type’s transforms: the * quads the extraction read off the referent ({@link referentHopFields}) go * in, and what the target’s transforms make of them comes back in their place. + * A lookup’s referent named by IRI alone is seeded for its presence leaves, + * through the type’s own transforms too where the lookup names its own type. * `undefined` where no such reference names a target with transforms. * * The batch is buffered to find the referents, so memory stays bounded by the @@ -414,33 +419,41 @@ function referentTransforms( readonly QuadTransform[] >, ): QuadTransform | undefined { - // Per target with transforms: the reference fields naming it, and the - // aliases its referents carry. A target’s aliases are its own, so no quad - // belongs to two targets. + // Per target with transforms other than the type itself: the reference + // fields naming it, and the aliases its referents carry. A target’s aliases + // are its own, so no quad belongs to two targets. const routes = new Map< RootType, { fields: Set; aliases: Set } >(); - for (const { alias, field } of searchedReferences(searchType, schema)) { + // Per reference counting presence leaves: the one target (`searchSchema` + // refuses several) its referents named by IRI alone are seeded through. + const countedTargetByField = new Map(); + for (const { alias, field } of describedReferences(searchType, schema)) { for (const target of referencedTargetsOf(field, schema)) { - // A reference to its own type: the type’s transform saw the batch. - if ( - target !== searchType && - transformsByClass.get(target.class) !== undefined - ) { - const route = routes.get(target) ?? { - fields: new Set(), - aliases: new Set(), - }; - route.fields.add(alias); - for (const hopField of referentHopFields(field, target)) { - route.aliases.add(irAlias(target, hopField)); - } - routes.set(target, route); + if (transformsByClass.get(target.class) === undefined) { + continue; + } + if (physicalFields(field).leafExists.length > 0) { + countedTargetByField.set(alias, target); } + // A reference to its own type: the type’s transform saw the batch, its + // referents’ quads included. + if (target === searchType) { + continue; + } + const route = routes.get(target) ?? { + fields: new Set(), + aliases: new Set(), + }; + route.fields.add(alias); + for (const hopField of referentHopFields(field, target)) { + route.aliases.add(irAlias(target, hopField)); + } + routes.set(target, route); } } - if (routes.size === 0) { + if (routes.size === 0 && countedTargetByField.size === 0) { return undefined; } const targetsByField = new Map(); @@ -455,16 +468,27 @@ function referentTransforms( } return async function* referentTransforms(quads, context) { const batch: Quad[] = []; + // Every node the batch states anything of. + const subjects = new Set(); // Each target’s referents in this batch: the objects of the references - // naming it. + // naming it – and, by IRI, the ones a presence leaf is counted for. const referents = new Map>(); + const counted = new Map>(); for await (const quad of quads) { batch.push(quad); + subjects.add(termKey(quad.subject)); for (const target of targetsByField.get(quad.predicate.value) ?? []) { const keys = referents.get(target) ?? new Set(); keys.add(termKey(quad.object)); referents.set(target, keys); } + const countedTarget = countedTargetByField.get(quad.predicate.value); + if (countedTarget !== undefined && quad.object.termType === 'NamedNode') { + const nodes = + counted.get(countedTarget) ?? new Map(); + nodes.set(termKey(quad.object), quad.object); + counted.set(countedTarget, nodes); + } } // One pass sorting each quad into its target’s input, or keeping it. const inputs = new Map(); @@ -481,24 +505,68 @@ function referentTransforms( yield quad; } } + // A referent a presence leaf is counted for, and that its target’s + // transforms would see nothing of, enters them as ` rdf:type + // `: a transform describing a node from its key then sees it. A + // referent of another target is described by its input; one of the type’s + // own by anything the batch states of it, which the type’s transforms + // have already seen. The seed itself is not kept. + const seeds = new Map>(); + for (const [target, nodes] of counted) { + const input = inputs.get(target) ?? []; + const described = + target === searchType + ? subjects + : new Set(input.map((quad) => termKey(quad.subject))); + const seeded = new Set(); + for (const [key, node] of nodes) { + if (!described.has(key)) { + input.push( + factory.quad(node, RDF_TYPE, factory.namedNode(target.class)), + ); + seeded.add(key); + } + } + seeds.set(target, seeded); + inputs.set(target, input); + } for (const [target, input] of inputs) { + if (input.length === 0) { + continue; + } let stream: AsyncIterable = streamOf(input); for (const transform of transformsByClass.get( target.class, ) as readonly QuadTransform[]) { stream = transform(stream, context); } - yield* stream; + const seeded = seeds.get(target); + for await (const quad of stream) { + if ( + seeded?.has(termKey(quad.subject)) && + quad.predicate.equals(RDF_TYPE) && + quad.object.value === target.class + ) { + continue; + } + yield quad; + } } }; } -/** The type’s `searchable` reference fields, each with the IR Alias whose +const factory = DataFactory; +const RDF_TYPE = factory.namedNode( + 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type', +); + +/** The type’s reference fields whose referents are described – `searchable` + * ones, and lookups declaring presence leaves – each with the IR Alias whose * objects are its referents: its own, and those of the inline referents it * nests. A graph that leaves an edge out points at the endpoint through the * inline field itself, so an endpoint field is reached through both. * `searchSchema` refuses an inline cycle, so the descent ends. */ -function searchedReferences( +function describedReferences( owner: SearchType, schema: SearchSchema, ): { alias: string; field: SearchField }[] { @@ -511,13 +579,17 @@ function searchedReferences( ) as ReferenceType; const endpointField = endpointFieldOf(referenceType, field.ref.identity); return [ - ...searchedReferences(referenceType, schema), + ...describedReferences(referenceType, schema), ...(endpointField?.searchable === undefined ? [] : [{ alias: irAlias(owner, field), field: endpointField }]), ]; } - return field.kind === 'reference' && field.searchable !== undefined + // A reference’s referents are described when it is searched by their + // labels, or counted for a presence leaf. + return field.kind === 'reference' && + (field.searchable !== undefined || + physicalFields(field).leafExists.length > 0) ? [{ alias: irAlias(owner, field), field }] : []; }); diff --git a/packages/search-pipeline/test/extraction.test.ts b/packages/search-pipeline/test/extraction.test.ts index 97632407..aaf89996 100644 --- a/packages/search-pipeline/test/extraction.test.ts +++ b/packages/search-pipeline/test/extraction.test.ts @@ -20,7 +20,11 @@ import { referenceTypeNamed, } from '@lde/search/adapter'; import type { SearchSchema, SearchType } from '@lde/search'; -import { extractionQuery, extractionQueryString } from '../src/extraction.js'; +import { + extractionQuery, + extractionQueryString, + referentHopFields, +} from '../src/extraction.js'; const SCHEMA = 'https://schema.org/'; const RDF_TYPE = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type'; @@ -569,6 +573,26 @@ describe('references into a keyed type', () => { expect(group.patterns[1]).toMatchObject({ subType: 'optional' }); }); + it('reads a presence leaf that is also the key once', () => { + const counting = { + ...work.fields[0], + filterable: true, + searchable: { weight: 1 }, + ref: { + strategy: 'lookup', + target: 'Place', + presenceLeaves: ['_sameAs'], + }, + } as const; + const countingWork = defineSearchType({ ...work, fields: [counting] }); + searchSchema(countingWork, place); + + expect(referentHopFields(counting, place)).toEqual([ + place.fields[1], + place.fields[0], + ]); + }); + it('reads no label for a lookup that is not searchable', () => { expect( templatePredicates(extractionQuery(work, keyedSchema)), diff --git a/packages/search-pipeline/test/fixtures/lookup-presence-sample.ttl b/packages/search-pipeline/test/fixtures/lookup-presence-sample.ttl new file mode 100644 index 00000000..eb294efc --- /dev/null +++ b/packages/search-pipeline/test/fixtures/lookup-presence-sample.ttl @@ -0,0 +1,43 @@ +@prefix schema: . + +# Works naming places, for a presence leaf (`latitude`) counted on the work. + +# Named by IRI alone; the place transform describes GeoNames places from +# their IRI, coordinates included. + a schema:CreativeWork ; + schema:name "Geonames" ; + schema:contentLocation . + +# Named by IRI alone, and no transform describes it. + a schema:CreativeWork ; + schema:name "Unknown" ; + schema:contentLocation . + +# The work’s own graph states the place’s coordinates. + a schema:CreativeWork ; + schema:name "Stated" ; + schema:contentLocation . + schema:latitude 51.37 . + +# No place at all. + a schema:CreativeWork ; + schema:name "Bare" . + +# A place stated as a blank node: nothing to key a transform on. + a schema:CreativeWork ; + schema:name "Blank" ; + schema:contentLocation [ schema:latitude 50.85 ] . + +# Places within places, for a presence leaf on a lookup into its own type. + a schema:Place ; + schema:name "Village" ; + schema:containedInPlace . + a schema:Place ; + schema:name "Hamlet" ; + schema:containedInPlace . + a schema:Place ; + schema:name "Region" ; + schema:latitude 50.9 . + a schema:Place ; + schema:name "Isolated" ; + schema:containedInPlace . diff --git a/packages/search-pipeline/test/lookup-presence.integration.test.ts b/packages/search-pipeline/test/lookup-presence.integration.test.ts new file mode 100644 index 00000000..56cbd89d --- /dev/null +++ b/packages/search-pipeline/test/lookup-presence.integration.test.ts @@ -0,0 +1,288 @@ +import { fileURLToPath } from 'node:url'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { Dataset, Distribution } from '@lde/dataset'; +import type { QuadTransform, ReaderContext } from '@lde/pipeline'; +import { defineSearchType, searchSchema } from '@lde/search'; +import { irAlias } from '@lde/search/adapter'; +import { DataFactory } from 'n3'; +import { + startSparqlEndpoint, + teardownSparqlEndpoint, +} from '@lde/local-sparql-endpoint'; +import { searchStages, selectByClass } from '../src/search-stages.js'; + +const SCHEMA = 'https://schema.org/'; +const GEONAMES = 'https://sws.geonames.org/'; + +/** + * A lookup’s presence leaf, counted at index time: the extraction reads the + * leaf one hop from the root, the target type’s transform runs on the + * referent in the batch, and the projection writes whether any referent holds + * a value. A referent the graph names by IRI alone enters the transform all + * the same – the type’s own, for a lookup into its own type – so a transform + * describing a place from its key supplies its coordinates. A blank-node + * referent is no id the lookup stores, so it does not count. + */ +describe('a lookup’s presence leaf, through its target’s transform', () => { + const port = 3020; + const distribution = Distribution.sparql( + new URL(`http://localhost:${port}/sparql`), + ); + const dataset = new Dataset({ + iri: new URL('http://example.org/dataset/lookup-presence'), + distributions: [distribution], + }); + + const place = defineSearchType({ + name: 'Place', + class: `${SCHEMA}Place`, + labelField: 'name', + fields: [ + { + name: 'name', + kind: 'text', + path: `<${SCHEMA}name>`, + locales: ['und'], + output: true, + searchable: { weight: 1 }, + }, + { + name: 'latitude', + kind: 'number', + path: `<${SCHEMA}latitude>`, + output: true, + }, + ], + }); + const work = defineSearchType({ + name: 'Work', + class: `${SCHEMA}CreativeWork`, + labelField: 'name', + fields: [ + { + name: 'name', + kind: 'text', + path: `<${SCHEMA}name>`, + locales: ['und'], + output: true, + searchable: { weight: 1 }, + }, + { + name: 'contentLocation', + kind: 'reference', + path: `<${SCHEMA}contentLocation>`, + output: true, + filterable: true, + ref: { + strategy: 'lookup', + target: 'Place', + presenceLeaves: ['latitude'], + }, + }, + ], + }); + const schema = searchSchema(work, place); + + // The deployment’s place transform: a GeoNames place is described from its + // IRI, coordinates included – whatever else the batch says of it. + const latitudeAlias = irAlias(place, place.fields[1]); + const seen: string[] = []; + const geonamesCoordinates: QuadTransform = async function* ( + quads, + ) { + const subjects = new Set(); + for await (const quad of quads) { + subjects.add(quad.subject.value); + yield quad; + } + seen.push(...subjects); + for (const subject of subjects) { + if (subject.startsWith(GEONAMES)) { + yield DataFactory.quad( + DataFactory.namedNode(subject), + DataFactory.namedNode(latitudeAlias), + DataFactory.literal('51.37'), + ); + } + } + }; + + beforeAll(async () => { + await startSparqlEndpoint( + port, + fileURLToPath( + new URL('./fixtures/lookup-presence-sample.ttl', import.meta.url), + ), + ); + }, 60_000); + + afterAll(async () => { + await teardownSparqlEndpoint(); + }); + + async function project(withTransform: boolean, batchSize?: number) { + const [stage] = searchStages({ + schema, + types: [ + { + searchType: work, + rootVariable: 'root', + itemSelector: selectByClass(work), + ...(batchSize !== undefined && { batchSize }), + }, + { + searchType: place, + rootVariable: 'root', + itemSelector: selectByClass(place), + ...(withTransform && { transform: [geonamesCoordinates] }), + }, + ], + }); + const documents = new Map>(); + await stage.run(dataset, distribution, { + write: async (_dataset, items) => { + for await (const item of items) { + documents.set(String(item.document.name_und), item.document); + } + }, + }); + return documents; + } + + it('counts a referent the transform describes from its IRI alone', async () => { + const documents = await project(true); + + expect(documents.get('Geonames')?.contentLocation_latitude_exists).toBe( + true, + ); + expect(documents.get('Unknown')?.contentLocation_latitude_exists).toBe( + false, + ); + expect(documents.get('Stated')?.contentLocation_latitude_exists).toBe(true); + expect(documents.get('Bare')?.contentLocation_latitude_exists).toBe(false); + // Every referent entered the transform, the IRI-only ones included… + expect(seen).toEqual( + expect.arrayContaining([ + 'https://sws.geonames.org/2745707', + 'https://a/place/unknown', + 'https://a/place/stated', + ]), + ); + // …and what seeded them is not kept: the work still names one place. + expect(documents.get('Geonames')?.contentLocation).toBe( + 'https://sws.geonames.org/2745707', + ); + }); + + it('reads the leaf the work’s own graph states, without a transform', async () => { + const documents = await project(false); + + expect(documents.get('Geonames')?.contentLocation_latitude_exists).toBe( + false, + ); + expect(documents.get('Stated')?.contentLocation_latitude_exists).toBe(true); + }); + + it('counts per batch, a batch whose referents are all named by IRI alone included', async () => { + const documents = await project(true, 1); + + expect(documents.get('Geonames')?.contentLocation_latitude_exists).toBe( + true, + ); + expect(documents.get('Unknown')?.contentLocation_latitude_exists).toBe( + false, + ); + // A blank node is no id the lookup stores, so it does not count – not + // even its own statement. + expect(documents.get('Blank')?.contentLocation_latitude_exists).toBe(false); + }); + + describe('on a lookup into its own type', () => { + const region = defineSearchType({ + ...place, + fields: [ + ...place.fields, + { + name: 'containedInPlace', + kind: 'reference', + path: `<${SCHEMA}containedInPlace>`, + output: true, + filterable: true, + ref: { + strategy: 'lookup', + target: 'Place', + presenceLeaves: ['latitude'], + }, + }, + ], + }); + + async function projectRegions(batchSize?: number) { + const [stage] = searchStages({ + schema: searchSchema(region), + types: [ + { + searchType: region, + rootVariable: 'root', + itemSelector: selectByClass(region), + transform: [geonamesCoordinates], + ...(batchSize !== undefined && { batchSize }), + }, + ], + }); + const documents = new Map>(); + await stage.run(dataset, distribution, { + write: async (_dataset, items) => { + for await (const item of items) { + documents.set(String(item.document.name_und), item.document); + } + }, + }); + return documents; + } + + it('seeds its referents through the type’s own transforms', async () => { + seen.length = 0; + const documents = await projectRegions(); + + // Named by IRI alone, and described by the type’s own transform. + expect(documents.get('Village')?.containedInPlace_latitude_exists).toBe( + true, + ); + // Described in the batch, so read as stated and not seeded. + expect(documents.get('Hamlet')?.containedInPlace_latitude_exists).toBe( + true, + ); + expect(documents.get('Isolated')?.containedInPlace_latitude_exists).toBe( + false, + ); + expect(seen).toEqual( + expect.arrayContaining([ + 'https://sws.geonames.org/2751283', + 'https://a/place/nowhere', + ]), + ); + expect(seen.filter((iri) => iri === 'https://a/place/region')).toEqual([ + 'https://a/place/region', + ]); + // The seed is not kept: the place still names one place. + expect(documents.get('Village')?.containedInPlace).toBe( + 'https://sws.geonames.org/2751283', + ); + }); + + it('seeds nothing in a batch that describes every referent', async () => { + const documents = await projectRegions(1); + + expect(documents.get('Village')?.containedInPlace_latitude_exists).toBe( + true, + ); + expect(documents.get('Hamlet')?.containedInPlace_latitude_exists).toBe( + true, + ); + expect(documents.get('Isolated')?.containedInPlace_latitude_exists).toBe( + false, + ); + }); + }); +}); diff --git a/packages/search-typesense/src/query-compiler.ts b/packages/search-typesense/src/query-compiler.ts index 7730bba2..b6ec346d 100644 --- a/packages/search-typesense/src/query-compiler.ts +++ b/packages/search-typesense/src/query-compiler.ts @@ -566,12 +566,19 @@ function compileLeaf( return UNUSABLE; } // Typesense cannot test a field for holding a value, so presence reads the - // boolean companion the projection writes on every root document – never - // inside an entry, so under a nesting prefix there is nothing to read. + // boolean companion the projection writes on every root document – the + // field’s own, or a lookup’s for one of its presence leaves – never inside + // an entry, so under a nesting prefix there is nothing to read. // `assertValidQuery` rejects both first; this is the guard for a hand-built // query. if ('exists' in criterion) { - const exists = physicalFields(field).exists; + const { leaf } = criterion; + const exists = + leaf === undefined + ? physicalFields(field).exists + : physicalFields(field).leafExists.find( + (companion) => companion.leaf === leaf, + )?.exists; return exists === undefined || prefix !== '' ? UNUSABLE : `${exists}:=${criterion.exists}`; diff --git a/packages/search-typesense/test/lookup-presence.integration.test.ts b/packages/search-typesense/test/lookup-presence.integration.test.ts new file mode 100644 index 00000000..76d84559 --- /dev/null +++ b/packages/search-typesense/test/lookup-presence.integration.test.ts @@ -0,0 +1,248 @@ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import type { Client } from 'typesense'; +import { + searchSchema, + type Filter, + type SearchEngine, + type SearchQuery, + type SearchType, +} from '@lde/search'; +import { describeSearchEngineContract } from '@lde/search/testing'; +import { buildCollectionDefinition } from '../src/collection-definition.js'; +import { createTypesenseSearchEngine } from '../src/search.js'; +import { TypesenseContainer } from './typesense-container.js'; + +/** + * Presence on a lookup’s leaf – “a work with a place that has coordinates” – + * pinned against a real Typesense. The root document stores only the places’ + * ids, so the projection writes one boolean per work, + * `contentLocation_latitude_exists`, true when any referent holds a value; + * the filter and the facet read it, and a bucket counts each work once. The + * reference’s own facet on its ids stands beside it, capped by its own limit. + */ +describe('a root document filtered and faceted on whether a referent’s leaf has a value', () => { + const container = new TypesenseContainer(); + let client: Client; + let engine: SearchEngine; + + const kessel = 'https://ex/place/kessel'; + const venlo = 'https://sws.geonames.org/2745707'; + const roermond = 'https://sws.geonames.org/2748812'; + const place: SearchType = { + name: 'Place', + class: 'https://schema.org/Place', + fields: [ + { + name: 'label', + kind: 'text', + path: 'https://schema.org/name', + locales: ['nl'], + output: true, + searchable: { weight: 1 }, + }, + { + name: 'latitude', + kind: 'number', + path: 'https://schema.org/latitude', + output: true, + }, + { + name: 'longitude', + kind: 'number', + path: 'https://schema.org/longitude', + output: true, + }, + ], + }; + const work = { + name: 'Work', + class: 'https://schema.org/CreativeWork', + fields: [ + { + name: 'title', + kind: 'keyword', + path: 'https://schema.org/name', + output: true, + filterable: true, + sortable: true, + }, + { + name: 'contentLocation', + kind: 'reference', + path: 'https://schema.org/contentLocation', + array: true, + output: true, + filterable: true, + facetable: true, + ref: { + strategy: 'lookup', + target: 'Place', + presenceLeaves: ['latitude', 'longitude'], + }, + }, + ], + } as const satisfies SearchType; + const schema = searchSchema(work, place); + const facet = 'contentLocation.latitude.exists'; + const base: SearchQuery = { + where: [], + orderBy: [{ field: 'title', direction: 'asc' }], + limit: 10, + offset: 0, + facets: [facet], + locale: 'nl', + }; + const located = (exists: boolean, leaf = 'latitude'): Filter => ({ + or: [{ field: 'contentLocation', leaf, exists }], + }); + const counts = (result: { facets: Record }, name = facet) => + Object.fromEntries( + ( + result.facets[name] as { + buckets: { value: string; count: number; is?: boolean }[]; + } + ).buckets.map((bucket) => [ + String(bucket.is ?? bucket.value), + bucket.count, + ]), + ); + const titles = (result: { hits: readonly { id: string }[] }) => + result.hits.map((hit) => hit.id.replace('https://ex/', '')); + + beforeAll(async () => { + client = await container.start(); + await client.collections().create( + buildCollectionDefinition(work, { + collectionNameFor: () => 'works', + schema, + }), + ); + await client.collections().create( + buildCollectionDefinition(place, { + collectionNameFor: () => 'places', + schema, + }), + ); + const results = await client + .collections('works') + .documents() + .import( + [ + { + // Two places, one with coordinates: the work has one, and counts + // once. + id: 'https://ex/mixed', + title: 'mixed', + contentLocation: [kessel, venlo], + contentLocation_latitude_exists: true, + contentLocation_longitude_exists: true, + }, + { + id: 'https://ex/geonames', + title: 'geonames', + contentLocation: [roermond, venlo], + contentLocation_latitude_exists: true, + contentLocation_longitude_exists: false, + }, + { + id: 'https://ex/unlocated', + title: 'unlocated', + contentLocation: [kessel], + contentLocation_latitude_exists: false, + contentLocation_longitude_exists: false, + }, + // No place at all. + { + id: 'https://ex/bare', + title: 'bare', + contentLocation_latitude_exists: false, + contentLocation_longitude_exists: false, + }, + ], + { action: 'create' }, + ); + expect(results.every((result) => result.success)).toBe(true); + engine = createTypesenseSearchEngine(client, schema, { + collections: { Work: 'works', Place: 'places' }, + }); + }, 120_000); + + afterAll(async () => { + await container.stop(); + }); + + describeSearchEngineContract( + 'a Typesense engine serving a lookup leaf’s presence', + () => engine, + ); + + it('counts each work once, under whether any of its places has the leaf', async () => { + expect(counts(await engine.search(work, base))).toEqual({ + true: 2, + false: 2, + }); + }); + + it('filters on any referent holding a value, and on none holding one', async () => { + const some = await engine.search(work, { + ...base, + where: [located(true)], + }); + const none = await engine.search(work, { + ...base, + where: [located(false)], + }); + + expect(titles(some)).toEqual(['geonames', 'mixed']); + expect(titles(none)).toEqual(['bare', 'unlocated']); + }); + + it('ANDs two leaves, each counted without its own selection', async () => { + const longitude = 'contentLocation.longitude.exists'; + const result = await engine.search(work, { + ...base, + facets: [facet, longitude], + where: [located(true), located(true, 'longitude')], + }); + + expect(titles(result)).toEqual(['mixed']); + expect(counts(result)).toEqual({ true: 1 }); + expect(counts(result, longitude)).toEqual({ true: 1, false: 1 }); + }); + + it('counts without its own selection, and filters the hits by it', async () => { + const result = await engine.search(work, { + ...base, + where: [located(false)], + }); + + expect(titles(result)).toEqual(['bare', 'unlocated']); + expect(counts(result)).toEqual({ true: 2, false: 2 }); + }); + + it('counts the reference’s own values beside it, the limit capping those alone', async () => { + const result = await engine.search(work, { + ...base, + facets: ['contentLocation', facet], + facetLimits: { contentLocation: 1 }, + }); + + // Venlo is in two works, Kessel in two – one bucket, of the top count. + expect( + (result.facets.contentLocation as { buckets: unknown[] } | undefined) + ?.buckets, + ).toHaveLength(1); + expect(counts(result)).toEqual({ true: 2, false: 2 }); + }); + + it('counts the presence under a selection of the reference’s own ids, which it does not own', async () => { + const result = await engine.search(work, { + ...base, + facets: ['contentLocation', facet], + where: [{ or: [{ field: 'contentLocation', in: [kessel] }] }], + }); + + expect(titles(result)).toEqual(['mixed', 'unlocated']); + expect(counts(result)).toEqual({ true: 1, false: 1 }); + }); +}); diff --git a/packages/search-typesense/test/query-compiler.test.ts b/packages/search-typesense/test/query-compiler.test.ts index 42aca64b..6da5da8e 100644 --- a/packages/search-typesense/test/query-compiler.test.ts +++ b/packages/search-typesense/test/query-compiler.test.ts @@ -943,6 +943,26 @@ describe('`exists` welded to an inline entry', () => { ]), ).toBe('id:=[]'); expect(filterBy([{ field: 'format', exists: true }])).toBe('id:=[]'); + // Nor a reference the type does not declare. + expect( + buildSearchParams( + { + ...base, + where: [ + { + or: [ + { + field: 'nope', + entry: [{ field: 'thumbnailUrl', exists: true }], + }, + ], + }, + ], + }, + work, + { schema: mediaSchema }, + ).filter_by, + ).toBe('id:=[]'); }); it('compiles to a term matching nothing under a nesting, where no boolean is written', () => { @@ -1001,3 +1021,58 @@ describe('`exists` welded to an inline entry', () => { ).toBe('id:=[]'); }); }); + +describe('`exists` on a lookup’s leaf', () => { + const place: SearchType = { + name: 'Place', + class: 'https://schema.org/Place', + fields: [ + { + name: 'label', + kind: 'text', + path: 'https://schema.org/name', + locales: ['nl'], + output: true, + searchable: { weight: 1 }, + }, + { name: 'latitude', kind: 'number', path: 'https://schema.org/latitude' }, + ], + }; + const work: SearchType = { + name: 'Work', + class: 'https://schema.org/CreativeWork', + fields: [ + { + name: 'contentLocation', + kind: 'reference', + filterable: true, + ref: { + strategy: 'lookup', + target: 'Place', + presenceLeaves: ['latitude'], + }, + }, + ], + }; + const placeSchema = searchSchema(work, place); + const filterBy = (criterion: Record) => + buildSearchParams( + { ...base, where: [{ or: [criterion] } as never] }, + work, + { schema: placeSchema }, + ).filter_by; + + it('reads the root document’s boolean, a plain criterion', () => { + expect( + filterBy({ field: 'contentLocation', leaf: 'latitude', exists: true }), + ).toBe('contentLocation_latitude_exists:=true'); + }); + + it('compiles to a term matching nothing on a leaf named no presence', () => { + // `assertValidQuery` rejects it first; this is the guard for a direct + // caller. + expect( + filterBy({ field: 'contentLocation', leaf: 'label', exists: true }), + ).toBe('id:=[]'); + }); +}); diff --git a/packages/search-typesense/vite.config.ts b/packages/search-typesense/vite.config.ts index 06da98e0..6138942b 100644 --- a/packages/search-typesense/vite.config.ts +++ b/packages/search-typesense/vite.config.ts @@ -22,10 +22,10 @@ export default mergeConfig( // by-name map for the shared `rootTypeNamed`: one COVERED arrow fewer // over one function fewer, which lowers the ratio without uncovering // anything. - functions: 99.38, + functions: 99.4, // Lines, branches and statements moved the same way when the range // bound reading went to `@lde/search` as `storedRangeBound`. - lines: 99.59, + lines: 99.68, // Re-anchored for the projection lookup: its guards against a // projection naming what no lookup reaches are unreachable through // the port, since `assertValidQuery` rejects such a query first. @@ -34,8 +34,8 @@ export default mergeConfig( // companion: covered branches deleted, nothing uncovered. And when // a field declaring presence alone went back to being internal, // deleting the covered branches that kept its value unindexed. - branches: 97.3, - statements: 99.6, + branches: 97.48, + statements: 99.69, }, }, }, diff --git a/packages/search/src/project.ts b/packages/search/src/project.ts index bd437512..52be1c03 100644 --- a/packages/search/src/project.ts +++ b/packages/search/src/project.ts @@ -151,7 +151,7 @@ export function projectDocument( ) as SearchDocument; // After every field, so a presence reads what a later derive left too; on // the root alone, since presence is declared only on a Root Type’s own field - // and on its inline references’ leaves. + // and on its references’ leaves. for (const field of searchType.fields) { if (field.presence === true) { document[physicalFields(field).exists as string] = holdsPresence( @@ -159,7 +159,7 @@ export function projectDocument( field, ); } - writeLeafPresence(document, field, schema); + writeLeafPresence(document, node, field, searchType, schema, context); } pruneInternalFields(document, searchType, schema); return document; @@ -493,37 +493,79 @@ function holdsPresence(document: ProjectedNode, field: SearchField): boolean { } /** - * Write whether any entry of an inline reference holds a value for each of - * its {@link ReferenceStrategy.presenceLeaves presence leaves}: one boolean - * per root document, `false` with no entries at all. Read before pruning, so - * a leaf declaring no Role of its own still counts. Without a schema the - * entries are not projected, so there is nothing to count – and nothing is - * written. + * Write whether any referent of a reference holds a value for each of its + * {@link ReferenceStrategy.presenceLeaves presence leaves}: one boolean per + * root document, `false` with no referents at all. Without a schema there is + * no type to read a leaf through, so nothing is written. + * + * An inline reference’s entries are read as projected, before pruning, so a + * leaf declaring no Role of its own still counts. A lookup stores only ids, so + * its referents are read off the frame instead: each leaf is projected from + * the referent’s node as its target declares it – the quads the extraction + * read one hop from the root, after the target’s transforms + * (`@lde/search-pipeline`). Only a referent with an IRI counts: a blank node + * is no id the lookup stores. One the frame names by IRI alone holds no value. */ function writeLeafPresence( document: ProjectedNode, + node: FramedNode, field: SearchField, + searchType: SearchType, schema: SearchSchema | undefined, + context: ProjectionContext, ): void { const leaves = physicalFields(field).leafExists; - const referenceType = - schema === undefined || leaves.length === 0 || !isInlineReference(field) - ? undefined - : referenceTypeNamed(schema, field.ref.typeName); - if (referenceType === undefined) { + if (schema === undefined || leaves.length === 0) { return; } - const stored = document[field.name]; - const entries = ( - stored === undefined ? [] : Array.isArray(stored) ? stored : [stored] - ) as ProjectedNode[]; + if (isInlineReference(field)) { + // Guaranteed declared: `searchSchema` resolves every inline `typeName`, + // and every presence leaf against it. + const referenceType = referenceTypeNamed( + schema, + field.ref.typeName, + ) as ReferenceType; + const stored = document[field.name]; + const entries = ( + stored === undefined ? [] : Array.isArray(stored) ? stored : [stored] + ) as ProjectedNode[]; + for (const { leaf: name, exists } of leaves) { + const leaf = fieldNamed(referenceType, name) as SearchField; + document[exists] = entries.some((entry) => holdsPresence(entry, leaf)); + } + return; + } + const referents = valuesOf(node, irAlias(searchType, field)).filter( + (value) => isObject(value) && iriString(value) !== undefined, + ) as FramedNode[]; + // `searchSchema` holds presence leaves to a lookup naming one target, and + // that target to declaring each leaf. + const [target] = referencedTargetsOf(field, schema); for (const { leaf: name, exists } of leaves) { - // `searchSchema` has resolved every presence leaf against the type. - const leaf = fieldNamed(referenceType, name) as SearchField; - document[exists] = entries.some((entry) => holdsPresence(entry, leaf)); + const leaf = fieldNamed(target, name) as SearchField; + document[exists] = referents.some((referent) => { + const read: ProjectedNode = {}; + applyField( + read, + referent, + leaf, + target, + schema, + context, + ignoreUnread, + true, + ); + return holdsPresence(read, leaf); + }); } } +/** A sink for a leaf read only to tell whether it holds a value: the + * referent’s own document reports what it cannot read. */ +function ignoreUnread(): void { + // The referent’s own document reports it. +} + /** Whether a stored value is one: neither `undefined` nor `null`, nor an * empty list. */ function holdsValue(value: unknown): boolean { diff --git a/packages/search/src/query.ts b/packages/search/src/query.ts index 8d22275b..24fd54ce 100644 --- a/packages/search/src/query.ts +++ b/packages/search/src/query.ts @@ -4,6 +4,7 @@ import { filterOperatorFor, ID_FIELD, holdsNoIds, + isInlineReference, nestedReferenceType, physicalFields, presenceFacetName, @@ -147,8 +148,20 @@ export type Criterion = * declaring {@link SearchFieldBase.presence}, whatever its kind, beside the * operator its kind fixes – on the searched type’s own field, not yet through * a join path or inside a weld. + * + * With a `leaf`, whether **any referent** of the `lookup` the field names + * holds a value for that leaf, one of the lookup’s + * {@link ReferenceStrategy.presenceLeaves presence leaves}: + * `{ field: 'contentLocation', leaf: 'latitude', exists: true }`. Like the + * field’s own presence it is recorded on the root document, so it is a plain + * criterion, combined with others like any. An inline reference’s presence + * leaf is instead selected by a weld of that one condition, as its other + * leaves are. */ -export type PresenceCriterion = CriterionBase & { readonly exists: boolean }; +export type PresenceCriterion = CriterionBase & { + readonly exists: boolean; + readonly leaf?: string; +}; /** * The conditions a criterion welds to **one entry** of the nested reference its @@ -312,8 +325,9 @@ export interface FacetSearch { * An `exists` criterion is on the axis `‹field›.exists`, which the field’s * presence facet owns. It owns nothing else: a range on the same field is * another axis, and so a presence facet counts under it. Welded alone to an - * entry, `exists` is on the axis `‹reference›.‹leaf›.exists`, owned by that - * presence leaf’s facet and never by a facet on the leaf’s values. + * entry – or naming a lookup’s `leaf` – `exists` is on the axis + * `‹reference›.‹leaf›.exists`, owned by that presence leaf’s facet and never + * by a facet on the leaf’s values. * * A default a surface adds to `where` (a valid-only `status`) is a clause * like any other: its facet counts across every value and reports the @@ -421,8 +435,9 @@ export interface QueryIssue { | 'inverted-range' /** * An `exists` criterion on a field that declares no - * {@link SearchFieldBase.presence} – or, welded to one entry, on a leaf - * its inline reference does not name among its + * {@link SearchFieldBase.presence} – or on a leaf, welded to one entry of + * an inline reference or named by `leaf` on a lookup, that the reference + * does not name among its * {@link ReferenceStrategy.presenceLeaves presence leaves} – so no * document records whether it holds a value. */ @@ -465,10 +480,23 @@ function collectLeafIssues( return; } // Presence is a Role of its own: a field declaring it takes `exists` - // whether or not it is `filterable`, and whatever operator its kind fixes. + // whether or not it is `filterable`, and whatever operator its kind fixes – + // and a lookup takes it on each of its presence leaves. if ('exists' in criterion) { - if (declared.presence !== true) { - issues.push({ part: 'where', field, reason: 'no-presence' }); + const recorded = + criterion.leaf === undefined + ? declared.presence === true + : !isInlineReference(declared) && + physicalFields(declared).leafExists.some( + ({ leaf }) => leaf === criterion.leaf, + ); + if (!recorded) { + issues.push({ + part: 'where', + field: + criterion.leaf === undefined ? field : `${field}.${criterion.leaf}`, + reason: 'no-presence', + }); } return; } @@ -911,7 +939,11 @@ function axisOf(criterion: Criterion): string | undefined { // presence facet: a range on the same field is not its selection, nor is an // `exists` the selection of a facet on the field’s values. if ('exists' in criterion) { - return presenceFacetName(path.join('.')); + return presenceFacetName( + [...path, ...(criterion.leaf === undefined ? [] : [criterion.leaf])].join( + '.', + ), + ); } if (!isWelded(criterion)) { return path.join('.'); @@ -937,6 +969,19 @@ function axisOf(criterion: Criterion): string | undefined { */ export function selectionOf(facet: FacetDeclaration, value: string): Filter { const field = facet.field.name; + // A lookup’s presence leaf is recorded on the root document, and selected + // there, by the reference and the leaf. + if ( + facet.presence === true && + facet.reference !== undefined && + !isInlineReference(facet.reference) + ) { + return filterOn({ + field: facet.reference.name, + leaf: field, + exists: value === 'true', + }); + } const condition: Criterion = facet.presence === true ? { field, exists: value === 'true' } diff --git a/packages/search/src/schema.ts b/packages/search/src/schema.ts index c6c6b8f2..ebf9956f 100644 --- a/packages/search/src/schema.ts +++ b/packages/search/src/schema.ts @@ -155,10 +155,12 @@ export interface SearchFieldBase { * {@link isInternalField internal field} – projected, read for its * companion, then pruned, its value stored nowhere. * - * Only on a Root Type’s own field, and not yet on an inline reference, a - * `text` field, a `boolean` (which states its own absence as `false`) or a - * field that is already `facetable` ({@link validateSearchType} enforces - * all of them). + * Only on a Root Type’s own field, and not on an inline reference (whose + * entries’ leaves declare it through + * {@link ReferenceStrategy.presenceLeaves}), a `text` field or a `boolean`, + * which states its own absence as `false` ({@link validateSearchType} + * enforces all of them). A field that is also `facetable` serves both + * facets. */ readonly presence?: boolean; /** @@ -350,7 +352,23 @@ export type ReferenceStrategy = readonly literal?: LiteralKind; readonly identity?: never; readonly facetedLeaves?: never; - readonly presenceLeaves?: never; + /** + * The `name`s of **leaf fields** of the target the root document is + * filtered and faceted on by whether **any referent** holds a value – + * as an inline reference’s {@link presenceLeaves} are, read here from + * each referent’s own node one hop from the root, after the target’s + * transforms (`@lde/search-pipeline`). Only a referent named by IRI + * counts, since a blank node is no id the lookup stores; one the graph + * names by IRI alone, and no transform describes, holds no value. + * + * Only on a lookup naming one `target`, which must declare each leaf, + * with a `path`; neither on a `joinable` lookup, whose `where` already + * states a condition on the joined document, nor on one reading its + * literals as periods ({@link searchSchema} enforces all of it). A + * criterion selects the leaf by `exists`, naming the reference and the + * `leaf`: `{ field, leaf, exists }`. + */ + readonly presenceLeaves?: readonly string[]; } | { readonly strategy: 'inline'; @@ -420,10 +438,9 @@ export type ReferenceStrategy = * Declared on the reference, like {@link facetedLeaves}, because the * count is a fact about the root. The reference must be `filterable` and * sit on a Root Type; each leaf is neither `text`, `boolean`, an inline - * reference nor one of the reference’s faceted leaves; and every inline - * reference nesting the same reference type declares the same leaves, - * since they share one filter input ({@link searchSchema} enforces all - * of it). + * reference, nor named `buckets`, `totalValues` or `exists`, which name + * parts of the reference’s facet object ({@link searchSchema} enforces + * all of it). */ readonly presenceLeaves?: readonly string[]; }; @@ -1296,7 +1313,6 @@ function assertResolvableInlineReferences( assertFacetedLeaves(searchType, field, referenceType); } } - assertSharedPresenceLeaves(types); for (const searchType of types) { for (const field of searchType.fields) { assertServiceableLocalLookup(searchType, field); @@ -1496,11 +1512,6 @@ function assertFacetedLeaves( `${where} declares “facetedLeaves”, but “${searchType.name}” is a reference type: a faceted leaf counts root documents, and an entry is not one. Declare it on the Root Type’s reference instead.`, ); } - if (field.facetable === true) { - throw new Error( - `${where} is “facetable” and declares “facetedLeaves”: its facet and its leaves’ facets would both be named after “${field.name}”. Facet the reference or its leaves, not both.`, - ); - } if (field.filterable !== true) { throw new Error( `${where} declares “facetedLeaves” without “filterable”: a bucket is selected by a condition on one entry, which the reference must accept. Declare “filterable” on it.`, @@ -1512,6 +1523,11 @@ function assertFacetedLeaves( throw new Error(`${where} names faceted leaf “${name}” twice.`); } seen.add(name); + if (RESERVED_LEAF_NAMES.has(name)) { + throw new Error( + `${where} names faceted leaf “${name}”: “buckets”, “totalValues” and “exists” name parts of the reference’s facet object.`, + ); + } const leaf = fieldNamed(referenceType, name); if (leaf === undefined) { throw new Error( @@ -1542,90 +1558,56 @@ function assertFacetedLeaves( /** * {@link SearchFieldBase.presence Presence} counts what one field of a root - * document holds, and an inline reference’s + * document holds, and a reference’s * {@link ReferenceStrategy.presenceLeaves presence leaves} what any of its - * entries holds, each written to a companion. Throws the first refusal + * referents holds, each written to a companion. Throws the first refusal * {@link presenceIssues} files – resolving each presence leaf against the - * reference type it names – as a sentence linking the issue that lifts it, - * where there is one. + * types the reference names – as a sentence. */ function assertServiceablePresence(types: readonly SearchType[]): void { - const referenceTypes = new Map( - types - .filter((searchType) => searchType.class === undefined) - .map((searchType) => [searchType.name, searchType]), + const typesByName = new Map( + types.map((searchType) => [searchType.name, searchType]), ); for (const searchType of types) { - const [issue] = presenceIssues(searchType, referenceTypes); + const [issue] = presenceIssues(searchType, typesByName); if (issue === undefined) { continue; } const [name, leaf] = issue.field.split('.'); const at = `“${searchType.name}.${name}”`; const field = `Field ${at} declares “presence”`; - const leaves = `Inline reference ${at} declares “presenceLeaves”`; - const named = `Inline reference ${at} names presence leaf “${leaf}”`; - const objectFacet = - 'would have to become an object to hold the presence buckets beside the value buckets, a change https://github.com/ldelements/lde/issues/1090 makes. Declare one of the two until then.'; + const leaves = `Reference ${at} declares “presenceLeaves”`; + const named = `Reference ${at} names presence leaf “${leaf}”`; + // Only a presence leaf’s issue names a reference whose type it is read + // through; every sentence below is built, so it is read here alone. + const owner = + issue.reason === 'presence-leaf-unknown' + ? presenceLeafOwnerName(fieldNamed(searchType, name) as SearchField) + : undefined; throw new Error( { 'presence-on-reference-type': `${field}, but “${searchType.name}” is a reference type: presence is counted on a root document. To count whether any entry holds a value for this leaf, name it in the “presenceLeaves” of the Root Type’s inline reference instead.`, 'presence-with-inline-ref': `${field} on an inline reference, which stores entries rather than a value. To count whether any entry holds a value for one of their leaves, name the leaf in the reference’s “presenceLeaves”.`, 'presence-on-text': `${field} on a “text” field, which stores its values per language for display and search, so no one stored value says whether it has one. Declare presence on a “keyword” field over the same path.`, 'presence-on-boolean': `${field} on a “boolean”, which states its own absence as “false”: filter it with “is” instead.`, - 'presence-with-facetable': `${field} and is “facetable”: its facet ${objectFacet}`, 'presence-leaves-on-reference-type': `${leaves}, but “${searchType.name}” is a reference type: presence is counted on a root document, and an entry is not one. Declare it on the Root Type’s reference instead.`, - 'presence-leaves-not-inline': `${at} declares “presenceLeaves” but is no inline reference, so it has no entries to count; a lookup target’s leaf waits for https://github.com/ldelements/lde/issues/1090.`, - 'presence-leaves-without-filterable': `${leaves} without “filterable”: a bucket is selected by a condition on one entry, which the reference must accept. Declare “filterable” on it.`, - 'presence-leaves-with-facetable': `${leaves} and is “facetable”: its facet ${objectFacet}`, + 'presence-leaves-on-id-only': `${leaves} but names no type its referents are read through: declare it on an “inline” reference or a “lookup”.`, + 'presence-leaves-with-several-targets': `${leaves} and names several targets: a leaf is read through one target’s declaration, so declare presence leaves on a lookup naming one.`, + 'presence-leaves-with-joinable': `${leaves} and is “joinable”: its “where” already states a condition on the joined document, so a leaf’s presence is not offered there yet.`, + 'presence-leaves-with-literal': `${leaves} and reads its literals as periods: a literal has no leaves to count.`, + 'presence-leaves-without-filterable': `${leaves} without “filterable”: a bucket is selected by a condition on the reference, which it must accept. Declare “filterable” on it.`, 'presence-leaves-duplicate': `${leaves} and names one leaf twice.`, - 'presence-leaf-unknown': `${named}, which “${(fieldNamed(searchType, name) as { readonly ref?: { readonly typeName?: string } } | undefined)?.ref?.typeName}” does not declare.`, + 'presence-leaf-unknown': `${named}, which “${owner}” does not declare.`, + 'presence-leaf-reserved-name': `${named}: “buckets”, “totalValues” and “exists” name parts of the reference’s facet object.`, 'presence-leaf-on-text': `${named}, a “text”: it stores its values per language, so no one stored value says whether it has one.`, 'presence-leaf-on-boolean': `${named}, a “boolean”: it states its own absence as “false”.`, 'presence-leaf-inline-ref': `${named}, which is itself an inline reference: a leaf of an entry’s entries is not counted.`, - 'presence-leaf-faceted': `${named}, which is also one of its faceted leaves: the leaf’s facet ${objectFacet}`, + 'presence-leaf-without-path': `${named}, which declares no “path”: a lookup reads its referents’ leaves one hop from the root, so the leaf must be read from the graph.`, }[issue.reason as PresenceIssueReason], ); } } -/** - * Every inline reference that gets a filter input – a `filterable` one on a - * Root Type – and nests one reference type must declare the same - * {@link ReferenceStrategy.presenceLeaves presence leaves}: the reference - * type’s `where` input is shared by all of them, and which of its keys take - * `exists` is decided by those leaves. Two references disagreeing would offer - * `exists` where one of them cannot answer it. A temporary rule: per-reference - * filter inputs ([#1090](https://github.com/ldelements/lde/issues/1090)) lift - * it. - */ -function assertSharedPresenceLeaves(types: readonly SearchType[]): void { - const declared = new Map(); - for (const searchType of types) { - if (searchType.class === undefined) { - continue; - } - for (const field of searchType.fields) { - if (!isInlineReference(field) || field.filterable !== true) { - continue; - } - const at = `${searchType.name}.${field.name}`; - const leaves = physicalFields(field) - .leafExists.map(({ leaf }) => leaf) - .sort() - .join(', '); - const first = declared.get(field.ref.typeName); - if (first === undefined) { - declared.set(field.ref.typeName, { at, leaves }); - } else if (first.leaves !== leaves) { - throw new Error( - `Inline references “${first.at}” and “${at}” both nest “${field.ref.typeName}” but declare different “presenceLeaves” (${first.leaves || 'none'}; ${leaves || 'none'}): they share one filter input on “${field.ref.typeName}”, so declare the same leaves on both.`, - ); - } - } - } -} - /** The reasons {@link presenceIssues} files a refused presence under. */ type PresenceIssueReason = Extract< SearchTypeIssue['reason'], @@ -1635,15 +1617,16 @@ type PresenceIssueReason = Extract< /** * The issues a type’s presence declarations carry – a field’s * {@link SearchFieldBase.presence presence}, refused wherever a root - * document’s companion cannot answer it yet, and an inline reference’s + * document’s companion cannot answer it, and a reference’s * {@link ReferenceStrategy.presenceLeaves presence leaves}. One issue per * field, the first rule it breaks; a presence leaf’s is filed under - * `‹reference›.‹leaf›`. The leaves are checked against the reference type - * only where `referenceTypes` resolves it – one declaration cannot see it. + * `‹reference›.‹leaf›`. The leaves are checked against the types the + * reference names only where `typesByName` resolves them – one declaration + * cannot see them. */ function presenceIssues( searchType: SearchType, - referenceTypes?: ReadonlyMap, + typesByName?: ReadonlyMap, ): readonly SearchTypeIssue[] { return searchType.fields.flatMap((field): SearchTypeIssue[] => { const reason: PresenceIssueReason | undefined = @@ -1657,56 +1640,77 @@ function presenceIssues( ? 'presence-on-text' : field.kind === 'boolean' ? 'presence-on-boolean' - : field.facetable === true - ? 'presence-with-facetable' - : undefined; + : undefined; const issues: SearchTypeIssue[] = reason === undefined ? [] : [{ field: field.name, reason }]; - return [ - ...issues, - ...presenceLeavesIssues(searchType, field, referenceTypes), - ]; + return [...issues, ...presenceLeavesIssues(searchType, field, typesByName)]; }); } -/** The issues an inline reference’s +/** The name of the one type the presence leaves of an inline reference or a + * lookup are read through: the reference type, or the lookup’s single + * target. */ +function presenceLeafOwnerName(field: SearchField): string { + const ref = (field as FlatField).ref as NonNullable; + return ref.strategy === 'inline' + ? ref.typeName + : (labelSourceNamesOf(field as ReferenceField)[0] as string); +} + +/** The leaf names a reference’s facet object holds of its own. */ +const RESERVED_LEAF_NAMES: ReadonlySet = new Set([ + 'buckets', + 'totalValues', + 'exists', +]); + +/** The issues a reference’s * {@link ReferenceStrategy.presenceLeaves presence leaves} carry: the - * reference’s own first, then – where the reference type resolves – each + * reference’s own first, then – where the type it names resolves – each * leaf’s. */ function presenceLeavesIssues( searchType: SearchType, field: SearchField, - referenceTypes: ReadonlyMap | undefined, + typesByName: ReadonlyMap | undefined, ): readonly SearchTypeIssue[] { - const leaves = (field as FlatField).ref?.presenceLeaves; + const flat = field as FlatField; + const leaves = flat.ref?.presenceLeaves; if (leaves === undefined || leaves.length === 0) { return []; } + const lookup = flat.ref?.strategy === 'lookup'; const reason: PresenceIssueReason | undefined = searchType.class === undefined ? 'presence-leaves-on-reference-type' - : !isInlineReference(field) - ? 'presence-leaves-not-inline' - : field.filterable !== true - ? 'presence-leaves-without-filterable' - : field.facetable === true - ? 'presence-leaves-with-facetable' - : new Set(leaves).size !== leaves.length - ? 'presence-leaves-duplicate' - : undefined; + : flat.kind !== 'reference' || + (flat.ref?.strategy !== 'inline' && !lookup) + ? 'presence-leaves-on-id-only' + : lookup && labelSourceNamesOf(field as ReferenceField).length > 1 + ? 'presence-leaves-with-several-targets' + : flat.joinable === true + ? 'presence-leaves-with-joinable' + : literalKindOf(flat) !== undefined + ? 'presence-leaves-with-literal' + : field.filterable !== true + ? 'presence-leaves-without-filterable' + : new Set(leaves).size !== leaves.length + ? 'presence-leaves-duplicate' + : undefined; if (reason !== undefined) { return [{ field: field.name, reason }]; } - const referenceType = referenceTypes?.get( - (field as { readonly ref: { readonly typeName: string } }).ref.typeName, - ); - if (referenceType === undefined) { + // The reason checks above leave an inline reference or a lookup. + const owner = typesByName?.get(presenceLeafOwnerName(field)); + if (owner === undefined) { return []; } return leaves.flatMap((name): SearchTypeIssue[] => { - const leaf = fieldNamed(referenceType, name); - const leafReason: PresenceIssueReason | undefined = - leaf === undefined + const leaf = fieldNamed(owner, name); + const leafReason: PresenceIssueReason | undefined = RESERVED_LEAF_NAMES.has( + name, + ) + ? 'presence-leaf-reserved-name' + : leaf === undefined ? 'presence-leaf-unknown' : leaf.kind === 'text' ? 'presence-leaf-on-text' @@ -1714,8 +1718,8 @@ function presenceLeavesIssues( ? 'presence-leaf-on-boolean' : isInlineReference(leaf) ? 'presence-leaf-inline-ref' - : facetedLeavesOf(field).includes(name) - ? 'presence-leaf-faceted' + : lookup && leaf.path === undefined + ? 'presence-leaf-without-path' : undefined; return leafReason === undefined ? [] @@ -2277,17 +2281,19 @@ export interface SearchTypeIssue { | 'presence-with-inline-ref' | 'presence-on-text' | 'presence-on-boolean' - | 'presence-with-facetable' | 'presence-leaves-on-reference-type' - | 'presence-leaves-not-inline' + | 'presence-leaves-on-id-only' + | 'presence-leaves-with-several-targets' + | 'presence-leaves-with-joinable' + | 'presence-leaves-with-literal' | 'presence-leaves-without-filterable' - | 'presence-leaves-with-facetable' | 'presence-leaves-duplicate' | 'presence-leaf-unknown' + | 'presence-leaf-reserved-name' | 'presence-leaf-on-text' | 'presence-leaf-on-boolean' | 'presence-leaf-inline-ref' - | 'presence-leaf-faceted'; + | 'presence-leaf-without-path'; } /** @@ -2903,12 +2909,12 @@ export interface PhysicalFields { */ readonly exists: string | undefined; /** - * The **presence leaf companions** of an inline reference declaring - * {@link ReferenceStrategy.presenceLeaves presence leaves}: per leaf, - * `${name}_${leaf}_exists`, a top-level boolean of the root document stating - * whether any entry holds a value for the leaf, written on every document – - * `false` with no entries. Indexed and faceted; never served. Empty for - * every other field. + * The **presence leaf companions** of a reference – inline or `lookup` – + * declaring {@link ReferenceStrategy.presenceLeaves presence leaves}: per + * leaf, `${name}_${leaf}_exists`, a top-level boolean of the root document + * stating whether any entry or referent holds a value for the leaf, written + * on every document – `false` with none. Indexed and faceted; never served. + * Empty for every other field. */ readonly leafExists: readonly { readonly leaf: string; @@ -3152,14 +3158,11 @@ export function facetsOf( } /** The {@link ReferenceStrategy.facetedLeaves faceted leaves} an inline - * reference declares, or none. */ -function facetedLeavesOf(inlineReference: SearchField): readonly string[] { + * reference declares, or none – for any other field too. */ +function facetedLeavesOf(field: SearchField): readonly string[] { return ( - ( - inlineReference as { - readonly ref: { readonly facetedLeaves?: readonly string[] }; - } - ).ref.facetedLeaves ?? [] + (field as { readonly ref?: { readonly facetedLeaves?: readonly string[] } }) + .ref?.facetedLeaves ?? [] ); } @@ -3183,47 +3186,51 @@ function declaredFacets( }, ] : []; - // `searchSchema` refuses presence on an inline reference and on a facetable - // field, so a presence facet stands alone. - const exists = physicalFields(field, schema).exists; - if (exists !== undefined) { - return [ - { - name: presenceFacetName(field.name), - field, - engineField: exists, - presence: true, - }, - ]; - } - const referenceType = - schema === undefined ? undefined : nestedReferenceType(schema, field); - if (referenceType === undefined) { - return own; + const { exists, leafExists } = physicalFields(field, schema); + // A presence facet of the field’s own, beside its value facet if any. + const presence: FacetDeclaration[] = + exists === undefined + ? [] + : [ + { + name: presenceFacetName(field.name), + field, + engineField: exists, + presence: true, + }, + ]; + // The type a leaf is read through: the nested reference type, or a + // lookup’s one target (`searchSchema` refuses presence leaves on a lookup + // naming several). + const leafOwner: SearchType | undefined = + schema === undefined + ? undefined + : (nestedReferenceType(schema, field) ?? + referencedTargetsOf(field, schema)[0]); + if (leafOwner === undefined) { + return [...own, ...presence]; } - // Only an inline reference nests a reference type, and `searchSchema` - // has resolved every faceted and presence leaf against it. - const leaves = facetedLeavesOf(field); + // `searchSchema` has resolved every faceted and presence leaf against it; + // only an inline reference declares faceted leaves. return [ ...own, - ...leaves.map((leafName): FacetDeclaration => { + ...presence, + ...facetedLeavesOf(field).map((leafName): FacetDeclaration => { const path = nestedFieldName(field.name, leafName); return { name: path, - field: fieldNamed(referenceType, leafName) as SearchField, + field: fieldNamed(leafOwner, leafName) as SearchField, reference: field, engineField: path, }; }), - ...physicalFields(field, schema).leafExists.map( - ({ leaf, exists }): FacetDeclaration => ({ - name: presenceFacetName(nestedFieldName(field.name, leaf)), - field: fieldNamed(referenceType, leaf) as SearchField, - reference: field, - engineField: exists, - presence: true, - }), - ), + ...leafExists.map(({ leaf, exists: companion }): FacetDeclaration => ({ + name: presenceFacetName(nestedFieldName(field.name, leaf)), + field: fieldNamed(leafOwner, leaf) as SearchField, + reference: field, + engineField: companion, + presence: true, + })), ]; }); } @@ -3896,8 +3903,8 @@ export function physicalFields( } : undefined, exists: field.presence === true ? `${field.name}_exists` : undefined, - leafExists: (isInlineReference(field) - ? (field.ref.presenceLeaves ?? []) + leafExists: (field.kind === 'reference' + ? (field.ref?.presenceLeaves ?? []) : [] ).map((leaf) => ({ leaf, exists: `${field.name}_${leaf}_exists` })), }; diff --git a/packages/search/test/faceted-leaves.test.ts b/packages/search/test/faceted-leaves.test.ts index 9098e4d7..934914df 100644 --- a/packages/search/test/faceted-leaves.test.ts +++ b/packages/search/test/faceted-leaves.test.ts @@ -134,7 +134,7 @@ describe('declaring the leaves a root document is faceted on', () => { ).toThrow(/“creator”, which stores a nested object/); }); - it('rejects a reference that is facetable itself, whose facet shares its name', () => { + it('accepts a reference that is facetable itself, its facet beside its leaves’', () => { const person = defineSearchType({ name: 'Person', class: `${SCHEMA_ORG}Person`, @@ -174,9 +174,10 @@ describe('declaring the leaves a root document is faceted on', () => { ], } as never; - expect(() => searchSchema(withIdentity, role, person)).toThrow( - /is “facetable” and declares “facetedLeaves”/, - ); + const schema = searchSchema(withIdentity, role, person); + expect( + facetsOf(withIdentity as SearchType, schema).map((facet) => facet.name), + ).toEqual(['associatedMedia', 'associatedMedia.license']); }); it('rejects a reference that is not filterable', () => { diff --git a/packages/search/test/leaf-presence.test.ts b/packages/search/test/leaf-presence.test.ts index a8f3463f..3d87dec5 100644 --- a/packages/search/test/leaf-presence.test.ts +++ b/packages/search/test/leaf-presence.test.ts @@ -154,14 +154,24 @@ describe('declaring presence leaves', () => { ]); expect( issues({ - ref: { - strategy: 'lookup', - target: 'MediaObject', - presenceLeaves: ['thumbnailUrl'], - }, + ref: { strategy: 'idOnly', presenceLeaves: ['thumbnailUrl'] }, }), ).toEqual([ - { field: 'associatedMedia', reason: 'presence-leaves-not-inline' }, + { field: 'associatedMedia', reason: 'presence-leaves-on-id-only' }, + ]); + const lookup = { + strategy: 'lookup', + target: 'Place', + presenceLeaves: ['latitude'], + }; + expect(issues({ ref: lookup })).toEqual([]); + expect( + issues({ ref: lookup, labelSource: 'Place', joinable: true }), + ).toEqual([ + { field: 'associatedMedia', reason: 'presence-leaves-with-joinable' }, + ]); + expect(issues({ ref: { ...lookup, literal: 'period' } })).toEqual([ + { field: 'associatedMedia', reason: 'presence-leaves-with-literal' }, ]); expect(issues({ filterable: undefined })).toEqual([ { @@ -180,9 +190,7 @@ describe('declaring presence leaves', () => { ).toEqual([ { field: 'associatedMedia', reason: 'presence-leaves-duplicate' }, ]); - expect(issues({ facetable: true })).toEqual([ - { field: 'associatedMedia', reason: 'presence-leaves-with-facetable' }, - ]); + expect(issues({ facetable: true })).toEqual([]); // A leaf is resolved against the reference type, which one declaration // cannot see: `searchSchema` checks it. expect( @@ -197,17 +205,68 @@ describe('declaring presence leaves', () => { expect(issues({})).toEqual([]); }); - it('refuses a facetable reference, whose facet would collide with its leaves’ until #1090', () => { - expect(() => - searchSchema( - workWith( - { presenceLeaves: ['thumbnailUrl'], identity: 'license' }, - { facetable: true }, - ), - mediaObject, + it('accepts a facetable reference, whose own facet sits beside its leaves’', () => { + const licence = defineSearchType({ + name: 'Licence', + class: `${SCHEMA_ORG}CreativeWork`, + fields: [ + { + name: 'label', + kind: 'text', + path: `${SCHEMA_ORG}name`, + locales: ['nl'], + output: true, + searchable: { weight: 1 }, + }, + ], + }); + const licensed = defineSearchType({ + ...mediaObject, + fields: [ + ...mediaObject.fields.filter((field) => field.name !== 'license'), + { + name: 'license', + kind: 'reference', + path: `${SCHEMA_ORG}license`, + filterable: true, + ref: { strategy: 'lookup', target: 'Licence' }, + }, + ], + }); + const facetable = { + ...workWith( + { presenceLeaves: ['thumbnailUrl'], identity: 'license' }, + { facetable: true }, ), + class: `${SCHEMA_ORG}VisualArtwork`, + } as SearchType; + const schema = searchSchema(facetable, licensed, licence); + expect(facetsOf(facetable, schema).map((facet) => facet.name)).toEqual([ + 'associatedMedia', + 'associatedMedia.thumbnailUrl.exists', + ]); + }); + + it('refuses a leaf named like a part of the facet object', () => { + const named = defineSearchType({ + name: 'MediaObject', + fields: [ + { name: 'exists', kind: 'keyword', path: `${SCHEMA_ORG}exists` }, + { name: 'buckets', kind: 'keyword', path: `${SCHEMA_ORG}buckets` }, + ], + }); + expect(() => + searchSchema(workWith({ presenceLeaves: ['exists'] }), named), + ).toThrow( + /names presence leaf “exists”: “buckets”, “totalValues” and “exists”/, + ); + expect(() => + searchSchema(workWith({ facetedLeaves: ['buckets'] }), { + ...named, + fields: [{ ...named.fields[1], filterable: true }], + } as SearchType), ).toThrow( - /declares “presenceLeaves” and is “facetable”: its facet would have to become an object.*issues\/1090/, + /names faceted leaf “buckets”: “buckets”, “totalValues” and “exists”/, ); }); @@ -259,7 +318,7 @@ describe('declaring presence leaves', () => { }, mediaObject, ), - ).toThrow(/but is no inline reference.*issues\/1090/); + ).toThrow(/names no type its referents are read through/); }); it('refuses a leaf the reference type does not declare', () => { @@ -313,15 +372,16 @@ describe('declaring presence leaves', () => { ).toThrow(/presence leaf “thumbnail”, which is itself an inline reference/); }); - it('refuses a leaf that is also faceted on, until its facet can hold both', () => { - expect(() => - searchSchema( - workWith({ presenceLeaves: ['license'], facetedLeaves: ['license'] }), - mediaObject, + it('accepts a leaf counted both by value and by presence', () => { + const both = workWith({ + presenceLeaves: ['license'], + facetedLeaves: ['license'], + }); + expect( + facetsOf(both, searchSchema(both, mediaObject)).map( + (facet) => facet.name, ), - ).toThrow( - /presence leaf “license”, which is also one of its faceted leaves/, - ); + ).toEqual(['associatedMedia.license', 'associatedMedia.license.exists']); }); it('refuses one on a reference type, which has no documents to count', () => { @@ -347,55 +407,13 @@ describe('declaring presence leaves', () => { ); }); - it('refuses two references that nest one type with different leaves', () => { + it('accepts two references that nest one type with different leaves, each filtered by its own input', () => { const collection = { ...workWith({}), name: 'Collection', class: `${SCHEMA_ORG}Collection`, } as SearchType; - expect(() => searchSchema(work, collection, mediaObject)).toThrow( - /“Work.associatedMedia” and “Collection.associatedMedia” both nest “MediaObject” but declare different “presenceLeaves” \(thumbnailUrl; none\)/, - ); - expect(() => searchSchema(collection, work, mediaObject)).toThrow( - /declare different “presenceLeaves” \(none; thumbnailUrl\)/, - ); - }); - - it('compares only the references that get a filter input', () => { - // Not filterable: no filter input, so nothing to share. - const unfiltered = { - ...workWith({}, { filterable: undefined }), - name: 'Collection', - class: `${SCHEMA_ORG}Collection`, - } as SearchType; - expect(() => searchSchema(work, unfiltered, mediaObject)).not.toThrow(); - // On a Reference Type: its references are never filtered through. - const edge = defineSearchType({ - name: 'Edge', - fields: [ - { - name: 'media', - kind: 'reference', - path: `${SCHEMA_ORG}associatedMedia`, - output: true, - ref: { strategy: 'inline', typeName: 'MediaObject' }, - }, - ], - }); - const withEdge = defineSearchType({ - name: 'Agent', - class: `${SCHEMA_ORG}Person`, - fields: [ - { - name: 'edge', - kind: 'reference', - path: `${SCHEMA_ORG}subjectOf`, - output: true, - ref: { strategy: 'inline', typeName: 'Edge' }, - }, - ], - }); - expect(() => searchSchema(work, withEdge, edge, mediaObject)).not.toThrow(); + expect(() => searchSchema(work, collection, mediaObject)).not.toThrow(); }); }); diff --git a/packages/search/test/lookup-presence.test.ts b/packages/search/test/lookup-presence.test.ts new file mode 100644 index 00000000..d7edb5ee --- /dev/null +++ b/packages/search/test/lookup-presence.test.ts @@ -0,0 +1,255 @@ +import { describe, expect, it } from 'vitest'; +import { projectDocument } from '../src/project.js'; +import { + defineSearchType, + facetNamed, + physicalFields, + searchSchema, + type FacetDeclaration, + type SearchType, +} from '../src/schema.js'; +import { + facetSearches, + selectionOf, + validateQuery, + type SearchQuery, +} from '../src/query.js'; + +const SCHEMA_ORG = 'https://schema.org/'; +const alias = (type: string, field: string) => `urn:lde:${type}/${field}`; + +const placeOf = (name: string, className: string) => + defineSearchType({ + name, + class: className, + fields: [ + { + name: 'label', + kind: 'text', + path: `${SCHEMA_ORG}name`, + locales: ['nl'], + output: true, + searchable: { weight: 1 }, + }, + { + name: 'latitude', + kind: 'number', + path: `${SCHEMA_ORG}latitude`, + output: true, + }, + ], + }); +const place = placeOf('Place', `${SCHEMA_ORG}Place`); +const landform = placeOf('Landform', `${SCHEMA_ORG}Landform`); + +const workWith = (ref: Record): SearchType => + ({ + name: 'Work', + class: `${SCHEMA_ORG}CreativeWork`, + fields: [ + { + name: 'contentLocation', + kind: 'reference', + path: `${SCHEMA_ORG}contentLocation`, + array: true, + output: true, + filterable: true, + ref: { + strategy: 'lookup', + target: 'Place', + presenceLeaves: ['latitude'], + ...ref, + }, + }, + ], + }) as never; +const work = workWith({}); +const schema = searchSchema(work, place); +const browse: SearchQuery = { + where: [], + orderBy: [], + limit: 10, + offset: 0, + facets: [], + locale: 'nl', +}; + +describe('presence leaves on a lookup', () => { + it('names one boolean per leaf on the root document', () => { + expect(physicalFields(work.fields[0]).leafExists).toEqual([ + { leaf: 'latitude', exists: 'contentLocation_latitude_exists' }, + ]); + }); + + it('serves the facet `‹reference›.‹leaf›.exists`, read through the target', () => { + expect(facetNamed(work, 'contentLocation.latitude.exists', schema)).toEqual( + { + name: 'contentLocation.latitude.exists', + field: place.fields[1], + reference: work.fields[0], + engineField: 'contentLocation_latitude_exists', + presence: true, + }, + ); + }); + + it('is refused on a lookup naming several targets', () => { + expect(() => + searchSchema( + workWith({ target: ['Place', 'Landform'] }), + place, + landform, + ), + ).toThrow( + /Reference “Work.contentLocation” declares “presenceLeaves” and names several targets/, + ); + }); + + it('requires the target to declare the leaf, with a path', () => { + const unlocated = defineSearchType({ ...place, fields: [place.fields[0]] }); + expect(() => searchSchema(work, unlocated)).toThrow( + /names presence leaf “latitude”, which “Place” does not declare/, + ); + const derived = defineSearchType({ + ...place, + fields: [ + place.fields[0], + { name: 'latitude', kind: 'number', output: true, derive: () => 1 }, + ], + }); + expect(() => searchSchema(work, derived)).toThrow( + /names presence leaf “latitude”, which declares no “path”/, + ); + }); + + it('is refused on a joinable lookup, whose `where` joins', () => { + expect(() => + searchSchema( + { + ...work, + fields: [{ ...work.fields[0], joinable: true }], + } as SearchType, + place, + ), + ).toThrow(/declares “presenceLeaves” and is “joinable”/); + }); +}); + +describe('projecting a lookup’s presence leaves', () => { + const project = (referents: unknown[] | undefined) => + projectDocument( + { + '@id': 'https://ex/w', + ...(referents !== undefined && { + [alias('Work', 'contentLocation')]: referents, + }), + }, + work, + schema, + ); + + it('records a value when any referent’s node states the leaf', () => { + const document = project([ + { '@id': 'https://ex/p/1' }, + { + '@id': 'https://ex/p/2', + [alias('Place', 'latitude')]: { '@value': '51.37' }, + }, + ]); + + expect(document.contentLocation_latitude_exists).toBe(true); + expect(document.contentLocation).toEqual([ + 'https://ex/p/1', + 'https://ex/p/2', + ]); + }); + + it('records no value for a referent named by IRI alone, or none at all', () => { + expect(project(['https://ex/p/1']).contentLocation_latitude_exists).toBe( + false, + ); + expect( + project([ + { + '@id': 'https://ex/p/1', + [alias('Place', 'latitude')]: { '@value': 'not a number' }, + }, + ]).contentLocation_latitude_exists, + ).toBe(false); + expect(project(undefined).contentLocation_latitude_exists).toBe(false); + }); + + it('counts no blank-node referent, which the lookup stores no id for', () => { + const document = project([ + { [alias('Place', 'latitude')]: { '@value': '51.37' } }, + { '@id': '_:b0', [alias('Place', 'latitude')]: { '@value': '51.37' } }, + ]); + + expect(document.contentLocation_latitude_exists).toBe(false); + expect(document.contentLocation).toBeUndefined(); + }); +}); + +describe('`exists` on a lookup’s leaf', () => { + const where = (...criteria: Record[]) => + validateQuery( + { + ...browse, + where: criteria.map((criterion) => ({ or: [criterion] }) as never), + }, + work, + schema, + ); + + it('is a plain criterion naming the reference and the leaf', () => { + expect( + where({ field: 'contentLocation', leaf: 'latitude', exists: true }), + ).toEqual([]); + }); + + it('is refused on a leaf the lookup counts no presence for', () => { + expect( + where({ field: 'contentLocation', leaf: 'label', exists: true }), + ).toEqual([ + { + part: 'where', + field: 'contentLocation.label', + reason: 'no-presence', + }, + ]); + }); + + it('welds nothing: a lookup has no entries', () => { + expect( + where({ + field: 'contentLocation', + entry: [{ field: 'latitude', exists: true }], + }), + ).toEqual([ + { part: 'where', field: 'contentLocation', reason: 'not-weldable' }, + ]); + }); + + it('is selected, and owned, on the leaf’s presence axis', () => { + const facet = facetNamed( + work, + 'contentLocation.latitude.exists', + schema, + ) as FacetDeclaration; + const selection = selectionOf(facet, 'false'); + expect(selection).toEqual({ + or: [{ field: 'contentLocation', leaf: 'latitude', exists: false }], + }); + + const [search] = facetSearches({ + ...browse, + where: [selection], + facets: ['contentLocation.latitude.exists'], + }); + expect(search?.own).toBe('contentLocation.latitude.exists'); + expect(search?.query.where).toEqual([]); + expect(search?.selected).toEqual({ + 'contentLocation.latitude.exists': ['false'], + }); + }); +}); diff --git a/packages/search/test/presence.test.ts b/packages/search/test/presence.test.ts index 8b0ddc4f..266f6491 100644 --- a/packages/search/test/presence.test.ts +++ b/packages/search/test/presence.test.ts @@ -184,10 +184,14 @@ describe('declaring presence', () => { ); }); - it('refuses a facetable field until its facet can hold both', () => { - expect(() => withField({ facetable: true })).toThrow( - /and is “facetable”: its facet would have to become an object/, - ); + it('counts a facetable field’s presence beside its values', () => { + const schema = withField({ facetable: true, filterable: true }); + const thing = [...schema.values()][0] as SearchType; + + expect(facetsOf(thing, schema).map((facet) => facet.name)).toEqual([ + 'field', + 'field.exists', + ]); }); it('files each refusal under a reason, for a declaration built outside TypeScript', () => { @@ -214,9 +218,7 @@ describe('declaring presence', () => { expect(declared({ kind: 'boolean' })).toEqual([ { field: 'field', reason: 'presence-on-boolean' }, ]); - expect(declared({ facetable: true })).toEqual([ - { field: 'field', reason: 'presence-with-facetable' }, - ]); + expect(declared({ facetable: true })).toEqual([]); expect(declared({})).toEqual([]); }); });