diff --git a/AGENTS.md b/AGENTS.md index 40433b54..c791b36a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -86,7 +86,7 @@ Each package uses conditional exports with a `development` condition for local d - Test files use `.test.ts` suffix in `test/` directory - Fixtures in `test/fixtures/` - HTTP mocking with Nock -- Tests that start a local SPARQL endpoint (`@lde/local-sparql-endpoint`) must use unique ports across packages to avoid conflicts when Nx runs tests in parallel. Current port allocations: `dataset-registry-client` (3002), `pipeline` sparqlQuery (3001), `pipeline` executor (3003), `pipeline` provenance store (3004), `pipeline-void` namespace-normalization (3005), `search-pipeline` extraction round-trip (3007), `search-pipeline` searchIndexerPipeline end-to-end (3008), `search-pipeline` registry-sourced extraction (3009), `search-pipeline` joins end-to-end (3010), `search-pipeline` keyed roots (3011), `search-pipeline` nested fan-out (3012), `pipeline-void` per-property partitions (3013), `pipeline-void` subject URI spaces (3014), `pipeline-void` distinct counts (3016), `change-detection` content hash reader (3017), `sparql-qlever` remote mode (3015 for QLever, 7070 for the task agent) +- Tests that start a local SPARQL endpoint (`@lde/local-sparql-endpoint`) must use unique ports across packages to avoid conflicts when Nx runs tests in parallel. Current port allocations: `dataset-registry-client` (3002), `pipeline` sparqlQuery (3001), `pipeline` executor (3003), `pipeline` provenance store (3004), `pipeline-void` namespace-normalization (3005), `search-pipeline` extraction round-trip (3007), `search-pipeline` searchIndexerPipeline end-to-end (3008), `search-pipeline` registry-sourced extraction (3009), `search-pipeline` joins end-to-end (3010), `search-pipeline` keyed roots (3011), `search-pipeline` nested fan-out (3012), `pipeline-void` per-property partitions (3013), `pipeline-void` subject URI spaces (3014), `pipeline-void` distinct counts (3016), `change-detection` content hash reader (3017), `search-pipeline` referents’ label transforms (3019), `search-pipeline` lookup presence (3020), `sparql-qlever` remote mode (3015 for QLever, 7070 for the task agent) ### Key Dependencies diff --git a/docs/decisions/0040-filter-and-facet-a-field-on-whether-it-has-a-value.md b/docs/decisions/0040-filter-and-facet-a-field-on-whether-it-has-a-value.md index f254cba2..898ab066 100644 --- a/docs/decisions/0040-filter-and-facet-a-field-on-whether-it-has-a-value.md +++ b/docs/decisions/0040-filter-and-facet-a-field-on-whether-it-has-a-value.md @@ -9,7 +9,9 @@ Accepted Supersedes the last consequence of [ADR 36](0036-facet-a-root-document-on-its-entries-leaves.md) for a Root Type’s own field. Extended to an inline entry’s leaf by -[ADR 41](0041-filter-and-facet-a-root-document-on-whether-an-entrys-leaf-has-a-value.md). +[ADR 41](0041-filter-and-facet-a-root-document-on-whether-an-entrys-leaf-has-a-value.md); +its refusal of a facetable field lifted by +[ADR 42](0042-count-a-referents-leaf-and-a-fields-facets-in-one-object.md). ## Context diff --git a/docs/decisions/0041-filter-and-facet-a-root-document-on-whether-an-entrys-leaf-has-a-value.md b/docs/decisions/0041-filter-and-facet-a-root-document-on-whether-an-entrys-leaf-has-a-value.md index 401e5da4..806eee25 100644 --- a/docs/decisions/0041-filter-and-facet-a-root-document-on-whether-an-entrys-leaf-has-a-value.md +++ b/docs/decisions/0041-filter-and-facet-a-root-document-on-whether-an-entrys-leaf-has-a-value.md @@ -7,7 +7,9 @@ Date: 2026-11-16 Accepted Extends [ADR 40](0040-filter-and-facet-a-field-on-whether-it-has-a-value.md) -to a leaf of an inline reference. +to a leaf of an inline reference. Extended to a lookup’s referents, and its +refusals for the facet shape lifted, by +[ADR 42](0042-count-a-referents-leaf-and-a-fields-facets-in-one-object.md). ## Context diff --git a/docs/decisions/0042-count-a-referents-leaf-and-a-fields-facets-in-one-object.md b/docs/decisions/0042-count-a-referents-leaf-and-a-fields-facets-in-one-object.md new file mode 100644 index 00000000..914cd619 --- /dev/null +++ b/docs/decisions/0042-count-a-referents-leaf-and-a-fields-facets-in-one-object.md @@ -0,0 +1,124 @@ +# 42. Count a referent’s leaf, and a field’s facets in one object + +Date: 2026-11-16 + +## Status + +Accepted + +Extends [ADR 39](0039-apply-a-label-sources-transforms-to-its-referents.md) +and [ADR 41](0041-filter-and-facet-a-root-document-on-whether-an-entrys-leaf-has-a-value.md); +lifts the refusals ADR 36, ADR 40 and ADR 41 made for the facet shape. + +## Context + +Linked Open Limburg’s map view filters works on whether a place they name has +coordinates. `contentLocation` is a lookup into `Place`, whose coordinates +come from GeoNames enrichment, not from the work’s graph. The root document +stores only the places’ ids, so LOL guesses from each place’s key: a GeoNames +IRI counts as located. When a GeoNames lookup fails, the place is indexed +without coordinates and the work still says it has them. + +ADR 41 counts a leaf of an inline reference’s entries, which the root +document stores. A lookup’s referents are other documents, possibly from +other datasets, so reading the leaf off the target’s collection would need a +pass after every dataset, or an order across them. + +Separately, the facets object had one key per facetable field, typed as a +bucket list. ADR 36, ADR 40 and ADR 41 therefore refused anything that would +give one key two meanings: presence on a facetable field, leaves on a +facetable reference, a leaf faceted both by value and by presence. + +## Decision + +- **A lookup declares `presenceLeaves`** naming leaves of its target, as an + inline reference does since ADR 41: + `ref: { strategy: 'lookup', target: 'Place', presenceLeaves: ['latitude'] }`. + One boolean per root document, `contentLocation_latitude_exists`, in + `PhysicalFields.leafExists`. Only on a lookup naming one target: a leaf is + read through one declaration. +- **Counted at index time, in the referring batch.** The extraction reads each + presence leaf one hop from the root, beside the target’s key and label. The + stage passes the referents through the target type’s transforms, as ADR 39 + does for labels, and the projection reads the leaf off each referent’s node + as the target declares it. No pass after all datasets: what the root’s graph + states, or a transform can say from the referent’s key, is counted; + anything else is not, and the work counts as holding none – so a failed + lookup gives `false`, not the guess’s `true`. Only a referent named by IRI + counts: a blank node is no id the lookup stores. +- **A referent named by IRI alone is seeded.** ADR 39 hands a transform only + the quads the batch has, so such a referent reached no transform. A referent + a presence leaf is counted for enters its target’s transforms as + ` rdf:type `, and the seed is dropped from what comes + back. A transform describing a node from its key, as LOL’s place transform + does, then supplies the leaf. A lookup into its own type seeds through that + type’s transforms too, though they saw the batch: they never saw a referent + the batch states nothing of. +- **The target declares the leaf, with a `path`.** A leaf is read off the + referent’s node through the target’s declaration, not derived. +- **A plain criterion on the root selects it**, naming the reference and the + leaf: `{ field: 'contentLocation', leaf: 'latitude', exists: true }`. The + value is the root document’s, so it combines with other criteria like any, + and each key of a lookup’s GraphQL `where` is one such criterion, ANDed like + sibling keys – two presence leaves in one `where` are two conditions. A + lookup takes no weld: it has no entries. +- **One facet object per field.** A field counting more than its own values + is served as `‹Type›‹Field›Facet`: its own values in `buckets` and + `totalValues` (a numeric field’s range facet parts instead), capped by the + field’s `limit`; `exists` for its presence; one key per leaf, a plain facet + type for a leaf counted by value only and `‹Type›‹Field›‹Leaf›Facet` for + one whose presence is counted. A field counting only its own values keeps + `IRIFacet`, `ValueFacet` or its range facet type. A leaf is not named + `buckets`, `totalValues` or `exists`. +- **One filter input per reference counting leaves.** `‹Type›‹Field›Filter` + holds the reference’s `in` and a `where`, `‹Type›‹Field›Where`, naming its + filterable entry leaves and its presence leaves. Two references nesting one + Reference Type may then declare different leaves, which ADR 41 refused. + Adding leaves to a reference therefore changes its filter type, from the + shared `‹Edge›Filter` or `‹Target›Filter` to its own – a breaking change + for a client whose variables name the old type. + +## Consequences + +- LOL declares `presenceLeaves: ['latitude']` on `contentLocation` and + `locationCreated`, and drops its key-based guess. +- Lifted: presence on a facetable field (ADR 40), leaves on a facetable + reference (ADR 36, ADR 41), a leaf faceted by value and by presence (ADR + 41), and one set of presence leaves per Reference Type (ADR 41). +- Breaking at the GraphQL surface: ADR 36’s `‹Type›‹Reference›Facets` and ADR + 40 and 41’s `…Facets` objects are renamed `…Facet`; a field that gains a + leaf or `exists` changes from `IRIFacet`/`ValueFacet` to its own object; a + reference counting leaves takes `‹Type›‹Field›Filter` instead of + `‹Edge›Filter` or `‹Target›Filter`, so the refined discovery strategy no + longer reaches it – and a reference that gains its first leaf later breaks + the variables typed with its old filter. +- Each part of a field’s facet object is charged as a facet key, so selecting + `buckets` with `totalValues` costs two where a plain facet costs one – an + overestimate, never a part that costs nothing. +- A target’s transform now also receives nodes carrying a type alone, once per + referring batch, and pays for any I/O it does there unless it caches. +- A value is a copy, changed only when the referring document is reindexed. +- Refused for now: presence leaves on a lookup naming several targets, on a + `joinable` lookup, whose `where` already joins, and on a lookup reading its + literals as periods. A lookup’s leaves faceted by value have no consumer + and are not offered. + +## Alternatives considered + +- **A pass after all datasets, from the collection being built**, as the + issue first proposed. Complete for referents described only in another + dataset’s graph, but a pass over every referring document per run, skipped + datasets included, outside the batch. +- **A Typesense join facet** (`facet_by=$places(latitude)`). On 30.2 it counted + the joined documents, not the roots, and returned nothing on a text query + without a filter. +- **A weld of one `exists` condition**, as for an inline entry: + `{ field: 'contentLocation', entry: [{ field: 'latitude', exists }] }`. + It pretends a lookup has entries, needs a special case wherever a weld is + read, and cannot hold two leaves, which a weld puts on one entry. +- **Counting through any of several targets.** A referent of either kind + would be read through its own target’s declaration, at the cost of a leaf + every target must declare alike; no consumer names several targets yet. +- **The reference’s own buckets under `id`**, beside its leaves. Every + facetable reference would change shape, where a field counting only its own + values needs none. diff --git a/docs/reference/search-api-graphql.md b/docs/reference/search-api-graphql.md index 9409e9ee..05386c98 100644 --- a/docs/reference/search-api-graphql.md +++ b/docs/reference/search-api-graphql.md @@ -124,8 +124,14 @@ fails this – accepts. The endpoint path defaults to `/graphql` | A joined reference (`lookup`), at any depth | 1 per target type | | A facet key | 1 | | A range facet’s `all` extent | 1 more | + | Each part of a `‹Type›‹Field›Facet` object | 1 | | A plain stored field or an inline reference | 0, at any depth | + In a field’s facet object every part selected is charged on its own – + `buckets`, `totalValues`, `exists`, a range facet’s `all`, `matching` and + `buckets`, and each part of a leaf’s – so `buckets` with `totalValues` + costs 2 where the same pair on a plain `IRIFacet` costs 1. + A lookup fetches the referents of a whole page in one request, and the facet keys of a search share one request, so `perPage` does not enter the cost. Fragments cost what their contents cost, a selection under `@skip` or @@ -280,22 +286,41 @@ role: { in: ["etser"] } } }` asks for _this person in this role_, where the without the keys of fields declaring presence alone. A type declaring none keeps one `‹Type›Where` for both. - An inline reference’s - [presence leaves](./search#presence-on-an-entry-s-leaf) take `exists` in its - reference type’s `‹Edge›Where`, typed as a field of the root’s own is - (`ExistsFilter`, or `‹Filter›OrExists` for a `filterable` leaf): - `where: { associatedMedia: { where: { thumbnailUrl: { exists: true } } } }`. - Whether any entry holds a value is recorded per document, so `exists` is - welded to nothing else: setting it beside another key of the same - `‹Edge›Where` is an error naming the rewrite into two conditions. Every - reference nesting one reference type declares the same presence leaves, so - `‹Edge›Where` stays shared; through a join, where no companion is read, a - reference with presence leaves takes `‹Edge›JoinedFilter`, without `exists`. + A reference counting leaves of its referents – an inline reference’s + [faceted](./search#faceting-on-an-entry-s-leaf) or + [presence leaves](./search#presence-on-an-entry-s-leaf), a lookup’s + [presence leaves](./search#on-a-lookup-s-referents) – takes a filter of its + own, **`‹Type›‹Field›Filter`**: `@oneOf` over `in` (the ids it holds, for a + lookup or an inline reference naming an identity) and `where`, typed + **`‹Type›‹Field›Where`**. For an inline reference that `where` names the + entries’ filterable leaves, as the shared `‹Edge›Where` does; for both + kinds it names each presence leaf, taking `exists` – `ExistsFilter`, or + `‹Filter›OrExists` for an inline entry’s filterable leaf: + `where: { contentLocation: { where: { latitude: { exists: true } } } }`. + A lookup’s `where` names its presence leaves alone, and each key is a plain + criterion on the root document, ANDed like sibling keys: + `{ latitude: { exists: true }, longitude: { exists: true } }` is two + conditions, each on whether any referent holds that leaf; inside an `or` + alternative it sets one key, as a joined condition does. In an inline + reference’s `where`, whether any entry holds a value is recorded per + document, so `exists` is welded to nothing else: setting it beside another + key of the same `where` is an error naming the rewrite into two conditions. + Through a join, where no companion is read, an inline reference with + presence leaves takes `‹Type›‹Field›JoinedFilter`, without `exists`, and a + lookup its plain `‹Target›Filter`. + + **Adding leaves changes the reference’s filter type.** A reference that + gains faceted or presence leaves moves from the shared `‹Edge›Filter` or + `‹Target›Filter` to its own `‹Type›‹Field›Filter`, so a client’s variables + typed with the old name stop validating – a breaking change to the API, + even where the reference’s `in` arm is unchanged. Like `‹Filter›OrExists`, + the per-reference filter is a type of its own, so the refined discovery + strategy does not reach it. - **`orderBy`**: `RELEVANCE` plus every `sortable` field, as an enum – field names SCREAMING_SNAKE_CASEd (`datePosted` → `DATE_POSTED`); `direction` defaults to `DESC`. -- **Facets**: a keyed object with one field per `facetable` field, typed by +- **Facets**: a keyed object with one key per field anything is counted on, typed by the field’s declaration: - a **reference** facet returns an `IRIFacet!` – `buckets: [IRIBucket!]!`, each a `value` (an `IRI`) + `count` + the resolved data `label`, and @@ -348,18 +373,33 @@ role: { in: ["etser"] } } }` asks for _this person in this role_, where the the engine counted documents for it, so a uniform result set yields one bucket, not two. - An inline reference’s [faceted leaves](./search#faceting-on-an-entry-s-leaf) - are served one level down, under the reference: `facets { associatedMedia { -license { buckets { value count } } } }`, typed `‹Type›‹Reference›Facets`. A presence - facet is served the same way, under the field it counts: `facets { -dateCreated { exists { value count } } }`, typed `‹Type›‹Field›Facets` - (`WorkDateCreatedFacets`) with `exists: [BooleanBucket!]!`. The object - leaves room for a field’s value buckets beside it, once a field may be both - `facetable` and declare presence. A presence leaf’s facet sits under the - reference, one level further down: `facets { associatedMedia { thumbnailUrl { -exists { value count } } } }`, typed `‹Type›‹Reference›‹Leaf›Facets` - (`WorkAssociatedMediaThumbnailUrlFacets`), beside the reference’s faceted - leaves. + **One object per field.** A field counting more than its own values – + its presence, or a leaf of its referents – is served as an object of its + own, **`‹Type›‹Field›Facet`**, holding everything counted on it: + + ```graphql + facets { + material(limit: 20) { buckets { value count } totalValues } + dateCreated { exists { value count } } + contentLocation(limit: 20) { + buckets { value count } + totalValues + latitude { exists { value count } } + } + associatedMedia { license(limit: 5) { buckets { value count } } } + } + ``` + + Its own values are `buckets` and `totalValues` (a numeric field’s `all`, + `matching` and `buckets` instead), capped by the field’s `limit`, which caps + nothing else; `exists` counts its presence. Each leaf is a key of its own: a + leaf counted only by value keeps its plain facet type (`license(limit): +IRIFacet!`), and a leaf whose presence is counted is an object again, + **`‹Type›‹Field›‹Leaf›Facet`** (`WorkContentLocationLatitudeFacet`), with + `exists` and – when it is faceted by value too – its `buckets`, + `totalValues` and `limit`. A field counting only its own values keeps + `IRIFacet`, `ValueFacet`, its range facet type or `[BooleanBucket!]!`. A + name a declaration took already is refused with a clear error. Selecting facet fields IS the request, and the whole selection is **batched per request**: one diff --git a/docs/reference/search-pipeline.md b/docs/reference/search-pipeline.md index 03441d24..fae24da5 100644 --- a/docs/reference/search-pipeline.md +++ b/docs/reference/search-pipeline.md @@ -398,6 +398,38 @@ readers: the stage’s projected documents into the write. A projected document is far heavier than a quad, so lower it where documents are large. +## Referents through their type’s transforms + +A stage passes the referents of its references through each target type’s own +`transforms`, inside the batch, so memory stays bounded by `batchSize`: the +quads the extraction read off each referent go in, under the target’s IR +Aliases, and what the transforms return replaces them. Two kinds of reference +are described this way: + +- a `searchable` reference, whose referents’ key and label are read, so it is + searched by the label the target’s transform leaves + ([ADR 39](../decisions/0039-apply-a-label-sources-transforms-to-its-referents.md)); +- a `lookup` declaring [presence leaves](./search#on-a-lookup-s-referents), + whose referents’ key and leaves are read, so the projection can write whether + any referent holds a leaf. + +A referent a presence leaf is counted for, named by IRI, and that the batch +states nothing about – the root’s graph names it by IRI alone – enters its +target’s transforms all the same, as ` rdf:type `, so a +transform that describes a node from its key (an Authority’s coordinates for a +GeoNames place) supplies the leaf. The seed is not kept. A transform on such a +target therefore also receives nodes carrying a type alone, or only a key and +a few leaves, once per referring batch; a transform doing I/O pays for that +unless it caches. A blank-node referent is not seeded: the lookup stores no id +for it, so it is not counted. + +A reference to the stage’s own type passes no referent’s quads through again, +since its transforms already saw the batch – but a lookup into its own type +declaring presence leaves still has the referents the batch states nothing +about seeded through them. A target wired with its own `readers` has no +transform to apply +([ADR 42](../decisions/0042-count-a-referents-leaf-and-a-fields-facets-in-one-object.md)). + ## Per-component isolation Each root type is an independent engine run – its own collection, alias and diff --git a/docs/reference/search-typesense.md b/docs/reference/search-typesense.md index 2d7ffc9e..6ba53139 100644 --- a/docs/reference/search-typesense.md +++ b/docs/reference/search-typesense.md @@ -139,11 +139,13 @@ value, so `exists` compiles to `${name}_exists:=true` or `:=false`, and the facet `‹field›.exists` counts the companion. The projection writes it on every document, `false` included, so the companion is not optional. A field declaring presence alone is internal, so the companion is all the collection -declares for it. An inline reference’s -[presence leaves](./search#presence-on-an-entry-s-leaf) each get a top-level -`${reference}_${leaf}_exists` of the same kind, true when any entry holds a -value: an `exists` welded alone to an entry compiles to it, never to a -condition inside the nested object. +declares for it. A reference’s presence leaves – an +[inline entry’s](./search#presence-on-an-entry-s-leaf) or a +[lookup’s referents’](./search#on-a-lookup-s-referents) – each get a top-level +`${reference}_${leaf}_exists` of the same kind, true when any entry or +referent holds a value: an `exists` welded alone to an inline reference, and a +lookup’s `{ field, leaf, exists }`, compile to it, never to a condition inside +the nested object. An `InPlaceRebuild` additionally refuses a type whose declared [dataset field](#provenance) inherits a policy: it enumerates the indexed diff --git a/docs/reference/search.md b/docs/reference/search.md index b11d8bdf..992a2ba0 100644 --- a/docs/reference/search.md +++ b/docs/reference/search.md @@ -601,7 +601,9 @@ label with an Authority’s, the referring document is searched by the Authority’s. A transform on a label source therefore also receives nodes that carry only those two fields, once per referring batch. A referent stated by IRI alone, with no triple of its own, gives a transform nothing to act on and adds -nothing to search. A copied label changes when the document is reindexed. +nothing to search – unless the reference also counts a +[presence leaf](#on-a-lookup-s-referents), which seeds it. A copied label +changes when the document is reindexed. A reference naming no label source still folds its IRIs, and a `derive`d or `from` reference reads no referent, so it is never searched by labels @@ -1205,8 +1207,10 @@ nobody’s own, as a clause spanning two fields is not `searchSchema` holds each leaf to what that round trip needs: it is declared on the reference type, `filterable`, and a `keyword`, `boolean` or `reference`; -the reference is `filterable`, not `facetable` itself (both facets would be -named after it), and sits on a Root Type. A `lookup` leaf whose target declares +the reference is `filterable` and sits on a Root Type, and a leaf is not named +`buckets`, `totalValues` or `exists`, which name parts of the reference’s +facet object. A reference that is `facetable` itself counts its own values +beside its leaves, in one facet object. A `lookup` leaf whose target declares a [facet policy](#facet-policy) is refused for now. A condition welded to one entry is owned by a faceted leaf, never by an @@ -1259,11 +1263,12 @@ declaring no presence (`no-presence`), which is also what a weld meets. `searchSchema` says why in full. It refuses it on a field of a Reference Type and on an inline reference, whose entries are counted through the reference’s -[presence leaves](#presence-on-an-entry-s-leaf) instead; on a `text` field, which stores its values per language; on a `boolean`, which -states its own absence as `false`; and on a field that is already `facetable`, -since its facet would have to become an object to hold both kinds of bucket -([#1090](https://github.com/ldelements/lde/issues/1090)) +[presence leaves](#presence-on-an-entry-s-leaf) instead; on a `text` field, +which stores its values per language; and on a `boolean`, which states its own +absence as `false` ([ADR 40](../decisions/0040-filter-and-facet-a-field-on-whether-it-has-a-value.md)). +A field that is also `facetable` counts both, its values and its presence +([ADR 42](../decisions/0042-count-a-referents-leaf-and-a-fields-facets-in-one-object.md)). ### Presence on an entry’s leaf @@ -1307,15 +1312,62 @@ holds a value is recorded per document, so it cannot hold of the same entry as another condition. Through a join it is refused like any `exists` (`joined-presence`). -The reference is `filterable`, sits on a Root Type and – for now – is not -`facetable` itself; each leaf is declared by the reference type, named once, -and neither `text`, `boolean`, an inline reference nor – for now – one of the -reference’s faceted leaves. Every `filterable` inline reference on a Root -Type nesting the same reference type declares the same presence leaves, -because they share the type’s filter input. The two “for now” refusals and the -shared leaves are lifted by per-field facet objects and per-reference filter -inputs ([#1090](https://github.com/ldelements/lde/issues/1090)) -([ADR 41](../decisions/0041-filter-and-facet-a-root-document-on-whether-an-entrys-leaf-has-a-value.md)). +The reference is `filterable` and sits on a Root Type; each leaf is declared +by the reference type, named once, neither `text`, `boolean` nor an inline +reference, and not named `buckets`, `totalValues` or `exists`. A leaf may be +faceted on by value and by presence at once, and the reference may be +`facetable` itself: everything counted on one field is served in one facet +object +([ADR 41](../decisions/0041-filter-and-facet-a-root-document-on-whether-an-entrys-leaf-has-a-value.md), +[ADR 42](../decisions/0042-count-a-referents-leaf-and-a-fields-facets-in-one-object.md)). + +#### On a lookup’s referents + +A `lookup` stores only its referents’ ids, so it declares `presenceLeaves` to +count a leaf of the documents it names – “a work with a place that has +coordinates”: + +```ts +{ + name: 'contentLocation', + kind: 'reference', + path: 'https://schema.org/contentLocation', + array: true, + filterable: true, + ref: { strategy: 'lookup', target: 'Place', presenceLeaves: ['latitude'] }, +} +``` + +It is counted at index time, in the referring batch: the extraction reads +each presence leaf one hop from the root, beside the target’s key and label, +the target type’s transforms run on the referents +([`@lde/search-pipeline`](./search-pipeline#referents-through-their-type-s-transforms)), +and the projection writes `contentLocation_latitude_exists` from what they +return. A referent the root’s graph states the leaf for counts; so does one a +transform describes from its IRI alone – the type’s own transforms, for a +lookup into its own type; one that neither states nor any transform describes +– a failed lookup, say – counts as holding none. Only a referent with an IRI +counts: a blank node is no id the lookup stores, so what the graph states of +it does not count either. + +The facet is `contentLocation.latitude.exists`. Since the value is recorded on +the root document, its criterion is a plain one on the root, naming the +reference and the leaf: +`{ field: 'contentLocation', leaf: 'latitude', exists: true }` +(GraphQL: `where: { contentLocation: { where: { latitude: { exists: true } } } }`). +It combines with other criteria like any: the keys of a lookup’s `where` are +ANDed as sibling keys are, so `{ latitude: { exists: true }, longitude: { +exists: true } }` asks for a work with a place that has a latitude and one +that has a longitude, each counted on its own. A weld (`entry`) on a lookup is +refused (`not-weldable`): a lookup has no entries. + +The lookup names one `target`, which declares each leaf, with a `path`: a leaf +is read off the referent’s own node, never derived, through that target’s +declaration. Refused on a lookup naming several targets +(`presence-leaves-with-several-targets`), on a `joinable` lookup, whose `where` +already joins, and on one reading its literals as periods. A reference’s +values faceted by a leaf of a lookup’s referents (`facetedLeaves` on a lookup) +are not offered. ### Facet policy diff --git a/package-lock.json b/package-lock.json index 1555ae54..58e92989 100644 --- a/package-lock.json +++ b/package-lock.json @@ -42129,11 +42129,11 @@ "@traqula/generator-sparql-1-1": "^1.3.1", "@traqula/parser-sparql-1-1": "^1.3.1", "@traqula/rules-sparql-1-1": "^1.2.0", + "n3": "^2.13.12", "tslib": "^2.3.0" }, "devDependencies": { "@lde/search-typesense": "^0.53.0", - "n3": "^2.13.12", "testcontainers": "^12.1.0", "typesense": "^3.2.0" }, @@ -42146,7 +42146,6 @@ "version": "2.13.12", "resolved": "https://registry.npmjs.org/n3/-/n3-2.13.12.tgz", "integrity": "sha512-z944NsyJi58WXcYevtNkUL2JhVr0KP6nvED9AgymQq5V35TDaNn1N8kO+OWJk/kj4PR0L/ilWB3d7/bWceg1CQ==", - "dev": true, "license": "MIT", "dependencies": { "buffer": "^6.0.3", diff --git a/packages/search-api-graphql/src/build-schema.ts b/packages/search-api-graphql/src/build-schema.ts index bf5fa0a4..e6ccd92c 100644 --- a/packages/search-api-graphql/src/build-schema.ts +++ b/packages/search-api-graphql/src/build-schema.ts @@ -16,6 +16,7 @@ import { responsePathAsArray, type GraphQLEnumValueConfigMap, type GraphQLFieldConfig, + type GraphQLArgumentConfig, type GraphQLInputFieldConfig, type GraphQLInputType, type GraphQLOutputType, @@ -24,6 +25,7 @@ import { import { NESTED_DOCUMENT_TYPE, type Criterion, + type FacetBucket, type Filter, type LocalizedValue, type NestedDocument, @@ -41,6 +43,7 @@ import { AND_KEY, assertValidQuery, facetsOf, + filterableFields, fieldNamed, labelSourceNamesOf, labelTargetNamesOf, @@ -59,6 +62,7 @@ import { nestedReferenceType, outputFields, pageForOffset, + isInlineReference, physicalFields, sortableFields, unixSecondsToIso, @@ -465,6 +469,13 @@ export function buildGraphQLSchema( }); const valueFacet = facetTypeOf('ValueFacet', valueBucket); const iriFacet = facetTypeOf('IRIFacet', iriBucket); + // The keys share one engine round-trip, but each is its own facet + // calculation in it. + const facetCost = costExtensions(operationCost); + const limitArgument: GraphQLArgumentConfig = { + type: GraphQLInt, + description: `At most this many buckets, between 1 and ${maxFacetLimit}: the values with the highest counts. A larger limit returns a superset of a smaller one; there is no next page. Omit it for the deployment’s default.`, + }; // A range facet’s extent, in the type its field’s `range` filter takes // (IntRange, FloatRange, DateRange), so a slider’s bounds feed straight back // into the filter that selects them. A date is stored as Unix seconds and @@ -861,6 +872,32 @@ export function buildGraphQLSchema( const joins = joinGraph(schema); const whereInputs = new Map(); const nestedFilters = new Map(); + const leafFilters = new Map(); + const leafCountingReferences = new Map< + SearchType, + ReadonlySet + >(); + + /** The references of a Root Type counting leaves of their referents – + * faceted or presence leaves – computed once per type. None for a + * Reference Type. */ + function leafCountingReferencesOf( + owner: SearchType, + ): ReadonlySet { + const existing = leafCountingReferences.get(owner); + if (existing !== undefined) { + return existing; + } + const counting = new Set( + owner.class === undefined + ? [] + : facetsOf(owner, schema).flatMap((facet) => + facet.reference === undefined ? [] : [facet.reference], + ), + ); + leafCountingReferences.set(owner, counting); + return counting; + } const referenceFilters = new Map(); /** The name a reference’s emitted type is keyed under: a `lookup`’s target @@ -1326,13 +1363,18 @@ export function buildGraphQLSchema( owner: SearchType, joined = false, ): GraphQLInputType { + // A reference counting leaves of its referents takes a filter of its own, + // whose `where` names exactly the keys a condition on it can take. + if (leafCountingReferencesOf(owner).has(field)) { + return leafFilterFor(field, owner as RootType, joined); + } // An inline reference is filterable in two ways at once, exactly as a // joinable one is: by the ids its entries hold (its identity), // and by a condition on an entry. Same input shape, because it is the same // question one hop out – only what the hop costs differs. const nested = nestedReferenceType(schema, field); if (nested !== undefined) { - return nestedFilterFor(nested, field, joined); + return nestedFilterFor(nested, field); } // A joinable reference is filterable in two ways at once – by the ids it // holds, and by a condition on the referent – so it earns a richer input @@ -1346,6 +1388,83 @@ export function buildGraphQLSchema( : plainWhereFieldType(field); } + /** + * The filter input of a reference counting leaves of its referents – + * faceted or presence leaves – `‹Type›‹Field›Filter`: `@oneOf` over its `in` + * (the ids it holds, for a lookup or an inline reference naming an + * identity) and a `where` of its own, `‹Type›‹Field›Where`. For an inline + * reference that `where` names the entries’ filterable leaves, as the + * shared `‹Edge›Where` does; for both kinds it names each presence leaf, + * taking `exists` – `ExistsFilter`, or `‹Filter›OrExists` for an inline + * entry’s filterable leaf. A lookup’s `where` holds its presence leaves + * alone, each recorded on the root document, so its keys combine with AND + * like any sibling keys rather than holding of one referent. + * + * Through a join no presence companion is read yet, so a reference with + * presence leaves takes `‹Type›‹Field›JoinedFilter` there, without `exists` + * – or, for a lookup, which then has nothing left to name, its plain + * filter. + */ + function leafFilterFor( + field: SearchField, + owner: RootType, + joined: boolean, + ): GraphQLInputType { + const nested = nestedReferenceType(schema, field); + const presenceLeaves = joined ? [] : presenceLeavesOf(field); + if (nested === undefined && presenceLeaves.length === 0) { + return plainWhereFieldType(field); + } + const base = `${owner.name}${capitalized(field.name)}`; + const variant = + joined && presenceLeavesOf(field).length > 0 ? `${base}Joined` : base; + const existing = leafFilters.get(variant); + if (existing !== undefined) { + return existing; + } + const at = `“${owner.name}.${field.name}”`; + const where = new GraphQLInputObjectType({ + name: claimTypeName(`${variant}Where`, `the condition on ${at}`), + description: + nested === undefined + ? 'Whether any referent holds a value for a leaf. Sibling keys are combined with AND.' + : 'Sibling keys are combined with AND, and all of them must hold of the SAME entry.', + fields: () => + nested === undefined + ? Object.fromEntries( + presenceLeaves.map((leaf) => [leaf, { type: existsFilter }]), + ) + : Object.fromEntries( + whereFieldsOf(nested, presenceLeaves).map((leaf) => [ + leaf.name, + { + type: presenceLeaves.includes(leaf.name) + ? presenceFilterFor(leaf, nested) + : whereFieldType(leaf, nested), + description: leaf.description, + }, + ]), + ), + }); + const ids = + nested === undefined || labelTargetNamesOf(field, schema).length > 0; + const created = new GraphQLInputObjectType({ + name: claimTypeName(`${variant}Filter`, `the filter on ${at}`), + description: ids + ? `A condition on ${at}: the ids it holds, or a condition on its referents.` + : `A condition on one ${at} entry.`, + isOneOf: true, + fields: () => ({ + ...(ids && { + in: { type: new GraphQLList(new GraphQLNonNull(iriScalar)) }, + }), + where: { type: where }, + }), + }); + leafFilters.set(variant, created); + return created; + } + /** * The filter input an inline reference takes: the ids its entries hold, or a * condition on one entry. `@oneOf`, so a criterion stays an atom. @@ -1354,45 +1473,22 @@ export function buildGraphQLSchema( * {@link ReferenceStrategy.identity identity} names, so a bucket a * consumer clicks feeds straight back in – the same round trip an ordinary * facetable reference already guarantees. A reference declaring no identity - * has no ids to filter, so it offers the `where` arm alone. + * has no ids to filter, so it offers the `where` arm alone. Shared by every + * reference nesting the type that counts no leaf of it + * ({@link leafFilterFor}). */ function nestedFilterFor( referenceType: ReferenceType, field: SearchField, - joined: boolean, ): GraphQLInputObjectType { - // Every reference nesting the type declares the same presence leaves - // (`searchSchema`), so the filter is shared – except through a join, which - // reads no presence companion yet and so gets a variant without `exists`. - const presenceLeaves = joined ? [] : presenceLeavesOf(field); - const variant = - joined && presenceLeavesOf(field).length > 0 - ? `${referenceType.name}Joined` - : referenceType.name; - const existing = nestedFilters.get(variant); + const existing = nestedFilters.get(referenceType.name); if (existing !== undefined) { return existing; } - // The join variants are names of their own, so they are claimed like - // every other name a declaration could clash with. - if (variant !== referenceType.name) { - claimTypeName( - `${variant}Filter`, - `the filter on “${referenceType.name}” through a join`, - ); - claimTypeName( - `${variant}Where`, - `the condition on one “${referenceType.name}” entry through a join`, - ); - } const identityTargets = labelTargetNamesOf(field, schema); - const nestedWhere = nestedWhereInputFor( - referenceType, - presenceLeaves, - variant, - ); + const nestedWhere = nestedWhereInputFor(referenceType); const created = new GraphQLInputObjectType({ - name: `${variant}Filter`, + name: `${referenceType.name}Filter`, description: identityTargets.length > 0 ? `A condition on ${referenceType.name}: the ids its entries reference, or a condition on one entry.` @@ -1405,7 +1501,7 @@ export function buildGraphQLSchema( where: { type: nestedWhere }, }), }); - nestedFilters.set(variant, created); + nestedFilters.set(referenceType.name, created); return created; } @@ -1414,33 +1510,22 @@ export function buildGraphQLSchema( * else. Deliberately without the `id` key every Root Type's `where` carries: * an entry is read, not addressed, so it has no document key to filter on * ({@link validateQuery} refuses one). - * - * A presence leaf of the references nesting the type takes `exists` too, - * as a field of the root’s own does: `ExistsFilter`, or `‹Filter›OrExists` - * for a `filterable` leaf. Whether any entry holds a value is the root - * document’s, so `exists` is welded to nothing else – a `where` setting it - * beside another key is refused with the rewrite ({@link - * weldedCriterionOf}). */ function nestedWhereInputFor( referenceType: ReferenceType, - presenceLeaves: readonly string[], - variant: string, ): GraphQLInputObjectType { // Memoized by its only caller, which is memoized itself – so this is built - // once per Reference Type (and variant) however many fields nest it. + // once per Reference Type however many fields nest it. return new GraphQLInputObjectType({ - name: `${variant}Where`, + name: `${referenceType.name}Where`, description: 'Sibling keys are combined with AND, and all of them must hold of the SAME entry.', fields: () => Object.fromEntries( - whereFieldsOf(referenceType, presenceLeaves).map((nestedField) => [ + filterableFields(referenceType).map((nestedField) => [ nestedField.name, { - type: presenceLeaves.includes(nestedField.name) - ? presenceFilterFor(nestedField, referenceType) - : whereFieldType(nestedField, referenceType), + type: whereFieldType(nestedField, referenceType), description: nestedField.description, }, ]), @@ -1800,179 +1885,250 @@ export function buildGraphQLSchema( } /** - * The keyed facets object for one type (only called with ≥ 1 facet). A - * faceted leaf is served one level down, under the reference it is reached - * through – `facets { associatedMedia { license } }` – so a facet is named - * by the same path as the value it counts. A presence facet is served one - * level down too, under the field it counts – `facets { dateCreated { - * exists } }` – in an object that has room for the field’s value buckets - * once a field may have both. + * The keyed facets object for one type (only called with ≥ 1 facet): one + * key per field anything is counted on. A field counting only its own values + * keeps its plain facet type (`IRIFacet`, `ValueFacet`, a range facet, a + * boolean bucket list). A field counting more – whether it holds a value, + * or a leaf of its referents – gets one object of its own, + * `‹Type›‹Field›Facet`, holding everything counted on it + * ({@link fieldFacetFor}). */ function facetsTypeFor( typeName: string, facets: readonly FacetDeclaration[], ): GraphQLObjectType { - const own: FacetDeclaration[] = []; - const byGroup = new Map(); + const groups = new Map(); for (const facet of facets) { - // A leaf’s facets, a presence leaf’s included, go under the reference; - // an own field’s presence facet under the field. - const group = - facet.reference?.name ?? - (facet.presence === true ? facet.field.name : undefined); - if (group === undefined) { - own.push(facet); - continue; + const name = facet.reference?.name ?? facet.field.name; + const group: FacetGroup = groups.get(name) ?? { leaves: new Map() }; + groups.set(name, group); + if (facet.reference !== undefined) { + const leaf = group.leaves.get(facet.field.name) ?? {}; + group.leaves.set(facet.field.name, leaf); + if (facet.presence === true) { + leaf.presence = facet; + } else { + leaf.values = facet; + } + } else if (facet.presence === true) { + group.presence = facet; + } else { + group.values = facet; } - const grouped = byGroup.get(group) ?? []; - grouped.push(facet); - byGroup.set(group, grouped); } - return new GraphQLObjectType({ name: `${typeName}Facets`, - fields: () => ({ - ...facetFieldsFor(own, typeName), - ...Object.fromEntries( - [...byGroup].map(([group, grouped]) => { - const groupBaseName = `${typeName}${capitalized(group)}`; - return [ - group, - { - type: new GraphQLNonNull( - new GraphQLObjectType({ - name: `${groupBaseName}Facets`, - fields: () => facetFieldsFor(grouped, groupBaseName), - }), - ), - // The source carries the facet loader; the grouped facets - // read it. - resolve: (source: Source) => source, - }, - ]; - }), + fields: () => + Object.fromEntries( + [...groups].map(([name, group]) => [ + name, + group.presence === undefined && group.leaves.size === 0 + ? valueFacetFieldFor(group.values as FacetDeclaration) + : fieldFacetFor(`${typeName}${capitalized(name)}`, group), + ]), ), - }), }); } /** - * One GraphQL field per facet, keyed by its field’s name – `exists` for an - * own field’s presence facet, served under the field – and loaded by the - * facet’s logical name. A presence leaf’s facet is one level further down, - * `‹leaf› { exists }`, in an object typed `‹Base›‹Leaf›Facets` after the - * base name of the object it sits in (`WorkAssociatedMedia`). + * The facet object of a field counting more than its own values, typed + * `‹Base›Facet` (`WorkContentLocationFacet`, `WorkDateCreatedFacet`): its + * own values in `buckets` and `totalValues` – or, for a numeric field, its + * range facet’s `all`, `matching` and `buckets` – capped by the field’s + * `limit`; `exists` for its presence; and one key per leaf of its + * referents, which is a plain facet type again for a leaf counting only its + * values, and `‹Base›‹Leaf›Facet` for one whose presence is counted. */ - function facetFieldsFor( - facets: readonly FacetDeclaration[], + function fieldFacetFor( baseName: string, - ): Record> { + group: FacetGroup, + ): GraphQLFieldConfig { + const values = group.values; + const range = values !== undefined && isRangeFacet(values.field); + const limited = values !== undefined && !range; const fields: Record< string, GraphQLFieldConfig > = {}; - // The keys share one engine round-trip, but each is its own facet - // calculation in it. - const facetCost = costExtensions(operationCost); - for (const { name, field, presence, reference: reachedThrough } of facets) { - // The batching into one engine dispatch and the degrade-to-empty - // error handling live in the loader (facet-batch.ts). - if (presence === true) { - const exists: GraphQLFieldConfig = { - type: nonNullListOf(booleanBucket), - extensions: facetCost, - description: reachedThrough - ? `Documents with an entry that holds a value for “${field.name}”, and documents with none.` - : `Documents that hold a value for “${field.name}”, and documents that hold none.`, - resolve: async (source: Source) => - (await (source.loadFacet as FacetLoader)(name, 'matching')).buckets, - }; - if (reachedThrough === undefined) { - fields.exists = exists; - continue; - } - fields[field.name] = { - type: new GraphQLNonNull( - new GraphQLObjectType({ - name: claimTypeName( - `${baseName}${capitalized(field.name)}Facets`, - `the presence facet of “${name}”`, - ), - fields: { exists }, - }), - ), - resolve: (source: Source) => source, - }; - continue; - } - if (isRangeFacet(field)) { - fields[field.name] = { - type: new GraphQLNonNull(rangeFacetTypeFor(field)), - extensions: facetCost, - // Resolved synchronously, so the scopes its fields load land in - // the same batch as the sibling facets. - resolve: (source: Source) => ({ - load: (scope: FacetScope) => - (source.loadFacet as FacetLoader)(name, scope), - }), - }; - continue; - } - if (field.kind === 'boolean') { - // Two values at most: nothing to cap and nothing to total. - fields[field.name] = { - type: nonNullListOf(booleanBucket), - extensions: facetCost, - resolve: async (source: Source) => - (await (source.loadFacet as FacetLoader)(name, 'matching')).buckets, + if (values !== undefined && range) { + // Each part is charged as the facet key it stands in for is – `all` + // already is, a facet query of its own. + const rangeFields = rangeFacetTypeFor( + values.field as NumericField, + ).toConfig().fields; + for (const [name, rangeField] of Object.entries(rangeFields)) { + fields[name] = { + ...rangeField, + extensions: { ...facetCost, ...rangeField.extensions }, }; - continue; } - // `IRI`-keyed buckets for a reference, literal keys otherwise: the - // reference test {@link whereFieldType} applies, so a bucket and the - // filter it feeds cannot disagree about what the field keys on. - const reference = field.kind === 'reference'; - fields[field.name] = { - type: new GraphQLNonNull(reference ? iriFacet : valueFacet), + } else if (values !== undefined) { + Object.assign(fields, valueFieldsFor(values)); + } + if (group.presence !== undefined) { + fields.exists = existsFieldFor(group.presence); + } + for (const [leafName, leaf] of group.leaves) { + fields[leafName] = + leaf.presence === undefined + ? valueFacetFieldFor(leaf.values as FacetDeclaration) + : fieldFacetFor(`${baseName}${capitalized(leafName)}`, { + values: leaf.values, + presence: leaf.presence, + leaves: new Map(), + }); + } + const name = `${baseName}Facet`; + return { + type: new GraphQLNonNull( + new GraphQLObjectType({ + name: claimTypeName(name, `the facet object of “${baseName}”`), + fields, + }), + ), + ...(limited && { args: { limit: limitArgument } }), + // Resolved synchronously, so what its keys load lands in the same batch + // as the sibling facets. A range facet’s keys read `load`. + resolve: (source: Source, args: { limit?: number | null }) => ({ + loadFacet: source.loadFacet, + limit: limited ? checkedLimit(args.limit) : undefined, + ...(range && { + load: (scope: FacetScope) => + (source.loadFacet as FacetLoader)( + (values as FacetDeclaration).name, + scope, + ), + }), + }), + }; + } + + /** A field counting only its own values, typed by its kind, as the facets + * object has always served it. */ + function valueFacetFieldFor( + facet: FacetDeclaration, + ): GraphQLFieldConfig { + const { name, field } = facet; + // The batching into one engine dispatch and the degrade-to-empty error + // handling live in the loader (facet-batch.ts). + if (isRangeFacet(field)) { + return { + type: new GraphQLNonNull(rangeFacetTypeFor(field)), extensions: facetCost, - args: { - limit: { - type: GraphQLInt, - description: `At most this many buckets, between 1 and ${maxFacetLimit}: the values with the highest counts. A larger limit returns a superset of a smaller one; there is no next page. Omit it for the deployment’s default.`, - }, - }, - resolve: async (source: Source, args: { limit?: number | null }) => { - const limit = args.limit ?? undefined; - if (limit !== undefined && (limit < 1 || limit > maxFacetLimit)) { - throw userError( - `limit must be between 1 and ${maxFacetLimit}; got ${limit}.`, - ); - } - const { buckets, totalValues } = await ( - source.loadFacet as FacetLoader - )(name, 'matching', limit); - return { - // A bucket keyed on something that is not an IRI is not a - // selection key – it cannot be sent back as the filter that - // selects it – so it is dropped rather than served under `IRI`. - // Dropping, not throwing: `value: IRI!` sits inside a chain of - // non-nulls up to the root field, so a raising coercion here - // would null the WHOLE response and take `items` with it, which - // is exactly what facet-batch.ts’s degradation contract forbids - // a supplementary sidebar count from doing. - buckets: reference - ? buckets.filter( - (bucket) => - typeof bucket.value === 'string' && - isAbsoluteIri(bucket.value), - ) - : buckets, - totalValues: totalValues ?? null, - }; - }, + // Resolved synchronously, so the scopes its fields load land in the + // same batch as the sibling facets. + resolve: (source: Source) => ({ + load: (scope: FacetScope) => + (source.loadFacet as FacetLoader)(name, scope), + }), }; } - return fields; + if (field.kind === 'boolean') { + // Two values at most: nothing to cap and nothing to total. + return { + type: nonNullListOf(booleanBucket), + extensions: facetCost, + resolve: async (source: Source) => + (await (source.loadFacet as FacetLoader)(name, 'matching')).buckets, + }; + } + return { + type: new GraphQLNonNull( + field.kind === 'reference' ? iriFacet : valueFacet, + ), + extensions: facetCost, + args: { limit: limitArgument }, + resolve: (source: Source, args: { limit?: number | null }) => + valueFacetOf(source, facet, checkedLimit(args.limit)), + }; + } + + /** + * A value facet’s `buckets` and `totalValues`, read through the loader. + * `IRI`-keyed buckets for a reference, literal keys otherwise: the reference + * test {@link whereFieldType} applies, so a bucket and the filter it feeds + * cannot disagree about what the field keys on. + */ + async function valueFacetOf( + source: Source, + facet: FacetDeclaration, + limit: number | undefined, + ): Promise<{ buckets: readonly FacetBucket[]; totalValues: number | null }> { + const { buckets, totalValues } = await (source.loadFacet as FacetLoader)( + facet.name, + 'matching', + limit, + ); + return { + // A bucket keyed on something that is not an IRI is not a selection key + // – it cannot be sent back as the filter that selects it – so it is + // dropped rather than served under `IRI`. Dropping, not throwing: + // `value: IRI!` sits inside a chain of non-nulls up to the root field, + // so a raising coercion here would null the WHOLE response and take + // `items` with it, which is exactly what facet-batch.ts’s degradation + // contract forbids a supplementary sidebar count from doing. + buckets: + facet.field.kind === 'reference' + ? buckets.filter( + (bucket) => + typeof bucket.value === 'string' && isAbsoluteIri(bucket.value), + ) + : buckets, + totalValues: totalValues ?? null, + }; + } + + /** `buckets` and `totalValues` of a field’s own values, inside its facet + * object, capped by the `limit` the field took ({@link fieldFacetFor}). */ + function valueFieldsFor( + facet: FacetDeclaration, + ): Record> { + const shape = ( + facet.field.kind === 'reference' ? iriFacet : valueFacet + ).toConfig().fields; + const read = (source: Source) => + valueFacetOf(source, facet, source.limit as number | undefined); + return { + buckets: { + ...shape.buckets, + extensions: facetCost, + resolve: async (source: Source) => (await read(source)).buckets, + }, + totalValues: { + ...shape.totalValues, + extensions: facetCost, + resolve: async (source: Source) => (await read(source)).totalValues, + }, + }; + } + + /** `exists`: the documents that hold a value – for a leaf, with a referent + * that holds one – and the documents that hold none. */ + function existsFieldFor( + facet: FacetDeclaration, + ): GraphQLFieldConfig { + return { + type: nonNullListOf(booleanBucket), + extensions: facetCost, + description: + facet.reference === undefined + ? `Documents that hold a value for “${facet.field.name}”, and documents that hold none.` + : `Documents with a referent that holds a value for “${facet.field.name}”, and documents with none.`, + resolve: async (source: Source) => + (await (source.loadFacet as FacetLoader)(facet.name, 'matching')) + .buckets, + }; + } + + /** A value facet’s `limit` argument, between 1 and `maxFacetLimit`. */ + function checkedLimit(limit: number | null | undefined): number | undefined { + const checked = limit ?? undefined; + if (checked !== undefined && (checked < 1 || checked > maxFacetLimit)) { + throw userError( + `limit must be between 1 and ${maxFacetLimit}; got ${checked}.`, + ); + } + return checked; } const queryFields: Record< @@ -2099,6 +2255,16 @@ function whereToFilters( // whole AND/OR mapping: which bucket a criterion lands in, never a combinator // inferred from how deeply it is nested. for (const entry of keyEntriesOf(where, searchType, joins, schema)) { + // A lookup’s presence leaves are recorded on this document, so each key + // of its `where` is a criterion of its own, ANDed like sibling keys. + if ('leafPresence' in entry) { + filters.push( + ...entry.leafPresence.map((criterion) => + filterOn(withPath(criterion, on)), + ), + ); + continue; + } if ('nested' in entry) { // A NESTING hop welds: every condition inside one edge's `where` must // hold of the SAME entry, which is the whole point of indexing an edge. @@ -2217,10 +2383,27 @@ function criteriaOf( ): Criterion[] { const criteria: Criterion[] = []; for (const entry of keyEntriesOf(keyed, searchType, joins, schema)) { - if (!('nested' in entry)) { + if ('criterion' in entry) { criteria.push(withPath(entry.criterion, on)); continue; } + // A lookup’s `where` ANDs its keys, so in an `or` alternative it must set + // exactly one, as a joined condition must. + if ('leafPresence' in entry) { + const [only, ...more] = entry.leafPresence; + if (only === undefined) { + throw userError( + `The condition on “${entry.name}” states no criterion, so it constrains nothing – and an “or” alternative that constrains nothing would make the whole disjunction match everything. Give it a condition, or leave the alternative out.`, + ); + } + if (more.length > 0) { + throw userError( + `The condition on “${entry.name}” states more than one criterion, so it is a conjunction – which cannot sit inside an “or”. Write one “or” alternative per criterion, or move the conjunction into “and”.`, + ); + } + criteria.push(withPath(only, on)); + continue; + } // A NESTING hop welds, in an `or` alternative exactly as anywhere else – // and it is the one place a weld belongs most naturally, since a welded // criterion IS an atom. Compiled as separate joined criteria it would be @@ -2260,10 +2443,17 @@ function criteriaOf( return criteria; } -/** One key of a `Where`/`Criterion` input, compiled: either a leaf criterion or - * a joined `where` to recurse into. */ +/** One key of a `Where`/`Criterion` input, compiled: a leaf criterion, a + * lookup’s presence-leaf criteria, or a joined or nested `where` to recurse + * into. */ type KeyEntry = | { readonly criterion: Criterion } + | { + readonly name: string; + // One criterion per presence leaf the lookup’s `where` sets, each on + // the root document’s companion. + readonly leafPresence: readonly Criterion[]; + } | { readonly name: string; readonly nested: Record; @@ -2312,6 +2502,27 @@ function keyEntriesOf( continue; } } + const nested = (value as { where?: Record | null }).where; + // A lookup’s `where` names its presence leaves, each a plain criterion on + // the root document’s companion – no entry, and no join. + if ( + !isInlineReference(field) && + presenceLeavesOf(field).length > 0 && + nested !== undefined && + nested !== null + ) { + entries.push({ + name: field.name, + leafPresence: presenceLeavesOf(field).flatMap((leaf): Criterion[] => { + const exists = (nested[leaf] as { exists?: boolean | null } | null) + ?.exists; + return exists === undefined || exists === null + ? [] + : [{ field: field.name, leaf, exists }]; + }), + }); + continue; + } // Both hop kinds take the same `@oneOf` input – `in` (the ids this field // holds) or `where` (a condition one hop out) – so both are read here, and // only what they resolve to differs: another collection, or this @@ -2319,18 +2530,14 @@ function keyEntriesOf( const target = nestedReferenceType(schema, field) ?? joins.resolve(searchType, [field.name]); - if (target !== undefined) { - const nested = (value as { where?: Record | null }) - .where; - if (nested !== undefined && nested !== null) { - entries.push({ - name: field.name, - nested, - target, - presenceLeaves: presenceLeavesOf(field), - }); - continue; - } + if (target !== undefined && nested !== undefined && nested !== null) { + entries.push({ + name: field.name, + nested, + target, + presenceLeaves: presenceLeavesOf(field), + }); + continue; } switch (filterOperatorFor(field)) { case 'in': @@ -2447,12 +2654,24 @@ function rangeInput( }); } -/** The presence leaves an inline reference declares, read off the physical - * companions that record them – none for any other field. */ +/** The presence leaves a reference declares, inline or `lookup`, read off + * the physical companions that record them – none for any other field. */ function presenceLeavesOf(field: SearchField): readonly string[] { return physicalFields(field).leafExists.map(({ leaf }) => leaf); } +/** What one field of a root type counts, gathered for its facet object: its + * own values, its presence, and – per leaf of its referents – the leaf’s + * values and presence. */ +interface FacetGroup { + values?: FacetDeclaration; + presence?: FacetDeclaration; + readonly leaves: Map< + string, + { values?: FacetDeclaration; presence?: FacetDeclaration } + >; +} + /** A name with its first letter upper-cased, as it is spelled inside a type * name: `associatedMedia` → `AssociatedMedia`. */ function capitalized(name: string): string { diff --git a/packages/search-api-graphql/test/faceted-leaves.test.ts b/packages/search-api-graphql/test/faceted-leaves.test.ts index d1807bf6..27ddb51c 100644 --- a/packages/search-api-graphql/test/faceted-leaves.test.ts +++ b/packages/search-api-graphql/test/faceted-leaves.test.ts @@ -78,10 +78,10 @@ describe('a root type faceted on its entries’ leaves', () => { const printed = printSchema(gqlSchema); expect(printed).toMatch( - /type WorkFacets \{\s*associatedMedia: WorkAssociatedMediaFacets!\s*\}/, + /type WorkFacets \{\s*associatedMedia: WorkAssociatedMediaFacet!\s*\}/, ); expect(printed).toMatch( - /type WorkAssociatedMediaFacets \{\s*license\([\s\S]*?\): IRIFacet!\s*\}/, + /type WorkAssociatedMediaFacet \{\s*license\([\s\S]*?\): IRIFacet!\s*\}/, ); }); diff --git a/packages/search-api-graphql/test/leaf-presence.test.ts b/packages/search-api-graphql/test/leaf-presence.test.ts index 5ce90ef4..a61d15d3 100644 --- a/packages/search-api-graphql/test/leaf-presence.test.ts +++ b/packages/search-api-graphql/test/leaf-presence.test.ts @@ -103,7 +103,7 @@ async function run(source: string) { describe('a root document filtered on whether an entry’s leaf has a value', () => { it('gives a presence leaf `exists` in the reference type’s where', () => { expect(printed).toMatch( - /input MediaObjectWhere \{[^}]*license: IRIFilter\n[^}]*thumbnailUrl: ExistsFilter\n[^}]*encodingFormat: KeywordFilterOrExists\n/, + /input WorkAssociatedMediaWhere \{[^}]*license: IRIFilter\n[^}]*thumbnailUrl: ExistsFilter\n[^}]*encodingFormat: KeywordFilterOrExists\n/, ); }); @@ -159,10 +159,10 @@ describe('a root document filtered on whether an entry’s leaf has a value', () describe('a root document faceted on whether an entry’s leaf has a value', () => { it('serves the facet under the leaf, beside the reference’s faceted leaves', () => { expect(printed).toMatch( - /type WorkAssociatedMediaFacets \{[^}]*\): IRIFacet!\s*thumbnailUrl: WorkAssociatedMediaThumbnailUrlFacets!\s*encodingFormat: WorkAssociatedMediaEncodingFormatFacets!\s*\}/, + /type WorkAssociatedMediaFacet \{[^}]*\): IRIFacet!\s*thumbnailUrl: WorkAssociatedMediaThumbnailUrlFacet!\s*encodingFormat: WorkAssociatedMediaEncodingFormatFacet!\s*\}/, ); expect(printed).toMatch( - /type WorkAssociatedMediaThumbnailUrlFacets \{[^}]*exists: \[BooleanBucket!\]!\s*\}/, + /type WorkAssociatedMediaThumbnailUrlFacet \{[^}]*exists: \[BooleanBucket!\]!\s*\}/, ); }); @@ -237,15 +237,15 @@ describe('a reference with presence leaves on a joined type', () => { it('reaches the entries through a join without `exists`, which no join reads', () => { expect(joinedPrinted).toMatch( - /input CollectionJoinedWhere \{[^}]*associatedMedia: MediaObjectJoinedFilter\n/, + /input CollectionJoinedWhere \{[^}]*associatedMedia: CollectionAssociatedMediaJoinedFilter\n/, ); - const where = /input MediaObjectJoinedWhere \{([^}]*)\}/.exec( + const where = /input CollectionAssociatedMediaJoinedWhere \{([^}]*)\}/.exec( joinedPrinted, ); expect(where?.[1]).toMatch(/encodingFormat: KeywordFilter\n/); expect(where?.[1]).not.toMatch(/thumbnailUrl|exists|OrExists/); expect(joinedPrinted).toMatch( - /input CollectionWhere \{[^}]*associatedMedia: MediaObjectFilter\n/, + /input CollectionWhere \{[^}]*associatedMedia: CollectionAssociatedMediaFilter\n/, ); }); }); @@ -273,13 +273,13 @@ describe('the names a presence leaf adds', () => { searchSchema( work, mediaObject, - clashing('WorkAssociatedMediaThumbnailUrlFacets'), + clashing('WorkAssociatedMediaThumbnailUrlFacet'), ), {}, ), ), ).toThrow( - /“WorkAssociatedMediaThumbnailUrlFacets”, the type name of the presence facet of “associatedMedia.thumbnailUrl.exists”, is already the name of another type/, + /“WorkAssociatedMediaThumbnailUrlFacet”, the type name of the facet object of “WorkAssociatedMediaThumbnailUrl”, is already the name of another type/, ); }); @@ -317,13 +317,13 @@ describe('the names a presence leaf adds', () => { labelled, item, mediaObject, - clashing('MediaObjectJoinedFilter'), + clashing('WorkAssociatedMediaJoinedFilter'), ), {}, ), ), ).toThrow( - /“MediaObjectJoinedFilter”, the type name of the filter on “MediaObject” through a join, is already the name of another type/, + /“WorkAssociatedMediaJoinedFilter”, the type name of the filter on “Work.associatedMedia”, is already the name of another type/, ); }); }); diff --git a/packages/search-api-graphql/test/lookup-presence.test.ts b/packages/search-api-graphql/test/lookup-presence.test.ts new file mode 100644 index 00000000..d1efdb42 --- /dev/null +++ b/packages/search-api-graphql/test/lookup-presence.test.ts @@ -0,0 +1,390 @@ +import { describe, expect, it } from 'vitest'; +import { graphql, printSchema } from 'graphql'; +import { + defineSearchType, + searchSchema, + type SearchEngine, + type SearchQuery, + type SearchResult, + type SearchType, +} from '@lde/search'; +import { buildGraphQLSchema, type SearchContext } from '../src/build-schema.js'; + +const SCHEMA_ORG = 'https://schema.org/'; +const VENLO = 'https://sws.geonames.org/2745707'; +const KESSEL = 'https://ex/place/kessel'; + +const place = defineSearchType({ + name: 'Place', + class: `${SCHEMA_ORG}Place`, + fields: [ + { + name: 'label', + kind: 'text', + path: `${SCHEMA_ORG}name`, + locales: ['nl'], + output: true, + searchable: { weight: 1 }, + }, + { + name: 'latitude', + kind: 'number', + path: `${SCHEMA_ORG}latitude`, + output: true, + }, + { + name: 'longitude', + kind: 'number', + path: `${SCHEMA_ORG}longitude`, + output: true, + }, + ], +}); + +const work = defineSearchType({ + name: 'Work', + class: `${SCHEMA_ORG}CreativeWork`, + fields: [ + { + name: 'label', + kind: 'text', + locales: ['nl'], + output: true, + searchable: { weight: 1 }, + }, + { name: 'material', kind: 'keyword', facetable: true, filterable: true }, + { + name: 'contentLocation', + kind: 'reference', + path: `${SCHEMA_ORG}contentLocation`, + array: true, + output: true, + filterable: true, + facetable: true, + ref: { + strategy: 'lookup', + target: 'Place', + presenceLeaves: ['latitude', 'longitude'], + }, + }, + { + name: 'locationCreated', + kind: 'reference', + path: `${SCHEMA_ORG}locationCreated`, + filterable: true, + ref: { strategy: 'lookup', target: 'Place' }, + }, + ], +}); + +const schema = searchSchema(work, place); +const gqlSchema = buildGraphQLSchema(schema, {}); +const printed = printSchema(gqlSchema); + +function recordingEngine() { + const searched: SearchQuery[] = []; + const faceted: SearchQuery[] = []; + const engine: SearchEngine = { + schema, + async search(_searchType: SearchType, query: SearchQuery) { + searched.push(query); + return { total: 0, hits: [], facets: {} } as SearchResult; + }, + async searchFacets(_searchType, queries) { + faceted.push(...queries); + return queries.map(() => ({ + facets: { + material: { buckets: [{ value: 'hout', count: 3 }], totalValues: 1 }, + contentLocation: { + buckets: [ + { value: VENLO, count: 2 }, + { value: KESSEL, count: 1 }, + ], + totalValues: 2, + }, + 'contentLocation.latitude.exists': { + buckets: [ + { value: 'true', count: 2, is: true }, + { value: 'false', count: 1, is: false }, + ], + }, + }, + })) as never; + }, + }; + return { engine, searched, faceted }; +} + +async function run(source: string) { + const recording = recordingEngine(); + const response = await graphql({ + schema: gqlSchema, + source, + contextValue: { + engine: recording.engine, + acceptLanguage: ['nl'], + } as SearchContext, + }); + return { response, ...recording }; +} + +describe('one facet object per field', () => { + it('keeps the plain facet type for a field counting only its own values', () => { + expect(printed).toMatch(/material\([^)]*\): ValueFacet!/); + }); + + it('gives a field counting more an object holding everything counted on it', () => { + expect(printed).toMatch( + /contentLocation\(\s*"""[^"]*"""\s*limit: Int\s*\): WorkContentLocationFacet!/, + ); + expect(printed).toMatch( + /type WorkContentLocationFacet \{[^}]*buckets: \[IRIBucket!\]![^}]*totalValues: Int\s*latitude: WorkContentLocationLatitudeFacet!\s*longitude: WorkContentLocationLongitudeFacet!\s*\}/, + ); + expect(printed).toMatch( + /type WorkContentLocationLatitudeFacet \{[^}]*exists: \[BooleanBucket!\]!\s*\}/, + ); + }); + + it('caps the field’s own buckets with its limit, and counts its leaf beside them', async () => { + const { response, faceted } = await run( + `{ works { facets { contentLocation(limit: 1) { buckets { value count } totalValues latitude { exists { value count } } } } } }`, + ); + + expect(response.errors).toBeUndefined(); + expect(faceted.map((query) => query.facets)).toEqual([ + ['contentLocation', 'contentLocation.latitude.exists'], + ]); + expect(faceted[0]?.facetLimits).toEqual({ contentLocation: 1 }); + expect(response.data).toEqual({ + works: { + facets: { + contentLocation: { + buckets: [ + { value: VENLO, count: 2 }, + { value: KESSEL, count: 1 }, + ], + totalValues: 2, + latitude: { + exists: [ + { value: true, count: 2 }, + { value: false, count: 1 }, + ], + }, + }, + }, + }, + }); + }); + + it('refuses a limit out of range', async () => { + const { response } = await run( + `{ works { facets { contentLocation(limit: 0) { totalValues } } } }`, + ); + + expect(response.errors?.[0]?.message).toMatch( + /limit must be between 1 and 100; got 0/, + ); + }); +}); + +describe('a lookup filtered on whether a referent’s leaf has a value', () => { + it('takes a filter of its own, beside its ids', () => { + expect(printed).toMatch(/contentLocation: WorkContentLocationFilter\n/); + expect(printed).toMatch( + /input WorkContentLocationFilter @oneOf \{[^}]*in: \[IRI!\][^}]*where: WorkContentLocationWhere\s*\}/, + ); + expect(printed).toMatch( + /input WorkContentLocationWhere \{[^}]*latitude: ExistsFilter\s*longitude: ExistsFilter\s*\}/, + ); + // A lookup counting no leaf keeps its target’s filter. + expect(printed).toMatch(/locationCreated: PlaceFilter\n/); + }); + + it('compiles `exists` on the leaf into a plain criterion on the root', async () => { + const { response, searched } = await run( + `{ works(where: { contentLocation: { where: { latitude: { exists: false } } } }) { pagination { total } } }`, + ); + + expect(response.errors).toBeUndefined(); + expect(searched[0]?.where).toEqual([ + { + or: [{ field: 'contentLocation', leaf: 'latitude', exists: false }], + }, + ]); + }); + + it('ANDs two leaves in one `where`, as sibling keys', async () => { + const { response, searched } = await run( + `{ works(where: { contentLocation: { where: { latitude: { exists: true }, longitude: { exists: false } } } }) { pagination { total } } }`, + ); + + expect(response.errors).toBeUndefined(); + expect(searched[0]?.where).toEqual([ + { or: [{ field: 'contentLocation', leaf: 'latitude', exists: true }] }, + { or: [{ field: 'contentLocation', leaf: 'longitude', exists: false }] }, + ]); + }); + + it('takes one leaf in an `or` alternative, and refuses two', async () => { + const one = await run( + `{ works(where: { or: [{ contentLocation: { where: { latitude: { exists: true } } } }, { material: { in: ["hout"] } }] }) { pagination { total } } }`, + ); + expect(one.response.errors).toBeUndefined(); + expect(one.searched[0]?.where).toEqual([ + { + or: [ + { field: 'contentLocation', leaf: 'latitude', exists: true }, + { field: 'material', in: ['hout'] }, + ], + }, + ]); + + const two = await run( + `{ works(where: { or: [{ contentLocation: { where: { latitude: { exists: true }, longitude: { exists: true } } } }] }) { pagination { total } } }`, + ); + expect(two.response.errors?.[0]?.message).toMatch( + /The condition on “contentLocation” states more than one criterion, so it is a conjunction/, + ); + + const none = await run( + `{ works(where: { or: [{ contentLocation: { where: {} } }] }) { pagination { total } } }`, + ); + expect(none.response.errors?.[0]?.message).toMatch( + /The condition on “contentLocation” states no criterion, so it constrains nothing/, + ); + }); + + it('still filters by the ids it holds', async () => { + const { response, searched } = await run( + `{ works(where: { contentLocation: { in: ["${VENLO}"] } }) { pagination { total } } }`, + ); + + expect(response.errors).toBeUndefined(); + expect(searched[0]?.where).toEqual([ + { or: [{ field: 'contentLocation', in: [VENLO] }] }, + ]); + }); +}); + +describe('a numeric field counted by its range and its presence', () => { + const dated = defineSearchType({ + name: 'Dated', + class: `${SCHEMA_ORG}Event`, + fields: [ + { + name: 'label', + kind: 'text', + locales: ['nl'], + output: true, + searchable: { weight: 1 }, + }, + { + name: 'startDate', + kind: 'date', + facetable: true, + filterable: true, + presence: true, + }, + ], + }); + const datedSchema = searchSchema(dated); + const datedPrinted = printSchema(buildGraphQLSchema(datedSchema, {})); + + it('holds the range facet’s parts and `exists` in one object', async () => { + expect(datedPrinted).toMatch( + /type DatedStartDateFacet \{[^}]*all: DateExtent[^}]*matching: DateExtent[^}]*buckets: \[DateBucket!\]![^}]*exists: \[BooleanBucket!\]!\s*\}/, + ); + const loaded: string[] = []; + const engine: SearchEngine = { + schema: datedSchema, + async search() { + return { total: 0, hits: [], facets: {} } as SearchResult; + }, + async searchFacets(_searchType, queries) { + loaded.push(...queries.flatMap((query) => query.facets)); + return queries.map(() => ({ + facets: { + startDate: { buckets: [], extent: { min: 0, max: 86400 } }, + 'startDate.exists': { + buckets: [{ value: 'true', count: 1, is: true }], + }, + }, + })) as never; + }, + }; + const response = await graphql({ + schema: buildGraphQLSchema(datedSchema, {}), + source: `{ dateds { facets { startDate { matching { min max } exists { value count } } } } }`, + contextValue: { engine, acceptLanguage: ['nl'] } as SearchContext, + }); + + expect(response.errors).toBeUndefined(); + expect(loaded).toEqual(['startDate', 'startDate.exists']); + expect(response.data).toEqual({ + dateds: { + facets: { + startDate: { + matching: { + min: '1970-01-01T00:00:00Z', + max: '1970-01-02T00:00:00Z', + }, + exists: [{ value: true, count: 1 }], + }, + }, + }, + }); + }); +}); + +describe('a lookup with presence leaves on a joined type', () => { + it('keeps its ids’ filter through the join, which reads no companion', () => { + const item = defineSearchType({ + name: 'Item', + class: `${SCHEMA_ORG}Product`, + fields: [ + { + name: 'work', + kind: 'reference', + filterable: true, + labelSource: 'Work', + joinable: true, + }, + ], + }); + const joinedPrinted = printSchema( + buildGraphQLSchema(searchSchema(work, place, item), {}), + ); + + expect(joinedPrinted).toMatch( + /input WorkJoinedWhere \{[^}]*contentLocation: PlaceFilter\n/, + ); + }); +}); + +describe('a keyword field counted by its values and its presence', () => { + it('serves its literal buckets beside `exists`', () => { + const tagged = defineSearchType({ + name: 'Tagged', + class: `${SCHEMA_ORG}Thing`, + fields: [ + { + name: 'label', + kind: 'text', + locales: ['nl'], + output: true, + searchable: { weight: 1 }, + }, + { + name: 'material', + kind: 'keyword', + facetable: true, + filterable: true, + presence: true, + }, + ], + }); + expect(printSchema(buildGraphQLSchema(searchSchema(tagged), {}))).toMatch( + /type TaggedMaterialFacet \{[^}]*buckets: \[ValueBucket!\]![^}]*totalValues: Int[^}]*exists: \[BooleanBucket!\]!\s*\}/, + ); + }); +}); diff --git a/packages/search-api-graphql/test/presence.test.ts b/packages/search-api-graphql/test/presence.test.ts index 5a7af6bc..53cd0675 100644 --- a/packages/search-api-graphql/test/presence.test.ts +++ b/packages/search-api-graphql/test/presence.test.ts @@ -165,9 +165,9 @@ describe('a field faceted on whether it has a value', () => { it('serves the facet under the field it counts', () => { const printed = printSchema(gqlSchema); - expect(printed).toMatch(/dateCreated: WorkDateCreatedFacets!/); + expect(printed).toMatch(/dateCreated: WorkDateCreatedFacet!/); expect(printed).toMatch( - /type WorkDateCreatedFacets \{[^}]*exists: \[BooleanBucket!\]!\s*\}/, + /type WorkDateCreatedFacet \{[^}]*exists: \[BooleanBucket!\]!\s*\}/, ); }); diff --git a/packages/search-api-graphql/test/query-cost.test.ts b/packages/search-api-graphql/test/query-cost.test.ts index 6ce4cbdb..8807e082 100644 --- a/packages/search-api-graphql/test/query-cost.test.ts +++ b/packages/search-api-graphql/test/query-cost.test.ts @@ -54,7 +54,12 @@ const place: SearchType = { output: true, searchable: { weight: 1 }, }, - { name: 'postalCode', kind: 'keyword', output: true }, + { + name: 'postalCode', + kind: 'keyword', + path: 'https://schema.org/postalCode', + output: true, + }, { name: 'containedIn', kind: 'reference', @@ -105,6 +110,36 @@ const work: SearchType = { output: true, ref: { strategy: 'lookup', target: ['Person', 'Organization'] }, }, + // Fields whose facet counts more than their own values, served as one + // object each. + { + name: 'material', + kind: 'keyword', + output: true, + filterable: true, + facetable: true, + presence: true, + }, + { + name: 'pages', + kind: 'integer', + output: true, + filterable: true, + facetable: true, + presence: true, + }, + { + name: 'location', + kind: 'reference', + output: true, + filterable: true, + facetable: true, + ref: { + strategy: 'lookup', + target: 'Place', + presenceLeaves: ['postalCode'], + }, + }, ], }; const schema = searchSchema(work, person, organization, place); @@ -201,6 +236,32 @@ describe('the cost of a query', () => { ).toBe(1 + 1); }); + it('charges each part of a field’s facet object it selects', async () => { + const facets = (selection: string) => + costOf(`{ creativeWorks(perPage: 0) { facets { ${selection} } } }`); + + expect(await facets('material { buckets { value } }')).toBe(1 + 1); + expect(await facets('material { totalValues }')).toBe(1 + 1); + expect(await facets('material { exists { value } }')).toBe(1 + 1); + // Each part is charged on its own, so the values’ two parts cost two. + expect( + await facets( + 'material { buckets { value } totalValues exists { value } }', + ), + ).toBe(1 + 3); + expect(await facets('pages { matching { min } }')).toBe(1 + 1); + expect(await facets('pages { buckets { value } }')).toBe(1 + 1); + expect(await facets('pages { all { min } exists { value } }')).toBe(1 + 2); + expect(await facets('location { postalCode { exists { value } } }')).toBe( + 1 + 1, + ); + expect( + await facets( + 'location { buckets { value } postalCode { exists { value } } }', + ), + ).toBe(1 + 2); + }); + it('charges a lookup inside a fragment like an inlined one', async () => { expect( await costOf( diff --git a/packages/search-api-graphql/vite.config.ts b/packages/search-api-graphql/vite.config.ts index 242dc709..453e092b 100644 --- a/packages/search-api-graphql/vite.config.ts +++ b/packages/search-api-graphql/vite.config.ts @@ -31,7 +31,7 @@ export default mergeConfig( // from a direct caller, since GraphQL validates the query first. // Re-anchored when facet grouping moved to `@lde/search`, taking its // covered branches with it. - branches: 99.05, + branches: 99.15, statements: 100, }, }, diff --git a/packages/search-pipeline/package.json b/packages/search-pipeline/package.json index 4451b4c1..0c050ce9 100644 --- a/packages/search-pipeline/package.json +++ b/packages/search-pipeline/package.json @@ -30,11 +30,11 @@ "@traqula/generator-sparql-1-1": "^1.3.1", "@traqula/parser-sparql-1-1": "^1.3.1", "@traqula/rules-sparql-1-1": "^1.2.0", + "n3": "^2.13.12", "tslib": "^2.3.0" }, "devDependencies": { "@lde/search-typesense": "^0.64.0", - "n3": "^2.13.12", "testcontainers": "^12.1.0", "typesense": "^3.2.0" }, diff --git a/packages/search-pipeline/src/extraction.ts b/packages/search-pipeline/src/extraction.ts index 3030aa50..ee40a98c 100644 --- a/packages/search-pipeline/src/extraction.ts +++ b/packages/search-pipeline/src/extraction.ts @@ -18,6 +18,7 @@ import { isInlineReference, labelFieldOf, localLookupTargetsOf, + physicalFields, referencedTargetsOf, referenceTypeNamed, } from '@lde/search/adapter'; @@ -143,20 +144,27 @@ export function extractionQueryString( /** * The fields a reference reads off a referent of a target it does not expand - * `local`ly: the target’s key field, where it is keyed, and – for a - * `searchable` reference – its label field, by which the reference is searched. + * `local`ly: the target’s key field, where it is keyed; – for a `searchable` + * reference – its label field, by which the reference is searched; and each of + * the reference’s presence leaves, whose companions say whether any referent + * holds one. Each field once, though it plays several of these parts. */ export function referentHopFields( field: SearchField, target: RootType, ): PathField[] { - return [ + const hopFields = [ keyedFieldOf(target), // `searchSchema` guarantees a searchable reference’s label a path. field.searchable === undefined ? undefined : (labelFieldOf(target) as PathField), + // …and a lookup’s target each presence leaf, with a path. + ...physicalFields(field).leafExists.map( + ({ leaf }) => fieldNamed(target, leaf) as PathField, + ), ].filter((hopField) => hopField !== undefined); + return [...new Set(hopFields)]; } /** A per-query counter minting distinct value/referent variable names. */ diff --git a/packages/search-pipeline/src/search-stages.ts b/packages/search-pipeline/src/search-stages.ts index 4361c82d..4fba1348 100644 --- a/packages/search-pipeline/src/search-stages.ts +++ b/packages/search-pipeline/src/search-stages.ts @@ -1,4 +1,5 @@ -import type { Quad, Term } from '@rdfjs/types'; +import type { NamedNode, Quad, Term } from '@rdfjs/types'; +import { DataFactory } from 'n3'; import { Distribution, type Dataset } from '@lde/dataset'; import { SparqlConstructReader, @@ -23,6 +24,7 @@ import { import { endpointFieldOf, irAlias, + physicalFields, isInlineReference, referencedTargetsOf, referenceTypeNamed, @@ -397,10 +399,13 @@ export function registrySource( export const defaultBatchSize = { endpoint: 10, imported: 100 } as const; /** - * A transform passing each referent of the type’s `searchable` references, - * its inline referents’ included, through its target type’s transforms: the + * A transform passing each referent of the type’s `searchable` references – + * its inline referents’ included – and of its lookups declaring presence + * leaves, through its target type’s transforms: the * quads the extraction read off the referent ({@link referentHopFields}) go * in, and what the target’s transforms make of them comes back in their place. + * A lookup’s referent named by IRI alone is seeded for its presence leaves, + * through the type’s own transforms too where the lookup names its own type. * `undefined` where no such reference names a target with transforms. * * The batch is buffered to find the referents, so memory stays bounded by the @@ -414,33 +419,41 @@ function referentTransforms( readonly QuadTransform[] >, ): QuadTransform | undefined { - // Per target with transforms: the reference fields naming it, and the - // aliases its referents carry. A target’s aliases are its own, so no quad - // belongs to two targets. + // Per target with transforms other than the type itself: the reference + // fields naming it, and the aliases its referents carry. A target’s aliases + // are its own, so no quad belongs to two targets. const routes = new Map< RootType, { fields: Set; aliases: Set } >(); - for (const { alias, field } of searchedReferences(searchType, schema)) { + // Per reference counting presence leaves: the one target (`searchSchema` + // refuses several) its referents named by IRI alone are seeded through. + const countedTargetByField = new Map(); + for (const { alias, field } of describedReferences(searchType, schema)) { for (const target of referencedTargetsOf(field, schema)) { - // A reference to its own type: the type’s transform saw the batch. - if ( - target !== searchType && - transformsByClass.get(target.class) !== undefined - ) { - const route = routes.get(target) ?? { - fields: new Set(), - aliases: new Set(), - }; - route.fields.add(alias); - for (const hopField of referentHopFields(field, target)) { - route.aliases.add(irAlias(target, hopField)); - } - routes.set(target, route); + if (transformsByClass.get(target.class) === undefined) { + continue; + } + if (physicalFields(field).leafExists.length > 0) { + countedTargetByField.set(alias, target); } + // A reference to its own type: the type’s transform saw the batch, its + // referents’ quads included. + if (target === searchType) { + continue; + } + const route = routes.get(target) ?? { + fields: new Set(), + aliases: new Set(), + }; + route.fields.add(alias); + for (const hopField of referentHopFields(field, target)) { + route.aliases.add(irAlias(target, hopField)); + } + routes.set(target, route); } } - if (routes.size === 0) { + if (routes.size === 0 && countedTargetByField.size === 0) { return undefined; } const targetsByField = new Map(); @@ -455,16 +468,27 @@ function referentTransforms( } return async function* referentTransforms(quads, context) { const batch: Quad[] = []; + // Every node the batch states anything of. + const subjects = new Set(); // Each target’s referents in this batch: the objects of the references - // naming it. + // naming it – and, by IRI, the ones a presence leaf is counted for. const referents = new Map>(); + const counted = new Map>(); for await (const quad of quads) { batch.push(quad); + subjects.add(termKey(quad.subject)); for (const target of targetsByField.get(quad.predicate.value) ?? []) { const keys = referents.get(target) ?? new Set(); keys.add(termKey(quad.object)); referents.set(target, keys); } + const countedTarget = countedTargetByField.get(quad.predicate.value); + if (countedTarget !== undefined && quad.object.termType === 'NamedNode') { + const nodes = + counted.get(countedTarget) ?? new Map(); + nodes.set(termKey(quad.object), quad.object); + counted.set(countedTarget, nodes); + } } // One pass sorting each quad into its target’s input, or keeping it. const inputs = new Map(); @@ -481,24 +505,68 @@ function referentTransforms( yield quad; } } + // A referent a presence leaf is counted for, and that its target’s + // transforms would see nothing of, enters them as ` rdf:type + // `: a transform describing a node from its key then sees it. A + // referent of another target is described by its input; one of the type’s + // own by anything the batch states of it, which the type’s transforms + // have already seen. The seed itself is not kept. + const seeds = new Map>(); + for (const [target, nodes] of counted) { + const input = inputs.get(target) ?? []; + const described = + target === searchType + ? subjects + : new Set(input.map((quad) => termKey(quad.subject))); + const seeded = new Set(); + for (const [key, node] of nodes) { + if (!described.has(key)) { + input.push( + factory.quad(node, RDF_TYPE, factory.namedNode(target.class)), + ); + seeded.add(key); + } + } + seeds.set(target, seeded); + inputs.set(target, input); + } for (const [target, input] of inputs) { + if (input.length === 0) { + continue; + } let stream: AsyncIterable = streamOf(input); for (const transform of transformsByClass.get( target.class, ) as readonly QuadTransform[]) { stream = transform(stream, context); } - yield* stream; + const seeded = seeds.get(target); + for await (const quad of stream) { + if ( + seeded?.has(termKey(quad.subject)) && + quad.predicate.equals(RDF_TYPE) && + quad.object.value === target.class + ) { + continue; + } + yield quad; + } } }; } -/** The type’s `searchable` reference fields, each with the IR Alias whose +const factory = DataFactory; +const RDF_TYPE = factory.namedNode( + 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type', +); + +/** The type’s reference fields whose referents are described – `searchable` + * ones, and lookups declaring presence leaves – each with the IR Alias whose * objects are its referents: its own, and those of the inline referents it * nests. A graph that leaves an edge out points at the endpoint through the * inline field itself, so an endpoint field is reached through both. * `searchSchema` refuses an inline cycle, so the descent ends. */ -function searchedReferences( +function describedReferences( owner: SearchType, schema: SearchSchema, ): { alias: string; field: SearchField }[] { @@ -511,13 +579,17 @@ function searchedReferences( ) as ReferenceType; const endpointField = endpointFieldOf(referenceType, field.ref.identity); return [ - ...searchedReferences(referenceType, schema), + ...describedReferences(referenceType, schema), ...(endpointField?.searchable === undefined ? [] : [{ alias: irAlias(owner, field), field: endpointField }]), ]; } - return field.kind === 'reference' && field.searchable !== undefined + // A reference’s referents are described when it is searched by their + // labels, or counted for a presence leaf. + return field.kind === 'reference' && + (field.searchable !== undefined || + physicalFields(field).leafExists.length > 0) ? [{ alias: irAlias(owner, field), field }] : []; }); diff --git a/packages/search-pipeline/test/extraction.test.ts b/packages/search-pipeline/test/extraction.test.ts index 97632407..aaf89996 100644 --- a/packages/search-pipeline/test/extraction.test.ts +++ b/packages/search-pipeline/test/extraction.test.ts @@ -20,7 +20,11 @@ import { referenceTypeNamed, } from '@lde/search/adapter'; import type { SearchSchema, SearchType } from '@lde/search'; -import { extractionQuery, extractionQueryString } from '../src/extraction.js'; +import { + extractionQuery, + extractionQueryString, + referentHopFields, +} from '../src/extraction.js'; const SCHEMA = 'https://schema.org/'; const RDF_TYPE = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type'; @@ -569,6 +573,26 @@ describe('references into a keyed type', () => { expect(group.patterns[1]).toMatchObject({ subType: 'optional' }); }); + it('reads a presence leaf that is also the key once', () => { + const counting = { + ...work.fields[0], + filterable: true, + searchable: { weight: 1 }, + ref: { + strategy: 'lookup', + target: 'Place', + presenceLeaves: ['_sameAs'], + }, + } as const; + const countingWork = defineSearchType({ ...work, fields: [counting] }); + searchSchema(countingWork, place); + + expect(referentHopFields(counting, place)).toEqual([ + place.fields[1], + place.fields[0], + ]); + }); + it('reads no label for a lookup that is not searchable', () => { expect( templatePredicates(extractionQuery(work, keyedSchema)), diff --git a/packages/search-pipeline/test/fixtures/lookup-presence-sample.ttl b/packages/search-pipeline/test/fixtures/lookup-presence-sample.ttl new file mode 100644 index 00000000..eb294efc --- /dev/null +++ b/packages/search-pipeline/test/fixtures/lookup-presence-sample.ttl @@ -0,0 +1,43 @@ +@prefix schema: . + +# Works naming places, for a presence leaf (`latitude`) counted on the work. + +# Named by IRI alone; the place transform describes GeoNames places from +# their IRI, coordinates included. + a schema:CreativeWork ; + schema:name "Geonames" ; + schema:contentLocation . + +# Named by IRI alone, and no transform describes it. + a schema:CreativeWork ; + schema:name "Unknown" ; + schema:contentLocation . + +# The work’s own graph states the place’s coordinates. + a schema:CreativeWork ; + schema:name "Stated" ; + schema:contentLocation . + schema:latitude 51.37 . + +# No place at all. + a schema:CreativeWork ; + schema:name "Bare" . + +# A place stated as a blank node: nothing to key a transform on. + a schema:CreativeWork ; + schema:name "Blank" ; + schema:contentLocation [ schema:latitude 50.85 ] . + +# Places within places, for a presence leaf on a lookup into its own type. + a schema:Place ; + schema:name "Village" ; + schema:containedInPlace . + a schema:Place ; + schema:name "Hamlet" ; + schema:containedInPlace . + a schema:Place ; + schema:name "Region" ; + schema:latitude 50.9 . + a schema:Place ; + schema:name "Isolated" ; + schema:containedInPlace . diff --git a/packages/search-pipeline/test/lookup-presence.integration.test.ts b/packages/search-pipeline/test/lookup-presence.integration.test.ts new file mode 100644 index 00000000..56cbd89d --- /dev/null +++ b/packages/search-pipeline/test/lookup-presence.integration.test.ts @@ -0,0 +1,288 @@ +import { fileURLToPath } from 'node:url'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { Dataset, Distribution } from '@lde/dataset'; +import type { QuadTransform, ReaderContext } from '@lde/pipeline'; +import { defineSearchType, searchSchema } from '@lde/search'; +import { irAlias } from '@lde/search/adapter'; +import { DataFactory } from 'n3'; +import { + startSparqlEndpoint, + teardownSparqlEndpoint, +} from '@lde/local-sparql-endpoint'; +import { searchStages, selectByClass } from '../src/search-stages.js'; + +const SCHEMA = 'https://schema.org/'; +const GEONAMES = 'https://sws.geonames.org/'; + +/** + * A lookup’s presence leaf, counted at index time: the extraction reads the + * leaf one hop from the root, the target type’s transform runs on the + * referent in the batch, and the projection writes whether any referent holds + * a value. A referent the graph names by IRI alone enters the transform all + * the same – the type’s own, for a lookup into its own type – so a transform + * describing a place from its key supplies its coordinates. A blank-node + * referent is no id the lookup stores, so it does not count. + */ +describe('a lookup’s presence leaf, through its target’s transform', () => { + const port = 3020; + const distribution = Distribution.sparql( + new URL(`http://localhost:${port}/sparql`), + ); + const dataset = new Dataset({ + iri: new URL('http://example.org/dataset/lookup-presence'), + distributions: [distribution], + }); + + const place = defineSearchType({ + name: 'Place', + class: `${SCHEMA}Place`, + labelField: 'name', + fields: [ + { + name: 'name', + kind: 'text', + path: `<${SCHEMA}name>`, + locales: ['und'], + output: true, + searchable: { weight: 1 }, + }, + { + name: 'latitude', + kind: 'number', + path: `<${SCHEMA}latitude>`, + output: true, + }, + ], + }); + const work = defineSearchType({ + name: 'Work', + class: `${SCHEMA}CreativeWork`, + labelField: 'name', + fields: [ + { + name: 'name', + kind: 'text', + path: `<${SCHEMA}name>`, + locales: ['und'], + output: true, + searchable: { weight: 1 }, + }, + { + name: 'contentLocation', + kind: 'reference', + path: `<${SCHEMA}contentLocation>`, + output: true, + filterable: true, + ref: { + strategy: 'lookup', + target: 'Place', + presenceLeaves: ['latitude'], + }, + }, + ], + }); + const schema = searchSchema(work, place); + + // The deployment’s place transform: a GeoNames place is described from its + // IRI, coordinates included – whatever else the batch says of it. + const latitudeAlias = irAlias(place, place.fields[1]); + const seen: string[] = []; + const geonamesCoordinates: QuadTransform = async function* ( + quads, + ) { + const subjects = new Set(); + for await (const quad of quads) { + subjects.add(quad.subject.value); + yield quad; + } + seen.push(...subjects); + for (const subject of subjects) { + if (subject.startsWith(GEONAMES)) { + yield DataFactory.quad( + DataFactory.namedNode(subject), + DataFactory.namedNode(latitudeAlias), + DataFactory.literal('51.37'), + ); + } + } + }; + + beforeAll(async () => { + await startSparqlEndpoint( + port, + fileURLToPath( + new URL('./fixtures/lookup-presence-sample.ttl', import.meta.url), + ), + ); + }, 60_000); + + afterAll(async () => { + await teardownSparqlEndpoint(); + }); + + async function project(withTransform: boolean, batchSize?: number) { + const [stage] = searchStages({ + schema, + types: [ + { + searchType: work, + rootVariable: 'root', + itemSelector: selectByClass(work), + ...(batchSize !== undefined && { batchSize }), + }, + { + searchType: place, + rootVariable: 'root', + itemSelector: selectByClass(place), + ...(withTransform && { transform: [geonamesCoordinates] }), + }, + ], + }); + const documents = new Map>(); + await stage.run(dataset, distribution, { + write: async (_dataset, items) => { + for await (const item of items) { + documents.set(String(item.document.name_und), item.document); + } + }, + }); + return documents; + } + + it('counts a referent the transform describes from its IRI alone', async () => { + const documents = await project(true); + + expect(documents.get('Geonames')?.contentLocation_latitude_exists).toBe( + true, + ); + expect(documents.get('Unknown')?.contentLocation_latitude_exists).toBe( + false, + ); + expect(documents.get('Stated')?.contentLocation_latitude_exists).toBe(true); + expect(documents.get('Bare')?.contentLocation_latitude_exists).toBe(false); + // Every referent entered the transform, the IRI-only ones included… + expect(seen).toEqual( + expect.arrayContaining([ + 'https://sws.geonames.org/2745707', + 'https://a/place/unknown', + 'https://a/place/stated', + ]), + ); + // …and what seeded them is not kept: the work still names one place. + expect(documents.get('Geonames')?.contentLocation).toBe( + 'https://sws.geonames.org/2745707', + ); + }); + + it('reads the leaf the work’s own graph states, without a transform', async () => { + const documents = await project(false); + + expect(documents.get('Geonames')?.contentLocation_latitude_exists).toBe( + false, + ); + expect(documents.get('Stated')?.contentLocation_latitude_exists).toBe(true); + }); + + it('counts per batch, a batch whose referents are all named by IRI alone included', async () => { + const documents = await project(true, 1); + + expect(documents.get('Geonames')?.contentLocation_latitude_exists).toBe( + true, + ); + expect(documents.get('Unknown')?.contentLocation_latitude_exists).toBe( + false, + ); + // A blank node is no id the lookup stores, so it does not count – not + // even its own statement. + expect(documents.get('Blank')?.contentLocation_latitude_exists).toBe(false); + }); + + describe('on a lookup into its own type', () => { + const region = defineSearchType({ + ...place, + fields: [ + ...place.fields, + { + name: 'containedInPlace', + kind: 'reference', + path: `<${SCHEMA}containedInPlace>`, + output: true, + filterable: true, + ref: { + strategy: 'lookup', + target: 'Place', + presenceLeaves: ['latitude'], + }, + }, + ], + }); + + async function projectRegions(batchSize?: number) { + const [stage] = searchStages({ + schema: searchSchema(region), + types: [ + { + searchType: region, + rootVariable: 'root', + itemSelector: selectByClass(region), + transform: [geonamesCoordinates], + ...(batchSize !== undefined && { batchSize }), + }, + ], + }); + const documents = new Map>(); + await stage.run(dataset, distribution, { + write: async (_dataset, items) => { + for await (const item of items) { + documents.set(String(item.document.name_und), item.document); + } + }, + }); + return documents; + } + + it('seeds its referents through the type’s own transforms', async () => { + seen.length = 0; + const documents = await projectRegions(); + + // Named by IRI alone, and described by the type’s own transform. + expect(documents.get('Village')?.containedInPlace_latitude_exists).toBe( + true, + ); + // Described in the batch, so read as stated and not seeded. + expect(documents.get('Hamlet')?.containedInPlace_latitude_exists).toBe( + true, + ); + expect(documents.get('Isolated')?.containedInPlace_latitude_exists).toBe( + false, + ); + expect(seen).toEqual( + expect.arrayContaining([ + 'https://sws.geonames.org/2751283', + 'https://a/place/nowhere', + ]), + ); + expect(seen.filter((iri) => iri === 'https://a/place/region')).toEqual([ + 'https://a/place/region', + ]); + // The seed is not kept: the place still names one place. + expect(documents.get('Village')?.containedInPlace).toBe( + 'https://sws.geonames.org/2751283', + ); + }); + + it('seeds nothing in a batch that describes every referent', async () => { + const documents = await projectRegions(1); + + expect(documents.get('Village')?.containedInPlace_latitude_exists).toBe( + true, + ); + expect(documents.get('Hamlet')?.containedInPlace_latitude_exists).toBe( + true, + ); + expect(documents.get('Isolated')?.containedInPlace_latitude_exists).toBe( + false, + ); + }); + }); +}); diff --git a/packages/search-typesense/src/query-compiler.ts b/packages/search-typesense/src/query-compiler.ts index 7730bba2..b6ec346d 100644 --- a/packages/search-typesense/src/query-compiler.ts +++ b/packages/search-typesense/src/query-compiler.ts @@ -566,12 +566,19 @@ function compileLeaf( return UNUSABLE; } // Typesense cannot test a field for holding a value, so presence reads the - // boolean companion the projection writes on every root document – never - // inside an entry, so under a nesting prefix there is nothing to read. + // boolean companion the projection writes on every root document – the + // field’s own, or a lookup’s for one of its presence leaves – never inside + // an entry, so under a nesting prefix there is nothing to read. // `assertValidQuery` rejects both first; this is the guard for a hand-built // query. if ('exists' in criterion) { - const exists = physicalFields(field).exists; + const { leaf } = criterion; + const exists = + leaf === undefined + ? physicalFields(field).exists + : physicalFields(field).leafExists.find( + (companion) => companion.leaf === leaf, + )?.exists; return exists === undefined || prefix !== '' ? UNUSABLE : `${exists}:=${criterion.exists}`; diff --git a/packages/search-typesense/test/lookup-presence.integration.test.ts b/packages/search-typesense/test/lookup-presence.integration.test.ts new file mode 100644 index 00000000..76d84559 --- /dev/null +++ b/packages/search-typesense/test/lookup-presence.integration.test.ts @@ -0,0 +1,248 @@ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import type { Client } from 'typesense'; +import { + searchSchema, + type Filter, + type SearchEngine, + type SearchQuery, + type SearchType, +} from '@lde/search'; +import { describeSearchEngineContract } from '@lde/search/testing'; +import { buildCollectionDefinition } from '../src/collection-definition.js'; +import { createTypesenseSearchEngine } from '../src/search.js'; +import { TypesenseContainer } from './typesense-container.js'; + +/** + * Presence on a lookup’s leaf – “a work with a place that has coordinates” – + * pinned against a real Typesense. The root document stores only the places’ + * ids, so the projection writes one boolean per work, + * `contentLocation_latitude_exists`, true when any referent holds a value; + * the filter and the facet read it, and a bucket counts each work once. The + * reference’s own facet on its ids stands beside it, capped by its own limit. + */ +describe('a root document filtered and faceted on whether a referent’s leaf has a value', () => { + const container = new TypesenseContainer(); + let client: Client; + let engine: SearchEngine; + + const kessel = 'https://ex/place/kessel'; + const venlo = 'https://sws.geonames.org/2745707'; + const roermond = 'https://sws.geonames.org/2748812'; + const place: SearchType = { + name: 'Place', + class: 'https://schema.org/Place', + fields: [ + { + name: 'label', + kind: 'text', + path: 'https://schema.org/name', + locales: ['nl'], + output: true, + searchable: { weight: 1 }, + }, + { + name: 'latitude', + kind: 'number', + path: 'https://schema.org/latitude', + output: true, + }, + { + name: 'longitude', + kind: 'number', + path: 'https://schema.org/longitude', + output: true, + }, + ], + }; + const work = { + name: 'Work', + class: 'https://schema.org/CreativeWork', + fields: [ + { + name: 'title', + kind: 'keyword', + path: 'https://schema.org/name', + output: true, + filterable: true, + sortable: true, + }, + { + name: 'contentLocation', + kind: 'reference', + path: 'https://schema.org/contentLocation', + array: true, + output: true, + filterable: true, + facetable: true, + ref: { + strategy: 'lookup', + target: 'Place', + presenceLeaves: ['latitude', 'longitude'], + }, + }, + ], + } as const satisfies SearchType; + const schema = searchSchema(work, place); + const facet = 'contentLocation.latitude.exists'; + const base: SearchQuery = { + where: [], + orderBy: [{ field: 'title', direction: 'asc' }], + limit: 10, + offset: 0, + facets: [facet], + locale: 'nl', + }; + const located = (exists: boolean, leaf = 'latitude'): Filter => ({ + or: [{ field: 'contentLocation', leaf, exists }], + }); + const counts = (result: { facets: Record }, name = facet) => + Object.fromEntries( + ( + result.facets[name] as { + buckets: { value: string; count: number; is?: boolean }[]; + } + ).buckets.map((bucket) => [ + String(bucket.is ?? bucket.value), + bucket.count, + ]), + ); + const titles = (result: { hits: readonly { id: string }[] }) => + result.hits.map((hit) => hit.id.replace('https://ex/', '')); + + beforeAll(async () => { + client = await container.start(); + await client.collections().create( + buildCollectionDefinition(work, { + collectionNameFor: () => 'works', + schema, + }), + ); + await client.collections().create( + buildCollectionDefinition(place, { + collectionNameFor: () => 'places', + schema, + }), + ); + const results = await client + .collections('works') + .documents() + .import( + [ + { + // Two places, one with coordinates: the work has one, and counts + // once. + id: 'https://ex/mixed', + title: 'mixed', + contentLocation: [kessel, venlo], + contentLocation_latitude_exists: true, + contentLocation_longitude_exists: true, + }, + { + id: 'https://ex/geonames', + title: 'geonames', + contentLocation: [roermond, venlo], + contentLocation_latitude_exists: true, + contentLocation_longitude_exists: false, + }, + { + id: 'https://ex/unlocated', + title: 'unlocated', + contentLocation: [kessel], + contentLocation_latitude_exists: false, + contentLocation_longitude_exists: false, + }, + // No place at all. + { + id: 'https://ex/bare', + title: 'bare', + contentLocation_latitude_exists: false, + contentLocation_longitude_exists: false, + }, + ], + { action: 'create' }, + ); + expect(results.every((result) => result.success)).toBe(true); + engine = createTypesenseSearchEngine(client, schema, { + collections: { Work: 'works', Place: 'places' }, + }); + }, 120_000); + + afterAll(async () => { + await container.stop(); + }); + + describeSearchEngineContract( + 'a Typesense engine serving a lookup leaf’s presence', + () => engine, + ); + + it('counts each work once, under whether any of its places has the leaf', async () => { + expect(counts(await engine.search(work, base))).toEqual({ + true: 2, + false: 2, + }); + }); + + it('filters on any referent holding a value, and on none holding one', async () => { + const some = await engine.search(work, { + ...base, + where: [located(true)], + }); + const none = await engine.search(work, { + ...base, + where: [located(false)], + }); + + expect(titles(some)).toEqual(['geonames', 'mixed']); + expect(titles(none)).toEqual(['bare', 'unlocated']); + }); + + it('ANDs two leaves, each counted without its own selection', async () => { + const longitude = 'contentLocation.longitude.exists'; + const result = await engine.search(work, { + ...base, + facets: [facet, longitude], + where: [located(true), located(true, 'longitude')], + }); + + expect(titles(result)).toEqual(['mixed']); + expect(counts(result)).toEqual({ true: 1 }); + expect(counts(result, longitude)).toEqual({ true: 1, false: 1 }); + }); + + it('counts without its own selection, and filters the hits by it', async () => { + const result = await engine.search(work, { + ...base, + where: [located(false)], + }); + + expect(titles(result)).toEqual(['bare', 'unlocated']); + expect(counts(result)).toEqual({ true: 2, false: 2 }); + }); + + it('counts the reference’s own values beside it, the limit capping those alone', async () => { + const result = await engine.search(work, { + ...base, + facets: ['contentLocation', facet], + facetLimits: { contentLocation: 1 }, + }); + + // Venlo is in two works, Kessel in two – one bucket, of the top count. + expect( + (result.facets.contentLocation as { buckets: unknown[] } | undefined) + ?.buckets, + ).toHaveLength(1); + expect(counts(result)).toEqual({ true: 2, false: 2 }); + }); + + it('counts the presence under a selection of the reference’s own ids, which it does not own', async () => { + const result = await engine.search(work, { + ...base, + facets: ['contentLocation', facet], + where: [{ or: [{ field: 'contentLocation', in: [kessel] }] }], + }); + + expect(titles(result)).toEqual(['mixed', 'unlocated']); + expect(counts(result)).toEqual({ true: 1, false: 1 }); + }); +}); diff --git a/packages/search-typesense/test/query-compiler.test.ts b/packages/search-typesense/test/query-compiler.test.ts index 42aca64b..6da5da8e 100644 --- a/packages/search-typesense/test/query-compiler.test.ts +++ b/packages/search-typesense/test/query-compiler.test.ts @@ -943,6 +943,26 @@ describe('`exists` welded to an inline entry', () => { ]), ).toBe('id:=[]'); expect(filterBy([{ field: 'format', exists: true }])).toBe('id:=[]'); + // Nor a reference the type does not declare. + expect( + buildSearchParams( + { + ...base, + where: [ + { + or: [ + { + field: 'nope', + entry: [{ field: 'thumbnailUrl', exists: true }], + }, + ], + }, + ], + }, + work, + { schema: mediaSchema }, + ).filter_by, + ).toBe('id:=[]'); }); it('compiles to a term matching nothing under a nesting, where no boolean is written', () => { @@ -1001,3 +1021,58 @@ describe('`exists` welded to an inline entry', () => { ).toBe('id:=[]'); }); }); + +describe('`exists` on a lookup’s leaf', () => { + const place: SearchType = { + name: 'Place', + class: 'https://schema.org/Place', + fields: [ + { + name: 'label', + kind: 'text', + path: 'https://schema.org/name', + locales: ['nl'], + output: true, + searchable: { weight: 1 }, + }, + { name: 'latitude', kind: 'number', path: 'https://schema.org/latitude' }, + ], + }; + const work: SearchType = { + name: 'Work', + class: 'https://schema.org/CreativeWork', + fields: [ + { + name: 'contentLocation', + kind: 'reference', + filterable: true, + ref: { + strategy: 'lookup', + target: 'Place', + presenceLeaves: ['latitude'], + }, + }, + ], + }; + const placeSchema = searchSchema(work, place); + const filterBy = (criterion: Record) => + buildSearchParams( + { ...base, where: [{ or: [criterion] } as never] }, + work, + { schema: placeSchema }, + ).filter_by; + + it('reads the root document’s boolean, a plain criterion', () => { + expect( + filterBy({ field: 'contentLocation', leaf: 'latitude', exists: true }), + ).toBe('contentLocation_latitude_exists:=true'); + }); + + it('compiles to a term matching nothing on a leaf named no presence', () => { + // `assertValidQuery` rejects it first; this is the guard for a direct + // caller. + expect( + filterBy({ field: 'contentLocation', leaf: 'label', exists: true }), + ).toBe('id:=[]'); + }); +}); diff --git a/packages/search-typesense/vite.config.ts b/packages/search-typesense/vite.config.ts index 06da98e0..6138942b 100644 --- a/packages/search-typesense/vite.config.ts +++ b/packages/search-typesense/vite.config.ts @@ -22,10 +22,10 @@ export default mergeConfig( // by-name map for the shared `rootTypeNamed`: one COVERED arrow fewer // over one function fewer, which lowers the ratio without uncovering // anything. - functions: 99.38, + functions: 99.4, // Lines, branches and statements moved the same way when the range // bound reading went to `@lde/search` as `storedRangeBound`. - lines: 99.59, + lines: 99.68, // Re-anchored for the projection lookup: its guards against a // projection naming what no lookup reaches are unreachable through // the port, since `assertValidQuery` rejects such a query first. @@ -34,8 +34,8 @@ export default mergeConfig( // companion: covered branches deleted, nothing uncovered. And when // a field declaring presence alone went back to being internal, // deleting the covered branches that kept its value unindexed. - branches: 97.3, - statements: 99.6, + branches: 97.48, + statements: 99.69, }, }, }, diff --git a/packages/search/src/project.ts b/packages/search/src/project.ts index bd437512..52be1c03 100644 --- a/packages/search/src/project.ts +++ b/packages/search/src/project.ts @@ -151,7 +151,7 @@ export function projectDocument( ) as SearchDocument; // After every field, so a presence reads what a later derive left too; on // the root alone, since presence is declared only on a Root Type’s own field - // and on its inline references’ leaves. + // and on its references’ leaves. for (const field of searchType.fields) { if (field.presence === true) { document[physicalFields(field).exists as string] = holdsPresence( @@ -159,7 +159,7 @@ export function projectDocument( field, ); } - writeLeafPresence(document, field, schema); + writeLeafPresence(document, node, field, searchType, schema, context); } pruneInternalFields(document, searchType, schema); return document; @@ -493,37 +493,79 @@ function holdsPresence(document: ProjectedNode, field: SearchField): boolean { } /** - * Write whether any entry of an inline reference holds a value for each of - * its {@link ReferenceStrategy.presenceLeaves presence leaves}: one boolean - * per root document, `false` with no entries at all. Read before pruning, so - * a leaf declaring no Role of its own still counts. Without a schema the - * entries are not projected, so there is nothing to count – and nothing is - * written. + * Write whether any referent of a reference holds a value for each of its + * {@link ReferenceStrategy.presenceLeaves presence leaves}: one boolean per + * root document, `false` with no referents at all. Without a schema there is + * no type to read a leaf through, so nothing is written. + * + * An inline reference’s entries are read as projected, before pruning, so a + * leaf declaring no Role of its own still counts. A lookup stores only ids, so + * its referents are read off the frame instead: each leaf is projected from + * the referent’s node as its target declares it – the quads the extraction + * read one hop from the root, after the target’s transforms + * (`@lde/search-pipeline`). Only a referent with an IRI counts: a blank node + * is no id the lookup stores. One the frame names by IRI alone holds no value. */ function writeLeafPresence( document: ProjectedNode, + node: FramedNode, field: SearchField, + searchType: SearchType, schema: SearchSchema | undefined, + context: ProjectionContext, ): void { const leaves = physicalFields(field).leafExists; - const referenceType = - schema === undefined || leaves.length === 0 || !isInlineReference(field) - ? undefined - : referenceTypeNamed(schema, field.ref.typeName); - if (referenceType === undefined) { + if (schema === undefined || leaves.length === 0) { return; } - const stored = document[field.name]; - const entries = ( - stored === undefined ? [] : Array.isArray(stored) ? stored : [stored] - ) as ProjectedNode[]; + if (isInlineReference(field)) { + // Guaranteed declared: `searchSchema` resolves every inline `typeName`, + // and every presence leaf against it. + const referenceType = referenceTypeNamed( + schema, + field.ref.typeName, + ) as ReferenceType; + const stored = document[field.name]; + const entries = ( + stored === undefined ? [] : Array.isArray(stored) ? stored : [stored] + ) as ProjectedNode[]; + for (const { leaf: name, exists } of leaves) { + const leaf = fieldNamed(referenceType, name) as SearchField; + document[exists] = entries.some((entry) => holdsPresence(entry, leaf)); + } + return; + } + const referents = valuesOf(node, irAlias(searchType, field)).filter( + (value) => isObject(value) && iriString(value) !== undefined, + ) as FramedNode[]; + // `searchSchema` holds presence leaves to a lookup naming one target, and + // that target to declaring each leaf. + const [target] = referencedTargetsOf(field, schema); for (const { leaf: name, exists } of leaves) { - // `searchSchema` has resolved every presence leaf against the type. - const leaf = fieldNamed(referenceType, name) as SearchField; - document[exists] = entries.some((entry) => holdsPresence(entry, leaf)); + const leaf = fieldNamed(target, name) as SearchField; + document[exists] = referents.some((referent) => { + const read: ProjectedNode = {}; + applyField( + read, + referent, + leaf, + target, + schema, + context, + ignoreUnread, + true, + ); + return holdsPresence(read, leaf); + }); } } +/** A sink for a leaf read only to tell whether it holds a value: the + * referent’s own document reports what it cannot read. */ +function ignoreUnread(): void { + // The referent’s own document reports it. +} + /** Whether a stored value is one: neither `undefined` nor `null`, nor an * empty list. */ function holdsValue(value: unknown): boolean { diff --git a/packages/search/src/query.ts b/packages/search/src/query.ts index 8d22275b..24fd54ce 100644 --- a/packages/search/src/query.ts +++ b/packages/search/src/query.ts @@ -4,6 +4,7 @@ import { filterOperatorFor, ID_FIELD, holdsNoIds, + isInlineReference, nestedReferenceType, physicalFields, presenceFacetName, @@ -147,8 +148,20 @@ export type Criterion = * declaring {@link SearchFieldBase.presence}, whatever its kind, beside the * operator its kind fixes – on the searched type’s own field, not yet through * a join path or inside a weld. + * + * With a `leaf`, whether **any referent** of the `lookup` the field names + * holds a value for that leaf, one of the lookup’s + * {@link ReferenceStrategy.presenceLeaves presence leaves}: + * `{ field: 'contentLocation', leaf: 'latitude', exists: true }`. Like the + * field’s own presence it is recorded on the root document, so it is a plain + * criterion, combined with others like any. An inline reference’s presence + * leaf is instead selected by a weld of that one condition, as its other + * leaves are. */ -export type PresenceCriterion = CriterionBase & { readonly exists: boolean }; +export type PresenceCriterion = CriterionBase & { + readonly exists: boolean; + readonly leaf?: string; +}; /** * The conditions a criterion welds to **one entry** of the nested reference its @@ -312,8 +325,9 @@ export interface FacetSearch { * An `exists` criterion is on the axis `‹field›.exists`, which the field’s * presence facet owns. It owns nothing else: a range on the same field is * another axis, and so a presence facet counts under it. Welded alone to an - * entry, `exists` is on the axis `‹reference›.‹leaf›.exists`, owned by that - * presence leaf’s facet and never by a facet on the leaf’s values. + * entry – or naming a lookup’s `leaf` – `exists` is on the axis + * `‹reference›.‹leaf›.exists`, owned by that presence leaf’s facet and never + * by a facet on the leaf’s values. * * A default a surface adds to `where` (a valid-only `status`) is a clause * like any other: its facet counts across every value and reports the @@ -421,8 +435,9 @@ export interface QueryIssue { | 'inverted-range' /** * An `exists` criterion on a field that declares no - * {@link SearchFieldBase.presence} – or, welded to one entry, on a leaf - * its inline reference does not name among its + * {@link SearchFieldBase.presence} – or on a leaf, welded to one entry of + * an inline reference or named by `leaf` on a lookup, that the reference + * does not name among its * {@link ReferenceStrategy.presenceLeaves presence leaves} – so no * document records whether it holds a value. */ @@ -465,10 +480,23 @@ function collectLeafIssues( return; } // Presence is a Role of its own: a field declaring it takes `exists` - // whether or not it is `filterable`, and whatever operator its kind fixes. + // whether or not it is `filterable`, and whatever operator its kind fixes – + // and a lookup takes it on each of its presence leaves. if ('exists' in criterion) { - if (declared.presence !== true) { - issues.push({ part: 'where', field, reason: 'no-presence' }); + const recorded = + criterion.leaf === undefined + ? declared.presence === true + : !isInlineReference(declared) && + physicalFields(declared).leafExists.some( + ({ leaf }) => leaf === criterion.leaf, + ); + if (!recorded) { + issues.push({ + part: 'where', + field: + criterion.leaf === undefined ? field : `${field}.${criterion.leaf}`, + reason: 'no-presence', + }); } return; } @@ -911,7 +939,11 @@ function axisOf(criterion: Criterion): string | undefined { // presence facet: a range on the same field is not its selection, nor is an // `exists` the selection of a facet on the field’s values. if ('exists' in criterion) { - return presenceFacetName(path.join('.')); + return presenceFacetName( + [...path, ...(criterion.leaf === undefined ? [] : [criterion.leaf])].join( + '.', + ), + ); } if (!isWelded(criterion)) { return path.join('.'); @@ -937,6 +969,19 @@ function axisOf(criterion: Criterion): string | undefined { */ export function selectionOf(facet: FacetDeclaration, value: string): Filter { const field = facet.field.name; + // A lookup’s presence leaf is recorded on the root document, and selected + // there, by the reference and the leaf. + if ( + facet.presence === true && + facet.reference !== undefined && + !isInlineReference(facet.reference) + ) { + return filterOn({ + field: facet.reference.name, + leaf: field, + exists: value === 'true', + }); + } const condition: Criterion = facet.presence === true ? { field, exists: value === 'true' } diff --git a/packages/search/src/schema.ts b/packages/search/src/schema.ts index c6c6b8f2..ebf9956f 100644 --- a/packages/search/src/schema.ts +++ b/packages/search/src/schema.ts @@ -155,10 +155,12 @@ export interface SearchFieldBase { * {@link isInternalField internal field} – projected, read for its * companion, then pruned, its value stored nowhere. * - * Only on a Root Type’s own field, and not yet on an inline reference, a - * `text` field, a `boolean` (which states its own absence as `false`) or a - * field that is already `facetable` ({@link validateSearchType} enforces - * all of them). + * Only on a Root Type’s own field, and not on an inline reference (whose + * entries’ leaves declare it through + * {@link ReferenceStrategy.presenceLeaves}), a `text` field or a `boolean`, + * which states its own absence as `false` ({@link validateSearchType} + * enforces all of them). A field that is also `facetable` serves both + * facets. */ readonly presence?: boolean; /** @@ -350,7 +352,23 @@ export type ReferenceStrategy = readonly literal?: LiteralKind; readonly identity?: never; readonly facetedLeaves?: never; - readonly presenceLeaves?: never; + /** + * The `name`s of **leaf fields** of the target the root document is + * filtered and faceted on by whether **any referent** holds a value – + * as an inline reference’s {@link presenceLeaves} are, read here from + * each referent’s own node one hop from the root, after the target’s + * transforms (`@lde/search-pipeline`). Only a referent named by IRI + * counts, since a blank node is no id the lookup stores; one the graph + * names by IRI alone, and no transform describes, holds no value. + * + * Only on a lookup naming one `target`, which must declare each leaf, + * with a `path`; neither on a `joinable` lookup, whose `where` already + * states a condition on the joined document, nor on one reading its + * literals as periods ({@link searchSchema} enforces all of it). A + * criterion selects the leaf by `exists`, naming the reference and the + * `leaf`: `{ field, leaf, exists }`. + */ + readonly presenceLeaves?: readonly string[]; } | { readonly strategy: 'inline'; @@ -420,10 +438,9 @@ export type ReferenceStrategy = * Declared on the reference, like {@link facetedLeaves}, because the * count is a fact about the root. The reference must be `filterable` and * sit on a Root Type; each leaf is neither `text`, `boolean`, an inline - * reference nor one of the reference’s faceted leaves; and every inline - * reference nesting the same reference type declares the same leaves, - * since they share one filter input ({@link searchSchema} enforces all - * of it). + * reference, nor named `buckets`, `totalValues` or `exists`, which name + * parts of the reference’s facet object ({@link searchSchema} enforces + * all of it). */ readonly presenceLeaves?: readonly string[]; }; @@ -1296,7 +1313,6 @@ function assertResolvableInlineReferences( assertFacetedLeaves(searchType, field, referenceType); } } - assertSharedPresenceLeaves(types); for (const searchType of types) { for (const field of searchType.fields) { assertServiceableLocalLookup(searchType, field); @@ -1496,11 +1512,6 @@ function assertFacetedLeaves( `${where} declares “facetedLeaves”, but “${searchType.name}” is a reference type: a faceted leaf counts root documents, and an entry is not one. Declare it on the Root Type’s reference instead.`, ); } - if (field.facetable === true) { - throw new Error( - `${where} is “facetable” and declares “facetedLeaves”: its facet and its leaves’ facets would both be named after “${field.name}”. Facet the reference or its leaves, not both.`, - ); - } if (field.filterable !== true) { throw new Error( `${where} declares “facetedLeaves” without “filterable”: a bucket is selected by a condition on one entry, which the reference must accept. Declare “filterable” on it.`, @@ -1512,6 +1523,11 @@ function assertFacetedLeaves( throw new Error(`${where} names faceted leaf “${name}” twice.`); } seen.add(name); + if (RESERVED_LEAF_NAMES.has(name)) { + throw new Error( + `${where} names faceted leaf “${name}”: “buckets”, “totalValues” and “exists” name parts of the reference’s facet object.`, + ); + } const leaf = fieldNamed(referenceType, name); if (leaf === undefined) { throw new Error( @@ -1542,90 +1558,56 @@ function assertFacetedLeaves( /** * {@link SearchFieldBase.presence Presence} counts what one field of a root - * document holds, and an inline reference’s + * document holds, and a reference’s * {@link ReferenceStrategy.presenceLeaves presence leaves} what any of its - * entries holds, each written to a companion. Throws the first refusal + * referents holds, each written to a companion. Throws the first refusal * {@link presenceIssues} files – resolving each presence leaf against the - * reference type it names – as a sentence linking the issue that lifts it, - * where there is one. + * types the reference names – as a sentence. */ function assertServiceablePresence(types: readonly SearchType[]): void { - const referenceTypes = new Map( - types - .filter((searchType) => searchType.class === undefined) - .map((searchType) => [searchType.name, searchType]), + const typesByName = new Map( + types.map((searchType) => [searchType.name, searchType]), ); for (const searchType of types) { - const [issue] = presenceIssues(searchType, referenceTypes); + const [issue] = presenceIssues(searchType, typesByName); if (issue === undefined) { continue; } const [name, leaf] = issue.field.split('.'); const at = `“${searchType.name}.${name}”`; const field = `Field ${at} declares “presence”`; - const leaves = `Inline reference ${at} declares “presenceLeaves”`; - const named = `Inline reference ${at} names presence leaf “${leaf}”`; - const objectFacet = - 'would have to become an object to hold the presence buckets beside the value buckets, a change https://github.com/ldelements/lde/issues/1090 makes. Declare one of the two until then.'; + const leaves = `Reference ${at} declares “presenceLeaves”`; + const named = `Reference ${at} names presence leaf “${leaf}”`; + // Only a presence leaf’s issue names a reference whose type it is read + // through; every sentence below is built, so it is read here alone. + const owner = + issue.reason === 'presence-leaf-unknown' + ? presenceLeafOwnerName(fieldNamed(searchType, name) as SearchField) + : undefined; throw new Error( { 'presence-on-reference-type': `${field}, but “${searchType.name}” is a reference type: presence is counted on a root document. To count whether any entry holds a value for this leaf, name it in the “presenceLeaves” of the Root Type’s inline reference instead.`, 'presence-with-inline-ref': `${field} on an inline reference, which stores entries rather than a value. To count whether any entry holds a value for one of their leaves, name the leaf in the reference’s “presenceLeaves”.`, 'presence-on-text': `${field} on a “text” field, which stores its values per language for display and search, so no one stored value says whether it has one. Declare presence on a “keyword” field over the same path.`, 'presence-on-boolean': `${field} on a “boolean”, which states its own absence as “false”: filter it with “is” instead.`, - 'presence-with-facetable': `${field} and is “facetable”: its facet ${objectFacet}`, 'presence-leaves-on-reference-type': `${leaves}, but “${searchType.name}” is a reference type: presence is counted on a root document, and an entry is not one. Declare it on the Root Type’s reference instead.`, - 'presence-leaves-not-inline': `${at} declares “presenceLeaves” but is no inline reference, so it has no entries to count; a lookup target’s leaf waits for https://github.com/ldelements/lde/issues/1090.`, - 'presence-leaves-without-filterable': `${leaves} without “filterable”: a bucket is selected by a condition on one entry, which the reference must accept. Declare “filterable” on it.`, - 'presence-leaves-with-facetable': `${leaves} and is “facetable”: its facet ${objectFacet}`, + 'presence-leaves-on-id-only': `${leaves} but names no type its referents are read through: declare it on an “inline” reference or a “lookup”.`, + 'presence-leaves-with-several-targets': `${leaves} and names several targets: a leaf is read through one target’s declaration, so declare presence leaves on a lookup naming one.`, + 'presence-leaves-with-joinable': `${leaves} and is “joinable”: its “where” already states a condition on the joined document, so a leaf’s presence is not offered there yet.`, + 'presence-leaves-with-literal': `${leaves} and reads its literals as periods: a literal has no leaves to count.`, + 'presence-leaves-without-filterable': `${leaves} without “filterable”: a bucket is selected by a condition on the reference, which it must accept. Declare “filterable” on it.`, 'presence-leaves-duplicate': `${leaves} and names one leaf twice.`, - 'presence-leaf-unknown': `${named}, which “${(fieldNamed(searchType, name) as { readonly ref?: { readonly typeName?: string } } | undefined)?.ref?.typeName}” does not declare.`, + 'presence-leaf-unknown': `${named}, which “${owner}” does not declare.`, + 'presence-leaf-reserved-name': `${named}: “buckets”, “totalValues” and “exists” name parts of the reference’s facet object.`, 'presence-leaf-on-text': `${named}, a “text”: it stores its values per language, so no one stored value says whether it has one.`, 'presence-leaf-on-boolean': `${named}, a “boolean”: it states its own absence as “false”.`, 'presence-leaf-inline-ref': `${named}, which is itself an inline reference: a leaf of an entry’s entries is not counted.`, - 'presence-leaf-faceted': `${named}, which is also one of its faceted leaves: the leaf’s facet ${objectFacet}`, + 'presence-leaf-without-path': `${named}, which declares no “path”: a lookup reads its referents’ leaves one hop from the root, so the leaf must be read from the graph.`, }[issue.reason as PresenceIssueReason], ); } } -/** - * Every inline reference that gets a filter input – a `filterable` one on a - * Root Type – and nests one reference type must declare the same - * {@link ReferenceStrategy.presenceLeaves presence leaves}: the reference - * type’s `where` input is shared by all of them, and which of its keys take - * `exists` is decided by those leaves. Two references disagreeing would offer - * `exists` where one of them cannot answer it. A temporary rule: per-reference - * filter inputs ([#1090](https://github.com/ldelements/lde/issues/1090)) lift - * it. - */ -function assertSharedPresenceLeaves(types: readonly SearchType[]): void { - const declared = new Map(); - for (const searchType of types) { - if (searchType.class === undefined) { - continue; - } - for (const field of searchType.fields) { - if (!isInlineReference(field) || field.filterable !== true) { - continue; - } - const at = `${searchType.name}.${field.name}`; - const leaves = physicalFields(field) - .leafExists.map(({ leaf }) => leaf) - .sort() - .join(', '); - const first = declared.get(field.ref.typeName); - if (first === undefined) { - declared.set(field.ref.typeName, { at, leaves }); - } else if (first.leaves !== leaves) { - throw new Error( - `Inline references “${first.at}” and “${at}” both nest “${field.ref.typeName}” but declare different “presenceLeaves” (${first.leaves || 'none'}; ${leaves || 'none'}): they share one filter input on “${field.ref.typeName}”, so declare the same leaves on both.`, - ); - } - } - } -} - /** The reasons {@link presenceIssues} files a refused presence under. */ type PresenceIssueReason = Extract< SearchTypeIssue['reason'], @@ -1635,15 +1617,16 @@ type PresenceIssueReason = Extract< /** * The issues a type’s presence declarations carry – a field’s * {@link SearchFieldBase.presence presence}, refused wherever a root - * document’s companion cannot answer it yet, and an inline reference’s + * document’s companion cannot answer it, and a reference’s * {@link ReferenceStrategy.presenceLeaves presence leaves}. One issue per * field, the first rule it breaks; a presence leaf’s is filed under - * `‹reference›.‹leaf›`. The leaves are checked against the reference type - * only where `referenceTypes` resolves it – one declaration cannot see it. + * `‹reference›.‹leaf›`. The leaves are checked against the types the + * reference names only where `typesByName` resolves them – one declaration + * cannot see them. */ function presenceIssues( searchType: SearchType, - referenceTypes?: ReadonlyMap, + typesByName?: ReadonlyMap, ): readonly SearchTypeIssue[] { return searchType.fields.flatMap((field): SearchTypeIssue[] => { const reason: PresenceIssueReason | undefined = @@ -1657,56 +1640,77 @@ function presenceIssues( ? 'presence-on-text' : field.kind === 'boolean' ? 'presence-on-boolean' - : field.facetable === true - ? 'presence-with-facetable' - : undefined; + : undefined; const issues: SearchTypeIssue[] = reason === undefined ? [] : [{ field: field.name, reason }]; - return [ - ...issues, - ...presenceLeavesIssues(searchType, field, referenceTypes), - ]; + return [...issues, ...presenceLeavesIssues(searchType, field, typesByName)]; }); } -/** The issues an inline reference’s +/** The name of the one type the presence leaves of an inline reference or a + * lookup are read through: the reference type, or the lookup’s single + * target. */ +function presenceLeafOwnerName(field: SearchField): string { + const ref = (field as FlatField).ref as NonNullable; + return ref.strategy === 'inline' + ? ref.typeName + : (labelSourceNamesOf(field as ReferenceField)[0] as string); +} + +/** The leaf names a reference’s facet object holds of its own. */ +const RESERVED_LEAF_NAMES: ReadonlySet = new Set([ + 'buckets', + 'totalValues', + 'exists', +]); + +/** The issues a reference’s * {@link ReferenceStrategy.presenceLeaves presence leaves} carry: the - * reference’s own first, then – where the reference type resolves – each + * reference’s own first, then – where the type it names resolves – each * leaf’s. */ function presenceLeavesIssues( searchType: SearchType, field: SearchField, - referenceTypes: ReadonlyMap | undefined, + typesByName: ReadonlyMap | undefined, ): readonly SearchTypeIssue[] { - const leaves = (field as FlatField).ref?.presenceLeaves; + const flat = field as FlatField; + const leaves = flat.ref?.presenceLeaves; if (leaves === undefined || leaves.length === 0) { return []; } + const lookup = flat.ref?.strategy === 'lookup'; const reason: PresenceIssueReason | undefined = searchType.class === undefined ? 'presence-leaves-on-reference-type' - : !isInlineReference(field) - ? 'presence-leaves-not-inline' - : field.filterable !== true - ? 'presence-leaves-without-filterable' - : field.facetable === true - ? 'presence-leaves-with-facetable' - : new Set(leaves).size !== leaves.length - ? 'presence-leaves-duplicate' - : undefined; + : flat.kind !== 'reference' || + (flat.ref?.strategy !== 'inline' && !lookup) + ? 'presence-leaves-on-id-only' + : lookup && labelSourceNamesOf(field as ReferenceField).length > 1 + ? 'presence-leaves-with-several-targets' + : flat.joinable === true + ? 'presence-leaves-with-joinable' + : literalKindOf(flat) !== undefined + ? 'presence-leaves-with-literal' + : field.filterable !== true + ? 'presence-leaves-without-filterable' + : new Set(leaves).size !== leaves.length + ? 'presence-leaves-duplicate' + : undefined; if (reason !== undefined) { return [{ field: field.name, reason }]; } - const referenceType = referenceTypes?.get( - (field as { readonly ref: { readonly typeName: string } }).ref.typeName, - ); - if (referenceType === undefined) { + // The reason checks above leave an inline reference or a lookup. + const owner = typesByName?.get(presenceLeafOwnerName(field)); + if (owner === undefined) { return []; } return leaves.flatMap((name): SearchTypeIssue[] => { - const leaf = fieldNamed(referenceType, name); - const leafReason: PresenceIssueReason | undefined = - leaf === undefined + const leaf = fieldNamed(owner, name); + const leafReason: PresenceIssueReason | undefined = RESERVED_LEAF_NAMES.has( + name, + ) + ? 'presence-leaf-reserved-name' + : leaf === undefined ? 'presence-leaf-unknown' : leaf.kind === 'text' ? 'presence-leaf-on-text' @@ -1714,8 +1718,8 @@ function presenceLeavesIssues( ? 'presence-leaf-on-boolean' : isInlineReference(leaf) ? 'presence-leaf-inline-ref' - : facetedLeavesOf(field).includes(name) - ? 'presence-leaf-faceted' + : lookup && leaf.path === undefined + ? 'presence-leaf-without-path' : undefined; return leafReason === undefined ? [] @@ -2277,17 +2281,19 @@ export interface SearchTypeIssue { | 'presence-with-inline-ref' | 'presence-on-text' | 'presence-on-boolean' - | 'presence-with-facetable' | 'presence-leaves-on-reference-type' - | 'presence-leaves-not-inline' + | 'presence-leaves-on-id-only' + | 'presence-leaves-with-several-targets' + | 'presence-leaves-with-joinable' + | 'presence-leaves-with-literal' | 'presence-leaves-without-filterable' - | 'presence-leaves-with-facetable' | 'presence-leaves-duplicate' | 'presence-leaf-unknown' + | 'presence-leaf-reserved-name' | 'presence-leaf-on-text' | 'presence-leaf-on-boolean' | 'presence-leaf-inline-ref' - | 'presence-leaf-faceted'; + | 'presence-leaf-without-path'; } /** @@ -2903,12 +2909,12 @@ export interface PhysicalFields { */ readonly exists: string | undefined; /** - * The **presence leaf companions** of an inline reference declaring - * {@link ReferenceStrategy.presenceLeaves presence leaves}: per leaf, - * `${name}_${leaf}_exists`, a top-level boolean of the root document stating - * whether any entry holds a value for the leaf, written on every document – - * `false` with no entries. Indexed and faceted; never served. Empty for - * every other field. + * The **presence leaf companions** of a reference – inline or `lookup` – + * declaring {@link ReferenceStrategy.presenceLeaves presence leaves}: per + * leaf, `${name}_${leaf}_exists`, a top-level boolean of the root document + * stating whether any entry or referent holds a value for the leaf, written + * on every document – `false` with none. Indexed and faceted; never served. + * Empty for every other field. */ readonly leafExists: readonly { readonly leaf: string; @@ -3152,14 +3158,11 @@ export function facetsOf( } /** The {@link ReferenceStrategy.facetedLeaves faceted leaves} an inline - * reference declares, or none. */ -function facetedLeavesOf(inlineReference: SearchField): readonly string[] { + * reference declares, or none – for any other field too. */ +function facetedLeavesOf(field: SearchField): readonly string[] { return ( - ( - inlineReference as { - readonly ref: { readonly facetedLeaves?: readonly string[] }; - } - ).ref.facetedLeaves ?? [] + (field as { readonly ref?: { readonly facetedLeaves?: readonly string[] } }) + .ref?.facetedLeaves ?? [] ); } @@ -3183,47 +3186,51 @@ function declaredFacets( }, ] : []; - // `searchSchema` refuses presence on an inline reference and on a facetable - // field, so a presence facet stands alone. - const exists = physicalFields(field, schema).exists; - if (exists !== undefined) { - return [ - { - name: presenceFacetName(field.name), - field, - engineField: exists, - presence: true, - }, - ]; - } - const referenceType = - schema === undefined ? undefined : nestedReferenceType(schema, field); - if (referenceType === undefined) { - return own; + const { exists, leafExists } = physicalFields(field, schema); + // A presence facet of the field’s own, beside its value facet if any. + const presence: FacetDeclaration[] = + exists === undefined + ? [] + : [ + { + name: presenceFacetName(field.name), + field, + engineField: exists, + presence: true, + }, + ]; + // The type a leaf is read through: the nested reference type, or a + // lookup’s one target (`searchSchema` refuses presence leaves on a lookup + // naming several). + const leafOwner: SearchType | undefined = + schema === undefined + ? undefined + : (nestedReferenceType(schema, field) ?? + referencedTargetsOf(field, schema)[0]); + if (leafOwner === undefined) { + return [...own, ...presence]; } - // Only an inline reference nests a reference type, and `searchSchema` - // has resolved every faceted and presence leaf against it. - const leaves = facetedLeavesOf(field); + // `searchSchema` has resolved every faceted and presence leaf against it; + // only an inline reference declares faceted leaves. return [ ...own, - ...leaves.map((leafName): FacetDeclaration => { + ...presence, + ...facetedLeavesOf(field).map((leafName): FacetDeclaration => { const path = nestedFieldName(field.name, leafName); return { name: path, - field: fieldNamed(referenceType, leafName) as SearchField, + field: fieldNamed(leafOwner, leafName) as SearchField, reference: field, engineField: path, }; }), - ...physicalFields(field, schema).leafExists.map( - ({ leaf, exists }): FacetDeclaration => ({ - name: presenceFacetName(nestedFieldName(field.name, leaf)), - field: fieldNamed(referenceType, leaf) as SearchField, - reference: field, - engineField: exists, - presence: true, - }), - ), + ...leafExists.map(({ leaf, exists: companion }): FacetDeclaration => ({ + name: presenceFacetName(nestedFieldName(field.name, leaf)), + field: fieldNamed(leafOwner, leaf) as SearchField, + reference: field, + engineField: companion, + presence: true, + })), ]; }); } @@ -3896,8 +3903,8 @@ export function physicalFields( } : undefined, exists: field.presence === true ? `${field.name}_exists` : undefined, - leafExists: (isInlineReference(field) - ? (field.ref.presenceLeaves ?? []) + leafExists: (field.kind === 'reference' + ? (field.ref?.presenceLeaves ?? []) : [] ).map((leaf) => ({ leaf, exists: `${field.name}_${leaf}_exists` })), }; diff --git a/packages/search/test/faceted-leaves.test.ts b/packages/search/test/faceted-leaves.test.ts index 9098e4d7..934914df 100644 --- a/packages/search/test/faceted-leaves.test.ts +++ b/packages/search/test/faceted-leaves.test.ts @@ -134,7 +134,7 @@ describe('declaring the leaves a root document is faceted on', () => { ).toThrow(/“creator”, which stores a nested object/); }); - it('rejects a reference that is facetable itself, whose facet shares its name', () => { + it('accepts a reference that is facetable itself, its facet beside its leaves’', () => { const person = defineSearchType({ name: 'Person', class: `${SCHEMA_ORG}Person`, @@ -174,9 +174,10 @@ describe('declaring the leaves a root document is faceted on', () => { ], } as never; - expect(() => searchSchema(withIdentity, role, person)).toThrow( - /is “facetable” and declares “facetedLeaves”/, - ); + const schema = searchSchema(withIdentity, role, person); + expect( + facetsOf(withIdentity as SearchType, schema).map((facet) => facet.name), + ).toEqual(['associatedMedia', 'associatedMedia.license']); }); it('rejects a reference that is not filterable', () => { diff --git a/packages/search/test/leaf-presence.test.ts b/packages/search/test/leaf-presence.test.ts index a8f3463f..3d87dec5 100644 --- a/packages/search/test/leaf-presence.test.ts +++ b/packages/search/test/leaf-presence.test.ts @@ -154,14 +154,24 @@ describe('declaring presence leaves', () => { ]); expect( issues({ - ref: { - strategy: 'lookup', - target: 'MediaObject', - presenceLeaves: ['thumbnailUrl'], - }, + ref: { strategy: 'idOnly', presenceLeaves: ['thumbnailUrl'] }, }), ).toEqual([ - { field: 'associatedMedia', reason: 'presence-leaves-not-inline' }, + { field: 'associatedMedia', reason: 'presence-leaves-on-id-only' }, + ]); + const lookup = { + strategy: 'lookup', + target: 'Place', + presenceLeaves: ['latitude'], + }; + expect(issues({ ref: lookup })).toEqual([]); + expect( + issues({ ref: lookup, labelSource: 'Place', joinable: true }), + ).toEqual([ + { field: 'associatedMedia', reason: 'presence-leaves-with-joinable' }, + ]); + expect(issues({ ref: { ...lookup, literal: 'period' } })).toEqual([ + { field: 'associatedMedia', reason: 'presence-leaves-with-literal' }, ]); expect(issues({ filterable: undefined })).toEqual([ { @@ -180,9 +190,7 @@ describe('declaring presence leaves', () => { ).toEqual([ { field: 'associatedMedia', reason: 'presence-leaves-duplicate' }, ]); - expect(issues({ facetable: true })).toEqual([ - { field: 'associatedMedia', reason: 'presence-leaves-with-facetable' }, - ]); + expect(issues({ facetable: true })).toEqual([]); // A leaf is resolved against the reference type, which one declaration // cannot see: `searchSchema` checks it. expect( @@ -197,17 +205,68 @@ describe('declaring presence leaves', () => { expect(issues({})).toEqual([]); }); - it('refuses a facetable reference, whose facet would collide with its leaves’ until #1090', () => { - expect(() => - searchSchema( - workWith( - { presenceLeaves: ['thumbnailUrl'], identity: 'license' }, - { facetable: true }, - ), - mediaObject, + it('accepts a facetable reference, whose own facet sits beside its leaves’', () => { + const licence = defineSearchType({ + name: 'Licence', + class: `${SCHEMA_ORG}CreativeWork`, + fields: [ + { + name: 'label', + kind: 'text', + path: `${SCHEMA_ORG}name`, + locales: ['nl'], + output: true, + searchable: { weight: 1 }, + }, + ], + }); + const licensed = defineSearchType({ + ...mediaObject, + fields: [ + ...mediaObject.fields.filter((field) => field.name !== 'license'), + { + name: 'license', + kind: 'reference', + path: `${SCHEMA_ORG}license`, + filterable: true, + ref: { strategy: 'lookup', target: 'Licence' }, + }, + ], + }); + const facetable = { + ...workWith( + { presenceLeaves: ['thumbnailUrl'], identity: 'license' }, + { facetable: true }, ), + class: `${SCHEMA_ORG}VisualArtwork`, + } as SearchType; + const schema = searchSchema(facetable, licensed, licence); + expect(facetsOf(facetable, schema).map((facet) => facet.name)).toEqual([ + 'associatedMedia', + 'associatedMedia.thumbnailUrl.exists', + ]); + }); + + it('refuses a leaf named like a part of the facet object', () => { + const named = defineSearchType({ + name: 'MediaObject', + fields: [ + { name: 'exists', kind: 'keyword', path: `${SCHEMA_ORG}exists` }, + { name: 'buckets', kind: 'keyword', path: `${SCHEMA_ORG}buckets` }, + ], + }); + expect(() => + searchSchema(workWith({ presenceLeaves: ['exists'] }), named), + ).toThrow( + /names presence leaf “exists”: “buckets”, “totalValues” and “exists”/, + ); + expect(() => + searchSchema(workWith({ facetedLeaves: ['buckets'] }), { + ...named, + fields: [{ ...named.fields[1], filterable: true }], + } as SearchType), ).toThrow( - /declares “presenceLeaves” and is “facetable”: its facet would have to become an object.*issues\/1090/, + /names faceted leaf “buckets”: “buckets”, “totalValues” and “exists”/, ); }); @@ -259,7 +318,7 @@ describe('declaring presence leaves', () => { }, mediaObject, ), - ).toThrow(/but is no inline reference.*issues\/1090/); + ).toThrow(/names no type its referents are read through/); }); it('refuses a leaf the reference type does not declare', () => { @@ -313,15 +372,16 @@ describe('declaring presence leaves', () => { ).toThrow(/presence leaf “thumbnail”, which is itself an inline reference/); }); - it('refuses a leaf that is also faceted on, until its facet can hold both', () => { - expect(() => - searchSchema( - workWith({ presenceLeaves: ['license'], facetedLeaves: ['license'] }), - mediaObject, + it('accepts a leaf counted both by value and by presence', () => { + const both = workWith({ + presenceLeaves: ['license'], + facetedLeaves: ['license'], + }); + expect( + facetsOf(both, searchSchema(both, mediaObject)).map( + (facet) => facet.name, ), - ).toThrow( - /presence leaf “license”, which is also one of its faceted leaves/, - ); + ).toEqual(['associatedMedia.license', 'associatedMedia.license.exists']); }); it('refuses one on a reference type, which has no documents to count', () => { @@ -347,55 +407,13 @@ describe('declaring presence leaves', () => { ); }); - it('refuses two references that nest one type with different leaves', () => { + it('accepts two references that nest one type with different leaves, each filtered by its own input', () => { const collection = { ...workWith({}), name: 'Collection', class: `${SCHEMA_ORG}Collection`, } as SearchType; - expect(() => searchSchema(work, collection, mediaObject)).toThrow( - /“Work.associatedMedia” and “Collection.associatedMedia” both nest “MediaObject” but declare different “presenceLeaves” \(thumbnailUrl; none\)/, - ); - expect(() => searchSchema(collection, work, mediaObject)).toThrow( - /declare different “presenceLeaves” \(none; thumbnailUrl\)/, - ); - }); - - it('compares only the references that get a filter input', () => { - // Not filterable: no filter input, so nothing to share. - const unfiltered = { - ...workWith({}, { filterable: undefined }), - name: 'Collection', - class: `${SCHEMA_ORG}Collection`, - } as SearchType; - expect(() => searchSchema(work, unfiltered, mediaObject)).not.toThrow(); - // On a Reference Type: its references are never filtered through. - const edge = defineSearchType({ - name: 'Edge', - fields: [ - { - name: 'media', - kind: 'reference', - path: `${SCHEMA_ORG}associatedMedia`, - output: true, - ref: { strategy: 'inline', typeName: 'MediaObject' }, - }, - ], - }); - const withEdge = defineSearchType({ - name: 'Agent', - class: `${SCHEMA_ORG}Person`, - fields: [ - { - name: 'edge', - kind: 'reference', - path: `${SCHEMA_ORG}subjectOf`, - output: true, - ref: { strategy: 'inline', typeName: 'Edge' }, - }, - ], - }); - expect(() => searchSchema(work, withEdge, edge, mediaObject)).not.toThrow(); + expect(() => searchSchema(work, collection, mediaObject)).not.toThrow(); }); }); diff --git a/packages/search/test/lookup-presence.test.ts b/packages/search/test/lookup-presence.test.ts new file mode 100644 index 00000000..d7edb5ee --- /dev/null +++ b/packages/search/test/lookup-presence.test.ts @@ -0,0 +1,255 @@ +import { describe, expect, it } from 'vitest'; +import { projectDocument } from '../src/project.js'; +import { + defineSearchType, + facetNamed, + physicalFields, + searchSchema, + type FacetDeclaration, + type SearchType, +} from '../src/schema.js'; +import { + facetSearches, + selectionOf, + validateQuery, + type SearchQuery, +} from '../src/query.js'; + +const SCHEMA_ORG = 'https://schema.org/'; +const alias = (type: string, field: string) => `urn:lde:${type}/${field}`; + +const placeOf = (name: string, className: string) => + defineSearchType({ + name, + class: className, + fields: [ + { + name: 'label', + kind: 'text', + path: `${SCHEMA_ORG}name`, + locales: ['nl'], + output: true, + searchable: { weight: 1 }, + }, + { + name: 'latitude', + kind: 'number', + path: `${SCHEMA_ORG}latitude`, + output: true, + }, + ], + }); +const place = placeOf('Place', `${SCHEMA_ORG}Place`); +const landform = placeOf('Landform', `${SCHEMA_ORG}Landform`); + +const workWith = (ref: Record): SearchType => + ({ + name: 'Work', + class: `${SCHEMA_ORG}CreativeWork`, + fields: [ + { + name: 'contentLocation', + kind: 'reference', + path: `${SCHEMA_ORG}contentLocation`, + array: true, + output: true, + filterable: true, + ref: { + strategy: 'lookup', + target: 'Place', + presenceLeaves: ['latitude'], + ...ref, + }, + }, + ], + }) as never; +const work = workWith({}); +const schema = searchSchema(work, place); +const browse: SearchQuery = { + where: [], + orderBy: [], + limit: 10, + offset: 0, + facets: [], + locale: 'nl', +}; + +describe('presence leaves on a lookup', () => { + it('names one boolean per leaf on the root document', () => { + expect(physicalFields(work.fields[0]).leafExists).toEqual([ + { leaf: 'latitude', exists: 'contentLocation_latitude_exists' }, + ]); + }); + + it('serves the facet `‹reference›.‹leaf›.exists`, read through the target', () => { + expect(facetNamed(work, 'contentLocation.latitude.exists', schema)).toEqual( + { + name: 'contentLocation.latitude.exists', + field: place.fields[1], + reference: work.fields[0], + engineField: 'contentLocation_latitude_exists', + presence: true, + }, + ); + }); + + it('is refused on a lookup naming several targets', () => { + expect(() => + searchSchema( + workWith({ target: ['Place', 'Landform'] }), + place, + landform, + ), + ).toThrow( + /Reference “Work.contentLocation” declares “presenceLeaves” and names several targets/, + ); + }); + + it('requires the target to declare the leaf, with a path', () => { + const unlocated = defineSearchType({ ...place, fields: [place.fields[0]] }); + expect(() => searchSchema(work, unlocated)).toThrow( + /names presence leaf “latitude”, which “Place” does not declare/, + ); + const derived = defineSearchType({ + ...place, + fields: [ + place.fields[0], + { name: 'latitude', kind: 'number', output: true, derive: () => 1 }, + ], + }); + expect(() => searchSchema(work, derived)).toThrow( + /names presence leaf “latitude”, which declares no “path”/, + ); + }); + + it('is refused on a joinable lookup, whose `where` joins', () => { + expect(() => + searchSchema( + { + ...work, + fields: [{ ...work.fields[0], joinable: true }], + } as SearchType, + place, + ), + ).toThrow(/declares “presenceLeaves” and is “joinable”/); + }); +}); + +describe('projecting a lookup’s presence leaves', () => { + const project = (referents: unknown[] | undefined) => + projectDocument( + { + '@id': 'https://ex/w', + ...(referents !== undefined && { + [alias('Work', 'contentLocation')]: referents, + }), + }, + work, + schema, + ); + + it('records a value when any referent’s node states the leaf', () => { + const document = project([ + { '@id': 'https://ex/p/1' }, + { + '@id': 'https://ex/p/2', + [alias('Place', 'latitude')]: { '@value': '51.37' }, + }, + ]); + + expect(document.contentLocation_latitude_exists).toBe(true); + expect(document.contentLocation).toEqual([ + 'https://ex/p/1', + 'https://ex/p/2', + ]); + }); + + it('records no value for a referent named by IRI alone, or none at all', () => { + expect(project(['https://ex/p/1']).contentLocation_latitude_exists).toBe( + false, + ); + expect( + project([ + { + '@id': 'https://ex/p/1', + [alias('Place', 'latitude')]: { '@value': 'not a number' }, + }, + ]).contentLocation_latitude_exists, + ).toBe(false); + expect(project(undefined).contentLocation_latitude_exists).toBe(false); + }); + + it('counts no blank-node referent, which the lookup stores no id for', () => { + const document = project([ + { [alias('Place', 'latitude')]: { '@value': '51.37' } }, + { '@id': '_:b0', [alias('Place', 'latitude')]: { '@value': '51.37' } }, + ]); + + expect(document.contentLocation_latitude_exists).toBe(false); + expect(document.contentLocation).toBeUndefined(); + }); +}); + +describe('`exists` on a lookup’s leaf', () => { + const where = (...criteria: Record[]) => + validateQuery( + { + ...browse, + where: criteria.map((criterion) => ({ or: [criterion] }) as never), + }, + work, + schema, + ); + + it('is a plain criterion naming the reference and the leaf', () => { + expect( + where({ field: 'contentLocation', leaf: 'latitude', exists: true }), + ).toEqual([]); + }); + + it('is refused on a leaf the lookup counts no presence for', () => { + expect( + where({ field: 'contentLocation', leaf: 'label', exists: true }), + ).toEqual([ + { + part: 'where', + field: 'contentLocation.label', + reason: 'no-presence', + }, + ]); + }); + + it('welds nothing: a lookup has no entries', () => { + expect( + where({ + field: 'contentLocation', + entry: [{ field: 'latitude', exists: true }], + }), + ).toEqual([ + { part: 'where', field: 'contentLocation', reason: 'not-weldable' }, + ]); + }); + + it('is selected, and owned, on the leaf’s presence axis', () => { + const facet = facetNamed( + work, + 'contentLocation.latitude.exists', + schema, + ) as FacetDeclaration; + const selection = selectionOf(facet, 'false'); + expect(selection).toEqual({ + or: [{ field: 'contentLocation', leaf: 'latitude', exists: false }], + }); + + const [search] = facetSearches({ + ...browse, + where: [selection], + facets: ['contentLocation.latitude.exists'], + }); + expect(search?.own).toBe('contentLocation.latitude.exists'); + expect(search?.query.where).toEqual([]); + expect(search?.selected).toEqual({ + 'contentLocation.latitude.exists': ['false'], + }); + }); +}); diff --git a/packages/search/test/presence.test.ts b/packages/search/test/presence.test.ts index 8b0ddc4f..266f6491 100644 --- a/packages/search/test/presence.test.ts +++ b/packages/search/test/presence.test.ts @@ -184,10 +184,14 @@ describe('declaring presence', () => { ); }); - it('refuses a facetable field until its facet can hold both', () => { - expect(() => withField({ facetable: true })).toThrow( - /and is “facetable”: its facet would have to become an object/, - ); + it('counts a facetable field’s presence beside its values', () => { + const schema = withField({ facetable: true, filterable: true }); + const thing = [...schema.values()][0] as SearchType; + + expect(facetsOf(thing, schema).map((facet) => facet.name)).toEqual([ + 'field', + 'field.exists', + ]); }); it('files each refusal under a reason, for a declaration built outside TypeScript', () => { @@ -214,9 +218,7 @@ describe('declaring presence', () => { expect(declared({ kind: 'boolean' })).toEqual([ { field: 'field', reason: 'presence-on-boolean' }, ]); - expect(declared({ facetable: true })).toEqual([ - { field: 'field', reason: 'presence-with-facetable' }, - ]); + expect(declared({ facetable: true })).toEqual([]); expect(declared({})).toEqual([]); }); });