From c8a479bc4e7dab1a0e75e5c1e096f23156977958 Mon Sep 17 00:00:00 2001 From: David de Boer Date: Mon, 16 Nov 2026 17:07:56 +0100 Subject: [PATCH 1/2] =?UTF-8?q?feat(search-pipeline)!:=20apply=20a=20label?= =?UTF-8?q?=20source=E2=80=99s=20transforms=20to=20its=20referents?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Pass the key and label quads of each searchable reference’s referents through the target type’s own transforms, inside the batch - Search the reference by the labels those transforms leave, as the target’s documents carry them - Record the decision in ADR 39 BREAKING CHANGE: a root type’s transforms also run on its referents in the stages of types whose searchable references name it, receiving nodes that carry only the key and label field. Make such a transform cope with that input, or drop searchable from the referring reference. --- ...bel-sources-transforms-to-its-referents.md | 63 ++++++ docs/reference/search.md | 23 ++- package-lock.json | 2 +- packages/search-pipeline/package.json | 2 +- packages/search-pipeline/src/extraction.ts | 35 ++-- .../src/search-indexer-pipeline.ts | 3 + packages/search-pipeline/src/search-stages.ts | 165 +++++++++++++++- .../test/keyed-roots.integration.test.ts | 184 +++++++++++++++++- 8 files changed, 451 insertions(+), 26 deletions(-) create mode 100644 docs/decisions/0039-apply-a-label-sources-transforms-to-its-referents.md diff --git a/docs/decisions/0039-apply-a-label-sources-transforms-to-its-referents.md b/docs/decisions/0039-apply-a-label-sources-transforms-to-its-referents.md new file mode 100644 index 00000000..bc714a24 --- /dev/null +++ b/docs/decisions/0039-apply-a-label-sources-transforms-to-its-referents.md @@ -0,0 +1,63 @@ +# 39. Apply a label source’s transforms to its referents + +Date: 2026-11-16 + +## Status + +Accepted + +Amends [ADR 38](0038-search-a-document-by-its-referents-labels.md). + +## Context + +ADR 38 searches a reference by the labels the referring graph states for its +referents. A label a transform gives the target’s documents never reaches the +referrer. In Linked Open Limburg, the `Term` type’s transform replaces a keyed +term’s content with its Authority’s, so the Term document for a Cultuurhistorische +Thesaurus concept is called “boerenbont”, while a work referencing it is +searched only by what its own publisher calls the term – nothing, where the +publisher’s label is untagged and so outside the label field’s locales +([limburg/lol#318](https://codeberg.org/limburg/lol/issues/318), +[lde#1058](https://github.com/ldelements/lde/issues/1058)). + +## Decision + +Before projection, a stage passes the referents of its `searchable` +references, its inline referents’ included, through each target type’s own +transforms. + +- **Only the referent’s key and label quads go in**, under the target’s IR + Aliases, and what the transforms return replaces them. The referent’s + `rdf:type`, and every quad the referrer reads for its own fields, stay out. +- **It runs inside the batch**, after the stage’s own transforms, so memory + stays bounded by `batchSize` ([ADR 12](0012-bound-memory-by-the-unit-of-work-not-the-input.md)). +- **A reference to its own type is skipped**: the type’s transform already + sees the batch. A target wired with its own `readers` has no transform to + apply. + +## Consequences + +- The referrer is searched by the labels the target’s transform leaves. Where it + replaces a publisher’s label with an Authority’s, the publisher’s is no longer + searched, as it is no longer on the target’s document. +- No stage order is needed, and a referent is covered whichever dataset its + target document comes from, as long as the referring graph states its key or + label. One described only in another dataset’s graph still adds nothing. +- A transform on a label source must cope with nodes carrying only a key and a + label, and runs once per referring batch as well as for its own roots. A + transform doing I/O pays for it there too, unless it caches. +- A referent stated by IRI alone has no quad for a transform to act on, so it + adds nothing to search. +- Labels are copies, changed only when the referrer is reindexed. + +## Alternatives considered + +- **Copy labels from the target’s projected documents, held in memory for the + dataset pass.** No transform runs twice, but the index grows with the dataset, + which ADR 12 rules out; it also needs label sources’ stages to run first and + misses referents from other datasets. +- **The same index on disk.** Bounded memory, but a store to manage per run for + a copy the transform can recompute. +- **Patch referrers in the writer before commit**, paging through the + collections. Bounded and complete, but it rewrites whole collections through + the engine on every run, outside the engine-neutral projection. diff --git a/docs/reference/search.md b/docs/reference/search.md index 7b36912f..19fe9f66 100644 --- a/docs/reference/search.md +++ b/docs/reference/search.md @@ -592,14 +592,21 @@ points at the term “Starluster” matches the query `starluster`. The extracti reads each referent’s label field off the referent, one hop from the document, beside its key. The projection folds the labels in the label field’s own `locales` into the reference’s `${name}_search` companion, so a query still -searches one collection. The label is the one the document’s own graph states -for its referent. A referent stated by IRI alone, or one whose label only an -enrichment or another dataset supplies, adds nothing to search, and the label -field must therefore have a `path`. A copied label changes when the document is -reindexed. A reference naming no label source still folds its IRIs, and a -`derive`d or `from` reference reads no referent, so it is never searched by -labels -([ADR 38](../decisions/0038-search-a-document-by-its-referents-labels.md)). +searches one collection. The label field must therefore have a `path`. + +Before projecting, `@lde/search-pipeline` passes each referent’s key and label, +as read, through the target type’s own transforms, so the reference is searched +by the labels the target’s documents get: where a transform replaces a term’s +label with an Authority’s, the referring document is searched by the +Authority’s. A transform on a label source therefore also receives nodes that +carry only those two fields, once per referring batch. A referent stated by IRI +alone, with no triple of its own, gives a transform nothing to act on and adds +nothing to search. A copied label changes when the document is reindexed. + +A reference naming no label source still folds its IRIs, and a `derive`d or +`from` reference reads no referent, so it is never searched by labels +([ADR 38](../decisions/0038-search-a-document-by-its-referents-labels.md), +[ADR 39](../decisions/0039-apply-a-label-sources-transforms-to-its-referents.md)). The resolved label carries **one word** wherever it surfaces: on the label source’s own type, on the reference that resolves against it (`dataset { id diff --git a/package-lock.json b/package-lock.json index 3f7a9e00..1555ae54 100644 --- a/package-lock.json +++ b/package-lock.json @@ -42125,6 +42125,7 @@ "license": "MIT", "dependencies": { "@lde/search": "^0.41.3", + "@rdfjs/types": "^2.0.1", "@traqula/generator-sparql-1-1": "^1.3.1", "@traqula/parser-sparql-1-1": "^1.3.1", "@traqula/rules-sparql-1-1": "^1.2.0", @@ -42132,7 +42133,6 @@ }, "devDependencies": { "@lde/search-typesense": "^0.53.0", - "@rdfjs/types": "^2.0.1", "n3": "^2.13.12", "testcontainers": "^12.1.0", "typesense": "^3.2.0" diff --git a/packages/search-pipeline/package.json b/packages/search-pipeline/package.json index 7d946484..fc84c478 100644 --- a/packages/search-pipeline/package.json +++ b/packages/search-pipeline/package.json @@ -26,6 +26,7 @@ ], "dependencies": { "@lde/search": "^0.46.0", + "@rdfjs/types": "^2.0.1", "@traqula/generator-sparql-1-1": "^1.3.1", "@traqula/parser-sparql-1-1": "^1.3.1", "@traqula/rules-sparql-1-1": "^1.2.0", @@ -33,7 +34,6 @@ }, "devDependencies": { "@lde/search-typesense": "^0.61.1", - "@rdfjs/types": "^2.0.1", "n3": "^2.13.12", "testcontainers": "^12.1.0", "typesense": "^3.2.0" diff --git a/packages/search-pipeline/src/extraction.ts b/packages/search-pipeline/src/extraction.ts index c2897213..554cfdee 100644 --- a/packages/search-pipeline/src/extraction.ts +++ b/packages/search-pipeline/src/extraction.ts @@ -450,19 +450,10 @@ function buildReferentHops( // is one of them, so emitting the hop as well would state it twice. // A searchable reference is searched by its referents’ labels, so it // reads them off the referent beside the key. - const hopFields = [ - keyedFieldOf(target), - // `searchSchema` guarantees a searchable reference’s label a path. - field.searchable === undefined - ? undefined - : (labelFieldOf(target) as PathField), - ]; - for (const hopField of hopFields) { - if (hopField !== undefined) { - const built = buildFieldHop(target, hopField, referent, counter); - template.push(built.triple); - expansions.push(built.pattern); - } + for (const hopField of referentHopFields(field, target)) { + const built = buildFieldHop(target, hopField, referent, counter); + template.push(built.triple); + expansions.push(built.pattern); } continue; } @@ -495,6 +486,24 @@ function localTargetsOf( * reads the same declarations rather than a rule of its own: a reference that * names no target keeps the node IRI, and needs no hop. */ +/** + * The fields a reference reads off a referent of a target it does not expand + * `local`ly: the target’s key field, where it is keyed, and – for a + * `searchable` reference – its label field, by which the reference is searched. + */ +export function referentHopFields( + field: SearchField, + target: RootType, +): PathField[] { + return [ + keyedFieldOf(target), + // `searchSchema` guarantees a searchable reference’s label a path. + field.searchable === undefined + ? undefined + : (labelFieldOf(target) as PathField), + ].filter((hopField) => hopField !== undefined); +} + function keyedFieldOf(target: RootType): PathField | undefined { if (target.key === undefined) { return undefined; diff --git a/packages/search-pipeline/src/search-indexer-pipeline.ts b/packages/search-pipeline/src/search-indexer-pipeline.ts index a39ddf10..d9c848d7 100644 --- a/packages/search-pipeline/src/search-indexer-pipeline.ts +++ b/packages/search-pipeline/src/search-indexer-pipeline.ts @@ -110,6 +110,9 @@ export interface SearchIndexerPipelineOptions { * Named by `name` rather than by class IRI, like {@link registryTypes}; a * name the schema does not declare throws at wiring time rather than * attaching nothing and shipping an unenriched collection. + * + * A type’s transforms also run on its referents in the stages of the types + * searching it by its labels ({@link SearchStageType.transform}). */ transforms?: Readonly< Record< diff --git a/packages/search-pipeline/src/search-stages.ts b/packages/search-pipeline/src/search-stages.ts index e47a1ab8..6074918c 100644 --- a/packages/search-pipeline/src/search-stages.ts +++ b/packages/search-pipeline/src/search-stages.ts @@ -1,3 +1,4 @@ +import type { Quad, Term } from '@rdfjs/types'; import { Distribution, type Dataset } from '@lde/dataset'; import { SparqlConstructReader, @@ -15,8 +16,15 @@ import { type RootType, type SearchDocument, type SearchSchema, + type SearchType, } from '@lde/search'; -import { extractionQueryString } from './extraction.js'; +import { + irAlias, + isInlineReference, + referencedTargetsOf, + referenceTypeNamed, +} from '@lde/search/adapter'; +import { extractionQueryString, referentHopFields } from './extraction.js'; import type { TypedSearchDocument } from './typed-search-document.js'; /** One root type’s stage in a search pipeline. */ @@ -67,6 +75,15 @@ export interface SearchStageType { * A field a transform fills must still declare a `path`: projection skips a * field with neither a `path` nor a `derive`, so a transform-minted IR Alias * is otherwise never read. + * + * **It also runs on this type’s referents in other stages.** Where another + * type’s `searchable` reference names this type as its target, that stage + * hands the transform(s) each referent its batch read – the referent’s key + * and label field under this type’s IR Aliases, and nothing else – so the + * reference is searched by the labels this type’s documents get. A transform + * must therefore cope with a node carrying only those two fields, once per + * referring batch. A reference from this type to itself gets no separate + * pass: this transform already sees its referents’ quads in the batch. */ transform?: QuadTransform | QuadTransform[]; /** @@ -126,6 +143,12 @@ export interface SearchStagesOptions { * projection as `ProjectionContext.datasetDocument`, so a `derive` can copy a * field of the dataset onto each document. * + * Before projection, a stage passes the referents of its `searchable` + * references through their target type’s + * {@link SearchStageType.transform transform(s)}, so a reference is searched + * by the labels the target’s documents get. A stage with its own `readers` + * gets no such pass. + * * Wire the result as `new Pipeline({ datasetSelector, * stages: searchStages(...), writers: searchIndexWriter(...) })` – one terminal, * N stages. @@ -148,6 +171,16 @@ export function searchStages( ...types.filter(isDatasetType), ...types.filter((type) => !isDatasetType(type)), ]; + // Each type’s transforms, for the referring stages to apply to its + // referents. A type with its own readers has none here: `transform` + // attaches to the default reader only. + const transformsByName = new Map( + types.flatMap((type) => + type.transform === undefined + ? [] + : [[type.searchType.name, [type.transform].flat()] as const], + ), + ); return ordered.map((type) => { // Project with the schema’s OWN declaration object, whatever the caller // passed: `assertTypeInSchema` (inside `projectRoots`) is an identity check, @@ -180,7 +213,10 @@ export function searchStages( subjectVariable: rootVariable, }), }), - transform: type.transform, + transform: [ + type.transform ?? [], + referentTransforms(searchType, schema, transformsByName) ?? [], + ].flat(), } satisfies AttachedReader); return new Stage({ name: searchType.name, @@ -356,5 +392,130 @@ export function registrySource( /** {@link SearchStageType.batchSize} when the type sets none. */ export const defaultBatchSize = { endpoint: 10, imported: 100 } as const; +/** + * A transform passing each referent of the type’s `searchable` references, + * its inline referents’ included, through its target type’s transforms: the + * quads the extraction read off the referent ({@link referentHopFields}) go + * in, and what the target’s transforms make of them comes back in their place. + * `undefined` where no such reference names a target with transforms. + * + * The batch is buffered to find the referents, so memory stays bounded by the + * stage’s `batchSize`. + */ +function referentTransforms( + searchType: RootType, + schema: SearchSchema, + transformsByName: ReadonlyMap< + string, + readonly QuadTransform[] + >, +): QuadTransform | undefined { + // Per target with transforms: the reference fields naming it, and the + // aliases its referents carry. A target’s aliases are its own, so no quad + // belongs to two targets. + const routes = new Map< + RootType, + { fields: Set; aliases: Set } + >(); + for (const { owner, field } of searchedReferences(searchType, schema)) { + for (const target of referencedTargetsOf(field, schema)) { + // A reference to its own type: the type’s transform saw the batch. + if ( + target !== searchType && + transformsByName.get(target.name) !== undefined + ) { + const route = routes.get(target) ?? { + fields: new Set(), + aliases: new Set(), + }; + route.fields.add(irAlias(owner, field)); + for (const hopField of referentHopFields(field, target)) { + route.aliases.add(irAlias(target, hopField)); + } + routes.set(target, route); + } + } + } + if (routes.size === 0) { + return undefined; + } + const targetsByField = new Map(); + const targetByAlias = new Map(); + for (const [target, { fields, aliases }] of routes) { + for (const field of fields) { + targetsByField.set(field, [...(targetsByField.get(field) ?? []), target]); + } + for (const alias of aliases) { + targetByAlias.set(alias, target); + } + } + return async function* referentTransforms(quads, context) { + const batch: Quad[] = []; + // Each target’s referents in this batch: the objects of the references + // naming it. + const referents = new Map>(); + for await (const quad of quads) { + batch.push(quad); + for (const target of targetsByField.get(quad.predicate.value) ?? []) { + const keys = referents.get(target) ?? new Set(); + keys.add(termKey(quad.object)); + referents.set(target, keys); + } + } + // One pass sorting each quad into its target’s input, or keeping it. + const inputs = new Map(); + for (const quad of batch) { + const target = targetByAlias.get(quad.predicate.value); + if ( + target !== undefined && + referents.get(target)?.has(termKey(quad.subject)) + ) { + inputs.set(target, [...(inputs.get(target) ?? []), quad]); + } else { + yield quad; + } + } + for (const [target, input] of inputs) { + let stream: AsyncIterable = streamOf(input); + for (const transform of transformsByName.get( + target.name, + ) as readonly QuadTransform[]) { + stream = transform(stream, context); + } + yield* stream; + } + }; +} + +/** The type’s `searchable` reference fields, with the type declaring each: + * its own, and those of the inline referents it nests. `searchSchema` + * refuses an inline cycle, so the descent ends. */ +function searchedReferences( + owner: SearchType, + schema: SearchSchema, +): { owner: SearchType; field: SearchType['fields'][number] }[] { + return owner.fields.flatMap((field) => { + if (isInlineReference(field)) { + // Guaranteed declared: `searchSchema` resolves every inline `typeName`. + return searchedReferences( + referenceTypeNamed(schema, field.ref.typeName) as SearchType, + schema, + ); + } + return field.kind === 'reference' && field.searchable !== undefined + ? [{ owner, field }] + : []; + }); +} + +/** A term’s identity as a string: a blank node never equals an IRI. */ +function termKey(term: Term): string { + return `${term.termType}:${term.value}`; +} + +async function* streamOf(quads: readonly Quad[]): AsyncIterable { + yield* quads; +} + /** The class of a root type that describes the dataset being indexed. */ const DCAT_DATASET = 'http://www.w3.org/ns/dcat#Dataset'; diff --git a/packages/search-pipeline/test/keyed-roots.integration.test.ts b/packages/search-pipeline/test/keyed-roots.integration.test.ts index 206581df..62aeeaa3 100644 --- a/packages/search-pipeline/test/keyed-roots.integration.test.ts +++ b/packages/search-pipeline/test/keyed-roots.integration.test.ts @@ -1,8 +1,14 @@ import { fileURLToPath } from 'node:url'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { Dataset, Distribution } from '@lde/dataset'; -import type { DatasetWriter } from '@lde/pipeline'; +import type { + DatasetWriter, + QuadTransform, + ReaderContext, +} from '@lde/pipeline'; import { defineSearchType, searchSchema, type RootType } from '@lde/search'; +import { irAlias } from '@lde/search/adapter'; +import { DataFactory } from 'n3'; import { startSparqlEndpoint, teardownSparqlEndpoint, @@ -106,6 +112,7 @@ describe('a root type keyed on a declared field, end to end', () => { * over roots selected by class, and collect the projected documents. */ async function project( searchType: RootType, + placeTransform?: QuadTransform, ): Promise[]> { const [stage] = searchStages({ schema, @@ -115,6 +122,17 @@ describe('a root type keyed on a declared field, end to end', () => { rootVariable: 'root', itemSelector: selectByClass(searchType), }, + // Declared so its transform is known; only the first type’s stage runs. + ...(placeTransform === undefined + ? [] + : [ + { + searchType: place, + rootVariable: 'root', + itemSelector: selectByClass(place), + transform: placeTransform, + }, + ]), ], }); const documents: Record[] = []; @@ -186,4 +204,168 @@ describe('a root type keyed on a declared field, end to end', () => { expect(byName['Werk 2'].locationCreated_search).toBe('kessel'); expect(byName['Werk 3'].locationCreated_search).toBe('roermond'); }); + + it('searches each work by its place’s label as the place’s transform makes it', async () => { + // A transform on `Place` replacing Kessel’s name, as an Authority’s + // description replaces a publisher’s. + const nameAlias = irAlias(place, place.fields[0]); + const renameKessel: QuadTransform = async function* (quads) { + for await (const quad of quads) { + yield quad.subject.value === 'https://a/place/kessel' && + quad.predicate.value === nameAlias + ? DataFactory.quad( + quad.subject, + quad.predicate, + DataFactory.literal('Kessel aan de Maas'), + ) + : quad; + } + }; + const byName = Object.fromEntries( + (await project(creativeWork, renameKessel)).map((document) => [ + document.name_nl, + document, + ]), + ); + + // No `Place` stage ran: the work’s own stage applied the transform. + expect(byName['Werk 2'].locationCreated_search).toBe('kessel aan de maas'); + expect(byName['Werk 2'].locationCreated).toBe('https://a/place/kessel'); + expect(byName['Werk 3'].locationCreated_search).toBe('roermond'); + expect(byName['Werk 1'].locationCreated).toBe(`${GEONAMES}2745707`); + }); +}); + +describe('a searchable reference’s referents, through their type’s transform', () => { + const port = 3019; + const distribution = Distribution.sparql( + new URL(`http://localhost:${port}/sparql`), + ); + const dataset = new Dataset({ + iri: new URL('http://example.org/dataset/referents'), + distributions: [distribution], + }); + + // An unkeyed place: a referent carries its label alone. + const plainPlace = defineSearchType({ + name: 'PlainPlace', + class: `${SCHEMA}Place`, + labelField: 'name', + fields: [ + { + name: 'name', + kind: 'text', + path: `<${SCHEMA}name>`, + locales: ['und'], + output: true, + searchable: { weight: 1 }, + }, + ], + }); + // A label source no transform is declared for. + const untransformed = defineSearchType({ + ...plainPlace, + name: 'UntransformedPlace', + class: 'https://example.org/UntransformedPlace', + }); + const reference = (name: string, target: string) => + ({ + name, + kind: 'reference', + path: `<${SCHEMA}locationCreated>`, + output: true, + searchable: { weight: 1 }, + ref: { strategy: 'lookup', target }, + }) as const; + const work = defineSearchType({ + name: 'Work', + class: `${SCHEMA}CreativeWork`, + labelField: 'name', + fields: [ + { + name: 'name', + kind: 'text', + path: `<${SCHEMA}name>`, + locales: ['nl'], + output: true, + searchable: { weight: 5 }, + }, + // Two references naming one target. + reference('locationCreated', 'PlainPlace'), + reference('contentLocation', 'PlainPlace'), + reference('untransformed', 'UntransformedPlace'), + // A reference to its own type: its stage’s transform saw the batch. + { ...reference('isPartOf', 'Work'), path: `<${SCHEMA}isPartOf>` }, + ], + }); + const referentSchema = searchSchema(work, plainPlace, untransformed); + + beforeAll(async () => { + await startSparqlEndpoint( + port, + fileURLToPath( + new URL('./fixtures/keyed-places-sample.ttl', import.meta.url), + ), + ); + }, 60_000); + + afterAll(async () => { + await teardownSparqlEndpoint(); + }); + + it('applies the target’s transform after the stage’s own, for every reference naming it', async () => { + const seen: string[] = []; + const nameAlias = irAlias(plainPlace, plainPlace.fields[0]); + const upperCaseNames: QuadTransform = async function* ( + quads, + ) { + seen.push('place'); + for await (const quad of quads) { + yield quad.predicate.value === nameAlias + ? DataFactory.quad( + quad.subject, + quad.predicate, + DataFactory.literal(`${quad.object.value} (plaats)`), + ) + : quad; + } + }; + const ownTransform: QuadTransform = async function* (quads) { + seen.push('work'); + yield* quads; + }; + const [stage] = searchStages({ + schema: referentSchema, + types: [ + { + searchType: work, + rootVariable: 'root', + itemSelector: selectByClass(work), + transform: ownTransform, + }, + { + searchType: plainPlace, + rootVariable: 'root', + itemSelector: selectByClass(plainPlace), + transform: [upperCaseNames], + }, + ], + }); + const documents: Record[] = []; + await stage.run(dataset, distribution, { + write: async (_dataset, items) => { + for await (const item of items) { + documents.push(item.document); + } + }, + }); + const werk2 = documents.find((document) => document.name_nl === 'Werk 2'); + + // One batch: the stage’s own transform, then the referents’ pass. + expect(seen).toEqual(['work', 'place']); + expect(werk2?.locationCreated_search).toBe('kessel (plaats)'); + expect(werk2?.contentLocation_search).toBe('kessel (plaats)'); + // The same referent, read for a target with no transform: as stated. + expect(werk2?.untransformed_search).toBe('kessel'); + }); }); From 60cd6302fcab22a88af3905bea9f44dd591c7617 Mon Sep 17 00:00:00 2001 From: David de Boer Date: Mon, 16 Nov 2026 19:20:06 +0100 Subject: [PATCH 2/2] fix(search-pipeline): pass an endpoint stated without its edge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Route the referents of an inline reference whose graph leaves the edge out, read off the inline node itself, through the endpoint’s target - Key the referents’ transforms by class, as the schema resolves a class-equal lookalike - Append routed quads in place instead of copying the array per quad - Move referentHopFields above the private helpers and restore keyedFieldOf’s doc comment --- packages/search-pipeline/src/extraction.ts | 36 ++++----- packages/search-pipeline/src/search-stages.ts | 51 ++++++++----- .../test/keyed-roots.integration.test.ts | 73 +++++++++++++++++++ 3 files changed, 124 insertions(+), 36 deletions(-) diff --git a/packages/search-pipeline/src/extraction.ts b/packages/search-pipeline/src/extraction.ts index 554cfdee..3030aa50 100644 --- a/packages/search-pipeline/src/extraction.ts +++ b/packages/search-pipeline/src/extraction.ts @@ -141,6 +141,24 @@ export function extractionQueryString( return generator.generate(extractionQuery(searchType, schema, options)); } +/** + * The fields a reference reads off a referent of a target it does not expand + * `local`ly: the target’s key field, where it is keyed, and – for a + * `searchable` reference – its label field, by which the reference is searched. + */ +export function referentHopFields( + field: SearchField, + target: RootType, +): PathField[] { + return [ + keyedFieldOf(target), + // `searchSchema` guarantees a searchable reference’s label a path. + field.searchable === undefined + ? undefined + : (labelFieldOf(target) as PathField), + ].filter((hopField) => hopField !== undefined); +} + /** A per-query counter minting distinct value/referent variable names. */ interface VariableCounter { next: number; @@ -486,24 +504,6 @@ function localTargetsOf( * reads the same declarations rather than a rule of its own: a reference that * names no target keeps the node IRI, and needs no hop. */ -/** - * The fields a reference reads off a referent of a target it does not expand - * `local`ly: the target’s key field, where it is keyed, and – for a - * `searchable` reference – its label field, by which the reference is searched. - */ -export function referentHopFields( - field: SearchField, - target: RootType, -): PathField[] { - return [ - keyedFieldOf(target), - // `searchSchema` guarantees a searchable reference’s label a path. - field.searchable === undefined - ? undefined - : (labelFieldOf(target) as PathField), - ].filter((hopField) => hopField !== undefined); -} - function keyedFieldOf(target: RootType): PathField | undefined { if (target.key === undefined) { return undefined; diff --git a/packages/search-pipeline/src/search-stages.ts b/packages/search-pipeline/src/search-stages.ts index 6074918c..4361c82d 100644 --- a/packages/search-pipeline/src/search-stages.ts +++ b/packages/search-pipeline/src/search-stages.ts @@ -13,12 +13,15 @@ import { } from '@lde/pipeline'; import { projectRoots, + type ReferenceType, type RootType, type SearchDocument, + type SearchField, type SearchSchema, type SearchType, } from '@lde/search'; import { + endpointFieldOf, irAlias, isInlineReference, referencedTargetsOf, @@ -174,11 +177,12 @@ export function searchStages( // Each type’s transforms, for the referring stages to apply to its // referents. A type with its own readers has none here: `transform` // attaches to the default reader only. - const transformsByName = new Map( + const transformsByClass = new Map( types.flatMap((type) => type.transform === undefined ? [] - : [[type.searchType.name, [type.transform].flat()] as const], + : // By class, as the schema resolves a class-equal lookalike. + [[type.searchType.class, [type.transform].flat()] as const], ), ); return ordered.map((type) => { @@ -215,7 +219,7 @@ export function searchStages( }), transform: [ type.transform ?? [], - referentTransforms(searchType, schema, transformsByName) ?? [], + referentTransforms(searchType, schema, transformsByClass) ?? [], ].flat(), } satisfies AttachedReader); return new Stage({ @@ -405,7 +409,7 @@ export const defaultBatchSize = { endpoint: 10, imported: 100 } as const; function referentTransforms( searchType: RootType, schema: SearchSchema, - transformsByName: ReadonlyMap< + transformsByClass: ReadonlyMap< string, readonly QuadTransform[] >, @@ -417,18 +421,18 @@ function referentTransforms( RootType, { fields: Set; aliases: Set } >(); - for (const { owner, field } of searchedReferences(searchType, schema)) { + for (const { alias, field } of searchedReferences(searchType, schema)) { for (const target of referencedTargetsOf(field, schema)) { // A reference to its own type: the type’s transform saw the batch. if ( target !== searchType && - transformsByName.get(target.name) !== undefined + transformsByClass.get(target.class) !== undefined ) { const route = routes.get(target) ?? { fields: new Set(), aliases: new Set(), }; - route.fields.add(irAlias(owner, field)); + route.fields.add(alias); for (const hopField of referentHopFields(field, target)) { route.aliases.add(irAlias(target, hopField)); } @@ -470,15 +474,17 @@ function referentTransforms( target !== undefined && referents.get(target)?.has(termKey(quad.subject)) ) { - inputs.set(target, [...(inputs.get(target) ?? []), quad]); + const input = inputs.get(target) ?? []; + input.push(quad); + inputs.set(target, input); } else { yield quad; } } for (const [target, input] of inputs) { let stream: AsyncIterable = streamOf(input); - for (const transform of transformsByName.get( - target.name, + for (const transform of transformsByClass.get( + target.class, ) as readonly QuadTransform[]) { stream = transform(stream, context); } @@ -487,23 +493,32 @@ function referentTransforms( }; } -/** The type’s `searchable` reference fields, with the type declaring each: - * its own, and those of the inline referents it nests. `searchSchema` - * refuses an inline cycle, so the descent ends. */ +/** The type’s `searchable` reference fields, each with the IR Alias whose + * objects are its referents: its own, and those of the inline referents it + * nests. A graph that leaves an edge out points at the endpoint through the + * inline field itself, so an endpoint field is reached through both. + * `searchSchema` refuses an inline cycle, so the descent ends. */ function searchedReferences( owner: SearchType, schema: SearchSchema, -): { owner: SearchType; field: SearchType['fields'][number] }[] { +): { alias: string; field: SearchField }[] { return owner.fields.flatMap((field) => { if (isInlineReference(field)) { // Guaranteed declared: `searchSchema` resolves every inline `typeName`. - return searchedReferences( - referenceTypeNamed(schema, field.ref.typeName) as SearchType, + const referenceType = referenceTypeNamed( schema, - ); + field.ref.typeName, + ) as ReferenceType; + const endpointField = endpointFieldOf(referenceType, field.ref.identity); + return [ + ...searchedReferences(referenceType, schema), + ...(endpointField?.searchable === undefined + ? [] + : [{ alias: irAlias(owner, field), field: endpointField }]), + ]; } return field.kind === 'reference' && field.searchable !== undefined - ? [{ owner, field }] + ? [{ alias: irAlias(owner, field), field }] : []; }); } diff --git a/packages/search-pipeline/test/keyed-roots.integration.test.ts b/packages/search-pipeline/test/keyed-roots.integration.test.ts index 62aeeaa3..486d51df 100644 --- a/packages/search-pipeline/test/keyed-roots.integration.test.ts +++ b/packages/search-pipeline/test/keyed-roots.integration.test.ts @@ -368,4 +368,77 @@ describe('a searchable reference’s referents, through their type’s transform // The same referent, read for a target with no transform: as stated. expect(werk2?.untransformed_search).toBe('kessel'); }); + + it('applies it to an endpoint the graph states without its edge', async () => { + const placeRole = defineSearchType({ + name: 'PlaceRole', + edgeClass: `${SCHEMA}Role`, + fields: [ + { + name: 'roleName', + kind: 'keyword', + output: true, + path: `<${SCHEMA}roleName>`, + }, + { ...reference('place', 'PlainPlace'), filterable: true }, + ], + }); + const roleWork = defineSearchType({ + ...work, + fields: [ + work.fields[0], + { + name: 'created', + kind: 'reference', + output: true, + path: `<${SCHEMA}locationCreated>`, + facetable: true, + ref: { strategy: 'inline', typeName: 'PlaceRole', identity: 'place' }, + }, + ], + }); + const nameAlias = irAlias(plainPlace, plainPlace.fields[0]); + const [stage] = searchStages({ + schema: searchSchema(roleWork, placeRole, plainPlace), + types: [ + { + searchType: roleWork, + rootVariable: 'root', + itemSelector: selectByClass(roleWork), + }, + { + searchType: plainPlace, + rootVariable: 'root', + itemSelector: selectByClass(plainPlace), + transform: async function* (quads) { + for await (const quad of quads) { + yield quad.predicate.value === nameAlias + ? DataFactory.quad( + quad.subject, + quad.predicate, + DataFactory.literal(`${quad.object.value} (plaats)`), + ) + : quad; + } + }, + }, + ], + }); + const documents: Record[] = []; + await stage.run(dataset, distribution, { + write: async (_dataset, items) => { + for await (const item of items) { + documents.push(item.document); + } + }, + }); + + // The work points straight at the place: no Role in between. + expect( + documents.find((document) => document.name_nl === 'Werk 2')?.created, + ).toEqual({ + place: 'https://a/place/kessel', + place_search: 'kessel (plaats)', + }); + }); });