From 3d52ebe5e9d01b8eb883eb074572794a20122878 Mon Sep 17 00:00:00 2001 From: Daan Meijers Date: Wed, 11 Nov 2026 15:14:38 +0100 Subject: [PATCH 1/6] feat: make intermediate chain stage output optional --- docs/guide/chain-stage-outputs.md | 22 ++++- docs/reference/pipeline.md | 6 +- packages/pipeline/src/pipeline.ts | 24 ++++-- packages/pipeline/src/stage.ts | 26 ++++++ packages/pipeline/test/pipeline.test.ts | 104 ++++++++++++++++++++++++ packages/pipeline/test/stage.test.ts | 41 ++++++++++ packages/pipeline/vite.config.ts | 6 +- 7 files changed, 217 insertions(+), 12 deletions(-) diff --git a/docs/guide/chain-stage-outputs.md b/docs/guide/chain-stage-outputs.md index a764ba7d1..8c901af45 100644 --- a/docs/guide/chain-stage-outputs.md +++ b/docs/guide/chain-stage-outputs.md @@ -21,6 +21,26 @@ const parent = new Stage({ }); ``` +## Keep a stage’s output out of the result + +By default the output of every stage in the chain ends up in the pipeline’s output. If a stage’s output is only input for the next stage, set `emitOutput: false` on it: + +```typescript +const parent = new Stage({ + name: 'extract', + readers: await SparqlConstructReader.fromFile('extract.rq'), + emitOutput: false, // only 'aggregate' reads this + stages: [ + new Stage({ + name: 'aggregate', + readers: await SparqlConstructReader.fromFile('aggregate.rq'), + }), + ], +}); +``` + +The last stage in a chain must emit its output, because no stage reads it. + ## Implement a stage output resolver Between stages, the pipeline writes output to a scratch N-Triples file. A `StageOutputResolver` turns that file into the `Distribution` the next stage queries: @@ -70,7 +90,7 @@ For each dataset: 1. The parent stage runs against the dataset’s resolved distribution and writes N-Triples to a scratch file under `//`. 2. `resolve()` turns that file into a `Distribution`; the first sub-stage runs against it, writing its own scratch file. Each further sub-stage chains off the previous one’s output the same way. -3. The scratch files of the whole chain are concatenated and written to the pipeline’s [writers](../reference/pipeline#writer) under the parent stage’s name. +3. The scratch files of the chain’s stages that emit output are concatenated and written to the pipeline’s [writers](../reference/pipeline#writer) under the parent stage’s name. 4. `cleanup()` runs when the chain finishes, on success and on failure. Two constraints apply: diff --git a/docs/reference/pipeline.md b/docs/reference/pipeline.md index d6b73b0d8..5e026369c 100644 --- a/docs/reference/pipeline.md +++ b/docs/reference/pipeline.md @@ -211,7 +211,11 @@ new Pipeline({ }); ``` -Because the scratch files are N-Triples, chains are quad-only: a stage with `stages` cannot `project`. A chained stage that returns `NotSupported` fails the chain rather than being skipped. See the how-to guide [Chain stage outputs](../guide/chain-stage-outputs). +Because the scratch files are N-Triples, chains are quad-only: a stage with `stages` cannot `project`. A chained stage that returns `NotSupported` fails the chain rather than being skipped. + +Set `emitOutput: false` on a stage whose output should only feed the next stage in the chain – a working graph a child queries, say. The stage still writes its scratch file for the next stage to query, but the chain leaves it out of what it writes to the pipeline’s writers. Because nothing would read the output otherwise, the last chained stage and a top-level stage without `stages` cannot set it; their constructors throw. + +See the how-to guide [Chain stage outputs](../guide/chain-stage-outputs). ### Item Selector diff --git a/packages/pipeline/src/pipeline.ts b/packages/pipeline/src/pipeline.ts index e2aac7bf6..a864cc268 100644 --- a/packages/pipeline/src/pipeline.ts +++ b/packages/pipeline/src/pipeline.ts @@ -440,6 +440,16 @@ export class Pipeline { if (hasSubStages && !options.chaining) { throw new Error('chaining is required when any stage has sub-stages'); } + // No stage reads a non-chained stage's output. Checked here because a + // stage cannot tell whether it is top-level or a chained child. + const silentStage = options.stages.find( + (stage) => !stage.emitOutput && stage.stages.length === 0, + ); + if (silentStage) { + throw new Error( + `Stage '${silentStage.name}' sets 'emitOutput: false' but has no chained 'stages' – no stage reads its output, so it would be discarded.`, + ); + } if (options.provenanceStore && options.pipelineVersion === undefined) { throw new Error( @@ -1176,6 +1186,8 @@ export class Pipeline { timeout?: TimeoutPolicy, ): Promise { const { stageOutputResolver, outputDir } = this.chaining!; + // Scratch files of the stages that emit output: every stage writes one for + // the next stage to read, but only these reach the writers. const outputFiles: string[] = []; // Chaining is a quad-only concern: every stage in a chain writes N-Triples @@ -1217,23 +1229,21 @@ export class Pipeline { stage as unknown as Stage, distribution, ); - outputFiles.push(parentOutput); + if (stage.emitOutput) outputFiles.push(parentOutput); // 2. Chain through children. let currentDistribution = await stageOutputResolver.resolve(parentOutput); for (let i = 0; i < stage.stages.length; i++) { - const childOutput = await runScratchStage( - stage.stages[i], - currentDistribution, - ); - outputFiles.push(childOutput); + const child = stage.stages[i]; + const childOutput = await runScratchStage(child, currentDistribution); + if (child.emitOutput) outputFiles.push(childOutput); if (i < stage.stages.length - 1) { currentDistribution = await stageOutputResolver.resolve(childOutput); } } - // 3. Concatenate all output files → the open run, applying the plugins' + // 3. Concatenate the emitted output files → the open run, applying the plugins' // beforeStageWrite transforms once for the chain under the parent stage. // The concatenation is quads (per the quad-only contract above), written // to the `Out`-typed run writer – sound because chaining implies `Quad`. diff --git a/packages/pipeline/src/stage.ts b/packages/pipeline/src/stage.ts index dccc5e0bd..f029e405e 100644 --- a/packages/pipeline/src/stage.ts +++ b/packages/pipeline/src/stage.ts @@ -173,6 +173,20 @@ export interface StageOptions { sourceFor?: (dataset: Dataset, distribution: Distribution) => Distribution; /** Child stages that chain off this stage's output. */ stages?: Stage[]; + /** + * Whether this stage's output reaches the pipeline's writers. Set it to + * `false` on a stage in a chain whose output only feeds the next stage – a + * working graph a child queries, say: the stage still writes its scratch + * file, but the chain leaves it out of what it writes. + * + * Only meaningful inside a chain, and only where a next stage reads the + * output, so `false` is rejected on the last chained stage (by the parent's + * constructor) and on a top-level stage without `stages` (by the + * `Pipeline`'s) – either would discard the stage's work. + * + * @default true + */ + emitOutput?: boolean; /** * Treat a supported stage that produces no quads as a hard failure (throws), * rather than a legitimately empty result. @@ -232,6 +246,9 @@ export class Stage { /** Whether this stage reads a source of its own rather than the dataset’s * resolved distribution. @see {@link StageOptions.sourceFor} */ readonly sourcesOwnData: boolean; + /** Whether this stage's output reaches the pipeline's writers. + * @see {@link StageOptions.emitOutput} */ + readonly emitOutput: boolean; private readonly readers: NormalizedReader[]; private readonly itemSelector?: ItemSelector; private readonly batchSize: number; @@ -262,6 +279,14 @@ export class Stage { `Stage '${options.name}': chained stage '${sourcedChild.name}' declares 'sourceFor' – a chained stage reads its parent's output, so it cannot source its own data.`, ); } + // Nothing reads the last chained stage's output, so withholding it from the + // writers would discard the stage's work. + const lastChild = options.stages?.at(-1); + if (lastChild && !lastChild.emitOutput) { + throw new Error( + `Stage '${options.name}': the last chained stage '${lastChild.name}' sets 'emitOutput: false' – no stage reads its output, so it would be discarded.`, + ); + } this.name = options.name; this.stages = options.stages ?? []; this.readers = normalizeReaders(options.readers); @@ -274,6 +299,7 @@ export class Stage { this.queueCapacity = options.queueCapacity; this.sourceFor = options.sourceFor; this.sourcesOwnData = options.sourceFor !== undefined; + this.emitOutput = options.emitOutput ?? true; } /** The validator for this stage, if configured. */ diff --git a/packages/pipeline/test/pipeline.test.ts b/packages/pipeline/test/pipeline.test.ts index a6f34f14c..fc4d2468e 100644 --- a/packages/pipeline/test/pipeline.test.ts +++ b/packages/pipeline/test/pipeline.test.ts @@ -1122,6 +1122,110 @@ describe('Pipeline', () => { expect(stageOutputResolver.cleanup).toHaveBeenCalledTimes(1); }); + describe('emitOutput', () => { + /** A stage whose run writes one quad with subject `http://`. */ + function writingStage( + name: string, + options: { emitOutput?: boolean; stages?: Stage[] } = {}, + ): Stage { + const stage = new Stage({ name, readers: [], ...options }); + vi.spyOn(stage, 'run').mockImplementation( + async (staged, _distribution, stageWriter) => { + await stageWriter.write( + staged, + (async function* () { + yield DataFactory.quad( + DataFactory.namedNode(`http://${name}`), + DataFactory.namedNode('http://p'), + DataFactory.namedNode('http://o'), + ); + })(), + ); + }, + ); + return stage; + } + + /** Run `parent` as a chain and return the subjects the writers got. */ + async function writtenSubjects(parent: Stage): Promise { + const outputDir = await mkdtemp(join(tmpdir(), 'pipeline-emit-test-')); + const subjects: string[] = []; + writer.runWriter.write.mockImplementation( + async (_dataset: Dataset, quads: AsyncIterable) => { + for await (const quad of quads) subjects.push(quad.subject.value); + }, + ); + const pipeline = new Pipeline({ + datasetSelector: makeDatasetSelector(dataset), + stages: [parent], + writers: writer, + distributionResolver: makeResolver(makeResolvedDistribution()), + chaining: { + stageOutputResolver: makeStageOutputResolver(), + outputDir, + }, + }); + try { + await pipeline.run(); + } finally { + await rm(outputDir, { recursive: true, force: true }); + } + return subjects; + } + + it('writes the output of every stage in a chain by default', async () => { + const parent = writingStage('parent', { + stages: [writingStage('child')], + }); + + expect(await writtenSubjects(parent)).toEqual([ + 'http://parent', + 'http://child', + ]); + }); + + it('leaves a parent that does not emit output out of the writers, while its child still runs', async () => { + const child = writingStage('child'); + const parent = writingStage('parent', { + emitOutput: false, + stages: [child], + }); + + expect(await writtenSubjects(parent)).toEqual(['http://child']); + expect(child.run).toHaveBeenCalledTimes(1); + }); + + it('leaves a middle child that does not emit output out of the writers, while the next child still chains off it', async () => { + const middle = writingStage('middle', { emitOutput: false }); + const last = writingStage('last'); + const parent = writingStage('parent', { stages: [middle, last] }); + + expect(await writtenSubjects(parent)).toEqual([ + 'http://parent', + 'http://last', + ]); + expect(last.run).toHaveBeenCalledTimes(1); + }); + + it('rejects a top-level stage without sub-stages that does not emit output', () => { + const stage = new Stage({ + name: 'lonely', + readers: [], + emitOutput: false, + }); + + expect( + () => + new Pipeline({ + datasetSelector: makeDatasetSelector(dataset), + stages: [stage], + writers: writer, + distributionResolver: makeResolver(makeResolvedDistribution()), + }), + ).toThrow(/Stage 'lonely' sets 'emitOutput: false'/); + }); + }); + it('validates chaining is required for sub-stages', () => { const child = makeStage('child'); const parent = makeStage('parent', undefined, [child]); diff --git a/packages/pipeline/test/stage.test.ts b/packages/pipeline/test/stage.test.ts index 1be23c4d4..67305805d 100644 --- a/packages/pipeline/test/stage.test.ts +++ b/packages/pipeline/test/stage.test.ts @@ -1388,6 +1388,47 @@ describe('Stage', () => { ).toThrow(/chained stage 'child' declares 'sourceFor'/); }); + it('rejects a last chained stage that does not emit output', () => { + // Nothing reads the last child's output, so withholding it would + // discard the stage's work. + const middle = new Stage({ + name: 'middle', + readers: mockExecutor([]), + emitOutput: false, + }); + const last = new Stage({ + name: 'last', + readers: mockExecutor([]), + emitOutput: false, + }); + expect( + () => + new Stage({ + name: 'parent', + readers: mockExecutor([q1]), + stages: [middle, last], + }), + ).toThrow(/the last chained stage 'last' sets 'emitOutput: false'/); + }); + + it('allows a chained stage that does not emit output when a later stage reads it', () => { + const middle = new Stage({ + name: 'middle', + readers: mockExecutor([]), + emitOutput: false, + }); + const last = new Stage({ name: 'last', readers: mockExecutor([]) }); + expect( + () => + new Stage({ + name: 'parent', + readers: mockExecutor([q1]), + emitOutput: false, + stages: [middle, last], + }), + ).not.toThrow(); + }); + it('allows a chaining parent to source its own data', () => { const child = new Stage({ name: 'child', readers: mockExecutor([]) }); expect( diff --git a/packages/pipeline/vite.config.ts b/packages/pipeline/vite.config.ts index dd76dd1ac..d5906eaea 100644 --- a/packages/pipeline/vite.config.ts +++ b/packages/pipeline/vite.config.ts @@ -12,9 +12,9 @@ export default mergeConfig( thresholds: { autoUpdate: true, functions: 99.39, - lines: 98.68, - branches: 96.25, - statements: 98.45, + lines: 99.22, + branches: 96.32, + statements: 98.98, }, }, }, From 4db6c10c43bfc6ed9dc564590c00d4936f607bdd Mon Sep 17 00:00:00 2001 From: Daan Meijers Date: Wed, 11 Nov 2026 16:23:39 +0100 Subject: [PATCH 2/6] fix: minor test and doc tweaks --- docs/reference/pipeline.md | 2 +- packages/pipeline/test/stage.test.ts | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/reference/pipeline.md b/docs/reference/pipeline.md index 5e026369c..7b12838c9 100644 --- a/docs/reference/pipeline.md +++ b/docs/reference/pipeline.md @@ -213,7 +213,7 @@ new Pipeline({ Because the scratch files are N-Triples, chains are quad-only: a stage with `stages` cannot `project`. A chained stage that returns `NotSupported` fails the chain rather than being skipped. -Set `emitOutput: false` on a stage whose output should only feed the next stage in the chain – a working graph a child queries, say. The stage still writes its scratch file for the next stage to query, but the chain leaves it out of what it writes to the pipeline’s writers. Because nothing would read the output otherwise, the last chained stage and a top-level stage without `stages` cannot set it; their constructors throw. +Set `emitOutput: false` on a stage whose output should only feed the next stage in the chain – a working graph a child queries, say. The stage still writes its scratch file for the next stage to query, but the chain leaves it out of what it writes to the pipeline’s writers. Because nothing would read the output otherwise, the last chained stage and a top-level stage without `stages` cannot set it; the parent stage's or the Pipeline's constructor throws. See the how-to guide [Chain stage outputs](../guide/chain-stage-outputs). diff --git a/packages/pipeline/test/stage.test.ts b/packages/pipeline/test/stage.test.ts index 67305805d..6be350560 100644 --- a/packages/pipeline/test/stage.test.ts +++ b/packages/pipeline/test/stage.test.ts @@ -1394,7 +1394,6 @@ describe('Stage', () => { const middle = new Stage({ name: 'middle', readers: mockExecutor([]), - emitOutput: false, }); const last = new Stage({ name: 'last', From e8877f88d95cd02a01ce08052fe38bc1db9b2165 Mon Sep 17 00:00:00 2001 From: Daan Meijers Date: Wed, 18 Nov 2026 11:52:43 +0100 Subject: [PATCH 3/6] fix: replace emitOutput with chainOutput option --- docs/guide/chain-stage-outputs.md | 10 +++--- docs/reference/pipeline.md | 2 +- packages/pipeline/src/pipeline.ts | 31 +++++++--------- packages/pipeline/src/stage.ts | 37 ++++++++----------- packages/pipeline/test/pipeline.test.ts | 47 ++++++------------------- packages/pipeline/test/stage.test.ts | 40 --------------------- packages/pipeline/vite.config.ts | 4 +-- 7 files changed, 44 insertions(+), 127 deletions(-) diff --git a/docs/guide/chain-stage-outputs.md b/docs/guide/chain-stage-outputs.md index 8c901af45..ec938e41b 100644 --- a/docs/guide/chain-stage-outputs.md +++ b/docs/guide/chain-stage-outputs.md @@ -21,15 +21,15 @@ const parent = new Stage({ }); ``` -## Keep a stage’s output out of the result +## Write only the last stage’s output -By default the output of every stage in the chain ends up in the pipeline’s output. If a stage’s output is only input for the next stage, set `emitOutput: false` on it: +By default the output of every stage in the chain ends up in the pipeline’s output. If the earlier stages only produce input for the next stage, set `chainOutput: 'last'` on the parent stage to keep only the last chained stage’s output: ```typescript const parent = new Stage({ name: 'extract', readers: await SparqlConstructReader.fromFile('extract.rq'), - emitOutput: false, // only 'aggregate' reads this + chainOutput: 'last', // only 'aggregate' reaches the writers stages: [ new Stage({ name: 'aggregate', @@ -39,8 +39,6 @@ const parent = new Stage({ }); ``` -The last stage in a chain must emit its output, because no stage reads it. - ## Implement a stage output resolver Between stages, the pipeline writes output to a scratch N-Triples file. A `StageOutputResolver` turns that file into the `Distribution` the next stage queries: @@ -90,7 +88,7 @@ For each dataset: 1. The parent stage runs against the dataset’s resolved distribution and writes N-Triples to a scratch file under `//`. 2. `resolve()` turns that file into a `Distribution`; the first sub-stage runs against it, writing its own scratch file. Each further sub-stage chains off the previous one’s output the same way. -3. The scratch files of the chain’s stages that emit output are concatenated and written to the pipeline’s [writers](../reference/pipeline#writer) under the parent stage’s name. +3. The scratch files the parent’s `chainOutput` selects – all of them by default, or only the last stage’s – are concatenated and written to the pipeline’s [writers](../reference/pipeline#writer) under the parent stage’s name. 4. `cleanup()` runs when the chain finishes, on success and on failure. Two constraints apply: diff --git a/docs/reference/pipeline.md b/docs/reference/pipeline.md index 7b12838c9..41be18af8 100644 --- a/docs/reference/pipeline.md +++ b/docs/reference/pipeline.md @@ -213,7 +213,7 @@ new Pipeline({ Because the scratch files are N-Triples, chains are quad-only: a stage with `stages` cannot `project`. A chained stage that returns `NotSupported` fails the chain rather than being skipped. -Set `emitOutput: false` on a stage whose output should only feed the next stage in the chain – a working graph a child queries, say. The stage still writes its scratch file for the next stage to query, but the chain leaves it out of what it writes to the pipeline’s writers. Because nothing would read the output otherwise, the last chained stage and a top-level stage without `stages` cannot set it; the parent stage's or the Pipeline's constructor throws. +By default the output of every stage in the chain reaches the pipeline’s writers. Set `chainOutput: 'last'` on the parent stage when the earlier stages only produce working graphs for the next stage to query: they still write their scratch files, but only the last chained stage’s output is written. See the how-to guide [Chain stage outputs](../guide/chain-stage-outputs). diff --git a/packages/pipeline/src/pipeline.ts b/packages/pipeline/src/pipeline.ts index a864cc268..f2de08b6c 100644 --- a/packages/pipeline/src/pipeline.ts +++ b/packages/pipeline/src/pipeline.ts @@ -440,16 +440,6 @@ export class Pipeline { if (hasSubStages && !options.chaining) { throw new Error('chaining is required when any stage has sub-stages'); } - // No stage reads a non-chained stage's output. Checked here because a - // stage cannot tell whether it is top-level or a chained child. - const silentStage = options.stages.find( - (stage) => !stage.emitOutput && stage.stages.length === 0, - ); - if (silentStage) { - throw new Error( - `Stage '${silentStage.name}' sets 'emitOutput: false' but has no chained 'stages' – no stage reads its output, so it would be discarded.`, - ); - } if (options.provenanceStore && options.pipelineVersion === undefined) { throw new Error( @@ -1186,8 +1176,8 @@ export class Pipeline { timeout?: TimeoutPolicy, ): Promise { const { stageOutputResolver, outputDir } = this.chaining!; - // Scratch files of the stages that emit output: every stage writes one for - // the next stage to read, but only these reach the writers. + // Scratch files of every stage in the chain, in order; `chainOutput` + // decides which of them reach the writers. const outputFiles: string[] = []; // Chaining is a quad-only concern: every stage in a chain writes N-Triples @@ -1229,27 +1219,30 @@ export class Pipeline { stage as unknown as Stage, distribution, ); - if (stage.emitOutput) outputFiles.push(parentOutput); + outputFiles.push(parentOutput); // 2. Chain through children. let currentDistribution = await stageOutputResolver.resolve(parentOutput); for (let i = 0; i < stage.stages.length; i++) { const child = stage.stages[i]; const childOutput = await runScratchStage(child, currentDistribution); - if (child.emitOutput) outputFiles.push(childOutput); + outputFiles.push(childOutput); if (i < stage.stages.length - 1) { currentDistribution = await stageOutputResolver.resolve(childOutput); } } - // 3. Concatenate the emitted output files → the open run, applying the plugins' - // beforeStageWrite transforms once for the chain under the parent stage. - // The concatenation is quads (per the quad-only contract above), written - // to the `Out`-typed run writer – sound because chaining implies `Quad`. + // 3. Concatenate the output files `chainOutput` selects → the open run, + // applying the plugins' beforeStageWrite transforms once for the chain + // under the parent stage. The concatenation is quads (per the quad-only + // contract above), written to the `Out`-typed run writer – sound because + // chaining implies `Quad`. + const emitted = + stage.chainOutput === 'last' ? outputFiles.slice(-1) : outputFiles; await this.stageWriter(runWriter, stage.name).write( dataset, - this.readFiles(outputFiles) as unknown as AsyncIterable, + this.readFiles(emitted) as unknown as AsyncIterable, ); } finally { await stageOutputResolver.cleanup(); diff --git a/packages/pipeline/src/stage.ts b/packages/pipeline/src/stage.ts index f029e405e..8855b5a57 100644 --- a/packages/pipeline/src/stage.ts +++ b/packages/pipeline/src/stage.ts @@ -174,19 +174,20 @@ export interface StageOptions { /** Child stages that chain off this stage's output. */ stages?: Stage[]; /** - * Whether this stage's output reaches the pipeline's writers. Set it to - * `false` on a stage in a chain whose output only feeds the next stage – a - * working graph a child queries, say: the stage still writes its scratch - * file, but the chain leaves it out of what it writes. + * Which outputs of this stage's chain reach the pipeline's writers: * - * Only meaningful inside a chain, and only where a next stage reads the - * output, so `false` is rejected on the last chained stage (by the parent's - * constructor) and on a top-level stage without `stages` (by the - * `Pipeline`'s) – either would discard the stage's work. + * - `'all'` – this stage's output and that of every chained stage; + * - `'last'` – only the last chained stage's output. Set it when the earlier + * stages produce working graphs that only feed the next stage: they still + * write their scratch files for the next stage to query, but the chain + * leaves them out of what it writes. * - * @default true + * Only meaningful with {@link StageOptions.stages}: a stage without them is + * no chain, and its output is always written. + * + * @default 'all' */ - emitOutput?: boolean; + chainOutput?: 'all' | 'last'; /** * Treat a supported stage that produces no quads as a hard failure (throws), * rather than a legitimately empty result. @@ -246,9 +247,9 @@ export class Stage { /** Whether this stage reads a source of its own rather than the dataset’s * resolved distribution. @see {@link StageOptions.sourceFor} */ readonly sourcesOwnData: boolean; - /** Whether this stage's output reaches the pipeline's writers. - * @see {@link StageOptions.emitOutput} */ - readonly emitOutput: boolean; + /** Which outputs of this stage's chain reach the pipeline's writers. + * @see {@link StageOptions.chainOutput} */ + readonly chainOutput: 'all' | 'last'; private readonly readers: NormalizedReader[]; private readonly itemSelector?: ItemSelector; private readonly batchSize: number; @@ -279,14 +280,6 @@ export class Stage { `Stage '${options.name}': chained stage '${sourcedChild.name}' declares 'sourceFor' – a chained stage reads its parent's output, so it cannot source its own data.`, ); } - // Nothing reads the last chained stage's output, so withholding it from the - // writers would discard the stage's work. - const lastChild = options.stages?.at(-1); - if (lastChild && !lastChild.emitOutput) { - throw new Error( - `Stage '${options.name}': the last chained stage '${lastChild.name}' sets 'emitOutput: false' – no stage reads its output, so it would be discarded.`, - ); - } this.name = options.name; this.stages = options.stages ?? []; this.readers = normalizeReaders(options.readers); @@ -299,7 +292,7 @@ export class Stage { this.queueCapacity = options.queueCapacity; this.sourceFor = options.sourceFor; this.sourcesOwnData = options.sourceFor !== undefined; - this.emitOutput = options.emitOutput ?? true; + this.chainOutput = options.chainOutput ?? 'all'; } /** The validator for this stage, if configured. */ diff --git a/packages/pipeline/test/pipeline.test.ts b/packages/pipeline/test/pipeline.test.ts index fc4d2468e..7e8af0fcd 100644 --- a/packages/pipeline/test/pipeline.test.ts +++ b/packages/pipeline/test/pipeline.test.ts @@ -1122,11 +1122,11 @@ describe('Pipeline', () => { expect(stageOutputResolver.cleanup).toHaveBeenCalledTimes(1); }); - describe('emitOutput', () => { + describe('chainOutput', () => { /** A stage whose run writes one quad with subject `http://`. */ function writingStage( name: string, - options: { emitOutput?: boolean; stages?: Stage[] } = {}, + options: { chainOutput?: 'all' | 'last'; stages?: Stage[] } = {}, ): Stage { const stage = new Stage({ name, readers: [], ...options }); vi.spyOn(stage, 'run').mockImplementation( @@ -1184,46 +1184,19 @@ describe('Pipeline', () => { ]); }); - it('leaves a parent that does not emit output out of the writers, while its child still runs', async () => { - const child = writingStage('child'); + it('writes only the last chained stage’s output when set to last, while every stage still runs', async () => { + const middle = writingStage('middle'); + const last = writingStage('last'); const parent = writingStage('parent', { - emitOutput: false, - stages: [child], + chainOutput: 'last', + stages: [middle, last], }); - expect(await writtenSubjects(parent)).toEqual(['http://child']); - expect(child.run).toHaveBeenCalledTimes(1); - }); - - it('leaves a middle child that does not emit output out of the writers, while the next child still chains off it', async () => { - const middle = writingStage('middle', { emitOutput: false }); - const last = writingStage('last'); - const parent = writingStage('parent', { stages: [middle, last] }); - - expect(await writtenSubjects(parent)).toEqual([ - 'http://parent', - 'http://last', - ]); + expect(await writtenSubjects(parent)).toEqual(['http://last']); + expect(parent.run).toHaveBeenCalledTimes(1); + expect(middle.run).toHaveBeenCalledTimes(1); expect(last.run).toHaveBeenCalledTimes(1); }); - - it('rejects a top-level stage without sub-stages that does not emit output', () => { - const stage = new Stage({ - name: 'lonely', - readers: [], - emitOutput: false, - }); - - expect( - () => - new Pipeline({ - datasetSelector: makeDatasetSelector(dataset), - stages: [stage], - writers: writer, - distributionResolver: makeResolver(makeResolvedDistribution()), - }), - ).toThrow(/Stage 'lonely' sets 'emitOutput: false'/); - }); }); it('validates chaining is required for sub-stages', () => { diff --git a/packages/pipeline/test/stage.test.ts b/packages/pipeline/test/stage.test.ts index 6be350560..1be23c4d4 100644 --- a/packages/pipeline/test/stage.test.ts +++ b/packages/pipeline/test/stage.test.ts @@ -1388,46 +1388,6 @@ describe('Stage', () => { ).toThrow(/chained stage 'child' declares 'sourceFor'/); }); - it('rejects a last chained stage that does not emit output', () => { - // Nothing reads the last child's output, so withholding it would - // discard the stage's work. - const middle = new Stage({ - name: 'middle', - readers: mockExecutor([]), - }); - const last = new Stage({ - name: 'last', - readers: mockExecutor([]), - emitOutput: false, - }); - expect( - () => - new Stage({ - name: 'parent', - readers: mockExecutor([q1]), - stages: [middle, last], - }), - ).toThrow(/the last chained stage 'last' sets 'emitOutput: false'/); - }); - - it('allows a chained stage that does not emit output when a later stage reads it', () => { - const middle = new Stage({ - name: 'middle', - readers: mockExecutor([]), - emitOutput: false, - }); - const last = new Stage({ name: 'last', readers: mockExecutor([]) }); - expect( - () => - new Stage({ - name: 'parent', - readers: mockExecutor([q1]), - emitOutput: false, - stages: [middle, last], - }), - ).not.toThrow(); - }); - it('allows a chaining parent to source its own data', () => { const child = new Stage({ name: 'child', readers: mockExecutor([]) }); expect( diff --git a/packages/pipeline/vite.config.ts b/packages/pipeline/vite.config.ts index d5906eaea..54965bda1 100644 --- a/packages/pipeline/vite.config.ts +++ b/packages/pipeline/vite.config.ts @@ -13,8 +13,8 @@ export default mergeConfig( autoUpdate: true, functions: 99.39, lines: 99.22, - branches: 96.32, - statements: 98.98, + branches: 96.27, + statements: 98.97, }, }, }, From d6a199869a58b07eb8dc53f10ba9449778d09cf8 Mon Sep 17 00:00:00 2001 From: David de Boer Date: Fri, 20 Nov 2026 11:07:25 +0100 Subject: [PATCH 4/6] fix(pipeline): keep chain scratch output current and document both chain output patterns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A chained stage that produced no quads left no scratch file, so the chain crashed reading it – or, with a previous run’s file still on disk, wrote that stale output as current. Each chained stage now replaces its scratch file every run, leaving an empty one for empty output. The scratch files stay after the run for inspection, which the outputDir JSDoc now guarantees. chainOutput now requires stages, so setting it on a chained child fails instead of doing nothing. The guide documents the two patterns: enrich (every stage written) and normalise-then-map (only the last). --- docs/guide/chain-stage-outputs.md | 74 ++++++++-- docs/reference/pipeline.md | 9 +- packages/pipeline/src/pipeline.ts | 19 ++- packages/pipeline/src/stage.ts | 24 ++-- packages/pipeline/test/pipeline.test.ts | 182 ++++++++++++++++++++++-- packages/pipeline/test/stage.test.ts | 13 ++ packages/pipeline/vite.config.ts | 6 +- 7 files changed, 296 insertions(+), 31 deletions(-) diff --git a/docs/guide/chain-stage-outputs.md b/docs/guide/chain-stage-outputs.md index ec938e41b..daa83b67d 100644 --- a/docs/guide/chain-stage-outputs.md +++ b/docs/guide/chain-stage-outputs.md @@ -1,10 +1,10 @@ # Chain stage outputs -This guide shows how to feed one stage’s output to sub-stages that query it – for example an extraction stage whose result a second stage aggregates. +This guide shows how to feed one stage’s output to sub-stages that query it, and how to choose which stages’ output the chain writes. ## Declare sub-stages -Pass child stages in the parent’s `stages` option. Each child queries the previous stage’s output instead of the dataset’s own distribution: +Pass child stages in the parent’s `stages` option. Each child queries the previous stage’s output instead of the dataset’s own distribution. A chain is a line, not a tree: the second child queries the first child’s output, not the parent’s. To run several queries over the same output, give one child several readers: ```typescript import { SparqlConstructReader, Stage } from '@lde/pipeline'; @@ -21,24 +21,78 @@ const parent = new Stage({ }); ``` -## Write only the last stage’s output +## Choose what the chain writes -By default the output of every stage in the chain ends up in the pipeline’s output. If the earlier stages only produce input for the next stage, set `chainOutput: 'last'` on the parent stage to keep only the last chained stage’s output: +The parent’s `chainOutput` option decides which stages’ output reaches the pipeline’s [writers](../reference/pipeline#writer). Pick it by what the later stages do with the earlier output. + +### Enrich: write every stage + +Use the default, `chainOutput: 'all'`, when each stage adds to the result. A typical case is enrichment: the parent produces your data, and a chained stage selects the terms it links to and fetches their labels from another SPARQL endpoint. The chained stage’s item selector queries the parent’s output; its reader queries the other endpoint for the selected terms: ```typescript +import { Distribution } from '@lde/dataset'; +import { + SparqlConstructReader, + SparqlItemSelector, + Stage, +} from '@lde/pipeline'; +import type { Reader } from '@lde/pipeline'; + +const terminology = Distribution.sparql( + new URL('https://terminology.example.org/sparql'), +); +const labels = await SparqlConstructReader.fromFile('term-labels.rq'); + +// Query the terminology endpoint instead of the parent’s output. +const termLabels: Reader = { + read: (dataset, _distribution, options) => + labels.read(dataset, terminology, options), +}; + const parent = new Stage({ - name: 'extract', - readers: await SparqlConstructReader.fromFile('extract.rq'), - chainOutput: 'last', // only 'aggregate' reaches the writers + name: 'map', + readers: await SparqlConstructReader.fromFile('map.rq'), stages: [ new Stage({ - name: 'aggregate', - readers: await SparqlConstructReader.fromFile('aggregate.rq'), + name: 'enrich', + itemSelector: new SparqlItemSelector({ + query: + 'SELECT DISTINCT ?this WHERE { ?record ?this } ORDER BY ?this', + }), + readers: termLabels, + }), + ], +}); +``` + +The writers get the mapped data and the labels together. + +### Normalise, then map: write only the last stage + +Set `chainOutput: 'last'` when the earlier stages only produce input for the next one. A typical case is a shared mapping that expects its input in one shape: a first stage normalises each source to that shape, and the mapping reads the normalised output. The normalised data is scaffolding, so only the mapping’s output should reach the writers: + +```typescript +const parent = new Stage({ + name: 'normalise', + readers: await SparqlConstructReader.fromFile('normalise.rq'), + chainOutput: 'last', // only 'map' reaches the writers + stages: [ + new Stage({ + name: 'map', + readers: await SparqlConstructReader.fromFile('map.rq'), }), ], }); ``` +Without `'last'`, the writers would get the normalised data as well as the mapped data, and whatever reads the output would have to filter the normalised data back out. + +The stages `'last'` leaves out still run their [validation](../reference/pipeline#validation), and its results are reported – useful for checking that the normalised data has the shape the mapping expects. + +### Inspect the output between stages + +Whichever you choose, every stage in the chain keeps its own scratch file at `//.nt` after the run, including the stages `'last'` leaves out of what the chain writes. Compare one stage’s file with the next to see how a query acts on the previous one’s output. The next run of the dataset overwrites the files. + ## Implement a stage output resolver Between stages, the pipeline writes output to a scratch N-Triples file. A `StageOutputResolver` turns that file into the `Distribution` the next stage queries: @@ -88,7 +142,7 @@ For each dataset: 1. The parent stage runs against the dataset’s resolved distribution and writes N-Triples to a scratch file under `//`. 2. `resolve()` turns that file into a `Distribution`; the first sub-stage runs against it, writing its own scratch file. Each further sub-stage chains off the previous one’s output the same way. -3. The scratch files the parent’s `chainOutput` selects – all of them by default, or only the last stage’s – are concatenated and written to the pipeline’s [writers](../reference/pipeline#writer) under the parent stage’s name. +3. The scratch files the parent’s `chainOutput` selects – all of them by default, or only the last stage’s – are concatenated and written to the pipeline’s [writers](../reference/pipeline#writer) under the parent stage’s name. The scratch files stay on disk. 4. `cleanup()` runs when the chain finishes, on success and on failure. Two constraints apply: diff --git a/docs/reference/pipeline.md b/docs/reference/pipeline.md index 4de2f9acb..2ebad952f 100644 --- a/docs/reference/pipeline.md +++ b/docs/reference/pipeline.md @@ -203,7 +203,14 @@ new Pipeline({ Because the scratch files are N-Triples, chains are quad-only: a stage with `stages` cannot `project`. A chained stage that returns `NotSupported` fails the chain rather than being skipped. -By default the output of every stage in the chain reaches the pipeline’s writers. Set `chainOutput: 'last'` on the parent stage when the earlier stages only produce working graphs for the next stage to query: they still write their scratch files, but only the last chained stage’s output is written. +The parent stage’s `chainOutput` option decides which stages’ output reaches the writers: + +| `chainOutput` | Writes | Use when | +| ----------------- | ----------------------------- | -------------------------------------------------------------------------------------------------------------------------- | +| `'all'` (default) | every stage’s output | each stage adds to the result, such as a chained stage that enriches the parent’s output with labels from another endpoint | +| `'last'` | only the last chained stage’s | the earlier stages only produce input for the next, such as a normalisation that a shared mapping then reads | + +Either way, every stage’s scratch file stays at `//.nt` after the run, so you can inspect what each stage produced; the next run of the dataset overwrites it. See the how-to guide [Chain stage outputs](../guide/chain-stage-outputs). diff --git a/packages/pipeline/src/pipeline.ts b/packages/pipeline/src/pipeline.ts index 488b5802e..c5a6386b3 100644 --- a/packages/pipeline/src/pipeline.ts +++ b/packages/pipeline/src/pipeline.ts @@ -1,4 +1,6 @@ import { createReadStream } from 'node:fs'; +import { mkdir, rm, writeFile } from 'node:fs/promises'; +import { dirname } from 'node:path'; import { randomUUID } from 'node:crypto'; import { Dataset, Distribution } from '@lde/dataset'; import { @@ -146,6 +148,13 @@ export interface PipelineOptions { distributionResolver?: DistributionResolver; chaining?: { stageOutputResolver: StageOutputResolver; + /** + * Where every chained stage writes its output, as N-Triples at + * `//.nt`, for the next stage to query. + * The files stay after the run – including those a chain with + * `chainOutput: 'last'` leaves out of what it writes – so each step can be + * inspected; the next run of the dataset overwrites them. + */ outputDir: string; }; /** Reporter(s) handed every {@link PipelineEvent} of a run, in array order. */ @@ -1202,6 +1211,10 @@ export class Pipeline { outputDir: `${outputDir}/${chainedStage.name}`, format: 'n-triples', }); + const outputPath = scratchWriter.getOutputPath(dataset); + // The FileWriter writes no file for empty output, so remove the previous + // run's file first: it must never stand in for this run's output. + await rm(outputPath, { force: true }); const scratchRun = await scratchWriter.openRun(context); try { await this.runChainedStage( @@ -1219,7 +1232,11 @@ export class Pipeline { await scratchRun.abort(error); throw error; } - return scratchWriter.getOutputPath(dataset); + // A stage that produced nothing leaves an empty file, so the next stage + // and the writers read an empty graph rather than a missing file. + await mkdir(dirname(outputPath), { recursive: true }); + await writeFile(outputPath, '', { flag: 'a' }); + return outputPath; }; try { diff --git a/packages/pipeline/src/stage.ts b/packages/pipeline/src/stage.ts index ced494382..912f64660 100644 --- a/packages/pipeline/src/stage.ts +++ b/packages/pipeline/src/stage.ts @@ -190,16 +190,19 @@ export interface StageOptions { /** Child stages that chain off this stage's output. */ stages?: Stage[]; /** - * Which outputs of this stage's chain reach the pipeline's writers: + * Which outputs of this stage’s chain reach the pipeline’s writers: * - * - `'all'` – this stage's output and that of every chained stage; - * - `'last'` – only the last chained stage's output. Set it when the earlier - * stages produce working graphs that only feed the next stage: they still - * write their scratch files for the next stage to query, but the chain - * leaves them out of what it writes. + * - `'all'` – this stage’s output and that of every chained stage. Use it + * when each stage adds to the result, such as a chained stage that + * enriches this stage’s output with data from another source. + * - `'last'` – only the last chained stage’s output. Use it when the earlier + * stages produce working graphs that only feed the next stage, such as a + * normalisation that a shared mapping then reads: they still write their + * scratch files for the next stage to query, but the chain leaves them out + * of what it writes. Their validation still runs and is reported. * - * Only meaningful with {@link StageOptions.stages}: a stage without them is - * no chain, and its output is always written. + * Requires {@link StageOptions.stages}: a stage without them is no chain, and + * its output is always written. * * @default 'all' */ @@ -313,6 +316,11 @@ export class Stage { `Stage '${options.name}': 'project' cannot combine with chained 'stages' – a chained stage serializes to N-Triples, which a projected item cannot.`, ); } + if (options.chainOutput && !options.stages?.length) { + throw new Error( + `Stage '${options.name}': 'chainOutput' requires chained 'stages' – set it on the stage that starts the chain.`, + ); + } // A child is handed its parent's output as its distribution; a `sourceFor` // would substitute that away, silently reading elsewhere instead of // continuing the chain. Caught here rather than at run time, where it looks diff --git a/packages/pipeline/test/pipeline.test.ts b/packages/pipeline/test/pipeline.test.ts index 00007faa1..d55f2c6e7 100644 --- a/packages/pipeline/test/pipeline.test.ts +++ b/packages/pipeline/test/pipeline.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect, vi, beforeEach, type Mock } from 'vitest'; -import { mkdtemp, readdir, rm } from 'node:fs/promises'; +import { mkdtemp, readdir, readFile, rm } from 'node:fs/promises'; import { join } from 'node:path'; import { tmpdir } from 'node:os'; import { @@ -8,8 +8,12 @@ import { type PipelinePlugin, } from '../src/pipeline.js'; import { Dataset, Distribution } from '@lde/dataset'; -import { Stage } from '../src/stage.js'; -import { NoItemsSelected, NotSupported } from '../src/sparql/reader.js'; +import { Stage, type StageOptions } from '../src/stage.js'; +import { + NoItemsSelected, + NotSupported, + type Reader, +} from '../src/sparql/reader.js'; import { ResolvedDistribution, NoDistributionAvailable, @@ -1122,10 +1126,20 @@ describe('Pipeline', () => { }); describe('chainOutput', () => { - /** A stage whose run writes one quad with subject `http://`. */ + /** + * A stage whose run writes one quad with subject `http://`, or no + * quads when `empty` is set. + */ function writingStage( name: string, - options: { chainOutput?: 'all' | 'last'; stages?: Stage[] } = {}, + { + empty = false, + ...options + }: { + chainOutput?: StageOptions['chainOutput']; + stages?: Stage[]; + empty?: boolean; + } = {}, ): Stage { const stage = new Stage({ name, readers: [], ...options }); vi.spyOn(stage, 'run').mockImplementation( @@ -1133,6 +1147,7 @@ describe('Pipeline', () => { await stageWriter.write( staged, (async function* () { + if (empty) return; yield DataFactory.quad( DataFactory.namedNode(`http://${name}`), DataFactory.namedNode('http://p'), @@ -1145,9 +1160,17 @@ describe('Pipeline', () => { return stage; } - /** Run `parent` as a chain and return the subjects the writers got. */ - async function writtenSubjects(parent: Stage): Promise { - const outputDir = await mkdtemp(join(tmpdir(), 'pipeline-emit-test-')); + /** + * Run `parent` as a chain and return the subjects the writers got. + * `inspect` sees the scratch directory before it is removed. + */ + async function writtenSubjects( + parent: Stage, + inspect?: (outputDir: string) => Promise, + ): Promise { + const outputDir = await mkdtemp( + join(tmpdir(), 'pipeline-chain-output-test-'), + ); const subjects: string[] = []; writer.runWriter.write.mockImplementation( async (_dataset: Dataset, quads: AsyncIterable) => { @@ -1166,6 +1189,7 @@ describe('Pipeline', () => { }); try { await pipeline.run(); + await inspect?.(outputDir); } finally { await rm(outputDir, { recursive: true, force: true }); } @@ -1196,6 +1220,148 @@ describe('Pipeline', () => { expect(middle.run).toHaveBeenCalledTimes(1); expect(last.run).toHaveBeenCalledTimes(1); }); + + it('lets a chained stage select from its parent’s output and read another source', async () => { + const otherSource = Distribution.sparql( + new URL('http://other.example.org/sparql'), + ); + const otherSourceQuery = { + read: vi.fn( + async (_dataset, _distribution, options) => + (async function* () { + for (const row of options?.bindings ?? []) { + yield DataFactory.quad( + row['this'], + DataFactory.namedNode('http://label'), + DataFactory.literal('enriched'), + ); + } + })(), + ), + }; + const enrich = new Stage({ + name: 'enrich', + itemSelector: { + async *select() { + yield { this: DataFactory.namedNode('http://parent') }; + }, + }, + readers: { + read: (staged, _distribution, options) => + otherSourceQuery.read(staged, otherSource, options), + }, + }); + const parent = writingStage('parent', { stages: [enrich] }); + + expect(await writtenSubjects(parent)).toEqual([ + 'http://parent', + 'http://parent', + ]); + expect(otherSourceQuery.read).toHaveBeenCalledWith( + dataset, + otherSource, + expect.objectContaining({ + bindings: [{ this: DataFactory.namedNode('http://parent') }], + }), + ); + }); + + it('keeps the scratch file of every stage it leaves out of the written output', async () => { + const parent = writingStage('parent', { + chainOutput: 'last', + stages: [writingStage('middle'), writingStage('last')], + }); + + const scratch: Record = {}; + await writtenSubjects(parent, async (outputDir) => { + for (const name of ['parent', 'middle']) { + const [file] = await readdir(join(outputDir, name)); + scratch[name] = await readFile(join(outputDir, name, file), 'utf8'); + } + }); + + expect(scratch['parent']).toContain(''); + expect(scratch['middle']).toContain(''); + }); + + it('never writes a previous run’s output for a stage that produced nothing this run', async () => { + const outputDir = await mkdtemp( + join(tmpdir(), 'pipeline-chain-output-test-'), + ); + const subjects: string[] = []; + writer.runWriter.write.mockImplementation( + async (_dataset: Dataset, quads: AsyncIterable) => { + for await (const quad of quads) subjects.push(quad.subject.value); + }, + ); + const runChain = (last: Stage) => + new Pipeline({ + datasetSelector: makeDatasetSelector(dataset), + stages: [ + writingStage('parent', { chainOutput: 'last', stages: [last] }), + ], + writers: writer, + distributionResolver: makeResolver(makeResolvedDistribution()), + chaining: { + stageOutputResolver: makeStageOutputResolver(), + outputDir, + }, + }).run(); + + try { + await runChain(writingStage('last')); + subjects.length = 0; + await runChain(writingStage('last', { empty: true })); + } finally { + await rm(outputDir, { recursive: true, force: true }); + } + + expect(subjects).toEqual([]); + }); + + it('hands the next stage an empty file when a stage produced nothing', async () => { + const outputDir = await mkdtemp( + join(tmpdir(), 'pipeline-chain-output-test-'), + ); + const stageOutputResolver = makeStageOutputResolver(); + const resolvedContents: string[] = []; + stageOutputResolver.resolve.mockImplementation(async (outputPath) => { + resolvedContents.push(await readFile(outputPath, 'utf8')); + return Distribution.sparql( + new URL('http://resolved.example.org/sparql'), + ); + }); + const pipeline = new Pipeline({ + datasetSelector: makeDatasetSelector(dataset), + stages: [ + writingStage('parent', { + empty: true, + stages: [writingStage('child')], + }), + ], + writers: writer, + distributionResolver: makeResolver(makeResolvedDistribution()), + chaining: { stageOutputResolver, outputDir }, + }); + + try { + await pipeline.run(); + } finally { + await rm(outputDir, { recursive: true, force: true }); + } + + expect(resolvedContents).toEqual(['']); + }); + + it('writes nothing, without failing, when the last chained stage’s output is empty', async () => { + const parent = writingStage('parent', { + chainOutput: 'last', + stages: [writingStage('last', { empty: true })], + }); + + expect(await writtenSubjects(parent)).toEqual([]); + expect(writer.runWriter.flush).toHaveBeenCalledWith(dataset, 'success'); + }); }); it('validates chaining is required for sub-stages', () => { diff --git a/packages/pipeline/test/stage.test.ts b/packages/pipeline/test/stage.test.ts index 75fb129da..98246eec5 100644 --- a/packages/pipeline/test/stage.test.ts +++ b/packages/pipeline/test/stage.test.ts @@ -1303,6 +1303,19 @@ describe('Stage', () => { }); }); + describe('chainOutput', () => { + it('rejects chainOutput on a stage without chained stages at construction', () => { + expect( + () => + new Stage({ + name: 'aggregate', + readers: mockExecutor([q1]), + chainOutput: 'last', + }), + ).toThrow(/'chainOutput' requires chained 'stages'/); + }); + }); + describe('batchSize per source kind', () => { /** The batch size a stage hands its item selector when run with `options`. */ async function selectedBatchSize( diff --git a/packages/pipeline/vite.config.ts b/packages/pipeline/vite.config.ts index 5e76fa2bc..c8115ea95 100644 --- a/packages/pipeline/vite.config.ts +++ b/packages/pipeline/vite.config.ts @@ -12,9 +12,9 @@ export default mergeConfig( thresholds: { autoUpdate: true, functions: 99.4, - lines: 98.79, - branches: 96.75, - statements: 98.64, + lines: 99.32, + branches: 96.78, + statements: 99.15, }, }, }, From 188f783555900eaf2a26465be4b77bc016880b77 Mon Sep 17 00:00:00 2001 From: David de Boer Date: Fri, 20 Nov 2026 11:20:04 +0100 Subject: [PATCH 5/6] docs(pipeline): say that stageComplete counts quads a chain leaves out --- packages/pipeline/src/pipelineEvent.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/pipeline/src/pipelineEvent.ts b/packages/pipeline/src/pipelineEvent.ts index 155fca451..e0078f81f 100644 --- a/packages/pipeline/src/pipelineEvent.ts +++ b/packages/pipeline/src/pipelineEvent.ts @@ -199,6 +199,8 @@ export interface StageProgressEvent export interface StageCompleteEvent extends DatasetEventEnvelope<'stageComplete'> { stage: string; itemsProcessed: number; + /** The quads this stage generated – including those of a chained stage + * that `chainOutput: 'last'` leaves out of what the chain writes. */ quadsGenerated: number; durationMs: number; /** Where the stage spent its time. Always set by the pipeline; optional so From 0117b142c2686e40452727df22be7324076e2515 Mon Sep 17 00:00:00 2001 From: David de Boer Date: Fri, 20 Nov 2026 11:24:58 +0100 Subject: [PATCH 6/6] fix(pipeline): let a chain continue when a chained stage selects nothing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A chained stage whose item selector selected nothing failed the whole chain, so an enrichment stage that found no terms to enrich lost the dataset’s mapped output too. It now produces empty output and the chain continues; any other NotSupported still fails the chain. A stage now rejects chained stages that have stages of their own, which never ran, and two stages with one name in a chain, which overwrote each other’s scratch file. StageOutputResolver gains JSDoc, including that it may be handed an empty file. The enrichment example sets the batch size and concurrency for its remote endpoint. --- docs/guide/chain-stage-outputs.md | 11 +++++++-- docs/reference/pipeline.md | 4 ++-- packages/pipeline/src/pipeline.ts | 21 +++++++++-------- packages/pipeline/src/stage.ts | 24 ++++++++++++++++++++ packages/pipeline/src/stageOutputResolver.ts | 9 ++++++++ packages/pipeline/test/pipeline.test.ts | 9 ++++++++ packages/pipeline/test/stage.test.ts | 20 ++++++++++++++++ packages/pipeline/vite.config.ts | 6 ++--- 8 files changed, 88 insertions(+), 16 deletions(-) diff --git a/docs/guide/chain-stage-outputs.md b/docs/guide/chain-stage-outputs.md index daa83b67d..d27948add 100644 --- a/docs/guide/chain-stage-outputs.md +++ b/docs/guide/chain-stage-outputs.md @@ -60,12 +60,18 @@ const parent = new Stage({ 'SELECT DISTINCT ?this WHERE { ?record ?this } ORDER BY ?this', }), readers: termLabels, + // Chained stages are tuned for the pipeline’s own store; set + // what suits the remote endpoint. + batchSize: 50, + maxConcurrency: 2, }), ], }); ``` -The writers get the mapped data and the labels together. +The writers get the mapped data and the labels together. If the mapped data links to no terms, the enrich stage selects nothing and the writers get the mapped data alone. + +A chained stage runs with the batch size and concurrency meant for the store that serves its parent’s output. A stage whose reader queries a remote endpoint should set `batchSize` and `maxConcurrency` for that endpoint, as above. ### Normalise, then map: write only the last stage @@ -148,4 +154,5 @@ For each dataset: Two constraints apply: - Chains are quad-only: a stage with `stages` cannot `project`, because a chained stage serialises to N-Triples, which a projected item cannot (see [ADR 13](../decisions/0013-project-inside-the-batch-per-root-type)). -- A chained stage that returns `NotSupported` – for example, its item selector selects nothing – fails the whole chain instead of being skipped, because a later stage cannot chain off missing output. +- A chain is one line: a chained stage cannot have `stages` of its own, and the stages in a chain need distinct names, because each writes its scratch file under its name. +- A chained stage whose item selector selects nothing produces empty output, and the chain continues. Any other `NotSupported` – every reader returning it – fails the whole chain instead of being skipped, because the stage produced no output for the next one to read. diff --git a/docs/reference/pipeline.md b/docs/reference/pipeline.md index 2ebad952f..ffc242f40 100644 --- a/docs/reference/pipeline.md +++ b/docs/reference/pipeline.md @@ -183,7 +183,7 @@ Set it for scalar aggregates such as `SELECT (COUNT(*) AS ?n)`, which always ret #### Skipped stages -A stage that cannot run against a dataset is skipped, not failed: when every reader returns `NotSupported`, or the item selector yields no items, the pipeline reports `stageSkipped` and moves on. The selector is peeked before the writer’s first write, so a destructive writer – `SparqlUpdateWriter` issues `CLEAR GRAPH` on first write – does not clear anything for a stage that turns out to be empty. Inside a [chain](#chaining-stages), `NotSupported` fails the chain instead. +A stage that cannot run against a dataset is skipped, not failed: when every reader returns `NotSupported`, or the item selector yields no items, the pipeline reports `stageSkipped` and moves on. The selector is peeked before the writer’s first write, so a destructive writer – `SparqlUpdateWriter` issues `CLEAR GRAPH` on first write – does not clear anything for a stage that turns out to be empty. Inside a [chain](#chaining-stages), a stage whose selector yields no items produces empty output and the chain continues; any other `NotSupported` fails the chain. ### Chaining stages @@ -201,7 +201,7 @@ new Pipeline({ }); ``` -Because the scratch files are N-Triples, chains are quad-only: a stage with `stages` cannot `project`. A chained stage that returns `NotSupported` fails the chain rather than being skipped. +Because the scratch files are N-Triples, chains are quad-only: a stage with `stages` cannot `project`. A chained stage cannot have `stages` of its own, and the stages in a chain need distinct names. A chained stage whose selector yields no items produces empty output; any other `NotSupported` fails the chain rather than being skipped. The parent stage’s `chainOutput` option decides which stages’ output reaches the writers: diff --git a/packages/pipeline/src/pipeline.ts b/packages/pipeline/src/pipeline.ts index c5a6386b3..7b7ce8f96 100644 --- a/packages/pipeline/src/pipeline.ts +++ b/packages/pipeline/src/pipeline.ts @@ -1074,8 +1074,8 @@ export class Pipeline { } /** - * Run a stage with reporting and return whether it was supported. - * Returns `true` if the stage produced results, `false` if NotSupported. + * Run a stage with reporting. Returns the {@link NotSupported} reason when + * the stage was skipped, and nothing when it ran. */ private async runStage( runId: string, @@ -1086,7 +1086,7 @@ export class Pipeline { source: SourceKind, timeout?: TimeoutPolicy, unread?: UnreadValueTally, - ): Promise { + ): Promise { this.report(runId, { type: 'stageStart', dataset, stage: stage.name }); const stageStart = Date.now(); @@ -1140,7 +1140,7 @@ export class Pipeline { message: result.message, }, }); - return false; + return result; } this.report(runId, { @@ -1152,11 +1152,14 @@ export class Pipeline { durationMs: Date.now() - stageStart, ...summary, }); - - return true; + return undefined; } - /** Run a stage in chained mode, throwing if the stage is not supported. */ + /** + * Run a stage in chained mode. A stage whose item selector selects nothing + * produced empty output, which the next stage can still read; any other + * NotSupported throws, because the chain cannot continue without the stage. + */ private async runChainedStage( runId: string, dataset: Dataset, @@ -1166,7 +1169,7 @@ export class Pipeline { source: SourceKind, timeout?: TimeoutPolicy, ): Promise { - const supported = await this.runStage( + const skipped = await this.runStage( runId, dataset, distribution, @@ -1175,7 +1178,7 @@ export class Pipeline { source, timeout, ); - if (!supported) { + if (skipped && !(skipped instanceof NoItemsSelected)) { throw new Error( `Stage '${stage.name}' returned NotSupported in chained mode`, ); diff --git a/packages/pipeline/src/stage.ts b/packages/pipeline/src/stage.ts index 912f64660..1496f40d4 100644 --- a/packages/pipeline/src/stage.ts +++ b/packages/pipeline/src/stage.ts @@ -331,6 +331,30 @@ export class Stage { `Stage '${options.name}': chained stage '${sourcedChild.name}' declares 'sourceFor' – a chained stage reads its parent's output, so it cannot source its own data.`, ); } + // A chain is one line: the pipeline runs only the direct children, so a + // child's own `stages` would silently never run. + const nestingChild = options.stages?.find( + (child) => child.stages.length > 0, + ); + if (nestingChild) { + throw new Error( + `Stage '${options.name}': chained stage '${nestingChild.name}' declares 'stages' – a chain is one line, so list its stages under '${options.name}' instead.`, + ); + } + // Each stage in a chain writes its scratch file under its own name, so two + // stages with one name would overwrite each other's output. + const chainNames = [ + options.name, + ...(options.stages ?? []).map((child) => child.name), + ]; + const duplicateName = chainNames.find( + (name, index) => chainNames.indexOf(name) !== index, + ); + if (duplicateName) { + throw new Error( + `Stage '${options.name}': two stages in the chain are named '${duplicateName}' – each writes its scratch file under its name, so the names must differ.`, + ); + } this.name = options.name; this.stages = options.stages ?? []; this.readers = normalizeReaders(options.readers); diff --git a/packages/pipeline/src/stageOutputResolver.ts b/packages/pipeline/src/stageOutputResolver.ts index 3da1cda3c..3d39bd0cf 100644 --- a/packages/pipeline/src/stageOutputResolver.ts +++ b/packages/pipeline/src/stageOutputResolver.ts @@ -1,6 +1,15 @@ import { Distribution } from '@lde/dataset'; +/** + * Makes a chained stage’s scratch output queryable for the next stage in the + * chain. + */ export interface StageOutputResolver { + /** + * Serve the N-Triples file at `outputPath` as a {@link Distribution}. The + * file is empty when the stage produced nothing; serve it as an empty graph. + */ resolve(outputPath: string): Promise; + /** Release what {@link resolve} started, once the chain finishes. */ cleanup(): Promise; } diff --git a/packages/pipeline/test/pipeline.test.ts b/packages/pipeline/test/pipeline.test.ts index d55f2c6e7..8cdb5def3 100644 --- a/packages/pipeline/test/pipeline.test.ts +++ b/packages/pipeline/test/pipeline.test.ts @@ -1266,6 +1266,15 @@ describe('Pipeline', () => { ); }); + it('writes the parent’s output when a chained stage selects nothing', async () => { + const enrich = new Stage({ name: 'enrich', readers: [] }); + vi.spyOn(enrich, 'run').mockResolvedValue(new NoItemsSelected()); + const parent = writingStage('parent', { stages: [enrich] }); + + expect(await writtenSubjects(parent)).toEqual(['http://parent']); + expect(writer.runWriter.flush).toHaveBeenCalledWith(dataset, 'success'); + }); + it('keeps the scratch file of every stage it leaves out of the written output', async () => { const parent = writingStage('parent', { chainOutput: 'last', diff --git a/packages/pipeline/test/stage.test.ts b/packages/pipeline/test/stage.test.ts index 98246eec5..49cb13c53 100644 --- a/packages/pipeline/test/stage.test.ts +++ b/packages/pipeline/test/stage.test.ts @@ -1316,6 +1316,26 @@ describe('Stage', () => { }); }); + describe('chain shape', () => { + const stage = (name: string, stages?: Stage[]) => + new Stage({ name, readers: mockExecutor([q1]), stages }); + + it('rejects a chained stage that has stages of its own', () => { + expect(() => + stage('parent', [stage('child', [stage('grandchild')])]), + ).toThrow(/chained stage 'child' declares 'stages'/); + }); + + it('rejects two stages with one name in a chain', () => { + expect(() => stage('map', [stage('enrich'), stage('enrich')])).toThrow( + /two stages in the chain are named 'enrich'/, + ); + expect(() => stage('map', [stage('map')])).toThrow( + /two stages in the chain are named 'map'/, + ); + }); + }); + describe('batchSize per source kind', () => { /** The batch size a stage hands its item selector when run with `options`. */ async function selectedBatchSize( diff --git a/packages/pipeline/vite.config.ts b/packages/pipeline/vite.config.ts index c8115ea95..7103ee1e7 100644 --- a/packages/pipeline/vite.config.ts +++ b/packages/pipeline/vite.config.ts @@ -11,9 +11,9 @@ export default mergeConfig( coverage: { thresholds: { autoUpdate: true, - functions: 99.4, - lines: 99.32, - branches: 96.78, + functions: 99.41, + lines: 99.33, + branches: 96.82, statements: 99.15, }, },