diff --git a/docs/guide/chain-stage-outputs.md b/docs/guide/chain-stage-outputs.md index a764ba7d..d27948ad 100644 --- a/docs/guide/chain-stage-outputs.md +++ b/docs/guide/chain-stage-outputs.md @@ -1,10 +1,10 @@ # Chain stage outputs -This guide shows how to feed one stage’s output to sub-stages that query it – for example an extraction stage whose result a second stage aggregates. +This guide shows how to feed one stage’s output to sub-stages that query it, and how to choose which stages’ output the chain writes. ## Declare sub-stages -Pass child stages in the parent’s `stages` option. Each child queries the previous stage’s output instead of the dataset’s own distribution: +Pass child stages in the parent’s `stages` option. Each child queries the previous stage’s output instead of the dataset’s own distribution. A chain is a line, not a tree: the second child queries the first child’s output, not the parent’s. To run several queries over the same output, give one child several readers: ```typescript import { SparqlConstructReader, Stage } from '@lde/pipeline'; @@ -21,6 +21,84 @@ const parent = new Stage({ }); ``` +## Choose what the chain writes + +The parent’s `chainOutput` option decides which stages’ output reaches the pipeline’s [writers](../reference/pipeline#writer). Pick it by what the later stages do with the earlier output. + +### Enrich: write every stage + +Use the default, `chainOutput: 'all'`, when each stage adds to the result. A typical case is enrichment: the parent produces your data, and a chained stage selects the terms it links to and fetches their labels from another SPARQL endpoint. The chained stage’s item selector queries the parent’s output; its reader queries the other endpoint for the selected terms: + +```typescript +import { Distribution } from '@lde/dataset'; +import { + SparqlConstructReader, + SparqlItemSelector, + Stage, +} from '@lde/pipeline'; +import type { Reader } from '@lde/pipeline'; + +const terminology = Distribution.sparql( + new URL('https://terminology.example.org/sparql'), +); +const labels = await SparqlConstructReader.fromFile('term-labels.rq'); + +// Query the terminology endpoint instead of the parent’s output. +const termLabels: Reader = { + read: (dataset, _distribution, options) => + labels.read(dataset, terminology, options), +}; + +const parent = new Stage({ + name: 'map', + readers: await SparqlConstructReader.fromFile('map.rq'), + stages: [ + new Stage({ + name: 'enrich', + itemSelector: new SparqlItemSelector({ + query: + 'SELECT DISTINCT ?this WHERE { ?record ?this } ORDER BY ?this', + }), + readers: termLabels, + // Chained stages are tuned for the pipeline’s own store; set + // what suits the remote endpoint. + batchSize: 50, + maxConcurrency: 2, + }), + ], +}); +``` + +The writers get the mapped data and the labels together. If the mapped data links to no terms, the enrich stage selects nothing and the writers get the mapped data alone. + +A chained stage runs with the batch size and concurrency meant for the store that serves its parent’s output. A stage whose reader queries a remote endpoint should set `batchSize` and `maxConcurrency` for that endpoint, as above. + +### Normalise, then map: write only the last stage + +Set `chainOutput: 'last'` when the earlier stages only produce input for the next one. A typical case is a shared mapping that expects its input in one shape: a first stage normalises each source to that shape, and the mapping reads the normalised output. The normalised data is scaffolding, so only the mapping’s output should reach the writers: + +```typescript +const parent = new Stage({ + name: 'normalise', + readers: await SparqlConstructReader.fromFile('normalise.rq'), + chainOutput: 'last', // only 'map' reaches the writers + stages: [ + new Stage({ + name: 'map', + readers: await SparqlConstructReader.fromFile('map.rq'), + }), + ], +}); +``` + +Without `'last'`, the writers would get the normalised data as well as the mapped data, and whatever reads the output would have to filter the normalised data back out. + +The stages `'last'` leaves out still run their [validation](../reference/pipeline#validation), and its results are reported – useful for checking that the normalised data has the shape the mapping expects. + +### Inspect the output between stages + +Whichever you choose, every stage in the chain keeps its own scratch file at `//.nt` after the run, including the stages `'last'` leaves out of what the chain writes. Compare one stage’s file with the next to see how a query acts on the previous one’s output. The next run of the dataset overwrites the files. + ## Implement a stage output resolver Between stages, the pipeline writes output to a scratch N-Triples file. A `StageOutputResolver` turns that file into the `Distribution` the next stage queries: @@ -70,10 +148,11 @@ For each dataset: 1. The parent stage runs against the dataset’s resolved distribution and writes N-Triples to a scratch file under `//`. 2. `resolve()` turns that file into a `Distribution`; the first sub-stage runs against it, writing its own scratch file. Each further sub-stage chains off the previous one’s output the same way. -3. The scratch files of the whole chain are concatenated and written to the pipeline’s [writers](../reference/pipeline#writer) under the parent stage’s name. +3. The scratch files the parent’s `chainOutput` selects – all of them by default, or only the last stage’s – are concatenated and written to the pipeline’s [writers](../reference/pipeline#writer) under the parent stage’s name. The scratch files stay on disk. 4. `cleanup()` runs when the chain finishes, on success and on failure. Two constraints apply: - Chains are quad-only: a stage with `stages` cannot `project`, because a chained stage serialises to N-Triples, which a projected item cannot (see [ADR 13](../decisions/0013-project-inside-the-batch-per-root-type)). -- A chained stage that returns `NotSupported` – for example, its item selector selects nothing – fails the whole chain instead of being skipped, because a later stage cannot chain off missing output. +- A chain is one line: a chained stage cannot have `stages` of its own, and the stages in a chain need distinct names, because each writes its scratch file under its name. +- A chained stage whose item selector selects nothing produces empty output, and the chain continues. Any other `NotSupported` – every reader returning it – fails the whole chain instead of being skipped, because the stage produced no output for the next one to read. diff --git a/docs/reference/pipeline.md b/docs/reference/pipeline.md index f9b8d1b7..ffc242f4 100644 --- a/docs/reference/pipeline.md +++ b/docs/reference/pipeline.md @@ -183,7 +183,7 @@ Set it for scalar aggregates such as `SELECT (COUNT(*) AS ?n)`, which always ret #### Skipped stages -A stage that cannot run against a dataset is skipped, not failed: when every reader returns `NotSupported`, or the item selector yields no items, the pipeline reports `stageSkipped` and moves on. The selector is peeked before the writer’s first write, so a destructive writer – `SparqlUpdateWriter` issues `CLEAR GRAPH` on first write – does not clear anything for a stage that turns out to be empty. Inside a [chain](#chaining-stages), `NotSupported` fails the chain instead. +A stage that cannot run against a dataset is skipped, not failed: when every reader returns `NotSupported`, or the item selector yields no items, the pipeline reports `stageSkipped` and moves on. The selector is peeked before the writer’s first write, so a destructive writer – `SparqlUpdateWriter` issues `CLEAR GRAPH` on first write – does not clear anything for a stage that turns out to be empty. Inside a [chain](#chaining-stages), a stage whose selector yields no items produces empty output and the chain continues; any other `NotSupported` fails the chain. ### Chaining stages @@ -201,7 +201,18 @@ new Pipeline({ }); ``` -Because the scratch files are N-Triples, chains are quad-only: a stage with `stages` cannot `project`. A chained stage that returns `NotSupported` fails the chain rather than being skipped. See the how-to guide [Chain stage outputs](../guide/chain-stage-outputs). +Because the scratch files are N-Triples, chains are quad-only: a stage with `stages` cannot `project`. A chained stage cannot have `stages` of its own, and the stages in a chain need distinct names. A chained stage whose selector yields no items produces empty output; any other `NotSupported` fails the chain rather than being skipped. + +The parent stage’s `chainOutput` option decides which stages’ output reaches the writers: + +| `chainOutput` | Writes | Use when | +| ----------------- | ----------------------------- | -------------------------------------------------------------------------------------------------------------------------- | +| `'all'` (default) | every stage’s output | each stage adds to the result, such as a chained stage that enriches the parent’s output with labels from another endpoint | +| `'last'` | only the last chained stage’s | the earlier stages only produce input for the next, such as a normalisation that a shared mapping then reads | + +Either way, every stage’s scratch file stays at `//.nt` after the run, so you can inspect what each stage produced; the next run of the dataset overwrites it. + +See the how-to guide [Chain stage outputs](../guide/chain-stage-outputs). ### Item Selector diff --git a/packages/pipeline/src/pipeline.ts b/packages/pipeline/src/pipeline.ts index 28b9607a..7b7ce8f9 100644 --- a/packages/pipeline/src/pipeline.ts +++ b/packages/pipeline/src/pipeline.ts @@ -1,4 +1,6 @@ import { createReadStream } from 'node:fs'; +import { mkdir, rm, writeFile } from 'node:fs/promises'; +import { dirname } from 'node:path'; import { randomUUID } from 'node:crypto'; import { Dataset, Distribution } from '@lde/dataset'; import { @@ -146,6 +148,13 @@ export interface PipelineOptions { distributionResolver?: DistributionResolver; chaining?: { stageOutputResolver: StageOutputResolver; + /** + * Where every chained stage writes its output, as N-Triples at + * `//.nt`, for the next stage to query. + * The files stay after the run – including those a chain with + * `chainOutput: 'last'` leaves out of what it writes – so each step can be + * inspected; the next run of the dataset overwrites them. + */ outputDir: string; }; /** Reporter(s) handed every {@link PipelineEvent} of a run, in array order. */ @@ -1065,8 +1074,8 @@ export class Pipeline { } /** - * Run a stage with reporting and return whether it was supported. - * Returns `true` if the stage produced results, `false` if NotSupported. + * Run a stage with reporting. Returns the {@link NotSupported} reason when + * the stage was skipped, and nothing when it ran. */ private async runStage( runId: string, @@ -1077,7 +1086,7 @@ export class Pipeline { source: SourceKind, timeout?: TimeoutPolicy, unread?: UnreadValueTally, - ): Promise { + ): Promise { this.report(runId, { type: 'stageStart', dataset, stage: stage.name }); const stageStart = Date.now(); @@ -1131,7 +1140,7 @@ export class Pipeline { message: result.message, }, }); - return false; + return result; } this.report(runId, { @@ -1143,11 +1152,14 @@ export class Pipeline { durationMs: Date.now() - stageStart, ...summary, }); - - return true; + return undefined; } - /** Run a stage in chained mode, throwing if the stage is not supported. */ + /** + * Run a stage in chained mode. A stage whose item selector selects nothing + * produced empty output, which the next stage can still read; any other + * NotSupported throws, because the chain cannot continue without the stage. + */ private async runChainedStage( runId: string, dataset: Dataset, @@ -1157,7 +1169,7 @@ export class Pipeline { source: SourceKind, timeout?: TimeoutPolicy, ): Promise { - const supported = await this.runStage( + const skipped = await this.runStage( runId, dataset, distribution, @@ -1166,7 +1178,7 @@ export class Pipeline { source, timeout, ); - if (!supported) { + if (skipped && !(skipped instanceof NoItemsSelected)) { throw new Error( `Stage '${stage.name}' returned NotSupported in chained mode`, ); @@ -1183,6 +1195,8 @@ export class Pipeline { timeout?: TimeoutPolicy, ): Promise { const { stageOutputResolver, outputDir } = this.chaining!; + // Scratch files of every stage in the chain, in order; `chainOutput` + // decides which of them reach the writers. const outputFiles: string[] = []; // Chaining is a quad-only concern: every stage in a chain writes N-Triples @@ -1200,6 +1214,10 @@ export class Pipeline { outputDir: `${outputDir}/${chainedStage.name}`, format: 'n-triples', }); + const outputPath = scratchWriter.getOutputPath(dataset); + // The FileWriter writes no file for empty output, so remove the previous + // run's file first: it must never stand in for this run's output. + await rm(outputPath, { force: true }); const scratchRun = await scratchWriter.openRun(context); try { await this.runChainedStage( @@ -1217,7 +1235,11 @@ export class Pipeline { await scratchRun.abort(error); throw error; } - return scratchWriter.getOutputPath(dataset); + // A stage that produced nothing leaves an empty file, so the next stage + // and the writers read an empty graph rather than a missing file. + await mkdir(dirname(outputPath), { recursive: true }); + await writeFile(outputPath, '', { flag: 'a' }); + return outputPath; }; try { @@ -1246,13 +1268,16 @@ export class Pipeline { } } - // 3. Concatenate all output files → the open run, applying the plugins' - // beforeStageWrite transforms once for the chain under the parent stage. - // The concatenation is quads (per the quad-only contract above), written - // to the `Out`-typed run writer – sound because chaining implies `Quad`. + // 3. Concatenate the output files `chainOutput` selects → the open run, + // applying the plugins' beforeStageWrite transforms once for the chain + // under the parent stage. The concatenation is quads (per the quad-only + // contract above), written to the `Out`-typed run writer – sound because + // chaining implies `Quad`. + const emitted = + stage.chainOutput === 'last' ? outputFiles.slice(-1) : outputFiles; await this.stageWriter(runWriter, stage.name).write( dataset, - this.readFiles(outputFiles) as unknown as AsyncIterable, + this.readFiles(emitted) as unknown as AsyncIterable, ); } finally { await stageOutputResolver.cleanup(); diff --git a/packages/pipeline/src/pipelineEvent.ts b/packages/pipeline/src/pipelineEvent.ts index 155fca45..e0078f81 100644 --- a/packages/pipeline/src/pipelineEvent.ts +++ b/packages/pipeline/src/pipelineEvent.ts @@ -199,6 +199,8 @@ export interface StageProgressEvent export interface StageCompleteEvent extends DatasetEventEnvelope<'stageComplete'> { stage: string; itemsProcessed: number; + /** The quads this stage generated – including those of a chained stage + * that `chainOutput: 'last'` leaves out of what the chain writes. */ quadsGenerated: number; durationMs: number; /** Where the stage spent its time. Always set by the pipeline; optional so diff --git a/packages/pipeline/src/stage.ts b/packages/pipeline/src/stage.ts index 860ffee9..1496f40d 100644 --- a/packages/pipeline/src/stage.ts +++ b/packages/pipeline/src/stage.ts @@ -189,6 +189,24 @@ export interface StageOptions { sourceFor?: (dataset: Dataset, distribution: Distribution) => Distribution; /** Child stages that chain off this stage's output. */ stages?: Stage[]; + /** + * Which outputs of this stage’s chain reach the pipeline’s writers: + * + * - `'all'` – this stage’s output and that of every chained stage. Use it + * when each stage adds to the result, such as a chained stage that + * enriches this stage’s output with data from another source. + * - `'last'` – only the last chained stage’s output. Use it when the earlier + * stages produce working graphs that only feed the next stage, such as a + * normalisation that a shared mapping then reads: they still write their + * scratch files for the next stage to query, but the chain leaves them out + * of what it writes. Their validation still runs and is reported. + * + * Requires {@link StageOptions.stages}: a stage without them is no chain, and + * its output is always written. + * + * @default 'all' + */ + chainOutput?: 'all' | 'last'; /** * Treat a supported stage that produces no quads as a hard failure (throws), * rather than a legitimately empty result. @@ -272,6 +290,9 @@ export class Stage { /** Whether this stage reads a source of its own rather than the dataset’s * resolved distribution. @see {@link StageOptions.sourceFor} */ readonly sourcesOwnData: boolean; + /** Which outputs of this stage’s chain reach the pipeline’s writers. + * @see {@link StageOptions.chainOutput} */ + readonly chainOutput: 'all' | 'last'; /** Maximum concurrent in-flight SPARQL queries. @see {@link StageOptions.maxConcurrency} */ readonly maxConcurrency: number; private readonly readers: NormalizedReader[]; @@ -295,6 +316,11 @@ export class Stage { `Stage '${options.name}': 'project' cannot combine with chained 'stages' – a chained stage serializes to N-Triples, which a projected item cannot.`, ); } + if (options.chainOutput && !options.stages?.length) { + throw new Error( + `Stage '${options.name}': 'chainOutput' requires chained 'stages' – set it on the stage that starts the chain.`, + ); + } // A child is handed its parent's output as its distribution; a `sourceFor` // would substitute that away, silently reading elsewhere instead of // continuing the chain. Caught here rather than at run time, where it looks @@ -305,6 +331,30 @@ export class Stage { `Stage '${options.name}': chained stage '${sourcedChild.name}' declares 'sourceFor' – a chained stage reads its parent's output, so it cannot source its own data.`, ); } + // A chain is one line: the pipeline runs only the direct children, so a + // child's own `stages` would silently never run. + const nestingChild = options.stages?.find( + (child) => child.stages.length > 0, + ); + if (nestingChild) { + throw new Error( + `Stage '${options.name}': chained stage '${nestingChild.name}' declares 'stages' – a chain is one line, so list its stages under '${options.name}' instead.`, + ); + } + // Each stage in a chain writes its scratch file under its own name, so two + // stages with one name would overwrite each other's output. + const chainNames = [ + options.name, + ...(options.stages ?? []).map((child) => child.name), + ]; + const duplicateName = chainNames.find( + (name, index) => chainNames.indexOf(name) !== index, + ); + if (duplicateName) { + throw new Error( + `Stage '${options.name}': two stages in the chain are named '${duplicateName}' – each writes its scratch file under its name, so the names must differ.`, + ); + } this.name = options.name; this.stages = options.stages ?? []; this.readers = normalizeReaders(options.readers); @@ -319,6 +369,7 @@ export class Stage { this.queueCapacity = options.queueCapacity; this.sourceFor = options.sourceFor; this.sourcesOwnData = options.sourceFor !== undefined; + this.chainOutput = options.chainOutput ?? 'all'; } /** The validator for this stage, if configured. */ diff --git a/packages/pipeline/src/stageOutputResolver.ts b/packages/pipeline/src/stageOutputResolver.ts index 3da1cda3..3d39bd0c 100644 --- a/packages/pipeline/src/stageOutputResolver.ts +++ b/packages/pipeline/src/stageOutputResolver.ts @@ -1,6 +1,15 @@ import { Distribution } from '@lde/dataset'; +/** + * Makes a chained stage’s scratch output queryable for the next stage in the + * chain. + */ export interface StageOutputResolver { + /** + * Serve the N-Triples file at `outputPath` as a {@link Distribution}. The + * file is empty when the stage produced nothing; serve it as an empty graph. + */ resolve(outputPath: string): Promise; + /** Release what {@link resolve} started, once the chain finishes. */ cleanup(): Promise; } diff --git a/packages/pipeline/test/pipeline.test.ts b/packages/pipeline/test/pipeline.test.ts index 0570d665..8cdb5def 100644 --- a/packages/pipeline/test/pipeline.test.ts +++ b/packages/pipeline/test/pipeline.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect, vi, beforeEach, type Mock } from 'vitest'; -import { mkdtemp, readdir, rm } from 'node:fs/promises'; +import { mkdtemp, readdir, readFile, rm } from 'node:fs/promises'; import { join } from 'node:path'; import { tmpdir } from 'node:os'; import { @@ -8,8 +8,12 @@ import { type PipelinePlugin, } from '../src/pipeline.js'; import { Dataset, Distribution } from '@lde/dataset'; -import { Stage } from '../src/stage.js'; -import { NoItemsSelected, NotSupported } from '../src/sparql/reader.js'; +import { Stage, type StageOptions } from '../src/stage.js'; +import { + NoItemsSelected, + NotSupported, + type Reader, +} from '../src/sparql/reader.js'; import { ResolvedDistribution, NoDistributionAvailable, @@ -1121,6 +1125,254 @@ describe('Pipeline', () => { expect(stageOutputResolver.cleanup).toHaveBeenCalledTimes(1); }); + describe('chainOutput', () => { + /** + * A stage whose run writes one quad with subject `http://`, or no + * quads when `empty` is set. + */ + function writingStage( + name: string, + { + empty = false, + ...options + }: { + chainOutput?: StageOptions['chainOutput']; + stages?: Stage[]; + empty?: boolean; + } = {}, + ): Stage { + const stage = new Stage({ name, readers: [], ...options }); + vi.spyOn(stage, 'run').mockImplementation( + async (staged, _distribution, stageWriter) => { + await stageWriter.write( + staged, + (async function* () { + if (empty) return; + yield DataFactory.quad( + DataFactory.namedNode(`http://${name}`), + DataFactory.namedNode('http://p'), + DataFactory.namedNode('http://o'), + ); + })(), + ); + }, + ); + return stage; + } + + /** + * Run `parent` as a chain and return the subjects the writers got. + * `inspect` sees the scratch directory before it is removed. + */ + async function writtenSubjects( + parent: Stage, + inspect?: (outputDir: string) => Promise, + ): Promise { + const outputDir = await mkdtemp( + join(tmpdir(), 'pipeline-chain-output-test-'), + ); + const subjects: string[] = []; + writer.runWriter.write.mockImplementation( + async (_dataset: Dataset, quads: AsyncIterable) => { + for await (const quad of quads) subjects.push(quad.subject.value); + }, + ); + const pipeline = new Pipeline({ + datasetSelector: makeDatasetSelector(dataset), + stages: [parent], + writers: writer, + distributionResolver: makeResolver(makeResolvedDistribution()), + chaining: { + stageOutputResolver: makeStageOutputResolver(), + outputDir, + }, + }); + try { + await pipeline.run(); + await inspect?.(outputDir); + } finally { + await rm(outputDir, { recursive: true, force: true }); + } + return subjects; + } + + it('writes the output of every stage in a chain by default', async () => { + const parent = writingStage('parent', { + stages: [writingStage('child')], + }); + + expect(await writtenSubjects(parent)).toEqual([ + 'http://parent', + 'http://child', + ]); + }); + + it('writes only the last chained stage’s output when set to last, while every stage still runs', async () => { + const middle = writingStage('middle'); + const last = writingStage('last'); + const parent = writingStage('parent', { + chainOutput: 'last', + stages: [middle, last], + }); + + expect(await writtenSubjects(parent)).toEqual(['http://last']); + expect(parent.run).toHaveBeenCalledTimes(1); + expect(middle.run).toHaveBeenCalledTimes(1); + expect(last.run).toHaveBeenCalledTimes(1); + }); + + it('lets a chained stage select from its parent’s output and read another source', async () => { + const otherSource = Distribution.sparql( + new URL('http://other.example.org/sparql'), + ); + const otherSourceQuery = { + read: vi.fn( + async (_dataset, _distribution, options) => + (async function* () { + for (const row of options?.bindings ?? []) { + yield DataFactory.quad( + row['this'], + DataFactory.namedNode('http://label'), + DataFactory.literal('enriched'), + ); + } + })(), + ), + }; + const enrich = new Stage({ + name: 'enrich', + itemSelector: { + async *select() { + yield { this: DataFactory.namedNode('http://parent') }; + }, + }, + readers: { + read: (staged, _distribution, options) => + otherSourceQuery.read(staged, otherSource, options), + }, + }); + const parent = writingStage('parent', { stages: [enrich] }); + + expect(await writtenSubjects(parent)).toEqual([ + 'http://parent', + 'http://parent', + ]); + expect(otherSourceQuery.read).toHaveBeenCalledWith( + dataset, + otherSource, + expect.objectContaining({ + bindings: [{ this: DataFactory.namedNode('http://parent') }], + }), + ); + }); + + it('writes the parent’s output when a chained stage selects nothing', async () => { + const enrich = new Stage({ name: 'enrich', readers: [] }); + vi.spyOn(enrich, 'run').mockResolvedValue(new NoItemsSelected()); + const parent = writingStage('parent', { stages: [enrich] }); + + expect(await writtenSubjects(parent)).toEqual(['http://parent']); + expect(writer.runWriter.flush).toHaveBeenCalledWith(dataset, 'success'); + }); + + it('keeps the scratch file of every stage it leaves out of the written output', async () => { + const parent = writingStage('parent', { + chainOutput: 'last', + stages: [writingStage('middle'), writingStage('last')], + }); + + const scratch: Record = {}; + await writtenSubjects(parent, async (outputDir) => { + for (const name of ['parent', 'middle']) { + const [file] = await readdir(join(outputDir, name)); + scratch[name] = await readFile(join(outputDir, name, file), 'utf8'); + } + }); + + expect(scratch['parent']).toContain(''); + expect(scratch['middle']).toContain(''); + }); + + it('never writes a previous run’s output for a stage that produced nothing this run', async () => { + const outputDir = await mkdtemp( + join(tmpdir(), 'pipeline-chain-output-test-'), + ); + const subjects: string[] = []; + writer.runWriter.write.mockImplementation( + async (_dataset: Dataset, quads: AsyncIterable) => { + for await (const quad of quads) subjects.push(quad.subject.value); + }, + ); + const runChain = (last: Stage) => + new Pipeline({ + datasetSelector: makeDatasetSelector(dataset), + stages: [ + writingStage('parent', { chainOutput: 'last', stages: [last] }), + ], + writers: writer, + distributionResolver: makeResolver(makeResolvedDistribution()), + chaining: { + stageOutputResolver: makeStageOutputResolver(), + outputDir, + }, + }).run(); + + try { + await runChain(writingStage('last')); + subjects.length = 0; + await runChain(writingStage('last', { empty: true })); + } finally { + await rm(outputDir, { recursive: true, force: true }); + } + + expect(subjects).toEqual([]); + }); + + it('hands the next stage an empty file when a stage produced nothing', async () => { + const outputDir = await mkdtemp( + join(tmpdir(), 'pipeline-chain-output-test-'), + ); + const stageOutputResolver = makeStageOutputResolver(); + const resolvedContents: string[] = []; + stageOutputResolver.resolve.mockImplementation(async (outputPath) => { + resolvedContents.push(await readFile(outputPath, 'utf8')); + return Distribution.sparql( + new URL('http://resolved.example.org/sparql'), + ); + }); + const pipeline = new Pipeline({ + datasetSelector: makeDatasetSelector(dataset), + stages: [ + writingStage('parent', { + empty: true, + stages: [writingStage('child')], + }), + ], + writers: writer, + distributionResolver: makeResolver(makeResolvedDistribution()), + chaining: { stageOutputResolver, outputDir }, + }); + + try { + await pipeline.run(); + } finally { + await rm(outputDir, { recursive: true, force: true }); + } + + expect(resolvedContents).toEqual(['']); + }); + + it('writes nothing, without failing, when the last chained stage’s output is empty', async () => { + const parent = writingStage('parent', { + chainOutput: 'last', + stages: [writingStage('last', { empty: true })], + }); + + expect(await writtenSubjects(parent)).toEqual([]); + expect(writer.runWriter.flush).toHaveBeenCalledWith(dataset, 'success'); + }); + }); + it('validates chaining is required for sub-stages', () => { const child = makeStage('child'); const parent = makeStage('parent', undefined, [child]); diff --git a/packages/pipeline/test/stage.test.ts b/packages/pipeline/test/stage.test.ts index 75fb129d..49cb13c5 100644 --- a/packages/pipeline/test/stage.test.ts +++ b/packages/pipeline/test/stage.test.ts @@ -1303,6 +1303,39 @@ describe('Stage', () => { }); }); + describe('chainOutput', () => { + it('rejects chainOutput on a stage without chained stages at construction', () => { + expect( + () => + new Stage({ + name: 'aggregate', + readers: mockExecutor([q1]), + chainOutput: 'last', + }), + ).toThrow(/'chainOutput' requires chained 'stages'/); + }); + }); + + describe('chain shape', () => { + const stage = (name: string, stages?: Stage[]) => + new Stage({ name, readers: mockExecutor([q1]), stages }); + + it('rejects a chained stage that has stages of its own', () => { + expect(() => + stage('parent', [stage('child', [stage('grandchild')])]), + ).toThrow(/chained stage 'child' declares 'stages'/); + }); + + it('rejects two stages with one name in a chain', () => { + expect(() => stage('map', [stage('enrich'), stage('enrich')])).toThrow( + /two stages in the chain are named 'enrich'/, + ); + expect(() => stage('map', [stage('map')])).toThrow( + /two stages in the chain are named 'map'/, + ); + }); + }); + describe('batchSize per source kind', () => { /** The batch size a stage hands its item selector when run with `options`. */ async function selectedBatchSize( diff --git a/packages/pipeline/vite.config.ts b/packages/pipeline/vite.config.ts index 5e76fa2b..7103ee1e 100644 --- a/packages/pipeline/vite.config.ts +++ b/packages/pipeline/vite.config.ts @@ -11,10 +11,10 @@ export default mergeConfig( coverage: { thresholds: { autoUpdate: true, - functions: 99.4, - lines: 98.79, - branches: 96.75, - statements: 98.64, + functions: 99.41, + lines: 99.33, + branches: 96.82, + statements: 99.15, }, }, },