diff --git a/docs/guide/adapt-timeouts-to-endpoint-health.md b/docs/guide/adapt-timeouts-to-endpoint-health.md index cdde62eb..cae6f368 100644 --- a/docs/guide/adapt-timeouts-to-endpoint-health.md +++ b/docs/guide/adapt-timeouts-to-endpoint-health.md @@ -21,7 +21,9 @@ const pipeline = new Pipeline({ - `defaultMs` – the budget while an endpoint is healthy. - `tightenedMs` – the reduced budget once an endpoint is tightened; must be less than `defaultMs`. -- `tightenAfterTimeouts` – how many consecutive timeouts flip an endpoint from healthy to tightened. A single successful request flips it back. +- `tightenAfterTimeouts` – how many consecutive timeouts at `defaultMs` flip an endpoint from healthy to tightened. A single successful request flips it back. + +A timeout at the tightened budget says nothing about the endpoint’s health, so the policy re-probes it: the next request gets `defaultMs` while concurrent ones keep `tightenedMs`. If the probe succeeds, the endpoint relaxes, so a healthy endpoint whose heavy queries need longer than `tightenedMs` recovers on its own. Each probe that times out doubles the number of tightened timeouts before the next one, up to 64, so an endpoint that is really down rarely costs the full `defaultMs`. The pipeline invokes the factory once per dataset, so policy state resets at each dataset boundary and one bad dataset can’t poison the next. diff --git a/docs/reference/pipeline.md b/docs/reference/pipeline.md index 77c59cbf..f06ee323 100644 --- a/docs/reference/pipeline.md +++ b/docs/reference/pipeline.md @@ -346,7 +346,7 @@ By default, every SPARQL request uses the same 5-minute budget. When a pipeline A `TimeoutPolicy` decides the budget for each SPARQL request and observes the outcome. Two are built in: - **`ConstantTimeoutPolicy(timeoutMs)`** – returns the same budget for every request. The implicit default when `PipelineOptions.timeout` is omitted (`constantTimeoutPolicy(300_000)`). -- **`AdaptiveTimeoutPolicy({ defaultMs, tightenedMs, tightenAfterTimeouts })`** – per-endpoint state machine. Each endpoint is either _healthy_ (use `defaultMs`) or _tightened_ (use `tightenedMs`). After `tightenAfterTimeouts` consecutive `timeout` outcomes the endpoint flips to _tightened_; a single `ok` flips it back to _healthy_. +- **`AdaptiveTimeoutPolicy({ defaultMs, tightenedMs, tightenAfterTimeouts })`** – per-endpoint state machine. Each endpoint is either _healthy_ (use `defaultMs`) or _tightened_ (use `tightenedMs`). After `tightenAfterTimeouts` consecutive `timeout` outcomes at `defaultMs` the endpoint flips to _tightened_; a single `ok` flips it back to _healthy_. A timeout at `tightenedMs` doesn’t count: it may just be a heavy query on a healthy endpoint, so the next request re-probes at `defaultMs`, one probe at a time. If that probe succeeds the endpoint relaxes; if it times out, the endpoint stays tightened and the gap until the next probe doubles – 1, 2, 4… timeouts at `tightenedMs`, up to 64 – while a probe that fails with another error keeps the gap. A healthy-but-slow endpoint therefore loses one request rather than every heavy query until a cheap one happens to pass, and an endpoint that is genuinely down waits out the full budget on fewer and fewer requests. `PipelineOptions.timeout` accepts a `() => TimeoutPolicy` factory. The pipeline invokes it once per dataset, so policy state resets between datasets and one bad dataset can’t poison the next: @@ -373,7 +373,7 @@ Outcomes are classified as: Transitions are reported as `timeoutTightened` / `timeoutRelaxed` events; `ConsoleReporter` prints them as `↘ Tightened` / `↗ Relaxed` lines so operators can tell a fast-failed stage from an unexpected speedup. -Implement `TimeoutPolicy` directly for custom strategies (closing over shared state in the factory if you want it to span datasets). +Implement `TimeoutPolicy` directly for custom strategies (closing over shared state in the factory if you want it to span datasets). `beforeRequest` returns a `RequestBudget`, and the caller hands that same object back to `afterRequest` as `budget`, so a policy can tell by identity which request is reporting. Timeouts live at the pipeline level – neither `SparqlConstructReader` nor `SparqlItemSelector` accept their own `timeout` option. Per-endpoint state belongs in the adaptive policy, and per-stage budgets aren’t supported. Reusable stage facades ([@lde/pipeline-void](./pipeline-void), [@lde/pipeline-shacl-sampler](./pipeline-shacl-sampler)) follow the same convention. diff --git a/packages/pipeline/src/sparql/index.ts b/packages/pipeline/src/sparql/index.ts index eb37d89e..784a6a54 100644 --- a/packages/pipeline/src/sparql/index.ts +++ b/packages/pipeline/src/sparql/index.ts @@ -26,6 +26,7 @@ export { type AdaptiveTimeoutPolicyOptions, type AfterRequestContext, type BeforeRequestContext, + type RequestBudget, type TimeoutOutcome, type TimeoutPolicy, type TimeoutPolicyObserver, diff --git a/packages/pipeline/src/sparql/reader.ts b/packages/pipeline/src/sparql/reader.ts index f1ecaaea..6b4f9dde 100644 --- a/packages/pipeline/src/sparql/reader.ts +++ b/packages/pipeline/src/sparql/reader.ts @@ -367,10 +367,10 @@ export class SparqlConstructReader implements Reader { policy: TimeoutPolicy, response: ResponseNotes, ): Promise> { - const timeoutMs = policy.beforeRequest({ endpoint: endpointUrl }); - const fetcher = this.fetcherForAttempt(timeoutMs, response); + const budget = policy.beforeRequest({ endpoint: endpointUrl }); const start = Date.now(); try { + const fetcher = this.fetcherForAttempt(budget.timeoutMs, response); const quads = await this.fetchQuads( fetcher, endpointUrl.toString(), @@ -380,6 +380,7 @@ export class SparqlConstructReader implements Reader { endpoint: endpointUrl, outcome: 'ok', durationMs: Date.now() - start, + budget, }); return quads; } catch (error) { @@ -387,6 +388,7 @@ export class SparqlConstructReader implements Reader { endpoint: endpointUrl, outcome: classifyOutcome(error), durationMs: Date.now() - start, + budget, error, }); throw error; diff --git a/packages/pipeline/src/sparql/selector.ts b/packages/pipeline/src/sparql/selector.ts index 421e69f5..684e8f2d 100644 --- a/packages/pipeline/src/sparql/selector.ts +++ b/packages/pipeline/src/sparql/selector.ts @@ -280,11 +280,12 @@ export class SparqlItemSelector implements ItemSelector { paginatedQuery: string, policy: TimeoutPolicy, ): Promise[]> { - const timeoutMs = policy.beforeRequest({ endpoint }); - const fetcher = - this.userFetcher ?? new SparqlEndpointFetcher({ timeout: timeoutMs }); + const budget = policy.beforeRequest({ endpoint }); const start = Date.now(); try { + const fetcher = + this.userFetcher ?? + new SparqlEndpointFetcher({ timeout: budget.timeoutMs }); const stream = (await fetcher.fetchBindings( endpoint.toString(), paginatedQuery, @@ -297,6 +298,7 @@ export class SparqlItemSelector implements ItemSelector { endpoint, outcome: 'ok', durationMs: Date.now() - start, + budget, }); return page; } catch (error) { @@ -304,6 +306,7 @@ export class SparqlItemSelector implements ItemSelector { endpoint, outcome: classifyOutcome(error), durationMs: Date.now() - start, + budget, error, }); throw error; diff --git a/packages/pipeline/src/sparql/timeoutPolicy.ts b/packages/pipeline/src/sparql/timeoutPolicy.ts index df1b420b..aba0be0b 100644 --- a/packages/pipeline/src/sparql/timeoutPolicy.ts +++ b/packages/pipeline/src/sparql/timeoutPolicy.ts @@ -2,12 +2,12 @@ * Outcome of a single SPARQL request attempt, as reported back to a * {@link TimeoutPolicy} so it can adapt the budget for subsequent requests. * - * - `ok` — the request resolved successfully (the HTTP response was accepted + * - `ok` – the request resolved successfully (the HTTP response was accepted * and the body started streaming). - * - `timeout` — the per-call {@link AbortSignal} fired, or the endpoint + * - `timeout` – the per-call {@link AbortSignal} fired, or the endpoint * returned an HTTP 504 (upstream-reported timeout). Both are semantically * ‘the endpoint did not deliver in time’. - * - `error` — any other failure (4xx other than 504, parser errors, etc.). + * - `error` – any other failure (4xx other than 504, parser errors, etc.). * Neutral with respect to adaptive tightening. */ export type TimeoutOutcome = 'ok' | 'error' | 'timeout'; @@ -18,6 +18,17 @@ export interface BeforeRequestContext { endpoint: URL; } +/** + * Budget that {@link TimeoutPolicy.beforeRequest} hands out for one request + * attempt. Callers pass this same object back as + * {@link AfterRequestContext.budget}, so a policy can tell by identity which + * attempt is reporting. + */ +export interface RequestBudget { + /** Timeout to apply to the attempt, in milliseconds. */ + readonly timeoutMs: number; +} + /** Context passed to {@link TimeoutPolicy.afterRequest}. */ export interface AfterRequestContext { /** Endpoint URL the request was sent to. */ @@ -26,6 +37,11 @@ export interface AfterRequestContext { outcome: TimeoutOutcome; /** Wall-clock duration of the request attempt, in milliseconds. */ durationMs: number; + /** + * The object {@link TimeoutPolicy.beforeRequest} returned for this + * attempt – the same object, not a copy. + */ + budget: RequestBudget; /** The raw error, when {@link outcome} is `'error'` or `'timeout'`. */ error?: unknown; } @@ -33,7 +49,7 @@ export interface AfterRequestContext { /** * Decides the timeout budget for each SPARQL request and observes the * outcome. Implementations are free to adapt the budget based on recent - * behaviour — see {@link AdaptiveTimeoutPolicy} for the built-in adaptive + * behaviour – see {@link AdaptiveTimeoutPolicy} for the built-in adaptive * implementation, and {@link ConstantTimeoutPolicy} for fixed-budget * behaviour. * @@ -42,14 +58,15 @@ export interface AfterRequestContext { */ export interface TimeoutPolicy { /** - * Returns the timeout (in milliseconds) to apply to the upcoming request. - * Called once per attempt, retries included, so a retry can already use a - * tightened budget. + * Returns the budget to apply to the upcoming request. Called once per + * attempt, retries included, so a retry can already use a tightened + * budget. */ - beforeRequest(context: BeforeRequestContext): number; + beforeRequest(context: BeforeRequestContext): RequestBudget; /** - * Reports the outcome of the request that {@link beforeRequest} budgeted. - * Called once per attempt, regardless of outcome. + * Reports the outcome of the request that {@link beforeRequest} budgeted, + * passing back the budget it returned. Called once per attempt, + * regardless of outcome. */ afterRequest(context: AfterRequestContext): void; /** @@ -69,9 +86,9 @@ export interface TimeoutTransitionEvent { /** Budget in effect after the transition. */ toTimeoutMs: number; /** - * Number of consecutive timeouts observed at the moment of the - * transition. For a `relax` event, this is the run that ended in the - * `ok` that triggered relaxation. + * Number of consecutive timeouts at the default budget observed at the + * moment of the transition. For a `relax` event, this is the run that + * ended in the `ok` that triggered relaxation. */ consecutiveTimeouts: number; } @@ -101,12 +118,12 @@ export class ConstantTimeoutPolicy implements TimeoutPolicy { } } - beforeRequest(_context: BeforeRequestContext): number { - return this.timeoutMs; + beforeRequest(_context: BeforeRequestContext): RequestBudget { + return { timeoutMs: this.timeoutMs }; } afterRequest(_context: AfterRequestContext): void { - // Constant policy is stateless — outcomes never affect future budgets. + // Constant policy is stateless – outcomes never affect future budgets. } } @@ -126,12 +143,6 @@ export interface AdaptiveTimeoutPolicyOptions { tightenAfterTimeouts: number; } -interface EndpointState { - tightened: boolean; - /** Consecutive timeouts since the last `ok`. */ - consecutiveTimeouts: number; -} - /** * Adaptive per-endpoint policy: after {@link AdaptiveTimeoutPolicyOptions.tightenAfterTimeouts} * consecutive timeouts on the same endpoint, subsequent requests use the @@ -139,7 +150,20 @@ interface EndpointState { * full default budget. A single successful request relaxes the endpoint * back to the default budget. * - * State is in-memory and tied to the policy instance — Pipeline creates a + * A timeout at the tightened budget says nothing about the endpoint’s + * health – a healthy endpoint’s heavy query may simply need longer – so it + * does not count toward tightening. Instead, it brings a re-probe at the + * default budget closer, one probe at a time: concurrent requests keep the + * tightened budget until the probe reports. The first timeout at the + * tightened budget makes a probe due. If the probe succeeds, the endpoint + * relaxes; if it times out, the gap until the next probe doubles – 1, 2, 4… + * timeouts at the tightened budget, up to 64. A probe that fails with an + * error keeps the current gap. A healthy-but-slow endpoint therefore + * recovers after one lost request, while an endpoint that is genuinely + * down waits out the full default budget only on a shrinking share of + * requests. + * + * State is in-memory and tied to the policy instance – Pipeline creates a * fresh instance per dataset so one offending dataset doesn’t poison the * next. * @@ -182,19 +206,27 @@ export class AdaptiveTimeoutPolicy implements TimeoutPolicy { } } - beforeRequest(context: BeforeRequestContext): number { - const state = this.stateFor(context.endpoint); - return state.tightened ? this.options.tightenedMs : this.options.defaultMs; + beforeRequest(context: BeforeRequestContext): RequestBudget { + const { health } = this.stateFor(context.endpoint); + if (health.status === 'healthy') { + return { timeoutMs: this.options.defaultMs }; + } + if (health.probe.status === 'due') { + const budget = { timeoutMs: this.options.defaultMs }; + health.probe = { status: 'inFlight', budget }; + return budget; + } + return { timeoutMs: this.options.tightenedMs }; } afterRequest(context: AfterRequestContext): void { const state = this.stateFor(context.endpoint); + const { health } = state; if (context.outcome === 'ok') { - const wasTightened = state.tightened; const priorCount = state.consecutiveTimeouts; state.consecutiveTimeouts = 0; - state.tightened = false; - if (wasTightened) { + state.health = { status: 'healthy' }; + if (health.status === 'tightened') { this.notify('relax', { endpoint: context.endpoint, fromTimeoutMs: this.options.tightenedMs, @@ -204,13 +236,45 @@ export class AdaptiveTimeoutPolicy implements TimeoutPolicy { } return; } + if ( + health.status === 'tightened' && + health.probe.status === 'inFlight' && + health.probe.budget === context.budget + ) { + if (context.outcome === 'timeout') health.failedProbes += 1; + health.probe = { + status: 'waiting', + timeoutsUntilDue: timeoutsBetweenProbes(health.failedProbes), + }; + } if (context.outcome === 'timeout') { + if (context.budget.timeoutMs < this.options.defaultMs) { + // Too short a budget to judge the endpoint by: bring a re-probe + // closer instead. + if ( + health.status === 'tightened' && + health.probe.status === 'waiting' + ) { + health.probe.timeoutsUntilDue -= 1; + if (health.probe.timeoutsUntilDue === 0) { + health.probe = { status: 'due' }; + } + } + return; + } state.consecutiveTimeouts += 1; if ( - !state.tightened && + health.status === 'healthy' && state.consecutiveTimeouts >= this.options.tightenAfterTimeouts ) { - state.tightened = true; + state.health = { + status: 'tightened', + probe: { + status: 'waiting', + timeoutsUntilDue: timeoutsBetweenProbes(0), + }, + failedProbes: 0, + }; this.notify('tighten', { endpoint: context.endpoint, fromTimeoutMs: this.options.defaultMs, @@ -233,7 +297,7 @@ export class AdaptiveTimeoutPolicy implements TimeoutPolicy { const key = endpoint.toString(); let state = this.states.get(key); if (!state) { - state = { tightened: false, consecutiveTimeouts: 0 }; + state = { health: { status: 'healthy' }, consecutiveTimeouts: 0 }; this.states.set(key, state); } return state; @@ -278,3 +342,47 @@ export function adaptiveTimeoutPolicy( new AdaptiveTimeoutPolicy(options); return () => new AdaptiveTimeoutPolicy(options); } + +/** + * Upper bound on the number of timeouts at the tightened budget between two + * probes, so an endpoint that comes back late in a long run still gets + * re-probed. + */ +const maxTimeoutsBetweenProbes = 64; + +/** + * Timeouts at the tightened budget until the next probe is due, doubling + * with each failed probe. + */ +function timeoutsBetweenProbes(failedProbes: number): number { + return Math.min(2 ** failedProbes, maxTimeoutsBetweenProbes); +} + +/** + * Where a tightened endpoint’s next re-probe stands. + * + * - `waiting` – that many more timeouts at the tightened budget make a probe + * due. + * - `due` – the next request is the probe. + * - `inFlight` – the probe was handed out with this budget and has not + * reported yet; concurrent requests keep the tightened budget meanwhile. + */ +type Probe = + | { status: 'waiting'; timeoutsUntilDue: number } + | { status: 'due' } + | { status: 'inFlight'; budget: RequestBudget }; + +type EndpointHealth = + | { status: 'healthy' } + | { + status: 'tightened'; + probe: Probe; + /** Probes that timed out since the endpoint tightened. */ + failedProbes: number; + }; + +interface EndpointState { + health: EndpointHealth; + /** Consecutive timeouts at the default budget since the last `ok`. */ + consecutiveTimeouts: number; +} diff --git a/packages/pipeline/test/pipeline.test.ts b/packages/pipeline/test/pipeline.test.ts index 0193450b..0570d665 100644 --- a/packages/pipeline/test/pipeline.test.ts +++ b/packages/pipeline/test/pipeline.test.ts @@ -2871,7 +2871,7 @@ describe('Pipeline', () => { it('invokes the factory once per dataset', async () => { const factory = vi.fn().mockImplementation(() => ({ - beforeRequest: () => 300_000, + beforeRequest: () => ({ timeoutMs: 300_000 }), afterRequest: vi.fn(), })); @@ -2895,7 +2895,7 @@ describe('Pipeline', () => { const policies: unknown[] = []; const factory = vi.fn().mockImplementation(() => { const policy = { - beforeRequest: () => 300_000, + beforeRequest: () => ({ timeoutMs: 300_000 }), afterRequest: vi.fn(), }; policies.push(policy); @@ -2936,7 +2936,7 @@ describe('Pipeline', () => { consecutiveTimeouts: 0, }; const factory = vi.fn().mockImplementation(() => ({ - beforeRequest: () => 300_000, + beforeRequest: () => ({ timeoutMs: 300_000 }), afterRequest: vi.fn(), subscribe(observer: { onTighten?: (event: unknown) => void; diff --git a/packages/pipeline/test/sparql/reader.test.ts b/packages/pipeline/test/sparql/reader.test.ts index c0eeacb2..f0486e65 100644 --- a/packages/pipeline/test/sparql/reader.test.ts +++ b/packages/pipeline/test/sparql/reader.test.ts @@ -768,7 +768,7 @@ describe('SparqlConstructReader', () => { describe('timeout policy', () => { function recordingPolicy() { - const before = vi.fn().mockReturnValue(1234); + const before = vi.fn().mockReturnValue({ timeoutMs: 1234 }); const after = vi.fn(); return { beforeRequest: before, @@ -811,8 +811,13 @@ describe('SparqlConstructReader', () => { expect.objectContaining({ endpoint: distribution.accessUrl, outcome: 'ok', + budget: { timeoutMs: 1234 }, }), ); + // The policy tells attempts apart by identity, so no copy. + expect(policy.afterRequest.mock.calls[0][0].budget).toBe( + policy.beforeRequest.mock.results[0].value, + ); }); it('classifies HTTP 504 as outcome "timeout"', async () => { @@ -838,7 +843,10 @@ describe('SparqlConstructReader', () => { ).rejects.toThrow('504'); expect(policy.afterRequest).toHaveBeenCalledWith( - expect.objectContaining({ outcome: 'timeout' }), + expect.objectContaining({ + outcome: 'timeout', + budget: { timeoutMs: 1234 }, + }), ); }); @@ -862,7 +870,10 @@ describe('SparqlConstructReader', () => { ).rejects.toThrow(); expect(policy.afterRequest).toHaveBeenCalledWith( - expect.objectContaining({ outcome: 'timeout' }), + expect.objectContaining({ + outcome: 'timeout', + budget: { timeoutMs: 1234 }, + }), ); }); @@ -886,7 +897,10 @@ describe('SparqlConstructReader', () => { ).rejects.toThrow(); expect(policy.afterRequest).toHaveBeenCalledWith( - expect.objectContaining({ outcome: 'timeout' }), + expect.objectContaining({ + outcome: 'timeout', + budget: { timeoutMs: 1234 }, + }), ); }); @@ -912,7 +926,10 @@ describe('SparqlConstructReader', () => { ).rejects.toThrow(); expect(policy.afterRequest).toHaveBeenCalledWith( - expect.objectContaining({ outcome: 'error' }), + expect.objectContaining({ + outcome: 'error', + budget: { timeoutMs: 1234 }, + }), ); }); @@ -942,11 +959,14 @@ describe('SparqlConstructReader', () => { expect(policy.afterRequest).toHaveBeenCalledTimes(2); expect(policy.afterRequest).toHaveBeenNthCalledWith( 1, - expect.objectContaining({ outcome: 'timeout' }), + expect.objectContaining({ + outcome: 'timeout', + budget: { timeoutMs: 1234 }, + }), ); expect(policy.afterRequest).toHaveBeenNthCalledWith( 2, - expect.objectContaining({ outcome: 'ok' }), + expect.objectContaining({ outcome: 'ok', budget: { timeoutMs: 1234 } }), ); }); @@ -987,7 +1007,7 @@ describe('SparqlConstructReader', () => { }); const policy = { - beforeRequest: vi.fn().mockReturnValue(10), + beforeRequest: vi.fn().mockReturnValue({ timeoutMs: 10 }), afterRequest: vi.fn(), }; const reader = new SparqlConstructReader({ @@ -1003,7 +1023,10 @@ describe('SparqlConstructReader', () => { ).rejects.toThrow(); expect(policy.afterRequest).toHaveBeenCalledWith( - expect.objectContaining({ outcome: 'timeout' }), + expect.objectContaining({ + outcome: 'timeout', + budget: { timeoutMs: 10 }, + }), ); expect(slowFetch).toHaveBeenCalled(); const init = slowFetch.mock.calls[0][1]; diff --git a/packages/pipeline/test/sparql/selector.test.ts b/packages/pipeline/test/sparql/selector.test.ts index ecfb94f0..dbde488f 100644 --- a/packages/pipeline/test/sparql/selector.test.ts +++ b/packages/pipeline/test/sparql/selector.test.ts @@ -646,7 +646,7 @@ describe('SparqlItemSelector', () => { describe('timeout policy', () => { function recordingPolicy() { return { - beforeRequest: vi.fn().mockReturnValue(5_000), + beforeRequest: vi.fn().mockReturnValue({ timeoutMs: 5_000 }), afterRequest: vi.fn(), }; } @@ -681,7 +681,11 @@ describe('SparqlItemSelector', () => { }); expect(policy.afterRequest).toHaveBeenCalledTimes(2); expect(policy.afterRequest).toHaveBeenCalledWith( - expect.objectContaining({ outcome: 'ok' }), + expect.objectContaining({ outcome: 'ok', budget: { timeoutMs: 5000 } }), + ); + // The policy tells attempts apart by identity, so no copy. + expect(policy.afterRequest.mock.calls[0][0].budget).toBe( + policy.beforeRequest.mock.results[0].value, ); }); @@ -712,7 +716,10 @@ describe('SparqlItemSelector', () => { await expect(iterate()).rejects.toThrow('504'); expect(policy.afterRequest).toHaveBeenCalledWith( - expect.objectContaining({ outcome: 'timeout' }), + expect.objectContaining({ + outcome: 'timeout', + budget: { timeoutMs: 5000 }, + }), ); }); @@ -741,7 +748,10 @@ describe('SparqlItemSelector', () => { await expect(iterate()).rejects.toThrow('socket hang up'); expect(policy.afterRequest).toHaveBeenCalledTimes(1); expect(policy.afterRequest).toHaveBeenCalledWith( - expect.objectContaining({ outcome: 'error' }), + expect.objectContaining({ + outcome: 'error', + budget: { timeoutMs: 5000 }, + }), ); }); @@ -772,7 +782,10 @@ describe('SparqlItemSelector', () => { await expect(iterate()).rejects.toThrow(); expect(policy.afterRequest).toHaveBeenCalledWith( - expect.objectContaining({ outcome: 'error' }), + expect.objectContaining({ + outcome: 'error', + budget: { timeoutMs: 5000 }, + }), ); }); diff --git a/packages/pipeline/test/sparql/timeoutPolicy.test.ts b/packages/pipeline/test/sparql/timeoutPolicy.test.ts index 075c19d8..f8006bda 100644 --- a/packages/pipeline/test/sparql/timeoutPolicy.test.ts +++ b/packages/pipeline/test/sparql/timeoutPolicy.test.ts @@ -4,6 +4,8 @@ import { ConstantTimeoutPolicy, adaptiveTimeoutPolicy, constantTimeoutPolicy, + type RequestBudget, + type TimeoutOutcome, } from '../../src/sparql/timeoutPolicy.js'; const endpointA = new URL('https://example.org/a/sparql'); @@ -12,7 +14,7 @@ const endpointB = new URL('https://example.org/b/sparql'); describe('ConstantTimeoutPolicy', () => { it('returns the configured timeout from beforeRequest', () => { const policy = new ConstantTimeoutPolicy(1234); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1234); + expect(policy.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe(1234); }); it('ignores outcomes in afterRequest', () => { @@ -21,8 +23,9 @@ describe('ConstantTimeoutPolicy', () => { endpoint: endpointA, outcome: 'timeout', durationMs: 500, + budget: { timeoutMs: 1000 }, }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); + expect(policy.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe(1000); }); it('rejects non-positive timeouts', () => { @@ -39,7 +42,7 @@ describe('constantTimeoutPolicy factory', () => { const b = factory(); expect(a).toBeInstanceOf(ConstantTimeoutPolicy); expect(a).not.toBe(b); - expect(a.beforeRequest({ endpoint: endpointA })).toBe(5_000); + expect(a.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe(5_000); }); }); @@ -102,7 +105,9 @@ describe('AdaptiveTimeoutPolicy', () => { tightenedMs: 100, tightenAfterTimeouts: 2, }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); + expect(policy.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe( + 1000, + ); }); it('tightens after exactly tightenAfterTimeouts=1 consecutive timeouts', () => { @@ -111,13 +116,16 @@ describe('AdaptiveTimeoutPolicy', () => { tightenedMs: 100, tightenAfterTimeouts: 1, }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); + expect(policy.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe( + 1000, + ); policy.afterRequest({ endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + budget: { timeoutMs: 1000 }, }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); + expect(policy.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe(100); }); it('tightens after exactly tightenAfterTimeouts=2 consecutive timeouts', () => { @@ -130,14 +138,18 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + budget: { timeoutMs: 1000 }, }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); + expect(policy.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe( + 1000, + ); policy.afterRequest({ endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + budget: { timeoutMs: 1000 }, }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); + expect(policy.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe(100); }); it('tightens after exactly tightenAfterTimeouts=3 consecutive timeouts', () => { @@ -150,19 +162,24 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + budget: { timeoutMs: 1000 }, }); policy.afterRequest({ endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + budget: { timeoutMs: 1000 }, }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); + expect(policy.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe( + 1000, + ); policy.afterRequest({ endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + budget: { timeoutMs: 1000 }, }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); + expect(policy.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe(100); }); it('stays tightened on further timeouts', () => { @@ -176,9 +193,10 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + budget: { timeoutMs: 1000 }, }); } - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); + expect(policy.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe(100); }); it('relaxes to default on a single ok', () => { @@ -191,14 +209,18 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + budget: { timeoutMs: 1000 }, }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); + expect(policy.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe(100); policy.afterRequest({ endpoint: endpointA, outcome: 'ok', durationMs: 80, + budget: { timeoutMs: 1000 }, }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); + expect(policy.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe( + 1000, + ); }); it('resets the counter on ok so subsequent timeouts must accumulate again', () => { @@ -211,18 +233,23 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + budget: { timeoutMs: 1000 }, }); policy.afterRequest({ endpoint: endpointA, outcome: 'ok', durationMs: 80, + budget: { timeoutMs: 1000 }, }); policy.afterRequest({ endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + budget: { timeoutMs: 1000 }, }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); + expect(policy.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe( + 1000, + ); }); it('treats `error` outcomes as neutral (neither tighten nor relax)', () => { @@ -235,21 +262,26 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'error', durationMs: 50, + budget: { timeoutMs: 1000 }, }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); + expect(policy.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe( + 1000, + ); policy.afterRequest({ endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + budget: { timeoutMs: 1000 }, }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); + expect(policy.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe(100); // An `error` while tightened keeps state tightened. policy.afterRequest({ endpoint: endpointA, outcome: 'error', durationMs: 50, + budget: { timeoutMs: 1000 }, }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); + expect(policy.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe(100); }); it('isolates state per endpoint', () => { @@ -262,9 +294,212 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + budget: { timeoutMs: 1000 }, + }); + expect(policy.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe(100); + expect(policy.beforeRequest({ endpoint: endpointB }).timeoutMs).toBe( + 1000, + ); + }); + + describe('re-probing a tightened endpoint', () => { + const options = { + defaultMs: 1000, + tightenedMs: 100, + tightenAfterTimeouts: 1, + }; + + function request(policy: AdaptiveTimeoutPolicy): RequestBudget { + return policy.beforeRequest({ endpoint: endpointA }); + } + + function report( + policy: AdaptiveTimeoutPolicy, + budget: RequestBudget, + outcome: TimeoutOutcome, + ): void { + policy.afterRequest({ + endpoint: endpointA, + outcome, + durationMs: budget.timeoutMs, + budget, + }); + } + + function timeOutNextRequest(policy: AdaptiveTimeoutPolicy): void { + report(policy, request(policy), 'timeout'); + } + + function tightenedPolicy(): AdaptiveTimeoutPolicy { + const policy = new AdaptiveTimeoutPolicy(options); + timeOutNextRequest(policy); + return policy; + } + + /** + * Time out requests at the tightened budget until the policy hands out + * a probe; returns how many it took and the probe’s budget. + */ + function nextProbe(policy: AdaptiveTimeoutPolicy): { + timeouts: number; + probe: RequestBudget; + } { + for (let timeouts = 0; timeouts < 1000; timeouts++) { + const budget = request(policy); + if (budget.timeoutMs === options.defaultMs) { + return { timeouts, probe: budget }; + } + report(policy, budget, 'timeout'); + } + throw new Error('No probe handed out'); + } + + it('gives the next request the default budget after a timeout at the tightened budget', () => { + const policy = tightenedPolicy(); + const tightened = request(policy); + expect(tightened.timeoutMs).toBe(100); + report(policy, tightened, 'timeout'); + expect(request(policy).timeoutMs).toBe(1000); + // One probe at a time: concurrent requests keep the tightened budget. + expect(request(policy).timeoutMs).toBe(100); + }); + + it('hands out no second probe while one is in flight', () => { + const policy = tightenedPolicy(); + const first = request(policy); + const second = request(policy); + expect([first.timeoutMs, second.timeoutMs]).toEqual([100, 100]); + report(policy, first, 'timeout'); + expect(request(policy).timeoutMs).toBe(1000); + + // A concurrent request at the tightened budget times out while the + // probe is in flight: it schedules no other probe. + report(policy, second, 'timeout'); + expect(request(policy).timeoutMs).toBe(100); + }); + + it('does not mistake a request budgeted before tightening for the probe', () => { + const policy = new AdaptiveTimeoutPolicy({ + ...options, + tightenAfterTimeouts: 2, + }); + const budgetedWhileHealthy = request(policy); + timeOutNextRequest(policy); + timeOutNextRequest(policy); + const { probe } = nextProbe(policy); + + // The request from before tightening times out at the default budget, + // just like the probe would: the probe is still in flight. + report(policy, budgetedWhileHealthy, 'timeout'); + const tightened = request(policy); + report(policy, tightened, 'timeout'); + expect(request(policy).timeoutMs).toBe(100); + + report(policy, probe, 'timeout'); + expect(nextProbe(policy).timeouts).toBe(2); + }); + + it('releases the probe when it fails with an error, keeping the gap', () => { + const policy = tightenedPolicy(); + const { probe } = nextProbe(policy); + report(policy, probe, 'error'); + expect(nextProbe(policy).timeouts).toBe(1); + }); + + it('doubles the gap between probes after each timed-out probe, up to 64', () => { + const policy = tightenedPolicy(); + const gaps: number[] = []; + for (let attempt = 0; attempt < 8; attempt++) { + const { timeouts, probe } = nextProbe(policy); + gaps.push(timeouts); + report(policy, probe, 'timeout'); + } + expect(gaps).toEqual([1, 2, 4, 8, 16, 32, 64, 64]); + }); + + it('relaxes a healthy-but-slow endpoint when the probe succeeds', () => { + const policy = tightenedPolicy(); + const onRelax = vi.fn(); + policy.subscribe({ onRelax }); + const { probe } = nextProbe(policy); + report(policy, probe, 'ok'); + // The timeout at the tightened budget before the probe isn’t counted. + expect(onRelax).toHaveBeenCalledTimes(1); + expect(onRelax).toHaveBeenCalledWith( + expect.objectContaining({ consecutiveTimeouts: 1 }), + ); + expect(request(policy).timeoutMs).toBe(1000); + expect(request(policy).timeoutMs).toBe(1000); + }); + + it('resets the gap between probes once the endpoint relaxes', () => { + const policy = tightenedPolicy(); + report(policy, nextProbe(policy).probe, 'timeout'); + report(policy, nextProbe(policy).probe, 'ok'); + timeOutNextRequest(policy); + expect(nextProbe(policy).timeouts).toBe(1); + }); + + it('stays tightened when the probe times out at the default budget', () => { + const policy = tightenedPolicy(); + const onTighten = vi.fn(); + const onRelax = vi.fn(); + policy.subscribe({ onTighten, onRelax }); + const { probe } = nextProbe(policy); + report(policy, probe, 'timeout'); + expect(request(policy).timeoutMs).toBe(100); + expect(onTighten).not.toHaveBeenCalled(); + expect(onRelax).not.toHaveBeenCalled(); + }); + + it('ignores a late timeout at the tightened budget after the endpoint relaxed', () => { + const policy = tightenedPolicy(); + const succeeding = request(policy); + const timingOut = request(policy); + report(policy, succeeding, 'ok'); + report(policy, timingOut, 'timeout'); + expect(request(policy).timeoutMs).toBe(1000); + expect(request(policy).timeoutMs).toBe(1000); + }); + + it('ignores a probe that reports after the endpoint relaxed and tightened again', () => { + const policy = tightenedPolicy(); + const { probe: staleProbe } = nextProbe(policy); + report(policy, request(policy), 'ok'); + timeOutNextRequest(policy); + nextProbe(policy); + + report(policy, staleProbe, 'timeout'); + const tightened = request(policy); + report(policy, tightened, 'timeout'); + expect(request(policy).timeoutMs).toBe(100); + }); + + it('does not count timeouts at the tightened budget toward tightening', () => { + const policy = new AdaptiveTimeoutPolicy({ + ...options, + tightenAfterTimeouts: 2, + }); + timeOutNextRequest(policy); + timeOutNextRequest(policy); + const budgetedWhileTightened = request(policy); + expect(budgetedWhileTightened.timeoutMs).toBe(100); + report(policy, request(policy), 'ok'); + const onTighten = vi.fn(); + policy.subscribe({ onTighten }); + + // Healthy again: one timeout at the default budget, then the request + // budgeted while tightened times out at the tightened budget. + timeOutNextRequest(policy); + report(policy, budgetedWhileTightened, 'timeout'); + expect(request(policy).timeoutMs).toBe(1000); + expect(onTighten).not.toHaveBeenCalled(); + + timeOutNextRequest(policy); + expect(onTighten).toHaveBeenCalledWith( + expect.objectContaining({ consecutiveTimeouts: 2 }), + ); }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); - expect(policy.beforeRequest({ endpoint: endpointB })).toBe(1000); }); }); @@ -282,12 +517,14 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + budget: { timeoutMs: 1000 }, }); expect(onTighten).not.toHaveBeenCalled(); policy.afterRequest({ endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + budget: { timeoutMs: 1000 }, }); expect(onTighten).toHaveBeenCalledTimes(1); expect(onTighten).toHaveBeenCalledWith( @@ -313,16 +550,19 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + budget: { timeoutMs: 1000 }, }); policy.afterRequest({ endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + budget: { timeoutMs: 1000 }, }); policy.afterRequest({ endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + budget: { timeoutMs: 1000 }, }); expect(onTighten).toHaveBeenCalledTimes(1); }); @@ -339,11 +579,13 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + budget: { timeoutMs: 1000 }, }); policy.afterRequest({ endpoint: endpointA, outcome: 'ok', durationMs: 80, + budget: { timeoutMs: 1000 }, }); expect(onRelax).toHaveBeenCalledTimes(1); expect(onRelax).toHaveBeenCalledWith( @@ -367,6 +609,7 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'ok', durationMs: 80, + budget: { timeoutMs: 1000 }, }); expect(onRelax).not.toHaveBeenCalled(); }); @@ -384,6 +627,7 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + budget: { timeoutMs: 1000 }, }); expect(onTighten).not.toHaveBeenCalled(); }); @@ -414,8 +658,9 @@ describe('adaptiveTimeoutPolicy factory', () => { endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + budget: { timeoutMs: 1000 }, }); const b = factory(); - expect(b.beforeRequest({ endpoint: endpointA })).toBe(1000); + expect(b.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe(1000); }); }); diff --git a/packages/pipeline/vite.config.ts b/packages/pipeline/vite.config.ts index 2e9903d8..5e76fa2b 100644 --- a/packages/pipeline/vite.config.ts +++ b/packages/pipeline/vite.config.ts @@ -12,9 +12,9 @@ export default mergeConfig( thresholds: { autoUpdate: true, functions: 99.4, - lines: 98.78, - branches: 96.67, - statements: 98.62, + lines: 98.79, + branches: 96.75, + statements: 98.64, }, }, },