From 62d7ccfeb6065d1c83f9e2b4f178a182c096b101 Mon Sep 17 00:00:00 2001 From: David de Boer Date: Sat, 14 Nov 2026 13:03:30 +0100 Subject: [PATCH 1/3] fix(pipeline)!: re-probe a tightened endpoint at the default budget AdaptiveTimeoutPolicy kept an endpoint tightened as long as requests timed out, including timeouts at the tightened budget itself. A healthy endpoint whose heavy queries legitimately need longer than tightenedMs could then only recover when some cheap query happened to succeed, losing every heavy stage in between. A timeout at the tightened budget no longer counts toward tightening; instead the next request runs at defaultMs. The endpoint relaxes when that probe succeeds and stays tightened when it times out as well. BREAKING CHANGE: AfterRequestContext has a required timeoutMs property: the budget beforeRequest returned for the attempt. Code that calls TimeoutPolicy.afterRequest itself must pass it. --- .../adapt-timeouts-to-endpoint-health.md | 4 +- docs/reference/pipeline.md | 2 +- packages/pipeline/src/sparql/reader.ts | 2 + packages/pipeline/src/sparql/selector.ts | 2 + packages/pipeline/src/sparql/timeoutPolicy.ts | 55 +++++-- .../test/sparql/timeoutPolicy.test.ts | 149 ++++++++++++++++++ 6 files changed, 200 insertions(+), 14 deletions(-) diff --git a/docs/guide/adapt-timeouts-to-endpoint-health.md b/docs/guide/adapt-timeouts-to-endpoint-health.md index cdde62eb8..dca4e23b7 100644 --- a/docs/guide/adapt-timeouts-to-endpoint-health.md +++ b/docs/guide/adapt-timeouts-to-endpoint-health.md @@ -21,7 +21,9 @@ const pipeline = new Pipeline({ - `defaultMs` – the budget while an endpoint is healthy. - `tightenedMs` – the reduced budget once an endpoint is tightened; must be less than `defaultMs`. -- `tightenAfterTimeouts` – how many consecutive timeouts flip an endpoint from healthy to tightened. A single successful request flips it back. +- `tightenAfterTimeouts` – how many consecutive timeouts at `defaultMs` flip an endpoint from healthy to tightened. A single successful request flips it back. + +A timeout at the tightened budget doesn’t keep the endpoint tightened: the next request re-probes at `defaultMs`, so a healthy endpoint whose heavy queries need longer than `tightenedMs` relaxes again on its own. The pipeline invokes the factory once per dataset, so policy state resets at each dataset boundary and one bad dataset can’t poison the next. diff --git a/docs/reference/pipeline.md b/docs/reference/pipeline.md index 77c59cbf3..700322f62 100644 --- a/docs/reference/pipeline.md +++ b/docs/reference/pipeline.md @@ -346,7 +346,7 @@ By default, every SPARQL request uses the same 5-minute budget. When a pipeline A `TimeoutPolicy` decides the budget for each SPARQL request and observes the outcome. Two are built in: - **`ConstantTimeoutPolicy(timeoutMs)`** – returns the same budget for every request. The implicit default when `PipelineOptions.timeout` is omitted (`constantTimeoutPolicy(300_000)`). -- **`AdaptiveTimeoutPolicy({ defaultMs, tightenedMs, tightenAfterTimeouts })`** – per-endpoint state machine. Each endpoint is either _healthy_ (use `defaultMs`) or _tightened_ (use `tightenedMs`). After `tightenAfterTimeouts` consecutive `timeout` outcomes the endpoint flips to _tightened_; a single `ok` flips it back to _healthy_. +- **`AdaptiveTimeoutPolicy({ defaultMs, tightenedMs, tightenAfterTimeouts })`** – per-endpoint state machine. Each endpoint is either _healthy_ (use `defaultMs`) or _tightened_ (use `tightenedMs`). After `tightenAfterTimeouts` consecutive `timeout` outcomes at `defaultMs` the endpoint flips to _tightened_; a single `ok` flips it back to _healthy_. A timeout at `tightenedMs` doesn’t count: it may just be a heavy query on a healthy endpoint, so the next request re-probes at `defaultMs`. If that probe succeeds the endpoint relaxes; if it times out too, the endpoint stays tightened. A healthy-but-slow endpoint therefore loses one request rather than every heavy query until a cheap one happens to pass, while an endpoint that keeps timing out at `defaultMs` waits out the full budget on only every other request. `PipelineOptions.timeout` accepts a `() => TimeoutPolicy` factory. The pipeline invokes it once per dataset, so policy state resets between datasets and one bad dataset can’t poison the next: diff --git a/packages/pipeline/src/sparql/reader.ts b/packages/pipeline/src/sparql/reader.ts index f1ecaaea5..ec6d4e46d 100644 --- a/packages/pipeline/src/sparql/reader.ts +++ b/packages/pipeline/src/sparql/reader.ts @@ -380,6 +380,7 @@ export class SparqlConstructReader implements Reader { endpoint: endpointUrl, outcome: 'ok', durationMs: Date.now() - start, + timeoutMs, }); return quads; } catch (error) { @@ -387,6 +388,7 @@ export class SparqlConstructReader implements Reader { endpoint: endpointUrl, outcome: classifyOutcome(error), durationMs: Date.now() - start, + timeoutMs, error, }); throw error; diff --git a/packages/pipeline/src/sparql/selector.ts b/packages/pipeline/src/sparql/selector.ts index 421e69f56..2272b57b7 100644 --- a/packages/pipeline/src/sparql/selector.ts +++ b/packages/pipeline/src/sparql/selector.ts @@ -297,6 +297,7 @@ export class SparqlItemSelector implements ItemSelector { endpoint, outcome: 'ok', durationMs: Date.now() - start, + timeoutMs, }); return page; } catch (error) { @@ -304,6 +305,7 @@ export class SparqlItemSelector implements ItemSelector { endpoint, outcome: classifyOutcome(error), durationMs: Date.now() - start, + timeoutMs, error, }); throw error; diff --git a/packages/pipeline/src/sparql/timeoutPolicy.ts b/packages/pipeline/src/sparql/timeoutPolicy.ts index df1b420be..9cf71bce1 100644 --- a/packages/pipeline/src/sparql/timeoutPolicy.ts +++ b/packages/pipeline/src/sparql/timeoutPolicy.ts @@ -2,12 +2,12 @@ * Outcome of a single SPARQL request attempt, as reported back to a * {@link TimeoutPolicy} so it can adapt the budget for subsequent requests. * - * - `ok` — the request resolved successfully (the HTTP response was accepted + * - `ok` – the request resolved successfully (the HTTP response was accepted * and the body started streaming). - * - `timeout` — the per-call {@link AbortSignal} fired, or the endpoint + * - `timeout` – the per-call {@link AbortSignal} fired, or the endpoint * returned an HTTP 504 (upstream-reported timeout). Both are semantically * ‘the endpoint did not deliver in time’. - * - `error` — any other failure (4xx other than 504, parser errors, etc.). + * - `error` – any other failure (4xx other than 504, parser errors, etc.). * Neutral with respect to adaptive tightening. */ export type TimeoutOutcome = 'ok' | 'error' | 'timeout'; @@ -26,6 +26,11 @@ export interface AfterRequestContext { outcome: TimeoutOutcome; /** Wall-clock duration of the request attempt, in milliseconds. */ durationMs: number; + /** + * Budget, in milliseconds, that {@link TimeoutPolicy.beforeRequest} + * returned for this attempt. + */ + timeoutMs: number; /** The raw error, when {@link outcome} is `'error'` or `'timeout'`. */ error?: unknown; } @@ -33,7 +38,7 @@ export interface AfterRequestContext { /** * Decides the timeout budget for each SPARQL request and observes the * outcome. Implementations are free to adapt the budget based on recent - * behaviour — see {@link AdaptiveTimeoutPolicy} for the built-in adaptive + * behaviour – see {@link AdaptiveTimeoutPolicy} for the built-in adaptive * implementation, and {@link ConstantTimeoutPolicy} for fixed-budget * behaviour. * @@ -69,9 +74,9 @@ export interface TimeoutTransitionEvent { /** Budget in effect after the transition. */ toTimeoutMs: number; /** - * Number of consecutive timeouts observed at the moment of the - * transition. For a `relax` event, this is the run that ended in the - * `ok` that triggered relaxation. + * Number of consecutive timeouts at the default budget observed at the + * moment of the transition. For a `relax` event, this is the run that + * ended in the `ok` that triggered relaxation. */ consecutiveTimeouts: number; } @@ -106,7 +111,7 @@ export class ConstantTimeoutPolicy implements TimeoutPolicy { } afterRequest(_context: AfterRequestContext): void { - // Constant policy is stateless — outcomes never affect future budgets. + // Constant policy is stateless – outcomes never affect future budgets. } } @@ -128,8 +133,13 @@ export interface AdaptiveTimeoutPolicyOptions { interface EndpointState { tightened: boolean; - /** Consecutive timeouts since the last `ok`. */ + /** Consecutive timeouts at the default budget since the last `ok`. */ consecutiveTimeouts: number; + /** + * Whether the next request re-probes a tightened endpoint at the default + * budget. + */ + probeDue: boolean; } /** @@ -139,7 +149,17 @@ interface EndpointState { * full default budget. A single successful request relaxes the endpoint * back to the default budget. * - * State is in-memory and tied to the policy instance — Pipeline creates a + * A timeout at the tightened budget says nothing about the endpoint’s + * health – a healthy endpoint’s heavy query may simply need longer – so it + * does not count toward tightening. Instead, the next request re-probes + * the endpoint at the default budget: if that probe succeeds, the endpoint + * relaxes; if it times out too, the endpoint stays tightened until the + * next timeout at the tightened budget schedules another probe. A + * healthy-but-slow endpoint therefore recovers after one lost request, + * while an endpoint that keeps timing out at the default budget waits it + * out on only every other request. + * + * State is in-memory and tied to the policy instance – Pipeline creates a * fresh instance per dataset so one offending dataset doesn’t poison the * next. * @@ -184,7 +204,12 @@ export class AdaptiveTimeoutPolicy implements TimeoutPolicy { beforeRequest(context: BeforeRequestContext): number { const state = this.stateFor(context.endpoint); - return state.tightened ? this.options.tightenedMs : this.options.defaultMs; + if (!state.tightened) return this.options.defaultMs; + if (state.probeDue) { + state.probeDue = false; + return this.options.defaultMs; + } + return this.options.tightenedMs; } afterRequest(context: AfterRequestContext): void { @@ -194,6 +219,7 @@ export class AdaptiveTimeoutPolicy implements TimeoutPolicy { const priorCount = state.consecutiveTimeouts; state.consecutiveTimeouts = 0; state.tightened = false; + state.probeDue = false; if (wasTightened) { this.notify('relax', { endpoint: context.endpoint, @@ -205,6 +231,11 @@ export class AdaptiveTimeoutPolicy implements TimeoutPolicy { return; } if (context.outcome === 'timeout') { + if (context.timeoutMs < this.options.defaultMs) { + // Too short a budget to judge the endpoint by: re-probe it instead. + if (state.tightened) state.probeDue = true; + return; + } state.consecutiveTimeouts += 1; if ( !state.tightened && @@ -233,7 +264,7 @@ export class AdaptiveTimeoutPolicy implements TimeoutPolicy { const key = endpoint.toString(); let state = this.states.get(key); if (!state) { - state = { tightened: false, consecutiveTimeouts: 0 }; + state = { tightened: false, consecutiveTimeouts: 0, probeDue: false }; this.states.set(key, state); } return state; diff --git a/packages/pipeline/test/sparql/timeoutPolicy.test.ts b/packages/pipeline/test/sparql/timeoutPolicy.test.ts index 075c19d8d..57820a571 100644 --- a/packages/pipeline/test/sparql/timeoutPolicy.test.ts +++ b/packages/pipeline/test/sparql/timeoutPolicy.test.ts @@ -21,6 +21,7 @@ describe('ConstantTimeoutPolicy', () => { endpoint: endpointA, outcome: 'timeout', durationMs: 500, + timeoutMs: 1000, }); expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); }); @@ -116,6 +117,7 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + timeoutMs: 1000, }); expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); }); @@ -130,12 +132,14 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + timeoutMs: 1000, }); expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); policy.afterRequest({ endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + timeoutMs: 1000, }); expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); }); @@ -150,17 +154,20 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + timeoutMs: 1000, }); policy.afterRequest({ endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + timeoutMs: 1000, }); expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); policy.afterRequest({ endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + timeoutMs: 1000, }); expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); }); @@ -176,6 +183,7 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + timeoutMs: 1000, }); } expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); @@ -191,12 +199,14 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + timeoutMs: 1000, }); expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); policy.afterRequest({ endpoint: endpointA, outcome: 'ok', durationMs: 80, + timeoutMs: 1000, }); expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); }); @@ -211,16 +221,19 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + timeoutMs: 1000, }); policy.afterRequest({ endpoint: endpointA, outcome: 'ok', durationMs: 80, + timeoutMs: 1000, }); policy.afterRequest({ endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + timeoutMs: 1000, }); expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); }); @@ -235,12 +248,14 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'error', durationMs: 50, + timeoutMs: 1000, }); expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); policy.afterRequest({ endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + timeoutMs: 1000, }); expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); // An `error` while tightened keeps state tightened. @@ -248,6 +263,7 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'error', durationMs: 50, + timeoutMs: 1000, }); expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); }); @@ -262,10 +278,133 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + timeoutMs: 1000, }); expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); expect(policy.beforeRequest({ endpoint: endpointB })).toBe(1000); }); + + describe('re-probing a tightened endpoint', () => { + function tightenedPolicy(): AdaptiveTimeoutPolicy { + const policy = new AdaptiveTimeoutPolicy({ + defaultMs: 1000, + tightenedMs: 100, + tightenAfterTimeouts: 1, + }); + policy.afterRequest({ + endpoint: endpointA, + outcome: 'timeout', + durationMs: 1000, + timeoutMs: 1000, + }); + return policy; + } + + it('gives the next request the default budget after a timeout at the tightened budget', () => { + const policy = tightenedPolicy(); + expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); + policy.afterRequest({ + endpoint: endpointA, + outcome: 'timeout', + durationMs: 100, + timeoutMs: 100, + }); + expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); + // One probe at a time: concurrent requests keep the tightened budget. + expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); + }); + + it('relaxes a healthy-but-slow endpoint when the probe succeeds', () => { + const policy = tightenedPolicy(); + const onRelax = vi.fn(); + policy.subscribe({ onRelax }); + policy.afterRequest({ + endpoint: endpointA, + outcome: 'timeout', + durationMs: 100, + timeoutMs: 100, + }); + expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); + policy.afterRequest({ + endpoint: endpointA, + outcome: 'ok', + durationMs: 900, + timeoutMs: 1000, + }); + expect(onRelax).toHaveBeenCalledTimes(1); + expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); + expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); + }); + + it('stays tightened when the probe times out at the default budget', () => { + const policy = tightenedPolicy(); + const onTighten = vi.fn(); + const onRelax = vi.fn(); + policy.subscribe({ onTighten, onRelax }); + policy.afterRequest({ + endpoint: endpointA, + outcome: 'timeout', + durationMs: 100, + timeoutMs: 100, + }); + expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); + policy.afterRequest({ + endpoint: endpointA, + outcome: 'timeout', + durationMs: 1000, + timeoutMs: 1000, + }); + expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); + expect(onTighten).not.toHaveBeenCalled(); + expect(onRelax).not.toHaveBeenCalled(); + }); + + it('ignores a late timeout at the tightened budget after the endpoint relaxed', () => { + const policy = tightenedPolicy(); + expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); + policy.afterRequest({ + endpoint: endpointA, + outcome: 'ok', + durationMs: 50, + timeoutMs: 100, + }); + policy.afterRequest({ + endpoint: endpointA, + outcome: 'timeout', + durationMs: 100, + timeoutMs: 100, + }); + expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); + expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); + }); + + it('does not count timeouts at the tightened budget toward tightening', () => { + const policy = new AdaptiveTimeoutPolicy({ + defaultMs: 1000, + tightenedMs: 100, + tightenAfterTimeouts: 2, + }); + const onRelax = vi.fn(); + policy.subscribe({ onRelax }); + for (const timeoutMs of [1000, 1000, 100, 1000]) { + policy.afterRequest({ + endpoint: endpointA, + outcome: 'timeout', + durationMs: timeoutMs, + timeoutMs, + }); + } + policy.afterRequest({ + endpoint: endpointA, + outcome: 'ok', + durationMs: 900, + timeoutMs: 1000, + }); + expect(onRelax).toHaveBeenCalledWith( + expect.objectContaining({ consecutiveTimeouts: 3 }), + ); + }); + }); }); describe('transition events', () => { @@ -282,12 +421,14 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + timeoutMs: 1000, }); expect(onTighten).not.toHaveBeenCalled(); policy.afterRequest({ endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + timeoutMs: 1000, }); expect(onTighten).toHaveBeenCalledTimes(1); expect(onTighten).toHaveBeenCalledWith( @@ -313,16 +454,19 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + timeoutMs: 1000, }); policy.afterRequest({ endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + timeoutMs: 1000, }); policy.afterRequest({ endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + timeoutMs: 1000, }); expect(onTighten).toHaveBeenCalledTimes(1); }); @@ -339,11 +483,13 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + timeoutMs: 1000, }); policy.afterRequest({ endpoint: endpointA, outcome: 'ok', durationMs: 80, + timeoutMs: 1000, }); expect(onRelax).toHaveBeenCalledTimes(1); expect(onRelax).toHaveBeenCalledWith( @@ -367,6 +513,7 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'ok', durationMs: 80, + timeoutMs: 1000, }); expect(onRelax).not.toHaveBeenCalled(); }); @@ -384,6 +531,7 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + timeoutMs: 1000, }); expect(onTighten).not.toHaveBeenCalled(); }); @@ -414,6 +562,7 @@ describe('adaptiveTimeoutPolicy factory', () => { endpoint: endpointA, outcome: 'timeout', durationMs: 1000, + timeoutMs: 1000, }); const b = factory(); expect(b.beforeRequest({ endpoint: endpointA })).toBe(1000); From 6347299b6c0ef776b60c54c7514b277cab41988d Mon Sep 17 00:00:00 2001 From: David de Boer Date: Sun, 15 Nov 2026 12:51:45 +0100 Subject: [PATCH 2/3] fix(pipeline): hand out one re-probe at a time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Track a probe in flight so timeouts at the tightened budget can’t schedule a second probe while one is still running; any outcome at the default budget clears it. Assert that readers and selectors report the budget they were given, and state the dead-endpoint cost of re-probing plainly. --- docs/reference/pipeline.md | 2 +- packages/pipeline/src/sparql/timeoutPolicy.ts | 32 +++++++++--- packages/pipeline/test/sparql/reader.test.ts | 15 +++--- .../pipeline/test/sparql/selector.test.ts | 8 +-- .../test/sparql/timeoutPolicy.test.ts | 50 +++++++++++++++++++ packages/pipeline/vite.config.ts | 6 +-- 6 files changed, 90 insertions(+), 23 deletions(-) diff --git a/docs/reference/pipeline.md b/docs/reference/pipeline.md index 700322f62..c0a0e2a2a 100644 --- a/docs/reference/pipeline.md +++ b/docs/reference/pipeline.md @@ -346,7 +346,7 @@ By default, every SPARQL request uses the same 5-minute budget. When a pipeline A `TimeoutPolicy` decides the budget for each SPARQL request and observes the outcome. Two are built in: - **`ConstantTimeoutPolicy(timeoutMs)`** – returns the same budget for every request. The implicit default when `PipelineOptions.timeout` is omitted (`constantTimeoutPolicy(300_000)`). -- **`AdaptiveTimeoutPolicy({ defaultMs, tightenedMs, tightenAfterTimeouts })`** – per-endpoint state machine. Each endpoint is either _healthy_ (use `defaultMs`) or _tightened_ (use `tightenedMs`). After `tightenAfterTimeouts` consecutive `timeout` outcomes at `defaultMs` the endpoint flips to _tightened_; a single `ok` flips it back to _healthy_. A timeout at `tightenedMs` doesn’t count: it may just be a heavy query on a healthy endpoint, so the next request re-probes at `defaultMs`. If that probe succeeds the endpoint relaxes; if it times out too, the endpoint stays tightened. A healthy-but-slow endpoint therefore loses one request rather than every heavy query until a cheap one happens to pass, while an endpoint that keeps timing out at `defaultMs` waits out the full budget on only every other request. +- **`AdaptiveTimeoutPolicy({ defaultMs, tightenedMs, tightenAfterTimeouts })`** – per-endpoint state machine. Each endpoint is either _healthy_ (use `defaultMs`) or _tightened_ (use `tightenedMs`). After `tightenAfterTimeouts` consecutive `timeout` outcomes at `defaultMs` the endpoint flips to _tightened_; a single `ok` flips it back to _healthy_. A timeout at `tightenedMs` doesn’t count: it may just be a heavy query on a healthy endpoint, so the next request re-probes at `defaultMs`, one probe at a time. If that probe succeeds the endpoint relaxes; if it fails, the endpoint stays tightened. A healthy-but-slow endpoint therefore loses one request rather than every heavy query until a cheap one happens to pass. The price falls on an endpoint that is genuinely down: without re-probing, a tightened endpoint never waited out the full budget; now it does on about every other request. `PipelineOptions.timeout` accepts a `() => TimeoutPolicy` factory. The pipeline invokes it once per dataset, so policy state resets between datasets and one bad dataset can’t poison the next: diff --git a/packages/pipeline/src/sparql/timeoutPolicy.ts b/packages/pipeline/src/sparql/timeoutPolicy.ts index 9cf71bce1..c8ee9062c 100644 --- a/packages/pipeline/src/sparql/timeoutPolicy.ts +++ b/packages/pipeline/src/sparql/timeoutPolicy.ts @@ -140,6 +140,11 @@ interface EndpointState { * budget. */ probeDue: boolean; + /** + * Whether a probe has been handed out and its outcome not yet reported, + * so concurrent requests keep the tightened budget meanwhile. + */ + probeInFlight: boolean; } /** @@ -152,12 +157,14 @@ interface EndpointState { * A timeout at the tightened budget says nothing about the endpoint’s * health – a healthy endpoint’s heavy query may simply need longer – so it * does not count toward tightening. Instead, the next request re-probes - * the endpoint at the default budget: if that probe succeeds, the endpoint - * relaxes; if it times out too, the endpoint stays tightened until the - * next timeout at the tightened budget schedules another probe. A - * healthy-but-slow endpoint therefore recovers after one lost request, - * while an endpoint that keeps timing out at the default budget waits it - * out on only every other request. + * the endpoint at the default budget, one probe at a time: concurrent + * requests keep the tightened budget until the probe’s outcome is + * reported. If the probe succeeds, the endpoint relaxes; if it fails, the + * endpoint stays tightened until the next timeout at the tightened budget + * schedules another probe. A healthy-but-slow endpoint therefore recovers + * after one lost request. The price is paid by an endpoint that is + * genuinely down: without re-probing, a tightened endpoint never waited + * out the full default budget; now it does on about every other request. * * State is in-memory and tied to the policy instance – Pipeline creates a * fresh instance per dataset so one offending dataset doesn’t poison the @@ -207,6 +214,7 @@ export class AdaptiveTimeoutPolicy implements TimeoutPolicy { if (!state.tightened) return this.options.defaultMs; if (state.probeDue) { state.probeDue = false; + state.probeInFlight = true; return this.options.defaultMs; } return this.options.tightenedMs; @@ -214,6 +222,9 @@ export class AdaptiveTimeoutPolicy implements TimeoutPolicy { afterRequest(context: AfterRequestContext): void { const state = this.stateFor(context.endpoint); + if (context.timeoutMs === this.options.defaultMs) { + state.probeInFlight = false; + } if (context.outcome === 'ok') { const wasTightened = state.tightened; const priorCount = state.consecutiveTimeouts; @@ -233,7 +244,7 @@ export class AdaptiveTimeoutPolicy implements TimeoutPolicy { if (context.outcome === 'timeout') { if (context.timeoutMs < this.options.defaultMs) { // Too short a budget to judge the endpoint by: re-probe it instead. - if (state.tightened) state.probeDue = true; + if (state.tightened && !state.probeInFlight) state.probeDue = true; return; } state.consecutiveTimeouts += 1; @@ -264,7 +275,12 @@ export class AdaptiveTimeoutPolicy implements TimeoutPolicy { const key = endpoint.toString(); let state = this.states.get(key); if (!state) { - state = { tightened: false, consecutiveTimeouts: 0, probeDue: false }; + state = { + tightened: false, + consecutiveTimeouts: 0, + probeDue: false, + probeInFlight: false, + }; this.states.set(key, state); } return state; diff --git a/packages/pipeline/test/sparql/reader.test.ts b/packages/pipeline/test/sparql/reader.test.ts index c0eeacb28..6596a22bf 100644 --- a/packages/pipeline/test/sparql/reader.test.ts +++ b/packages/pipeline/test/sparql/reader.test.ts @@ -811,6 +811,7 @@ describe('SparqlConstructReader', () => { expect.objectContaining({ endpoint: distribution.accessUrl, outcome: 'ok', + timeoutMs: 1234, }), ); }); @@ -838,7 +839,7 @@ describe('SparqlConstructReader', () => { ).rejects.toThrow('504'); expect(policy.afterRequest).toHaveBeenCalledWith( - expect.objectContaining({ outcome: 'timeout' }), + expect.objectContaining({ outcome: 'timeout', timeoutMs: 1234 }), ); }); @@ -862,7 +863,7 @@ describe('SparqlConstructReader', () => { ).rejects.toThrow(); expect(policy.afterRequest).toHaveBeenCalledWith( - expect.objectContaining({ outcome: 'timeout' }), + expect.objectContaining({ outcome: 'timeout', timeoutMs: 1234 }), ); }); @@ -886,7 +887,7 @@ describe('SparqlConstructReader', () => { ).rejects.toThrow(); expect(policy.afterRequest).toHaveBeenCalledWith( - expect.objectContaining({ outcome: 'timeout' }), + expect.objectContaining({ outcome: 'timeout', timeoutMs: 1234 }), ); }); @@ -912,7 +913,7 @@ describe('SparqlConstructReader', () => { ).rejects.toThrow(); expect(policy.afterRequest).toHaveBeenCalledWith( - expect.objectContaining({ outcome: 'error' }), + expect.objectContaining({ outcome: 'error', timeoutMs: 1234 }), ); }); @@ -942,11 +943,11 @@ describe('SparqlConstructReader', () => { expect(policy.afterRequest).toHaveBeenCalledTimes(2); expect(policy.afterRequest).toHaveBeenNthCalledWith( 1, - expect.objectContaining({ outcome: 'timeout' }), + expect.objectContaining({ outcome: 'timeout', timeoutMs: 1234 }), ); expect(policy.afterRequest).toHaveBeenNthCalledWith( 2, - expect.objectContaining({ outcome: 'ok' }), + expect.objectContaining({ outcome: 'ok', timeoutMs: 1234 }), ); }); @@ -1003,7 +1004,7 @@ describe('SparqlConstructReader', () => { ).rejects.toThrow(); expect(policy.afterRequest).toHaveBeenCalledWith( - expect.objectContaining({ outcome: 'timeout' }), + expect.objectContaining({ outcome: 'timeout', timeoutMs: 10 }), ); expect(slowFetch).toHaveBeenCalled(); const init = slowFetch.mock.calls[0][1]; diff --git a/packages/pipeline/test/sparql/selector.test.ts b/packages/pipeline/test/sparql/selector.test.ts index ecfb94f0a..d5ae0e7d0 100644 --- a/packages/pipeline/test/sparql/selector.test.ts +++ b/packages/pipeline/test/sparql/selector.test.ts @@ -681,7 +681,7 @@ describe('SparqlItemSelector', () => { }); expect(policy.afterRequest).toHaveBeenCalledTimes(2); expect(policy.afterRequest).toHaveBeenCalledWith( - expect.objectContaining({ outcome: 'ok' }), + expect.objectContaining({ outcome: 'ok', timeoutMs: 5000 }), ); }); @@ -712,7 +712,7 @@ describe('SparqlItemSelector', () => { await expect(iterate()).rejects.toThrow('504'); expect(policy.afterRequest).toHaveBeenCalledWith( - expect.objectContaining({ outcome: 'timeout' }), + expect.objectContaining({ outcome: 'timeout', timeoutMs: 5000 }), ); }); @@ -741,7 +741,7 @@ describe('SparqlItemSelector', () => { await expect(iterate()).rejects.toThrow('socket hang up'); expect(policy.afterRequest).toHaveBeenCalledTimes(1); expect(policy.afterRequest).toHaveBeenCalledWith( - expect.objectContaining({ outcome: 'error' }), + expect.objectContaining({ outcome: 'error', timeoutMs: 5000 }), ); }); @@ -772,7 +772,7 @@ describe('SparqlItemSelector', () => { await expect(iterate()).rejects.toThrow(); expect(policy.afterRequest).toHaveBeenCalledWith( - expect.objectContaining({ outcome: 'error' }), + expect.objectContaining({ outcome: 'error', timeoutMs: 5000 }), ); }); diff --git a/packages/pipeline/test/sparql/timeoutPolicy.test.ts b/packages/pipeline/test/sparql/timeoutPolicy.test.ts index 57820a571..89b47299c 100644 --- a/packages/pipeline/test/sparql/timeoutPolicy.test.ts +++ b/packages/pipeline/test/sparql/timeoutPolicy.test.ts @@ -314,6 +314,56 @@ describe('AdaptiveTimeoutPolicy', () => { expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); }); + it('hands out no second probe while one is in flight, whatever its outcome', () => { + const policy = tightenedPolicy(); + const concurrentBudgets = [ + policy.beforeRequest({ endpoint: endpointA }), + policy.beforeRequest({ endpoint: endpointA }), + ]; + expect(concurrentBudgets).toEqual([100, 100]); + policy.afterRequest({ + endpoint: endpointA, + outcome: 'timeout', + durationMs: 100, + timeoutMs: 100, + }); + expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); + + // Concurrent requests at the tightened budget time out while the + // probe is in flight: none of them schedules another probe. + policy.afterRequest({ + endpoint: endpointA, + outcome: 'timeout', + durationMs: 100, + timeoutMs: 100, + }); + expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); + policy.afterRequest({ + endpoint: endpointA, + outcome: 'timeout', + durationMs: 100, + timeoutMs: 100, + }); + expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); + + // The probe fails with an error: no longer in flight, so the next + // tightened timeout schedules a new probe. + policy.afterRequest({ + endpoint: endpointA, + outcome: 'error', + durationMs: 50, + timeoutMs: 1000, + }); + expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); + policy.afterRequest({ + endpoint: endpointA, + outcome: 'timeout', + durationMs: 100, + timeoutMs: 100, + }); + expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); + }); + it('relaxes a healthy-but-slow endpoint when the probe succeeds', () => { const policy = tightenedPolicy(); const onRelax = vi.fn(); diff --git a/packages/pipeline/vite.config.ts b/packages/pipeline/vite.config.ts index 2e9903d83..9563d61d9 100644 --- a/packages/pipeline/vite.config.ts +++ b/packages/pipeline/vite.config.ts @@ -12,9 +12,9 @@ export default mergeConfig( thresholds: { autoUpdate: true, functions: 99.4, - lines: 98.78, - branches: 96.67, - statements: 98.62, + lines: 98.79, + branches: 96.72, + statements: 98.64, }, }, }, From 6fa5ee9ed27c4c5c948e5b9622d164c92bdb99f8 Mon Sep 17 00:00:00 2001 From: David de Boer Date: Mon, 16 Nov 2026 20:02:39 +0100 Subject: [PATCH 3/3] fix(pipeline)!: tell the probe by its budget and back off between probes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Have beforeRequest return a RequestBudget and afterRequest receive that same object, so the adaptive policy recognises its probe by identity; a request budgeted before tightening no longer releases the probe slot - Double the gap between probes after each timed-out probe (1, 2, 4… timeouts at the tightened budget, up to 64), so an endpoint that is down rarely waits out the full default budget; a probe that errors keeps the gap - Model endpoint health and probe state as one discriminated union instead of loose booleans BREAKING CHANGE: TimeoutPolicy.beforeRequest returns a RequestBudget ({ timeoutMs }) instead of a number, and AfterRequestContext.timeoutMs is replaced by budget, which must be the object beforeRequest returned. Custom policies return { timeoutMs } and read context.budget.timeoutMs. --- .../adapt-timeouts-to-endpoint-health.md | 2 +- docs/reference/pipeline.md | 4 +- packages/pipeline/src/sparql/index.ts | 1 + packages/pipeline/src/sparql/reader.ts | 8 +- packages/pipeline/src/sparql/selector.ts | 11 +- packages/pipeline/src/sparql/timeoutPolicy.ts | 185 ++++++--- packages/pipeline/test/pipeline.test.ts | 6 +- packages/pipeline/test/sparql/reader.test.ts | 42 +- .../pipeline/test/sparql/selector.test.ts | 23 +- .../test/sparql/timeoutPolicy.test.ts | 392 ++++++++++-------- packages/pipeline/vite.config.ts | 2 +- 11 files changed, 410 insertions(+), 266 deletions(-) diff --git a/docs/guide/adapt-timeouts-to-endpoint-health.md b/docs/guide/adapt-timeouts-to-endpoint-health.md index dca4e23b7..cae6f3686 100644 --- a/docs/guide/adapt-timeouts-to-endpoint-health.md +++ b/docs/guide/adapt-timeouts-to-endpoint-health.md @@ -23,7 +23,7 @@ const pipeline = new Pipeline({ - `tightenedMs` – the reduced budget once an endpoint is tightened; must be less than `defaultMs`. - `tightenAfterTimeouts` – how many consecutive timeouts at `defaultMs` flip an endpoint from healthy to tightened. A single successful request flips it back. -A timeout at the tightened budget doesn’t keep the endpoint tightened: the next request re-probes at `defaultMs`, so a healthy endpoint whose heavy queries need longer than `tightenedMs` relaxes again on its own. +A timeout at the tightened budget says nothing about the endpoint’s health, so the policy re-probes it: the next request gets `defaultMs` while concurrent ones keep `tightenedMs`. If the probe succeeds, the endpoint relaxes, so a healthy endpoint whose heavy queries need longer than `tightenedMs` recovers on its own. Each probe that times out doubles the number of tightened timeouts before the next one, up to 64, so an endpoint that is really down rarely costs the full `defaultMs`. The pipeline invokes the factory once per dataset, so policy state resets at each dataset boundary and one bad dataset can’t poison the next. diff --git a/docs/reference/pipeline.md b/docs/reference/pipeline.md index c0a0e2a2a..f06ee3236 100644 --- a/docs/reference/pipeline.md +++ b/docs/reference/pipeline.md @@ -346,7 +346,7 @@ By default, every SPARQL request uses the same 5-minute budget. When a pipeline A `TimeoutPolicy` decides the budget for each SPARQL request and observes the outcome. Two are built in: - **`ConstantTimeoutPolicy(timeoutMs)`** – returns the same budget for every request. The implicit default when `PipelineOptions.timeout` is omitted (`constantTimeoutPolicy(300_000)`). -- **`AdaptiveTimeoutPolicy({ defaultMs, tightenedMs, tightenAfterTimeouts })`** – per-endpoint state machine. Each endpoint is either _healthy_ (use `defaultMs`) or _tightened_ (use `tightenedMs`). After `tightenAfterTimeouts` consecutive `timeout` outcomes at `defaultMs` the endpoint flips to _tightened_; a single `ok` flips it back to _healthy_. A timeout at `tightenedMs` doesn’t count: it may just be a heavy query on a healthy endpoint, so the next request re-probes at `defaultMs`, one probe at a time. If that probe succeeds the endpoint relaxes; if it fails, the endpoint stays tightened. A healthy-but-slow endpoint therefore loses one request rather than every heavy query until a cheap one happens to pass. The price falls on an endpoint that is genuinely down: without re-probing, a tightened endpoint never waited out the full budget; now it does on about every other request. +- **`AdaptiveTimeoutPolicy({ defaultMs, tightenedMs, tightenAfterTimeouts })`** – per-endpoint state machine. Each endpoint is either _healthy_ (use `defaultMs`) or _tightened_ (use `tightenedMs`). After `tightenAfterTimeouts` consecutive `timeout` outcomes at `defaultMs` the endpoint flips to _tightened_; a single `ok` flips it back to _healthy_. A timeout at `tightenedMs` doesn’t count: it may just be a heavy query on a healthy endpoint, so the next request re-probes at `defaultMs`, one probe at a time. If that probe succeeds the endpoint relaxes; if it times out, the endpoint stays tightened and the gap until the next probe doubles – 1, 2, 4… timeouts at `tightenedMs`, up to 64 – while a probe that fails with another error keeps the gap. A healthy-but-slow endpoint therefore loses one request rather than every heavy query until a cheap one happens to pass, and an endpoint that is genuinely down waits out the full budget on fewer and fewer requests. `PipelineOptions.timeout` accepts a `() => TimeoutPolicy` factory. The pipeline invokes it once per dataset, so policy state resets between datasets and one bad dataset can’t poison the next: @@ -373,7 +373,7 @@ Outcomes are classified as: Transitions are reported as `timeoutTightened` / `timeoutRelaxed` events; `ConsoleReporter` prints them as `↘ Tightened` / `↗ Relaxed` lines so operators can tell a fast-failed stage from an unexpected speedup. -Implement `TimeoutPolicy` directly for custom strategies (closing over shared state in the factory if you want it to span datasets). +Implement `TimeoutPolicy` directly for custom strategies (closing over shared state in the factory if you want it to span datasets). `beforeRequest` returns a `RequestBudget`, and the caller hands that same object back to `afterRequest` as `budget`, so a policy can tell by identity which request is reporting. Timeouts live at the pipeline level – neither `SparqlConstructReader` nor `SparqlItemSelector` accept their own `timeout` option. Per-endpoint state belongs in the adaptive policy, and per-stage budgets aren’t supported. Reusable stage facades ([@lde/pipeline-void](./pipeline-void), [@lde/pipeline-shacl-sampler](./pipeline-shacl-sampler)) follow the same convention. diff --git a/packages/pipeline/src/sparql/index.ts b/packages/pipeline/src/sparql/index.ts index eb37d89e3..784a6a541 100644 --- a/packages/pipeline/src/sparql/index.ts +++ b/packages/pipeline/src/sparql/index.ts @@ -26,6 +26,7 @@ export { type AdaptiveTimeoutPolicyOptions, type AfterRequestContext, type BeforeRequestContext, + type RequestBudget, type TimeoutOutcome, type TimeoutPolicy, type TimeoutPolicyObserver, diff --git a/packages/pipeline/src/sparql/reader.ts b/packages/pipeline/src/sparql/reader.ts index ec6d4e46d..6b4f9dded 100644 --- a/packages/pipeline/src/sparql/reader.ts +++ b/packages/pipeline/src/sparql/reader.ts @@ -367,10 +367,10 @@ export class SparqlConstructReader implements Reader { policy: TimeoutPolicy, response: ResponseNotes, ): Promise> { - const timeoutMs = policy.beforeRequest({ endpoint: endpointUrl }); - const fetcher = this.fetcherForAttempt(timeoutMs, response); + const budget = policy.beforeRequest({ endpoint: endpointUrl }); const start = Date.now(); try { + const fetcher = this.fetcherForAttempt(budget.timeoutMs, response); const quads = await this.fetchQuads( fetcher, endpointUrl.toString(), @@ -380,7 +380,7 @@ export class SparqlConstructReader implements Reader { endpoint: endpointUrl, outcome: 'ok', durationMs: Date.now() - start, - timeoutMs, + budget, }); return quads; } catch (error) { @@ -388,7 +388,7 @@ export class SparqlConstructReader implements Reader { endpoint: endpointUrl, outcome: classifyOutcome(error), durationMs: Date.now() - start, - timeoutMs, + budget, error, }); throw error; diff --git a/packages/pipeline/src/sparql/selector.ts b/packages/pipeline/src/sparql/selector.ts index 2272b57b7..684e8f2d3 100644 --- a/packages/pipeline/src/sparql/selector.ts +++ b/packages/pipeline/src/sparql/selector.ts @@ -280,11 +280,12 @@ export class SparqlItemSelector implements ItemSelector { paginatedQuery: string, policy: TimeoutPolicy, ): Promise[]> { - const timeoutMs = policy.beforeRequest({ endpoint }); - const fetcher = - this.userFetcher ?? new SparqlEndpointFetcher({ timeout: timeoutMs }); + const budget = policy.beforeRequest({ endpoint }); const start = Date.now(); try { + const fetcher = + this.userFetcher ?? + new SparqlEndpointFetcher({ timeout: budget.timeoutMs }); const stream = (await fetcher.fetchBindings( endpoint.toString(), paginatedQuery, @@ -297,7 +298,7 @@ export class SparqlItemSelector implements ItemSelector { endpoint, outcome: 'ok', durationMs: Date.now() - start, - timeoutMs, + budget, }); return page; } catch (error) { @@ -305,7 +306,7 @@ export class SparqlItemSelector implements ItemSelector { endpoint, outcome: classifyOutcome(error), durationMs: Date.now() - start, - timeoutMs, + budget, error, }); throw error; diff --git a/packages/pipeline/src/sparql/timeoutPolicy.ts b/packages/pipeline/src/sparql/timeoutPolicy.ts index c8ee9062c..aba0be0b4 100644 --- a/packages/pipeline/src/sparql/timeoutPolicy.ts +++ b/packages/pipeline/src/sparql/timeoutPolicy.ts @@ -18,6 +18,17 @@ export interface BeforeRequestContext { endpoint: URL; } +/** + * Budget that {@link TimeoutPolicy.beforeRequest} hands out for one request + * attempt. Callers pass this same object back as + * {@link AfterRequestContext.budget}, so a policy can tell by identity which + * attempt is reporting. + */ +export interface RequestBudget { + /** Timeout to apply to the attempt, in milliseconds. */ + readonly timeoutMs: number; +} + /** Context passed to {@link TimeoutPolicy.afterRequest}. */ export interface AfterRequestContext { /** Endpoint URL the request was sent to. */ @@ -27,10 +38,10 @@ export interface AfterRequestContext { /** Wall-clock duration of the request attempt, in milliseconds. */ durationMs: number; /** - * Budget, in milliseconds, that {@link TimeoutPolicy.beforeRequest} - * returned for this attempt. + * The object {@link TimeoutPolicy.beforeRequest} returned for this + * attempt – the same object, not a copy. */ - timeoutMs: number; + budget: RequestBudget; /** The raw error, when {@link outcome} is `'error'` or `'timeout'`. */ error?: unknown; } @@ -47,14 +58,15 @@ export interface AfterRequestContext { */ export interface TimeoutPolicy { /** - * Returns the timeout (in milliseconds) to apply to the upcoming request. - * Called once per attempt, retries included, so a retry can already use a - * tightened budget. + * Returns the budget to apply to the upcoming request. Called once per + * attempt, retries included, so a retry can already use a tightened + * budget. */ - beforeRequest(context: BeforeRequestContext): number; + beforeRequest(context: BeforeRequestContext): RequestBudget; /** - * Reports the outcome of the request that {@link beforeRequest} budgeted. - * Called once per attempt, regardless of outcome. + * Reports the outcome of the request that {@link beforeRequest} budgeted, + * passing back the budget it returned. Called once per attempt, + * regardless of outcome. */ afterRequest(context: AfterRequestContext): void; /** @@ -106,8 +118,8 @@ export class ConstantTimeoutPolicy implements TimeoutPolicy { } } - beforeRequest(_context: BeforeRequestContext): number { - return this.timeoutMs; + beforeRequest(_context: BeforeRequestContext): RequestBudget { + return { timeoutMs: this.timeoutMs }; } afterRequest(_context: AfterRequestContext): void { @@ -131,22 +143,6 @@ export interface AdaptiveTimeoutPolicyOptions { tightenAfterTimeouts: number; } -interface EndpointState { - tightened: boolean; - /** Consecutive timeouts at the default budget since the last `ok`. */ - consecutiveTimeouts: number; - /** - * Whether the next request re-probes a tightened endpoint at the default - * budget. - */ - probeDue: boolean; - /** - * Whether a probe has been handed out and its outcome not yet reported, - * so concurrent requests keep the tightened budget meanwhile. - */ - probeInFlight: boolean; -} - /** * Adaptive per-endpoint policy: after {@link AdaptiveTimeoutPolicyOptions.tightenAfterTimeouts} * consecutive timeouts on the same endpoint, subsequent requests use the @@ -156,15 +152,16 @@ interface EndpointState { * * A timeout at the tightened budget says nothing about the endpoint’s * health – a healthy endpoint’s heavy query may simply need longer – so it - * does not count toward tightening. Instead, the next request re-probes - * the endpoint at the default budget, one probe at a time: concurrent - * requests keep the tightened budget until the probe’s outcome is - * reported. If the probe succeeds, the endpoint relaxes; if it fails, the - * endpoint stays tightened until the next timeout at the tightened budget - * schedules another probe. A healthy-but-slow endpoint therefore recovers - * after one lost request. The price is paid by an endpoint that is - * genuinely down: without re-probing, a tightened endpoint never waited - * out the full default budget; now it does on about every other request. + * does not count toward tightening. Instead, it brings a re-probe at the + * default budget closer, one probe at a time: concurrent requests keep the + * tightened budget until the probe reports. The first timeout at the + * tightened budget makes a probe due. If the probe succeeds, the endpoint + * relaxes; if it times out, the gap until the next probe doubles – 1, 2, 4… + * timeouts at the tightened budget, up to 64. A probe that fails with an + * error keeps the current gap. A healthy-but-slow endpoint therefore + * recovers after one lost request, while an endpoint that is genuinely + * down waits out the full default budget only on a shrinking share of + * requests. * * State is in-memory and tied to the policy instance – Pipeline creates a * fresh instance per dataset so one offending dataset doesn’t poison the @@ -209,29 +206,27 @@ export class AdaptiveTimeoutPolicy implements TimeoutPolicy { } } - beforeRequest(context: BeforeRequestContext): number { - const state = this.stateFor(context.endpoint); - if (!state.tightened) return this.options.defaultMs; - if (state.probeDue) { - state.probeDue = false; - state.probeInFlight = true; - return this.options.defaultMs; + beforeRequest(context: BeforeRequestContext): RequestBudget { + const { health } = this.stateFor(context.endpoint); + if (health.status === 'healthy') { + return { timeoutMs: this.options.defaultMs }; } - return this.options.tightenedMs; + if (health.probe.status === 'due') { + const budget = { timeoutMs: this.options.defaultMs }; + health.probe = { status: 'inFlight', budget }; + return budget; + } + return { timeoutMs: this.options.tightenedMs }; } afterRequest(context: AfterRequestContext): void { const state = this.stateFor(context.endpoint); - if (context.timeoutMs === this.options.defaultMs) { - state.probeInFlight = false; - } + const { health } = state; if (context.outcome === 'ok') { - const wasTightened = state.tightened; const priorCount = state.consecutiveTimeouts; state.consecutiveTimeouts = 0; - state.tightened = false; - state.probeDue = false; - if (wasTightened) { + state.health = { status: 'healthy' }; + if (health.status === 'tightened') { this.notify('relax', { endpoint: context.endpoint, fromTimeoutMs: this.options.tightenedMs, @@ -241,18 +236,45 @@ export class AdaptiveTimeoutPolicy implements TimeoutPolicy { } return; } + if ( + health.status === 'tightened' && + health.probe.status === 'inFlight' && + health.probe.budget === context.budget + ) { + if (context.outcome === 'timeout') health.failedProbes += 1; + health.probe = { + status: 'waiting', + timeoutsUntilDue: timeoutsBetweenProbes(health.failedProbes), + }; + } if (context.outcome === 'timeout') { - if (context.timeoutMs < this.options.defaultMs) { - // Too short a budget to judge the endpoint by: re-probe it instead. - if (state.tightened && !state.probeInFlight) state.probeDue = true; + if (context.budget.timeoutMs < this.options.defaultMs) { + // Too short a budget to judge the endpoint by: bring a re-probe + // closer instead. + if ( + health.status === 'tightened' && + health.probe.status === 'waiting' + ) { + health.probe.timeoutsUntilDue -= 1; + if (health.probe.timeoutsUntilDue === 0) { + health.probe = { status: 'due' }; + } + } return; } state.consecutiveTimeouts += 1; if ( - !state.tightened && + health.status === 'healthy' && state.consecutiveTimeouts >= this.options.tightenAfterTimeouts ) { - state.tightened = true; + state.health = { + status: 'tightened', + probe: { + status: 'waiting', + timeoutsUntilDue: timeoutsBetweenProbes(0), + }, + failedProbes: 0, + }; this.notify('tighten', { endpoint: context.endpoint, fromTimeoutMs: this.options.defaultMs, @@ -275,12 +297,7 @@ export class AdaptiveTimeoutPolicy implements TimeoutPolicy { const key = endpoint.toString(); let state = this.states.get(key); if (!state) { - state = { - tightened: false, - consecutiveTimeouts: 0, - probeDue: false, - probeInFlight: false, - }; + state = { health: { status: 'healthy' }, consecutiveTimeouts: 0 }; this.states.set(key, state); } return state; @@ -325,3 +342,47 @@ export function adaptiveTimeoutPolicy( new AdaptiveTimeoutPolicy(options); return () => new AdaptiveTimeoutPolicy(options); } + +/** + * Upper bound on the number of timeouts at the tightened budget between two + * probes, so an endpoint that comes back late in a long run still gets + * re-probed. + */ +const maxTimeoutsBetweenProbes = 64; + +/** + * Timeouts at the tightened budget until the next probe is due, doubling + * with each failed probe. + */ +function timeoutsBetweenProbes(failedProbes: number): number { + return Math.min(2 ** failedProbes, maxTimeoutsBetweenProbes); +} + +/** + * Where a tightened endpoint’s next re-probe stands. + * + * - `waiting` – that many more timeouts at the tightened budget make a probe + * due. + * - `due` – the next request is the probe. + * - `inFlight` – the probe was handed out with this budget and has not + * reported yet; concurrent requests keep the tightened budget meanwhile. + */ +type Probe = + | { status: 'waiting'; timeoutsUntilDue: number } + | { status: 'due' } + | { status: 'inFlight'; budget: RequestBudget }; + +type EndpointHealth = + | { status: 'healthy' } + | { + status: 'tightened'; + probe: Probe; + /** Probes that timed out since the endpoint tightened. */ + failedProbes: number; + }; + +interface EndpointState { + health: EndpointHealth; + /** Consecutive timeouts at the default budget since the last `ok`. */ + consecutiveTimeouts: number; +} diff --git a/packages/pipeline/test/pipeline.test.ts b/packages/pipeline/test/pipeline.test.ts index 0193450bf..0570d665d 100644 --- a/packages/pipeline/test/pipeline.test.ts +++ b/packages/pipeline/test/pipeline.test.ts @@ -2871,7 +2871,7 @@ describe('Pipeline', () => { it('invokes the factory once per dataset', async () => { const factory = vi.fn().mockImplementation(() => ({ - beforeRequest: () => 300_000, + beforeRequest: () => ({ timeoutMs: 300_000 }), afterRequest: vi.fn(), })); @@ -2895,7 +2895,7 @@ describe('Pipeline', () => { const policies: unknown[] = []; const factory = vi.fn().mockImplementation(() => { const policy = { - beforeRequest: () => 300_000, + beforeRequest: () => ({ timeoutMs: 300_000 }), afterRequest: vi.fn(), }; policies.push(policy); @@ -2936,7 +2936,7 @@ describe('Pipeline', () => { consecutiveTimeouts: 0, }; const factory = vi.fn().mockImplementation(() => ({ - beforeRequest: () => 300_000, + beforeRequest: () => ({ timeoutMs: 300_000 }), afterRequest: vi.fn(), subscribe(observer: { onTighten?: (event: unknown) => void; diff --git a/packages/pipeline/test/sparql/reader.test.ts b/packages/pipeline/test/sparql/reader.test.ts index 6596a22bf..f0486e656 100644 --- a/packages/pipeline/test/sparql/reader.test.ts +++ b/packages/pipeline/test/sparql/reader.test.ts @@ -768,7 +768,7 @@ describe('SparqlConstructReader', () => { describe('timeout policy', () => { function recordingPolicy() { - const before = vi.fn().mockReturnValue(1234); + const before = vi.fn().mockReturnValue({ timeoutMs: 1234 }); const after = vi.fn(); return { beforeRequest: before, @@ -811,9 +811,13 @@ describe('SparqlConstructReader', () => { expect.objectContaining({ endpoint: distribution.accessUrl, outcome: 'ok', - timeoutMs: 1234, + budget: { timeoutMs: 1234 }, }), ); + // The policy tells attempts apart by identity, so no copy. + expect(policy.afterRequest.mock.calls[0][0].budget).toBe( + policy.beforeRequest.mock.results[0].value, + ); }); it('classifies HTTP 504 as outcome "timeout"', async () => { @@ -839,7 +843,10 @@ describe('SparqlConstructReader', () => { ).rejects.toThrow('504'); expect(policy.afterRequest).toHaveBeenCalledWith( - expect.objectContaining({ outcome: 'timeout', timeoutMs: 1234 }), + expect.objectContaining({ + outcome: 'timeout', + budget: { timeoutMs: 1234 }, + }), ); }); @@ -863,7 +870,10 @@ describe('SparqlConstructReader', () => { ).rejects.toThrow(); expect(policy.afterRequest).toHaveBeenCalledWith( - expect.objectContaining({ outcome: 'timeout', timeoutMs: 1234 }), + expect.objectContaining({ + outcome: 'timeout', + budget: { timeoutMs: 1234 }, + }), ); }); @@ -887,7 +897,10 @@ describe('SparqlConstructReader', () => { ).rejects.toThrow(); expect(policy.afterRequest).toHaveBeenCalledWith( - expect.objectContaining({ outcome: 'timeout', timeoutMs: 1234 }), + expect.objectContaining({ + outcome: 'timeout', + budget: { timeoutMs: 1234 }, + }), ); }); @@ -913,7 +926,10 @@ describe('SparqlConstructReader', () => { ).rejects.toThrow(); expect(policy.afterRequest).toHaveBeenCalledWith( - expect.objectContaining({ outcome: 'error', timeoutMs: 1234 }), + expect.objectContaining({ + outcome: 'error', + budget: { timeoutMs: 1234 }, + }), ); }); @@ -943,11 +959,14 @@ describe('SparqlConstructReader', () => { expect(policy.afterRequest).toHaveBeenCalledTimes(2); expect(policy.afterRequest).toHaveBeenNthCalledWith( 1, - expect.objectContaining({ outcome: 'timeout', timeoutMs: 1234 }), + expect.objectContaining({ + outcome: 'timeout', + budget: { timeoutMs: 1234 }, + }), ); expect(policy.afterRequest).toHaveBeenNthCalledWith( 2, - expect.objectContaining({ outcome: 'ok', timeoutMs: 1234 }), + expect.objectContaining({ outcome: 'ok', budget: { timeoutMs: 1234 } }), ); }); @@ -988,7 +1007,7 @@ describe('SparqlConstructReader', () => { }); const policy = { - beforeRequest: vi.fn().mockReturnValue(10), + beforeRequest: vi.fn().mockReturnValue({ timeoutMs: 10 }), afterRequest: vi.fn(), }; const reader = new SparqlConstructReader({ @@ -1004,7 +1023,10 @@ describe('SparqlConstructReader', () => { ).rejects.toThrow(); expect(policy.afterRequest).toHaveBeenCalledWith( - expect.objectContaining({ outcome: 'timeout', timeoutMs: 10 }), + expect.objectContaining({ + outcome: 'timeout', + budget: { timeoutMs: 10 }, + }), ); expect(slowFetch).toHaveBeenCalled(); const init = slowFetch.mock.calls[0][1]; diff --git a/packages/pipeline/test/sparql/selector.test.ts b/packages/pipeline/test/sparql/selector.test.ts index d5ae0e7d0..dbde488fe 100644 --- a/packages/pipeline/test/sparql/selector.test.ts +++ b/packages/pipeline/test/sparql/selector.test.ts @@ -646,7 +646,7 @@ describe('SparqlItemSelector', () => { describe('timeout policy', () => { function recordingPolicy() { return { - beforeRequest: vi.fn().mockReturnValue(5_000), + beforeRequest: vi.fn().mockReturnValue({ timeoutMs: 5_000 }), afterRequest: vi.fn(), }; } @@ -681,7 +681,11 @@ describe('SparqlItemSelector', () => { }); expect(policy.afterRequest).toHaveBeenCalledTimes(2); expect(policy.afterRequest).toHaveBeenCalledWith( - expect.objectContaining({ outcome: 'ok', timeoutMs: 5000 }), + expect.objectContaining({ outcome: 'ok', budget: { timeoutMs: 5000 } }), + ); + // The policy tells attempts apart by identity, so no copy. + expect(policy.afterRequest.mock.calls[0][0].budget).toBe( + policy.beforeRequest.mock.results[0].value, ); }); @@ -712,7 +716,10 @@ describe('SparqlItemSelector', () => { await expect(iterate()).rejects.toThrow('504'); expect(policy.afterRequest).toHaveBeenCalledWith( - expect.objectContaining({ outcome: 'timeout', timeoutMs: 5000 }), + expect.objectContaining({ + outcome: 'timeout', + budget: { timeoutMs: 5000 }, + }), ); }); @@ -741,7 +748,10 @@ describe('SparqlItemSelector', () => { await expect(iterate()).rejects.toThrow('socket hang up'); expect(policy.afterRequest).toHaveBeenCalledTimes(1); expect(policy.afterRequest).toHaveBeenCalledWith( - expect.objectContaining({ outcome: 'error', timeoutMs: 5000 }), + expect.objectContaining({ + outcome: 'error', + budget: { timeoutMs: 5000 }, + }), ); }); @@ -772,7 +782,10 @@ describe('SparqlItemSelector', () => { await expect(iterate()).rejects.toThrow(); expect(policy.afterRequest).toHaveBeenCalledWith( - expect.objectContaining({ outcome: 'error', timeoutMs: 5000 }), + expect.objectContaining({ + outcome: 'error', + budget: { timeoutMs: 5000 }, + }), ); }); diff --git a/packages/pipeline/test/sparql/timeoutPolicy.test.ts b/packages/pipeline/test/sparql/timeoutPolicy.test.ts index 89b47299c..f8006bda9 100644 --- a/packages/pipeline/test/sparql/timeoutPolicy.test.ts +++ b/packages/pipeline/test/sparql/timeoutPolicy.test.ts @@ -4,6 +4,8 @@ import { ConstantTimeoutPolicy, adaptiveTimeoutPolicy, constantTimeoutPolicy, + type RequestBudget, + type TimeoutOutcome, } from '../../src/sparql/timeoutPolicy.js'; const endpointA = new URL('https://example.org/a/sparql'); @@ -12,7 +14,7 @@ const endpointB = new URL('https://example.org/b/sparql'); describe('ConstantTimeoutPolicy', () => { it('returns the configured timeout from beforeRequest', () => { const policy = new ConstantTimeoutPolicy(1234); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1234); + expect(policy.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe(1234); }); it('ignores outcomes in afterRequest', () => { @@ -21,9 +23,9 @@ describe('ConstantTimeoutPolicy', () => { endpoint: endpointA, outcome: 'timeout', durationMs: 500, - timeoutMs: 1000, + budget: { timeoutMs: 1000 }, }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); + expect(policy.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe(1000); }); it('rejects non-positive timeouts', () => { @@ -40,7 +42,7 @@ describe('constantTimeoutPolicy factory', () => { const b = factory(); expect(a).toBeInstanceOf(ConstantTimeoutPolicy); expect(a).not.toBe(b); - expect(a.beforeRequest({ endpoint: endpointA })).toBe(5_000); + expect(a.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe(5_000); }); }); @@ -103,7 +105,9 @@ describe('AdaptiveTimeoutPolicy', () => { tightenedMs: 100, tightenAfterTimeouts: 2, }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); + expect(policy.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe( + 1000, + ); }); it('tightens after exactly tightenAfterTimeouts=1 consecutive timeouts', () => { @@ -112,14 +116,16 @@ describe('AdaptiveTimeoutPolicy', () => { tightenedMs: 100, tightenAfterTimeouts: 1, }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); + expect(policy.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe( + 1000, + ); policy.afterRequest({ endpoint: endpointA, outcome: 'timeout', durationMs: 1000, - timeoutMs: 1000, + budget: { timeoutMs: 1000 }, }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); + expect(policy.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe(100); }); it('tightens after exactly tightenAfterTimeouts=2 consecutive timeouts', () => { @@ -132,16 +138,18 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'timeout', durationMs: 1000, - timeoutMs: 1000, + budget: { timeoutMs: 1000 }, }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); + expect(policy.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe( + 1000, + ); policy.afterRequest({ endpoint: endpointA, outcome: 'timeout', durationMs: 1000, - timeoutMs: 1000, + budget: { timeoutMs: 1000 }, }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); + expect(policy.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe(100); }); it('tightens after exactly tightenAfterTimeouts=3 consecutive timeouts', () => { @@ -154,22 +162,24 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'timeout', durationMs: 1000, - timeoutMs: 1000, + budget: { timeoutMs: 1000 }, }); policy.afterRequest({ endpoint: endpointA, outcome: 'timeout', durationMs: 1000, - timeoutMs: 1000, + budget: { timeoutMs: 1000 }, }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); + expect(policy.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe( + 1000, + ); policy.afterRequest({ endpoint: endpointA, outcome: 'timeout', durationMs: 1000, - timeoutMs: 1000, + budget: { timeoutMs: 1000 }, }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); + expect(policy.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe(100); }); it('stays tightened on further timeouts', () => { @@ -183,10 +193,10 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'timeout', durationMs: 1000, - timeoutMs: 1000, + budget: { timeoutMs: 1000 }, }); } - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); + expect(policy.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe(100); }); it('relaxes to default on a single ok', () => { @@ -199,16 +209,18 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'timeout', durationMs: 1000, - timeoutMs: 1000, + budget: { timeoutMs: 1000 }, }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); + expect(policy.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe(100); policy.afterRequest({ endpoint: endpointA, outcome: 'ok', durationMs: 80, - timeoutMs: 1000, + budget: { timeoutMs: 1000 }, }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); + expect(policy.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe( + 1000, + ); }); it('resets the counter on ok so subsequent timeouts must accumulate again', () => { @@ -221,21 +233,23 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'timeout', durationMs: 1000, - timeoutMs: 1000, + budget: { timeoutMs: 1000 }, }); policy.afterRequest({ endpoint: endpointA, outcome: 'ok', durationMs: 80, - timeoutMs: 1000, + budget: { timeoutMs: 1000 }, }); policy.afterRequest({ endpoint: endpointA, outcome: 'timeout', durationMs: 1000, - timeoutMs: 1000, + budget: { timeoutMs: 1000 }, }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); + expect(policy.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe( + 1000, + ); }); it('treats `error` outcomes as neutral (neither tighten nor relax)', () => { @@ -248,24 +262,26 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'error', durationMs: 50, - timeoutMs: 1000, + budget: { timeoutMs: 1000 }, }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); + expect(policy.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe( + 1000, + ); policy.afterRequest({ endpoint: endpointA, outcome: 'timeout', durationMs: 1000, - timeoutMs: 1000, + budget: { timeoutMs: 1000 }, }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); + expect(policy.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe(100); // An `error` while tightened keeps state tightened. policy.afterRequest({ endpoint: endpointA, outcome: 'error', durationMs: 50, - timeoutMs: 1000, + budget: { timeoutMs: 1000 }, }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); + expect(policy.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe(100); }); it('isolates state per endpoint', () => { @@ -278,112 +294,150 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'timeout', durationMs: 1000, - timeoutMs: 1000, + budget: { timeoutMs: 1000 }, }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); - expect(policy.beforeRequest({ endpoint: endpointB })).toBe(1000); + expect(policy.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe(100); + expect(policy.beforeRequest({ endpoint: endpointB }).timeoutMs).toBe( + 1000, + ); }); describe('re-probing a tightened endpoint', () => { - function tightenedPolicy(): AdaptiveTimeoutPolicy { - const policy = new AdaptiveTimeoutPolicy({ - defaultMs: 1000, - tightenedMs: 100, - tightenAfterTimeouts: 1, - }); + const options = { + defaultMs: 1000, + tightenedMs: 100, + tightenAfterTimeouts: 1, + }; + + function request(policy: AdaptiveTimeoutPolicy): RequestBudget { + return policy.beforeRequest({ endpoint: endpointA }); + } + + function report( + policy: AdaptiveTimeoutPolicy, + budget: RequestBudget, + outcome: TimeoutOutcome, + ): void { policy.afterRequest({ endpoint: endpointA, - outcome: 'timeout', - durationMs: 1000, - timeoutMs: 1000, + outcome, + durationMs: budget.timeoutMs, + budget, }); + } + + function timeOutNextRequest(policy: AdaptiveTimeoutPolicy): void { + report(policy, request(policy), 'timeout'); + } + + function tightenedPolicy(): AdaptiveTimeoutPolicy { + const policy = new AdaptiveTimeoutPolicy(options); + timeOutNextRequest(policy); return policy; } + /** + * Time out requests at the tightened budget until the policy hands out + * a probe; returns how many it took and the probe’s budget. + */ + function nextProbe(policy: AdaptiveTimeoutPolicy): { + timeouts: number; + probe: RequestBudget; + } { + for (let timeouts = 0; timeouts < 1000; timeouts++) { + const budget = request(policy); + if (budget.timeoutMs === options.defaultMs) { + return { timeouts, probe: budget }; + } + report(policy, budget, 'timeout'); + } + throw new Error('No probe handed out'); + } + it('gives the next request the default budget after a timeout at the tightened budget', () => { const policy = tightenedPolicy(); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); - policy.afterRequest({ - endpoint: endpointA, - outcome: 'timeout', - durationMs: 100, - timeoutMs: 100, - }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); + const tightened = request(policy); + expect(tightened.timeoutMs).toBe(100); + report(policy, tightened, 'timeout'); + expect(request(policy).timeoutMs).toBe(1000); // One probe at a time: concurrent requests keep the tightened budget. - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); + expect(request(policy).timeoutMs).toBe(100); }); - it('hands out no second probe while one is in flight, whatever its outcome', () => { + it('hands out no second probe while one is in flight', () => { const policy = tightenedPolicy(); - const concurrentBudgets = [ - policy.beforeRequest({ endpoint: endpointA }), - policy.beforeRequest({ endpoint: endpointA }), - ]; - expect(concurrentBudgets).toEqual([100, 100]); - policy.afterRequest({ - endpoint: endpointA, - outcome: 'timeout', - durationMs: 100, - timeoutMs: 100, - }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); + const first = request(policy); + const second = request(policy); + expect([first.timeoutMs, second.timeoutMs]).toEqual([100, 100]); + report(policy, first, 'timeout'); + expect(request(policy).timeoutMs).toBe(1000); - // Concurrent requests at the tightened budget time out while the - // probe is in flight: none of them schedules another probe. - policy.afterRequest({ - endpoint: endpointA, - outcome: 'timeout', - durationMs: 100, - timeoutMs: 100, - }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); - policy.afterRequest({ - endpoint: endpointA, - outcome: 'timeout', - durationMs: 100, - timeoutMs: 100, - }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); + // A concurrent request at the tightened budget times out while the + // probe is in flight: it schedules no other probe. + report(policy, second, 'timeout'); + expect(request(policy).timeoutMs).toBe(100); + }); - // The probe fails with an error: no longer in flight, so the next - // tightened timeout schedules a new probe. - policy.afterRequest({ - endpoint: endpointA, - outcome: 'error', - durationMs: 50, - timeoutMs: 1000, - }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); - policy.afterRequest({ - endpoint: endpointA, - outcome: 'timeout', - durationMs: 100, - timeoutMs: 100, + it('does not mistake a request budgeted before tightening for the probe', () => { + const policy = new AdaptiveTimeoutPolicy({ + ...options, + tightenAfterTimeouts: 2, }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); + const budgetedWhileHealthy = request(policy); + timeOutNextRequest(policy); + timeOutNextRequest(policy); + const { probe } = nextProbe(policy); + + // The request from before tightening times out at the default budget, + // just like the probe would: the probe is still in flight. + report(policy, budgetedWhileHealthy, 'timeout'); + const tightened = request(policy); + report(policy, tightened, 'timeout'); + expect(request(policy).timeoutMs).toBe(100); + + report(policy, probe, 'timeout'); + expect(nextProbe(policy).timeouts).toBe(2); + }); + + it('releases the probe when it fails with an error, keeping the gap', () => { + const policy = tightenedPolicy(); + const { probe } = nextProbe(policy); + report(policy, probe, 'error'); + expect(nextProbe(policy).timeouts).toBe(1); + }); + + it('doubles the gap between probes after each timed-out probe, up to 64', () => { + const policy = tightenedPolicy(); + const gaps: number[] = []; + for (let attempt = 0; attempt < 8; attempt++) { + const { timeouts, probe } = nextProbe(policy); + gaps.push(timeouts); + report(policy, probe, 'timeout'); + } + expect(gaps).toEqual([1, 2, 4, 8, 16, 32, 64, 64]); }); it('relaxes a healthy-but-slow endpoint when the probe succeeds', () => { const policy = tightenedPolicy(); const onRelax = vi.fn(); policy.subscribe({ onRelax }); - policy.afterRequest({ - endpoint: endpointA, - outcome: 'timeout', - durationMs: 100, - timeoutMs: 100, - }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); - policy.afterRequest({ - endpoint: endpointA, - outcome: 'ok', - durationMs: 900, - timeoutMs: 1000, - }); + const { probe } = nextProbe(policy); + report(policy, probe, 'ok'); + // The timeout at the tightened budget before the probe isn’t counted. expect(onRelax).toHaveBeenCalledTimes(1); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); + expect(onRelax).toHaveBeenCalledWith( + expect.objectContaining({ consecutiveTimeouts: 1 }), + ); + expect(request(policy).timeoutMs).toBe(1000); + expect(request(policy).timeoutMs).toBe(1000); + }); + + it('resets the gap between probes once the endpoint relaxes', () => { + const policy = tightenedPolicy(); + report(policy, nextProbe(policy).probe, 'timeout'); + report(policy, nextProbe(policy).probe, 'ok'); + timeOutNextRequest(policy); + expect(nextProbe(policy).timeouts).toBe(1); }); it('stays tightened when the probe times out at the default budget', () => { @@ -391,67 +445,59 @@ describe('AdaptiveTimeoutPolicy', () => { const onTighten = vi.fn(); const onRelax = vi.fn(); policy.subscribe({ onTighten, onRelax }); - policy.afterRequest({ - endpoint: endpointA, - outcome: 'timeout', - durationMs: 100, - timeoutMs: 100, - }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); - policy.afterRequest({ - endpoint: endpointA, - outcome: 'timeout', - durationMs: 1000, - timeoutMs: 1000, - }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); + const { probe } = nextProbe(policy); + report(policy, probe, 'timeout'); + expect(request(policy).timeoutMs).toBe(100); expect(onTighten).not.toHaveBeenCalled(); expect(onRelax).not.toHaveBeenCalled(); }); it('ignores a late timeout at the tightened budget after the endpoint relaxed', () => { const policy = tightenedPolicy(); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(100); - policy.afterRequest({ - endpoint: endpointA, - outcome: 'ok', - durationMs: 50, - timeoutMs: 100, - }); - policy.afterRequest({ - endpoint: endpointA, - outcome: 'timeout', - durationMs: 100, - timeoutMs: 100, - }); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); - expect(policy.beforeRequest({ endpoint: endpointA })).toBe(1000); + const succeeding = request(policy); + const timingOut = request(policy); + report(policy, succeeding, 'ok'); + report(policy, timingOut, 'timeout'); + expect(request(policy).timeoutMs).toBe(1000); + expect(request(policy).timeoutMs).toBe(1000); + }); + + it('ignores a probe that reports after the endpoint relaxed and tightened again', () => { + const policy = tightenedPolicy(); + const { probe: staleProbe } = nextProbe(policy); + report(policy, request(policy), 'ok'); + timeOutNextRequest(policy); + nextProbe(policy); + + report(policy, staleProbe, 'timeout'); + const tightened = request(policy); + report(policy, tightened, 'timeout'); + expect(request(policy).timeoutMs).toBe(100); }); it('does not count timeouts at the tightened budget toward tightening', () => { const policy = new AdaptiveTimeoutPolicy({ - defaultMs: 1000, - tightenedMs: 100, + ...options, tightenAfterTimeouts: 2, }); - const onRelax = vi.fn(); - policy.subscribe({ onRelax }); - for (const timeoutMs of [1000, 1000, 100, 1000]) { - policy.afterRequest({ - endpoint: endpointA, - outcome: 'timeout', - durationMs: timeoutMs, - timeoutMs, - }); - } - policy.afterRequest({ - endpoint: endpointA, - outcome: 'ok', - durationMs: 900, - timeoutMs: 1000, - }); - expect(onRelax).toHaveBeenCalledWith( - expect.objectContaining({ consecutiveTimeouts: 3 }), + timeOutNextRequest(policy); + timeOutNextRequest(policy); + const budgetedWhileTightened = request(policy); + expect(budgetedWhileTightened.timeoutMs).toBe(100); + report(policy, request(policy), 'ok'); + const onTighten = vi.fn(); + policy.subscribe({ onTighten }); + + // Healthy again: one timeout at the default budget, then the request + // budgeted while tightened times out at the tightened budget. + timeOutNextRequest(policy); + report(policy, budgetedWhileTightened, 'timeout'); + expect(request(policy).timeoutMs).toBe(1000); + expect(onTighten).not.toHaveBeenCalled(); + + timeOutNextRequest(policy); + expect(onTighten).toHaveBeenCalledWith( + expect.objectContaining({ consecutiveTimeouts: 2 }), ); }); }); @@ -471,14 +517,14 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'timeout', durationMs: 1000, - timeoutMs: 1000, + budget: { timeoutMs: 1000 }, }); expect(onTighten).not.toHaveBeenCalled(); policy.afterRequest({ endpoint: endpointA, outcome: 'timeout', durationMs: 1000, - timeoutMs: 1000, + budget: { timeoutMs: 1000 }, }); expect(onTighten).toHaveBeenCalledTimes(1); expect(onTighten).toHaveBeenCalledWith( @@ -504,19 +550,19 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'timeout', durationMs: 1000, - timeoutMs: 1000, + budget: { timeoutMs: 1000 }, }); policy.afterRequest({ endpoint: endpointA, outcome: 'timeout', durationMs: 1000, - timeoutMs: 1000, + budget: { timeoutMs: 1000 }, }); policy.afterRequest({ endpoint: endpointA, outcome: 'timeout', durationMs: 1000, - timeoutMs: 1000, + budget: { timeoutMs: 1000 }, }); expect(onTighten).toHaveBeenCalledTimes(1); }); @@ -533,13 +579,13 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'timeout', durationMs: 1000, - timeoutMs: 1000, + budget: { timeoutMs: 1000 }, }); policy.afterRequest({ endpoint: endpointA, outcome: 'ok', durationMs: 80, - timeoutMs: 1000, + budget: { timeoutMs: 1000 }, }); expect(onRelax).toHaveBeenCalledTimes(1); expect(onRelax).toHaveBeenCalledWith( @@ -563,7 +609,7 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'ok', durationMs: 80, - timeoutMs: 1000, + budget: { timeoutMs: 1000 }, }); expect(onRelax).not.toHaveBeenCalled(); }); @@ -581,7 +627,7 @@ describe('AdaptiveTimeoutPolicy', () => { endpoint: endpointA, outcome: 'timeout', durationMs: 1000, - timeoutMs: 1000, + budget: { timeoutMs: 1000 }, }); expect(onTighten).not.toHaveBeenCalled(); }); @@ -612,9 +658,9 @@ describe('adaptiveTimeoutPolicy factory', () => { endpoint: endpointA, outcome: 'timeout', durationMs: 1000, - timeoutMs: 1000, + budget: { timeoutMs: 1000 }, }); const b = factory(); - expect(b.beforeRequest({ endpoint: endpointA })).toBe(1000); + expect(b.beforeRequest({ endpoint: endpointA }).timeoutMs).toBe(1000); }); }); diff --git a/packages/pipeline/vite.config.ts b/packages/pipeline/vite.config.ts index 9563d61d9..5e76fa2bc 100644 --- a/packages/pipeline/vite.config.ts +++ b/packages/pipeline/vite.config.ts @@ -13,7 +13,7 @@ export default mergeConfig( autoUpdate: true, functions: 99.4, lines: 98.79, - branches: 96.72, + branches: 96.75, statements: 98.64, }, },