diff --git a/packages/core/src/__tests__/model-call-attempt.test.ts b/packages/core/src/__tests__/model-call-attempt.test.ts index f6b9e3389d..21b84414f9 100644 --- a/packages/core/src/__tests__/model-call-attempt.test.ts +++ b/packages/core/src/__tests__/model-call-attempt.test.ts @@ -59,6 +59,85 @@ function attempt(overrides: Partial = {}): ModelCallAttempt { } describe('ModelCallAttempt codec', () => { + test('accepts one bounded prepared-request observation on the canonical attempt', () => { + const decoded = decodeModelCallAttempt({ + ...attempt(), + requestObservation: { + schemaVersion: 1, + digest: `sha256:${'a'.repeat(64)}`, + bytes: 42, + segments: [ + { + kind: 'message', + index: 0, + cacheable: true, + comparison: 'exact', + digest: `sha256:${'b'.repeat(64)}`, + bytes: 21, + role: 'user', + }, + ], + }, + }); + + assert.equal(decoded.requestObservation?.segments[0]?.comparison, 'exact'); + }); + + test('rejects a prepared-request observation whose semantic segments are out of order', () => { + assert.throws(() => + decodeModelCallAttempt({ + ...attempt(), + requestObservation: { + schemaVersion: 1, + digest: `sha256:${'a'.repeat(64)}`, + bytes: 42, + segments: [ + { + kind: 'message', + index: 0, + cacheable: true, + comparison: 'exact', + digest: `sha256:${'b'.repeat(64)}`, + bytes: 21, + }, + { + kind: 'system_prompt', + index: 0, + cacheable: true, + comparison: 'exact', + digest: `sha256:${'c'.repeat(64)}`, + bytes: 21, + }, + ], + }, + }), + ); + }); + + test('rejects a bounded remainder that claims exact comparison', () => { + assert.throws(() => + decodeModelCallAttempt({ + ...attempt(), + requestObservation: { + schemaVersion: 1, + digest: `sha256:${'a'.repeat(64)}`, + bytes: 21, + segments: [ + { + kind: 'tool_schema', + index: 0, + cacheable: true, + comparison: 'exact', + digest: `sha256:${'b'.repeat(64)}`, + bytes: 21, + representedSegments: 1, + }, + ], + }, + }), + ); + }); + test('accepts bounded provider failure diagnostics on history compaction calls', () => { const decoded = decodeModelCallAttempt( attempt({ diff --git a/packages/core/src/agent-run.ts b/packages/core/src/agent-run.ts index 5c3816d57a..e85e0c5608 100644 --- a/packages/core/src/agent-run.ts +++ b/packages/core/src/agent-run.ts @@ -794,12 +794,14 @@ export interface AgentRunStore { sessionId: string, type: AgentRunProjectionKey, ): Promise; + /** Opaque revision of the canonical event ledger used to guard a derived repair. */ + readEventLedgerRevision?(sessionId: string): Promise; /** Rewrites derived state after the canonical event ledger repairs an absent or damaged projection. */ repairEventProjection?( sessionId: string, type: AgentRunProjectionKey, event: AgentRunEvent | null, - options?: { replaceEventId?: string }, + options: { ifLedgerRevision: string; replaceEventId?: string }, ): Promise; } diff --git a/packages/core/src/model-call-attempt.ts b/packages/core/src/model-call-attempt.ts index fad17c4ed2..efac9504c7 100644 --- a/packages/core/src/model-call-attempt.ts +++ b/packages/core/src/model-call-attempt.ts @@ -79,6 +79,55 @@ export type HistoryCompactRoute = (typeof HISTORY_COMPACT_ROUTES)[number]; /** Hard bound for provider-supplied diagnostic identifiers stored on an attempt. */ export const MODEL_CALL_DIAGNOSTIC_FIELD_MAX_LENGTH = 256; +export const PREPARED_REQUEST_OBSERVATION_SCHEMA_VERSION = 1 as const; +export const PREPARED_REQUEST_OBSERVATION_MAX_SEGMENTS = 256; +export const PREPARED_REQUEST_OBSERVATION_TEXT_MAX_LENGTH = 256; + +export type PreparedRequestObservationSegmentKind = + | 'tool_schema' + | 'system_prompt' + | 'message' + | 'provider_options'; + +/** + * One ordered semantic part of what Maka handed to the AI SDK model-call seam. + * + * `opaque` means the digest is useful for identity and auditing but MUST NOT be + * used to claim exact equality. It covers redacted content and bounded + * remainders that intentionally summarize more than one source segment. + */ +export interface PreparedRequestObservationSegment { + kind: PreparedRequestObservationSegmentKind; + index: number; + cacheable: boolean; + comparison: 'exact' | 'opaque'; + digest: string; + bytes: number; + /** Present only on an opaque bounded remainder; the value is the source-segment count. */ + representedSegments?: number; + role?: string; + label?: string; +} + +/** + * Bounded, secret-free observation of one prepared semantic model request. + * + * This is not the provider wire body. The full secret-free serialization stays + * in the private request artifact referenced by `captureArtifactId` when that + * sink is available. + */ +export interface PreparedRequestObservation { + schemaVersion: typeof PREPARED_REQUEST_OBSERVATION_SCHEMA_VERSION; + /** + * Identity of the complete secret-free normalized serialization. It does not + * prove semantic equality when any segment is `opaque`; continuity consumers + * must compare the ordered segment identity and each segment's `comparison`. + */ + digest: string; + bytes: number; + segments: PreparedRequestObservationSegment[]; +} + export interface ModelCallAttempt { schemaVersion: typeof MODEL_CALL_ATTEMPT_SCHEMA_VERSION; @@ -91,7 +140,7 @@ export interface ModelCallAttempt { logicalCallId: string; /** Idempotency key: appending the same `attemptId` twice records once. */ attemptId: string; - /** Tracker instance id, retained to join request-shape capture artifacts. */ + /** Tracker instance id, retained to join private prepared-request artifacts. */ traceId: string; /** @@ -116,8 +165,10 @@ export interface ModelCallAttempt { providerId: string; modelId: string; contextWindow?: number; - /** Join key for request-shape diagnostics; absent when capture is disabled. */ + /** Join key for the private prepared-request artifact, when best-effort persistence won the race. */ captureArtifactId?: string; + /** Semantic request actually prepared for this dispatched physical attempt. */ + requestObservation?: PreparedRequestObservation; startedAt: number; completedAt: number; @@ -175,6 +226,7 @@ const MODEL_CALL_ATTEMPT_SHAPE = defineObjectShape()( 'historyCompactRoute', 'contextWindow', 'captureArtifactId', + 'requestObservation', 'timeToFirstTokenMs', 'finishReason', 'errorClass', @@ -203,6 +255,24 @@ const TOKEN_FIELDS = [ 'reasoningTokens', ] as const satisfies readonly (keyof ModelCallAttempt)[]; +const PREPARED_REQUEST_OBSERVATION_SHAPE = defineObjectShape()( + ['schemaVersion', 'digest', 'bytes', 'segments'], + [], +); + +const PREPARED_REQUEST_OBSERVATION_SEGMENT_SHAPE = + defineObjectShape()( + ['kind', 'index', 'cacheable', 'comparison', 'digest', 'bytes'], + ['representedSegments', 'role', 'label'], + ); + +const PREPARED_REQUEST_SEGMENT_KINDS: readonly PreparedRequestObservationSegmentKind[] = [ + 'tool_schema', + 'system_prompt', + 'message', + 'provider_options', +]; + function isNonEmptyString(value: unknown): value is string { return typeof value === 'string' && value.length > 0; } @@ -235,6 +305,69 @@ function isOptionalHttpStatus(value: unknown): boolean { ); } +function isSha256Digest(value: unknown): value is string { + return typeof value === 'string' && /^sha256:[a-f0-9]{64}$/.test(value); +} + +function isOptionalBoundedText(value: unknown): boolean { + return ( + value === undefined || + (typeof value === 'string' && value.length <= PREPARED_REQUEST_OBSERVATION_TEXT_MAX_LENGTH) + ); +} + +function isPreparedRequestObservationSegment( + value: unknown, +): value is PreparedRequestObservationSegment { + if (!isRecord(value) || !hasExactShape(value, PREPARED_REQUEST_OBSERVATION_SEGMENT_SHAPE)) { + return false; + } + return ( + PREPARED_REQUEST_SEGMENT_KINDS.includes(value.kind as PreparedRequestObservationSegmentKind) && + isNonNegativeInteger(value.index) && + typeof value.cacheable === 'boolean' && + (value.comparison === 'exact' || value.comparison === 'opaque') && + isSha256Digest(value.digest) && + isNonNegativeInteger(value.bytes) && + (value.representedSegments === undefined || + (typeof value.representedSegments === 'number' && + Number.isSafeInteger(value.representedSegments) && + value.representedSegments > 0)) && + (value.representedSegments === undefined || value.comparison === 'opaque') && + isOptionalBoundedText(value.role) && + isOptionalBoundedText(value.label) + ); +} + +function isPreparedRequestObservation(value: unknown): value is PreparedRequestObservation { + if (!isRecord(value) || !hasExactShape(value, PREPARED_REQUEST_OBSERVATION_SHAPE)) return false; + return ( + value.schemaVersion === PREPARED_REQUEST_OBSERVATION_SCHEMA_VERSION && + isSha256Digest(value.digest) && + isNonNegativeInteger(value.bytes) && + Array.isArray(value.segments) && + value.segments.length <= PREPARED_REQUEST_OBSERVATION_MAX_SEGMENTS && + value.segments.every(isPreparedRequestObservationSegment) && + hasOrderedPreparedRequestSegments(value.segments) + ); +} + +function hasOrderedPreparedRequestSegments( + segments: readonly PreparedRequestObservationSegment[], +): boolean { + let previousKind = -1; + let previousIndex = -1; + for (const segment of segments) { + const kind = PREPARED_REQUEST_SEGMENT_KINDS.indexOf(segment.kind); + if (kind < previousKind) return false; + if (kind === previousKind && segment.index <= previousIndex) return false; + if (kind !== previousKind) previousIndex = -1; + previousKind = kind; + previousIndex = segment.index; + } + return true; +} + const PRICING_RATES_SHAPE = defineObjectShape()( ['modelKey', 'inputUsdPer1M', 'outputUsdPer1M'], ['cacheReadUsdPer1M', 'cacheWriteUsdPer1M'], @@ -286,6 +419,8 @@ export function decodeModelCallAttempt(value: unknown): ModelCallAttempt { isNonEmptyString(value.modelId) && isOptionalNonNegativeNumber(value.contextWindow) && isOptionalString(value.captureArtifactId) && + (value.requestObservation === undefined || + isPreparedRequestObservation(value.requestObservation)) && isFiniteNumber(value.startedAt) && isFiniteNumber(value.completedAt) && isNonNegativeNumber(value.latencyMs) && diff --git a/packages/runtime-host/src/__tests__/execution-model-composition.test.ts b/packages/runtime-host/src/__tests__/execution-model-composition.test.ts index 1e8d9e8766..83b6796d57 100644 --- a/packages/runtime-host/src/__tests__/execution-model-composition.test.ts +++ b/packages/runtime-host/src/__tests__/execution-model-composition.test.ts @@ -1631,6 +1631,7 @@ test('production Host executes a canonical ai-sdk Session against a real provide assert.equal(webSearchEnabled.kind, 'committed'); const execution = await openInteractiveExecutionStoresForWrite(owner.lease); + const usageStores = await openInteractiveUsageStoresForWrite(owner.lease); const session = await execution.sessionStore.create({ cwd: root, llmConnectionId: connection.connectionId, @@ -1807,14 +1808,30 @@ test('production Host executes a canonical ai-sdk Session against a real provide assert.equal(compactUsage.inputTokens, 7); assert.equal(compactUsage.outputTokens, 3); const capturedRequestCount = mainRequests.length + compactRequests.length; - const evidence = await waitForProviderEvidence(execution, session.id, capturedRequestCount); - assert.equal(evidence.captures.length, capturedRequestCount); - assert.equal(evidence.attempts.length, capturedRequestCount); + const attempts = await waitForCanonicalAttempts(usageStores, session.id, capturedRequestCount); + assert.equal(attempts.length, capturedRequestCount); + assert.ok(attempts.every((attempt) => attempt.requestObservation)); + const contextDiagnostics = await composition.handlers['context.diagnostics.query']( + { sessionId: session.id }, + connectionContext, + ); + assert.equal(contextDiagnostics.ok, true); + if (contextDiagnostics.ok) { + assert.equal(contextDiagnostics.result.status, 'available'); + if (contextDiagnostics.result.status === 'available') { + assert.ok( + contextDiagnostics.result.composition?.segments.some( + (segment) => segment.kind === 'messages', + ), + ); + } + } const artifacts = await openInteractiveArtifactStoreForWrite(owner.lease); - const artifactPage = await artifacts.listPage(session.id, { offset: 0, limit: 100 }); - const captureArtifacts = artifactPage.records.filter( - (artifact) => artifact.source === 'provider_request_capture', + const captureArtifacts = await waitForCaptureArtifacts( + artifacts, + session.id, + capturedRequestCount, ); assert.equal(captureArtifacts.length, capturedRequestCount); let summaryCaptureFound = false; @@ -1844,9 +1861,9 @@ test('production Host executes a canonical ai-sdk Session against a real provide failedStart, connectionContext, ); - assert.equal(failedTerminal.status, 'failed'); - assert.equal(provider.requests.length, requestsBeforeArtifactFailure); - assert.equal(drainRequests, 1); + assert.equal(failedTerminal.status, 'completed'); + assert.equal(provider.requests.length, requestsBeforeArtifactFailure + 1); + assert.equal(drainRequests, 0); } finally { try { await composition?.close(); @@ -3622,36 +3639,48 @@ async function waitForUsage( throw new Error('Hosted real-model usage attribution was not persisted'); } -async function waitForProviderEvidence( - execution: Awaited>, +async function waitForCanonicalAttempts( + usage: InteractiveUsageStoresWriter, sessionId: string, expectedRequests: number, -): Promise<{ captures: unknown[]; attempts: unknown[] }> { +): Promise { for (let attempt = 0; attempt < 100; attempt += 1) { - const runs = await execution.agentRunStore.listSessionRuns(sessionId); - const events = ( - await Promise.all(runs.map((run) => execution.agentRunStore.readEvents(sessionId, run.runId))) - ).flat(); - const captures = events.filter((event) => event.type === 'provider_request_captured'); - const attempts = events.filter((event) => event.type === 'provider_request_attempt_recorded'); - if (captures.length >= expectedRequests && attempts.length >= expectedRequests) { - return { captures, attempts }; - } + const page = await usage.modelCalls.modelCallAttempts( + { from: 0, to: Number.MAX_SAFE_INTEGER }, + sessionId, + ); + if (page.attempts.length >= expectedRequests) return page.attempts; await new Promise((resolve) => setTimeout(resolve, 10)); } - const runs = await execution.agentRunStore.listSessionRuns(sessionId); - const events = ( - await Promise.all(runs.map((run) => execution.agentRunStore.readEvents(sessionId, run.runId))) - ).flat(); + const page = await usage.modelCalls.modelCallAttempts( + { from: 0, to: Number.MAX_SAFE_INTEGER }, + sessionId, + ); throw new Error( - `Hosted provider request evidence was not persisted: ${JSON.stringify({ + `Hosted canonical model-call attempts were not persisted: ${JSON.stringify({ expectedRequests, - captures: events.filter((event) => event.type === 'provider_request_captured').length, - attempts: events.filter((event) => event.type === 'provider_request_attempt_recorded').length, + attempts: page.attempts.length, + unreadableRecords: page.unreadableRecords, })}`, ); } +async function waitForCaptureArtifacts( + artifacts: Awaited>, + sessionId: string, + expectedRequests: number, +) { + for (let attempt = 0; attempt < 100; attempt += 1) { + const page = await artifacts.listPage(sessionId, { offset: 0, limit: 100 }); + const captures = page.records.filter( + (artifact) => artifact.source === 'provider_request_capture', + ); + if (captures.length >= expectedRequests) return captures; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + throw new Error(`Hosted request artifacts did not reach ${expectedRequests}`); +} + async function waitForAutomaticMemoryRequestsToSettle( requests: readonly ProviderRequest[], ): Promise { diff --git a/packages/runtime-host/src/server/execution-model-composition.ts b/packages/runtime-host/src/server/execution-model-composition.ts index 3408e5cf50..7d48e228a0 100644 --- a/packages/runtime-host/src/server/execution-model-composition.ts +++ b/packages/runtime-host/src/server/execution-model-composition.ts @@ -36,7 +36,6 @@ import { } from '@maka/runtime/openai-codex-history-compactor'; import { buildPricingLookup, recordToolInvocation } from '@maka/runtime/telemetry'; import { buildProviderOptions, getAIModel } from '@maka/runtime/model-factory'; -import { createProviderRequestCaptureRecorder } from '@maka/runtime/provider-request-telemetry'; import { createProxiedFetchTransport, type ProxiedFetchProxy, @@ -269,32 +268,22 @@ export async function createHostAiSdkBackend(input: HostAiSdkBackendInput): Prom throw new Error('Canonical model-call accounting authority is unavailable'); } }; - let artifactDrainRequested = false; - const providerRequestCapture = input.context.recordProviderRequestCapture - ? createProviderRequestCaptureRecorder({ - persistArtifact: async (capture) => { - try { - const artifact = await persistProviderRequestCaptureArtifact(input.artifacts, { - sessionId: input.context.sessionId, - turnId: capture.turnId, - captureId: capture.captureId, - step: capture.step, - serializedRequest: capture.serializedRequest, - now: Date.now(), - }); - return { artifactId: artifact.id }; - } catch (error) { - if (!artifactDrainRequested) { - artifactDrainRequested = true; - input.requestDrain(); - } - throw error; - } - }, - recordLedger: input.context.recordProviderRequestCapture, - }) - : undefined; - const recordProviderRequestAttempt = input.context.recordProviderRequestAttempt ?? (() => {}); + const persistPreparedRequestArtifact = async (capture: { + turnId: string; + captureId: string; + step: number; + serializedRequest: string; + }): Promise<{ artifactId: string }> => { + const artifact = await persistProviderRequestCaptureArtifact(input.artifacts, { + sessionId: input.context.sessionId, + turnId: capture.turnId, + captureId: capture.captureId, + step: capture.step, + serializedRequest: capture.serializedRequest, + now: Date.now(), + }); + return { artifactId: artifact.id }; + }; const resolveRunPrompt = async (context: { readonly turnId: string; readonly emitSkillCatalogTrace?: (message: string, data?: Record) => void; @@ -448,18 +437,9 @@ export async function createHostAiSdkBackend(input: HostAiSdkBackendInput): Prom lookupPricing: pricing, recordModelCallAttempt, assertModelCallAccountingReady, + persistPreparedRequestArtifact, recordToolInvocation: (event) => recordToolInvocation({ repo: telemetry }, event), ...(input.runtimeCommitSink ? { runtimeCommitSink: input.runtimeCommitSink } : {}), - ...(providerRequestCapture - ? { - recordProviderRequestCapture: providerRequestCapture, - ...(input.context.recordProviderRequestAttempt - ? { - recordProviderRequestAttempt, - } - : {}), - } - : {}), newId: randomUUID, now: Date.now, }, diff --git a/packages/runtime/src/__tests__/ai-sdk-backend.test.ts b/packages/runtime/src/__tests__/ai-sdk-backend.test.ts index 435df61a85..f91e6af20a 100644 --- a/packages/runtime/src/__tests__/ai-sdk-backend.test.ts +++ b/packages/runtime/src/__tests__/ai-sdk-backend.test.ts @@ -58,11 +58,7 @@ import { import type { DurableSessionEventSink, MakaTool, ToolRuntime } from '../tool-runtime.js'; import { TOOL_SEARCH_NAME } from '../tool-availability.js'; import { buildNativeWebSearchTool } from '../native-web-search-tool.js'; -import { - canonicalizeToolSet, - computeRequestShapeDiagnostic, - findFirstChangedCacheableSegment, -} from '../request-shape.js'; +import { canonicalizeToolSet } from '../request-shape.js'; import { ARCHIVED_TOOL_RESULT_PLACEHOLDER_KIND, ARCHIVED_TOOL_RESULT_REWRITE_VERSION, @@ -83,10 +79,7 @@ import { } from '../sandbox-boundary-declaration.js'; import { FilesystemWorkerClientError } from '../filesystem-worker/client.js'; import { RunTrace } from '../run-trace.js'; -import type { - ProviderRequestAttemptRecord, - ProviderRequestCaptureRecord, -} from '../provider-request-telemetry.js'; +import type { PreparedRequestArtifactInput } from '../provider-request-telemetry.js'; import { decodeModelCallAttempt, type ModelCallAttempt } from '@maka/core/model-call-attempt'; import { buildLlmHistorySummarizer } from '../history-compact-summarizer.js'; import { createToolResultArchiveCapability } from '../tool-result-archive-capability.js'; @@ -5490,14 +5483,7 @@ describe('AiSdkBackend model history', () => { type FingerprintCase = { name: string; expectedCalls: number; - prepare?: (input: AiSdkBackendInput, backend: AiSdkBackend) => void; - change?: (input: AiSdkBackendInput, backend: AiSdkBackend) => void; - }; - const setRequestShapeHash = (backend: AiSdkBackend, requestShapeHash: string): void => { - const internals = backend as unknown as { - priorRequestShape: { requestShapeHash: string } | undefined; - }; - internals.priorRequestShape = { requestShapeHash }; + change?: (input: AiSdkBackendInput) => void; }; const cases = [ { @@ -5529,10 +5515,11 @@ describe('AiSdkBackend model history', () => { }, }, { - name: 'request shape change retries', + name: 'compaction route change retries', expectedCalls: 2, - prepare: (_input, backend) => setRequestShapeHash(backend, 'request-shape-before'), - change: (_input, backend) => setRequestShapeHash(backend, 'request-shape-after'), + change: (input) => { + input.historyCompactRoute = 'text_summary'; + }, }, ] satisfies readonly FingerprintCase[]; @@ -5582,14 +5569,12 @@ describe('AiSdkBackend model history', () => { text: 'recent', }), ]; - fingerprintCase.prepare?.(backendInput, backend); - const first = await backend.compactHistory({ turnId: 'turn-config-compact-1', runId: 'run-1', runtimeContext: history, }); - fingerprintCase.change?.(backendInput, backend); + fingerprintCase.change?.(backendInput); const repeated = await backend.compactHistory({ turnId: 'turn-config-compact-2', runId: 'run-2', @@ -8226,7 +8211,7 @@ describe('AiSdkBackend usage telemetry', () => { }, } as never); - await drain(backend.send({ turnId: 'turn-1', text: 'hi', context: [] })); + await drain(backend.send({ turnId: 'turn-1', runId: 'run-1', text: 'hi', context: [] })); assert.equal(usageCheckpoints.length, 1); assert.equal(usageCheckpoints[0]?.costUsd, undefined); @@ -8667,6 +8652,7 @@ describe('AiSdkBackend usage telemetry', () => { prefixChangeReason?: string; requestShapeHash?: string; requestShapeChangeReason?: string; + promptSegments?: unknown[]; } | undefined; const usageEvent = events.find((event) => event.type === 'token_usage') as @@ -8688,13 +8674,7 @@ describe('AiSdkBackend usage telemetry', () => { assert.equal(usageMessage?.reasoning, 2); assert.equal(usageMessage?.total, 17); assert.equal(usageMessage?.rawFinishReason, 'stop'); - assert.equal(usageMessage?.systemPromptHash, usageEvent?.systemPromptHash); - assert.ok(usageMessage?.systemPromptHash); assert.equal(usageMessage?.costUsd, expectedCostUsd); - assert.equal(usageMessage?.prefixChangeReason, 'first_turn'); - assert.equal(usageMessage?.requestShapeChangeReason, 'first_turn'); - assert.ok(usageMessage?.prefixHash); - assert.ok(usageMessage?.requestShapeHash); assert.equal(usageEvent?.input, 10); assert.equal(usageEvent?.output, 7); assert.equal(usageEvent?.cacheHitInput, 3); @@ -8706,156 +8686,24 @@ describe('AiSdkBackend usage telemetry', () => { assert.equal(usageEvent?.reasoning, 2); assert.equal(usageEvent?.total, 17); assert.equal(usageEvent?.rawFinishReason, 'stop'); - assert.equal(usageEvent?.systemPromptHash, usageMessage?.systemPromptHash); assert.equal(usageEvent?.costUsd, expectedCostUsd); - assert.equal(usageEvent?.prefixChangeReason, 'first_turn'); - assert.equal(usageEvent?.requestShapeChangeReason, 'first_turn'); - assert.ok(usageEvent?.prefixHash); - assert.ok(usageEvent?.requestShapeHash); - assert.equal(startTrace?.data?.prefixChangeReason, 'first_turn'); - assert.equal(startTrace?.data?.requestShapeChangeReason, 'first_turn'); - assert.ok(startTrace?.data?.prefixHash); - assert.ok(startTrace?.data?.requestShapeHash); - assert.equal(startTrace?.data?.systemPromptHash, usageMessage?.systemPromptHash); + for (const currentWriter of [ + usageMessage as Record, + usageEvent as unknown as Record, + startTrace?.data, + ]) { + assert.equal(currentWriter?.systemPromptHash, undefined); + assert.equal(currentWriter?.prefixHash, undefined); + assert.equal(currentWriter?.prefixChangeReason, undefined); + assert.equal(currentWriter?.requestShapeHash, undefined); + assert.equal(currentWriter?.requestShapeChangeReason, undefined); + assert.equal(currentWriter?.promptSegments, undefined); + } assert.equal(pricingLookupCalls, 1); }); }); -describe('AiSdkBackend request-shape diagnostics', () => { - test('classifies targeted request-shape changes', () => { - const tools = canonicalizeToolSet( - [testTool('Read', z.object({ path: z.string() }))], - testTool(INVALID_TOOL_NAME, z.object({ tool: z.string().optional() })), - ); - const baseInput = { - connection: connection(), - modelId: 'mock-model-id', - systemPrompt: 'durable system', - providerOptions: { temperature: 0 }, - providerTools: tools.providerTools, - activeTools: tools.activeTools, - priorMessages: [{ role: 'user' as const, content: 'hello' }], - }; - const base = computeRequestShapeDiagnostic(baseInput, undefined); - - assert.equal( - computeRequestShapeDiagnostic( - { - ...baseInput, - systemPrompt: 'changed system', - }, - base, - ).prefixChangeReason, - 'system_prompt_changed', - ); - assert.equal( - computeRequestShapeDiagnostic( - { - ...baseInput, - providerTools: canonicalizeToolSet( - [testTool('Read', z.object({ path: z.string(), offset: z.number().optional() }))], - testTool(INVALID_TOOL_NAME, z.object({ tool: z.string().optional() })), - ).providerTools, - }, - base, - ).prefixChangeReason, - 'tool_schema_changed', - ); - assert.equal( - computeRequestShapeDiagnostic( - { - ...baseInput, - providerOptions: { temperature: 1 }, - }, - base, - ).prefixChangeReason, - 'provider_options_changed', - ); - assert.equal( - computeRequestShapeDiagnostic( - { - ...baseInput, - modelId: 'other-model', - }, - base, - ).prefixChangeReason, - 'model_or_provider_changed', - ); - const historyChanged = computeRequestShapeDiagnostic( - { - ...baseInput, - priorMessages: [{ role: 'assistant' as const, content: 'hello' }], - }, - base, - ); - assert.equal(historyChanged.prefixChangeReason, 'stable'); - assert.equal(historyChanged.prefixHash, base.prefixHash); - assert.equal(historyChanged.requestShapeChangeReason, 'history_projection_changed'); - assert.notEqual(historyChanged.requestShapeHash, base.requestShapeHash); - }); - - test('tool-result output hydration changes request shape without changing durable prefix', () => { - const tools = canonicalizeToolSet( - [testTool('Read', z.object({ path: z.string() }))], - testTool(INVALID_TOOL_NAME, z.object({ tool: z.string().optional() })), - ); - const toolCallMessage: ModelMessage = { - role: 'assistant', - content: [ - { - type: 'tool-call', - toolCallId: 'tool-1', - toolName: 'Read', - input: { path: 'archive.txt' }, - }, - ], - }; - const placeholderToolResult: ModelMessage = { - role: 'tool', - content: [ - { - type: 'tool-result', - toolCallId: 'tool-1', - toolName: 'Read', - output: { type: 'text', value: '[archived placeholder]' }, - }, - ], - }; - const hydratedToolResult: ModelMessage = { - role: 'tool', - content: [ - { - type: 'tool-result', - toolCallId: 'tool-1', - toolName: 'Read', - output: { type: 'text', value: 'hydrated archive payload '.repeat(20) }, - }, - ], - }; - const baseInput = { - connection: connection(), - modelId: 'mock-model-id', - systemPrompt: 'durable system', - providerOptions: { temperature: 0 }, - providerTools: tools.providerTools, - activeTools: tools.activeTools, - priorMessages: [toolCallMessage, placeholderToolResult], - }; - const placeholder = computeRequestShapeDiagnostic(baseInput, undefined); - const hydrated = computeRequestShapeDiagnostic( - { - ...baseInput, - priorMessages: [toolCallMessage, hydratedToolResult], - }, - placeholder, - ); - - assert.equal(hydrated.prefixChangeReason, 'stable'); - assert.equal(hydrated.prefixHash, placeholder.prefixHash); - assert.equal(hydrated.requestShapeChangeReason, 'history_projection_changed'); - assert.notEqual(hydrated.requestShapeHash, placeholder.requestShapeHash); - }); - +describe('AiSdkBackend tool availability diagnostics', () => { test('tool canonicalization is independent of registration order and places invalid last', () => { const invalid = testTool(INVALID_TOOL_NAME, z.object({ tool: z.string().optional() })); const first = canonicalizeToolSet( @@ -8882,87 +8730,6 @@ describe('AiSdkBackend request-shape diagnostics', () => { second.providerTools.map((tool) => tool.name), ['Read', 'Write', INVALID_TOOL_NAME], ); - assert.equal( - computeRequestShapeDiagnostic( - { - connection: connection(), - modelId: 'mock-model-id', - providerTools: first.providerTools, - activeTools: first.activeTools, - priorMessages: [], - }, - undefined, - ).componentHashes.toolSchemaHash, - computeRequestShapeDiagnostic( - { - connection: connection(), - modelId: 'mock-model-id', - providerTools: second.providerTools, - activeTools: second.activeTools, - priorMessages: [], - }, - undefined, - ).componentHashes.toolSchemaHash, - ); - }); - - test('classifies strict enabled-group expansion as tool_source_enabled', () => { - const invalid = testTool(INVALID_TOOL_NAME, z.object({ tool: z.string().optional() })); - const initialTools = canonicalizeToolSet( - [ - testTool('Read', z.object({ path: z.string() })), - testTool(TOOL_SEARCH_NAME, z.object({ query: z.string() })), - ], - invalid, - ); - const expandedTools = canonicalizeToolSet( - [ - testTool('Read', z.object({ path: z.string() })), - testTool('WebFetch', z.object({ url: z.string() })), - testTool(TOOL_SEARCH_NAME, z.object({ query: z.string() })), - ], - invalid, - ); - const groupCatalog = { web: ['WebFetch'] }; - const first = computeRequestShapeDiagnostic( - { - connection: connection(), - modelId: 'mock-model-id', - providerTools: initialTools.providerTools, - activeTools: initialTools.activeTools, - priorMessages: [], - toolAvailability: { - mode: 'search', - enabledSourceIds: [], - availableSourceIds: ['web'], - connectorToolName: TOOL_SEARCH_NAME, - visibleToolNamesBySource: groupCatalog, - }, - }, - undefined, - ); - const second = computeRequestShapeDiagnostic( - { - connection: connection(), - modelId: 'mock-model-id', - providerTools: expandedTools.providerTools, - activeTools: expandedTools.activeTools, - priorMessages: [], - toolAvailability: { - mode: 'search', - enabledSourceIds: ['web'], - availableSourceIds: [], - connectorToolName: TOOL_SEARCH_NAME, - visibleToolNamesBySource: groupCatalog, - }, - }, - first, - ); - - assert.equal(second.prefixChangeReason, 'tool_schema_changed'); - assert.equal(second.requestShapeChangeReason, 'tool_schema_changed'); - assert.equal(second.toolSchemaChangeReason, 'tool_source_enabled'); - assert.notEqual(second.prefixHash, first.prefixHash); }); test('backend full mode keeps the complete tool surface and omits the connector', async () => { @@ -8991,14 +8758,11 @@ describe('AiSdkBackend request-shape diagnostics', () => { assert.deepEqual(modelToolNames(model), sortedModelToolNames(['Read', 'WebFetch'])); assert.equal(modelToolNames(model).includes(TOOL_SEARCH_NAME), false); - // toolCount tracks the model-visible (active) tools — the two real tools. - // The invalid fallback lives in providerTools but is never advertised, so - // it is not counted (toolCount is the wire-visible subset). const usageEvent = events.find( (event): event is Extract => event.type === 'token_usage', ); - assert.equal(toolSchemaPromptSegment(usageEvent)?.toolCount, 2); + assert.equal(usageEvent?.promptSegments, undefined); }); test('preserves the tool-call provider prefix across user turns', async () => { @@ -9135,7 +8899,7 @@ describe('AiSdkBackend context budget and prompt attribution', () => { ); }); - test('usage events include prompt segments and context budget diagnostics', async () => { + test('usage events keep context budget diagnostics without live prompt estimates', async () => { const model = completionModel(); const events: SessionEvent[] = []; const backend = createTestAiSdkBackend({ @@ -9210,30 +8974,15 @@ describe('AiSdkBackend context budget and prompt attribution', () => { assert.ok(usage); assert.equal(usage.contextBudget?.policyName, 'test-budget'); assert.equal(usage.contextBudget?.droppedTurns, 0); - assert.equal( - usage.promptSegments?.some((segment) => segment.kind === 'prior_history'), - true, - ); - assert.equal( - usage.promptSegments?.some((segment) => segment.kind === 'tool_schema'), - true, - ); - assert.equal( - usage.promptSegments?.some((segment) => segment.kind === 'current_user'), - true, - ); - assert.equal( - usage.promptSegments?.some((segment) => segment.kind === 'turn_tail'), - false, - ); + assert.equal(usage.promptSegments, undefined); }); }); describe('AiSdkBackend RunTrace', () => { for (const protocol of ['openai-compatible', 'anthropic-compatible'] as const) { test(`records ${protocol} multi-step requests and reconciles complete attempt usage`, async () => { - const captures: ProviderRequestCaptureRecord[] = []; - const attempts: ProviderRequestAttemptRecord[] = []; + const captures: PreparedRequestArtifactInput[] = []; + const attempts: ModelCallAttempt[] = []; const durable = durableTurnHarness('turn-1', 'hi'); let calls = 0; const usageFor = (step: number) => { @@ -9332,32 +9081,31 @@ describe('AiSdkBackend RunTrace', () => { loadTurnRuntimeEvents: durable.loadTurnRuntimeEvents, newId: idGenerator(), now: monotonicClock(), - recordProviderRequestCapture: async (capture) => { + persistPreparedRequestArtifact: async (capture) => { captures.push(capture); return { artifactId: `artifact-${captures.length}` }; }, - recordProviderRequestAttempt: (attempt) => { + recordModelCallAttempt: ({ attempt }) => { attempts.push(attempt); }, }); - const events = await drainDurably(backend.send(durable.input()), durable); + const events = await drainDurably(backend.send(durable.input({ runId: 'run-1' })), durable); assert.equal(captures.length, 2); assert.deepEqual( attempts.map(({ step, attempt, status }) => ({ step, attempt, status })), [ - { step: 0, attempt: 1, status: 'completed' }, - { step: 1, attempt: 1, status: 'completed' }, + { step: 0, attempt: 0, status: 'completed' }, + { step: 1, attempt: 0, status: 'completed' }, ], ); - assert.equal(findFirstChangedCacheableSegment(captures[1]!, captures[0]!)?.kind, 'message'); const aggregate = events.find( (event): event is Extract => event.type === 'token_usage', ); assert.ok(aggregate); - const sum = (field: keyof ProviderRequestAttemptRecord) => + const sum = (field: keyof ModelCallAttempt) => attempts.reduce( (total, attempt) => total + ((attempt[field] as number | undefined) ?? 0), 0, @@ -9370,12 +9118,16 @@ describe('AiSdkBackend RunTrace', () => { }); } - test('captures the prepared request before the provider call and records its physical attempt', async () => { - const captures: ProviderRequestCaptureRecord[] = []; - const attempts: ProviderRequestAttemptRecord[] = []; + test('observes the prepared request at dispatch and records its canonical attempt', async () => { + const captures: PreparedRequestArtifactInput[] = []; + const attempts: ModelCallAttempt[] = []; const model = new MockLanguageModelV4({ doStream: async () => { - assert.equal(captures.length, 1, 'capture must be durable before provider dispatch'); + assert.equal( + captures.length, + 1, + 'artifact persistence must start before provider dispatch', + ); return { stream: simulateReadableStream({ chunks: [ @@ -9414,28 +9166,32 @@ describe('AiSdkBackend RunTrace', () => { tools: [], newId: idGenerator(), now: monotonicClock(), - recordProviderRequestCapture: async (capture) => { + persistPreparedRequestArtifact: async (capture) => { captures.push(capture); return { artifactId: `artifact-${captures.length}` }; }, - recordProviderRequestAttempt: async (attempt) => { + recordModelCallAttempt: async ({ attempt }) => { attempts.push(attempt); }, }); const events: SessionEvent[] = []; - for await (const event of backend.send({ turnId: 'turn-1', text: 'hi', context: [] })) { + for await (const event of backend.send({ + turnId: 'turn-1', + runId: 'run-1', + text: 'hi', + context: [], + })) { events.push(event); } assert.equal(captures.length, 1); assert.equal(attempts.length, 1); assert.equal(attempts[0]?.step, 0); - assert.equal(attempts[0]?.attempt, 1); + assert.equal(attempts[0]?.attempt, 0); assert.equal(attempts[0]?.status, 'completed'); assert.equal(attempts[0]?.contextWindow, 200_000); - assert.equal(attempts[0]?.captureId, captures[0]?.captureId); - assert.equal(attempts[0]?.cacheMissInputSource, 'derived'); + assert.equal(attempts[0]?.cacheMissInputTokens, 4); assert.equal( events.find((event) => event.type === 'token_usage')?.providerRequestTraceId, captures[0]?.traceId, @@ -9544,7 +9300,7 @@ describe('AiSdkBackend RunTrace', () => { assert.equal(stored?.type === 'token_usage' && 'contextRemaining' in stored, false); }); - test('does not call the provider when prepared-request persistence fails', async () => { + test('continues the canonical call when private request persistence fails', async () => { const model = completionModel(); const backend = createTestAiSdkBackend({ sessionId: 'session-1', @@ -9557,10 +9313,9 @@ describe('AiSdkBackend RunTrace', () => { tools: [], newId: idGenerator(), now: monotonicClock(), - recordProviderRequestCapture: async () => { + persistPreparedRequestArtifact: async () => { throw new Error('capture unavailable'); }, - recordProviderRequestAttempt: () => {}, }); const events: SessionEvent[] = []; @@ -9568,14 +9323,14 @@ describe('AiSdkBackend RunTrace', () => { events.push(event); } - assert.equal(model.doStreamCalls.length, 0); + assert.equal(model.doStreamCalls.length, 1); assert.equal(events.at(-1)?.type, 'complete'); - assert.equal(events.find((event) => event.type === 'complete')?.stopReason, 'error'); + assert.equal(events.find((event) => event.type === 'complete')?.stopReason, 'end_turn'); }); test('disables hidden AI SDK retries and traces the one explicit Runtime retry', async () => { - const captures: ProviderRequestCaptureRecord[] = []; - const attempts: ProviderRequestAttemptRecord[] = []; + const captures: PreparedRequestArtifactInput[] = []; + const attempts: ModelCallAttempt[] = []; let calls = 0; const model = new MockLanguageModelV4({ doStream: async () => { @@ -9619,25 +9374,25 @@ describe('AiSdkBackend RunTrace', () => { tools: [], newId: idGenerator(), now: monotonicClock(), - recordProviderRequestCapture: async (capture) => { + persistPreparedRequestArtifact: async (capture) => { captures.push(capture); return { artifactId: 'artifact-1' }; }, - recordProviderRequestAttempt: (attempt) => { + recordModelCallAttempt: ({ attempt }) => { attempts.push(attempt); }, providerRetrySleep: async () => {}, }); - await drain(backend.send({ turnId: 'turn-1', text: 'hi', context: [] })); + await drain(backend.send({ turnId: 'turn-1', runId: 'run-1', text: 'hi', context: [] })); assert.equal(calls, 2); assert.equal(captures.length, 1); assert.deepEqual( attempts.map(({ attempt, status }) => ({ attempt, status })), [ - { attempt: 1, status: 'failed' }, - { attempt: 2, status: 'completed' }, + { attempt: 0, status: 'failed' }, + { attempt: 1, status: 'completed' }, ], ); }); @@ -15763,12 +15518,6 @@ function sortedModelToolNames(toolNames: readonly string[]): string[] { }); } -function toolSchemaPromptSegment( - carrier: { promptSegments?: readonly { kind: string; toolCount?: number }[] } | undefined, -): { toolCount?: number } | undefined { - return carrier?.promptSegments?.find((segment) => segment.kind === 'tool_schema'); -} - function sha256(text: string): string { return createHash('sha256').update(text).digest('hex'); } diff --git a/packages/runtime/src/__tests__/computer-use-provider-protocol.test.ts b/packages/runtime/src/__tests__/computer-use-provider-protocol.test.ts index e928514689..2d37b9a955 100644 --- a/packages/runtime/src/__tests__/computer-use-provider-protocol.test.ts +++ b/packages/runtime/src/__tests__/computer-use-provider-protocol.test.ts @@ -27,6 +27,7 @@ import type { LlmConnection } from '@maka/core/llm-connections'; import type { SessionEvent } from '@maka/core/events'; import type { SessionHeader, StoredMessage } from '@maka/core/session'; +import type { ModelCallAttempt } from '@maka/core/model-call-attempt'; import { AiSdkBackend } from '../ai-sdk-backend.js'; import { @@ -35,10 +36,7 @@ import { type CuObservation, } from '../computer-use-tools.js'; import { buildProviderOptions, getAIModel } from '../model-factory.js'; -import type { - ProviderRequestAttemptRecord, - ProviderRequestCaptureRecord, -} from '../provider-request-telemetry.js'; +import type { PreparedRequestArtifactInput } from '../provider-request-telemetry.js'; import { backfillRuntimeEventsFromStoredMessages } from '../runtime-event-backfill.js'; import { createDurableTurnHarness } from './durable-turn-harness.js'; import { createTestAiSdkBackend } from './execution-boundary-test-helpers.js'; @@ -105,8 +103,8 @@ describe('Anthropic-compatible Computer Use product loops', () => { text: 'Set the fixture field to provider-loop.', }); const requestBodies: Array> = []; - const captures: ProviderRequestCaptureRecord[] = []; - const attempts: ProviderRequestAttemptRecord[] = []; + const captures: PreparedRequestArtifactInput[] = []; + const attempts: ModelCallAttempt[] = []; const server = await startJsonServer(async (request, response) => { assert.equal(request.method, 'POST'); assert.equal(request.url, provider.expectedPath); @@ -164,16 +162,16 @@ describe('Anthropic-compatible Computer Use product loops', () => { loadTurnRuntimeEvents: durable.loadTurnRuntimeEvents, newId: idGenerator(), now: monotonicClock(), - recordProviderRequestCapture: async (capture) => { + persistPreparedRequestArtifact: async (capture) => { captures.push(capture); return { artifactId: `capture-artifact-${captures.length}` }; }, - recordProviderRequestAttempt: (attempt) => { + recordModelCallAttempt: ({ attempt }) => { attempts.push(attempt); }, }); - for await (const event of runtime.send(durable.sendInput())) { + for await (const event of runtime.send(durable.sendInput({ runId: 'run-1' }))) { durable.record(event); events.push(event); if (event.type === 'tool_result') { @@ -216,11 +214,8 @@ describe('Anthropic-compatible Computer Use product loops', () => { assert.equal(attempt.status, 'completed'); assert.equal(attempt.inputTokens, 15); assert.equal(attempt.cacheReadInputTokens, 4); - assert.equal(attempt.cacheReadInputSource, 'provider'); assert.equal(attempt.cacheWriteInputTokens, 1); - assert.equal(attempt.cacheWriteInputSource, 'provider'); assert.equal(attempt.cacheMissInputTokens, 10); - assert.equal(attempt.cacheMissInputSource, 'provider'); assert.equal(attempt.outputTokens, 5); } for (const body of requestBodies) { @@ -663,8 +658,8 @@ describe('OpenAI-compatible product loops', () => { text: 'Set the fixture field to provider-loop.', }); const requestBodies: Array> = []; - const captures: ProviderRequestCaptureRecord[] = []; - const attempts: ProviderRequestAttemptRecord[] = []; + const captures: PreparedRequestArtifactInput[] = []; + const attempts: ModelCallAttempt[] = []; const server = await startJsonServer(async (request, response) => { assert.equal(request.method, 'POST'); assert.equal(request.url, '/coding/v1/chat/completions'); @@ -714,16 +709,16 @@ describe('OpenAI-compatible product loops', () => { loadTurnRuntimeEvents: durable.loadTurnRuntimeEvents, newId: idGenerator(), now: monotonicClock(), - recordProviderRequestCapture: async (capture) => { + persistPreparedRequestArtifact: async (capture) => { captures.push(capture); return { artifactId: `capture-artifact-${captures.length}` }; }, - recordProviderRequestAttempt: (attempt) => { + recordModelCallAttempt: ({ attempt }) => { attempts.push(attempt); }, }); - for await (const event of runtime.send(durable.sendInput())) { + for await (const event of runtime.send(durable.sendInput({ runId: 'run-1' }))) { durable.record(event); events.push(event); } diff --git a/packages/runtime/src/__tests__/context-diagnostics.test.ts b/packages/runtime/src/__tests__/context-diagnostics.test.ts index d5e84bcacc..9354d762a3 100644 --- a/packages/runtime/src/__tests__/context-diagnostics.test.ts +++ b/packages/runtime/src/__tests__/context-diagnostics.test.ts @@ -22,6 +22,7 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { test } from 'node:test'; +import { DatabaseSync } from 'node:sqlite'; import type { AgentRunEvent, AgentRunHeader, @@ -30,6 +31,49 @@ import type { } from '@maka/core/agent-run'; import { createSqliteAgentRunStore } from '@maka/storage/agent-run-store'; import { readLatestContextDiagnostics } from '../context-diagnostics.js'; +import { readLatestContextSnapshot } from '../latest-context-snapshot.js'; + +test('rejects v2 snapshots that the canonical writer cannot produce', () => { + const base = { + schemaVersion: 2, + attemptId: 'attempt-1', + providerId: 'anthropic', + modelId: 'model', + completedAt: 10, + }; + const impossible = [ + { ...base, composition: { segments: [] } }, + { ...base, composition: { segments: [{ kind: 'messages', bytes: 0 }] } }, + { + ...base, + composition: { + segments: [{ kind: 'messages', bytes: 10 }], + tools: [{ name: 'Bash', bytes: 10 }], + }, + }, + { + ...base, + composition: { + segments: [{ kind: 'tool_definitions', bytes: 10 }], + remainingTools: { count: 0, bytes: 0 }, + }, + }, + { + ...base, + compaction: { + kind: 'history', + phase: 'pre_turn', + eventCount: 0, + turnCount: 0, + estimatedTokens: 10, + }, + }, + ]; + + for (const snapshot of impossible) { + assert.equal(readLatestContextSnapshot({ type: 'latest_context', data: snapshot }), undefined); + } +}); test('serves the sealed snapshot without reading a single run', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-context-diagnostics-')); @@ -60,6 +104,94 @@ test('serves the sealed snapshot without reading a single run', async () => { } }); +test('does not trust a pre-observation projection over its canonical attempt', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-context-diagnostics-')); + try { + const writer = createSqliteAgentRunStore(root); + await writer.createRun(runHeader('run-1', 1)); + const oldProjection = latestContext('attempt-1', 10); + oldProjection.snapshot.schemaVersion = 1; + oldProjection.snapshot.composition = { + segments: [{ kind: 'messages', bytes: 999 }], + tools: [], + }; + await writer.appendEvent( + 'session-1', + 'run-1', + meteringEvent('run-1', 'attempt-1', 10, 'model', 40, 200), + { durable: true, latestContext: oldProjection }, + ); + + const reader = createSqliteAgentRunStore(root); + const warm = await readLatestContextDiagnostics(reader, 'session-1'); + const cold = await readLatestContextDiagnostics( + { + listSessionRuns: (sessionId) => reader.listSessionRuns(sessionId), + readEvents: (sessionId, runId) => reader.readEvents(sessionId, runId), + }, + 'session-1', + ); + + assert.equal(warm.status, 'available'); + assert.equal(cold.status, 'available'); + if (warm.status !== 'available' || cold.status !== 'available') return; + assert.equal(warm.composition, undefined); + assert.equal(cold.composition, undefined); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test('upgrades exact-matched mixed-era composition into the current projection', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-context-diagnostics-')); + try { + const writer = createSqliteAgentRunStore(root); + await writer.createRun(runHeader('run-1', 1)); + const oldProjection = latestContext('attempt-1', 10); + oldProjection.snapshot.schemaVersion = 1; + await writer.appendEvent( + 'session-1', + 'run-1', + meteringEvent('run-1', 'attempt-1', 10, 'model', 40, 200), + { durable: true, latestContext: oldProjection }, + ); + await writer.appendEvent( + 'session-1', + 'run-1', + attemptEvent('run-1', 'attempt-1', 10, 'completed', 'model', 40, 200, [ + { + kind: 'tool_schema', + index: 0, + cacheable: true, + hash: 'legacy', + bytes: 700, + label: 'HistoricalTool', + }, + ]), + ); + + let scanned = 0; + const reader = createSqliteAgentRunStore(root); + const counted = countingStore(reader, () => { + scanned += 1; + }); + const upgraded = await readLatestContextDiagnostics(counted, 'session-1'); + + assert.equal(upgraded.status, 'available'); + if (upgraded.status !== 'available') return; + assert.deepEqual(upgraded.composition?.tools, [{ name: 'HistoricalTool', bytes: 700 }]); + + scanned = 0; + const warm = await readLatestContextDiagnostics(counted, 'session-1'); + assert.equal(warm.status, 'available'); + if (warm.status !== 'available') return; + assert.deepEqual(warm.composition, upgraded.composition); + assert.equal(scanned, 0, 'the mixed-era composition is sealed into the v2 projection'); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + test('a failed call does not replace the last good snapshot', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-context-diagnostics-')); try { @@ -133,9 +265,10 @@ test("a subagent's run never becomes the session's context", async () => { } }); -test('rebuilds a legacy ledger, then repairs it so the next read scans nothing', async () => { - // A session written before canonical metering sealed anything. The scan is - // the compatibility path; proving it happens ONCE needs two reads. +test('rebuilds a canonical observation, then repairs it so the next read scans nothing', async () => { + // The event is durable but predates a sealed projection. The first read + // rebuilds from its canonical observation; proving that happens once needs + // two reads. const root = await mkdtemp(join(tmpdir(), 'maka-context-diagnostics-')); try { const writer = createSqliteAgentRunStore(root); @@ -143,7 +276,19 @@ test('rebuilds a legacy ledger, then repairs it so the next read scans nothing', await writer.appendEvent( 'session-1', 'run-1', - meteringEvent('run-1', 'attempt-1', 20, 'model-new', 40, 200), + meteringEvent('run-1', 'attempt-1', 20, 'model-new', 40, 200, { + requestObservation: requestObservation([ + { + kind: 'tool_schema', + index: 0, + cacheable: true, + comparison: 'exact', + digest: `sha256:${'a'.repeat(64)}`, + bytes: 800, + label: 'Bash', + }, + ]), + }), ); await writer.appendEvent( 'session-1', @@ -176,6 +321,27 @@ test('rebuilds a legacy ledger, then repairs it so the next read scans nothing', } }); +test('rebuilds without repairing when the store lacks a ledger revision capability', async () => { + const base = runStore([ + { + header: runHeader('run-1', 1), + events: [meteringEvent('run-1', 'attempt-1', 20, 'model', 40, 200)], + }, + ]); + let repaired = false; + const store: Parameters[0] = { + ...base, + repairEventProjection: async () => { + repaired = true; + }, + }; + + const diagnostics = await readLatestContextDiagnostics(store, 'session-1'); + + assert.equal(diagnostics.status, 'available'); + assert.equal(repaired, false); +}); + test('reads a provider-only ledger that predates canonical metering', async () => { // No `model_call_attempt_recorded` anywhere. The provider attempt is the // only record of the request, and returning "no completed request" would @@ -219,6 +385,73 @@ test('a canonical record on the ledger keeps the legacy path out of it', async ( assert.equal(diagnostics.modelId, 'model-canonical'); }); +test('cold rebuild takes composition from the canonical attempt observation', async () => { + const store = runStore([ + { + header: runHeader('run-1', 1), + events: [ + meteringEvent('run-1', 'attempt-1', 10, 'model-canonical', 40, 200, { + requestObservation: requestObservation([ + { + kind: 'tool_schema', + index: 0, + cacheable: true, + comparison: 'exact', + digest: `sha256:${'a'.repeat(64)}`, + bytes: 800, + label: 'CanonicalTool', + }, + ]), + }), + attemptEvent('run-1', 'attempt-1', 10, 'completed', 'model-canonical', 40, 200, [ + { + kind: 'tool_schema', + index: 0, + cacheable: true, + hash: 'legacy', + bytes: 999, + label: 'BestEffortTool', + }, + ]), + ], + }, + ]); + + const diagnostics = await readLatestContextDiagnostics(store, 'session-1'); + + assert.equal(diagnostics.status, 'available'); + if (diagnostics.status !== 'available') return; + assert.deepEqual(diagnostics.composition?.tools, [{ name: 'CanonicalTool', bytes: 800 }]); +}); + +test('does not enrich a canonical attempt from an identity-mismatched provider row', async () => { + const store = runStore([ + { + header: runHeader('run-1', 1), + events: [ + meteringEvent('run-1', 'attempt-1', 10, 'model-canonical', 40, 200), + attemptEvent('run-1', 'attempt-1', 11, 'completed', 'model-other', 40, 200, [ + { + kind: 'tool_schema', + index: 0, + cacheable: true, + hash: 'legacy', + bytes: 999, + label: 'WrongRequestTool', + }, + ]), + ], + }, + ]); + + const diagnostics = await readLatestContextDiagnostics(store, 'session-1'); + + assert.equal(diagnostics.status, 'available'); + if (diagnostics.status !== 'available') return; + assert.equal(diagnostics.modelId, 'model-canonical'); + assert.equal(diagnostics.composition, undefined); +}); + test('a legacy request whose capture is missing reports no composition, not an older one', async () => { const store = runStore([ { @@ -408,8 +641,8 @@ test('a session confirmed to have nothing is answered from the projection, not r }); test('names at most the bounded number of tools, and accounts for the rest', async () => { - // The 257th tool used to fail the whole query at the wire decoder. The fold - // bounds it instead, so a large registry summarises rather than breaks. + // Historical provider-only ledgers can contain unbounded segment arrays. + // Their reader still bounds the diagnostic instead of rejecting the row. const segments = Array.from({ length: 300 }, (_, index) => ({ kind: 'tool_schema', index, @@ -421,10 +654,7 @@ test('names at most the bounded number of tools, and accounts for the rest', asy const store = runStore([ { header: runHeader('run-1', 1), - events: [ - meteringEvent('run-1', 'attempt-1', 10, 'model', 40, 200), - attemptEvent('run-1', 'attempt-1', 10, 'completed', 'model', 40, 200, segments), - ], + events: [attemptEvent('run-1', 'attempt-1', 10, 'completed', 'model', 40, 200, segments)], }, ]); @@ -509,7 +739,10 @@ test('a damaged projection is repaired, not preserved forever', async () => { ts: 10, data: { schemaVersion: 1, damaged: true }, }, - { replaceEventId: 'latest-context-attempt-1' }, + { + ifLedgerRevision: await writer.readEventLedgerRevision('session-1'), + replaceEventId: 'latest-context-attempt-1', + }, ); let scanned = 0; @@ -532,6 +765,237 @@ test('a damaged projection is repaired, not preserved forever', async () => { } }); +test('repairs malformed projection bytes from the canonical ledger', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-context-diagnostics-')); + try { + const writer = createSqliteAgentRunStore(root); + await writer.createRun(runHeader('run-1', 1)); + await writer.appendEvent( + 'session-1', + 'run-1', + meteringEvent('run-1', 'attempt-1', 10, 'model', 40, 200, { + requestObservation: requestObservation([ + { + kind: 'tool_schema', + index: 0, + cacheable: true, + comparison: 'exact', + digest: `sha256:${'a'.repeat(64)}`, + bytes: 800, + label: 'Bash', + }, + ]), + }), + { durable: true, latestContext: latestContext('attempt-1', 10) }, + ); + writer.close?.(); + + const database = new DatabaseSync(join(root, 'runtime.sqlite')); + try { + database + .prepare(` + UPDATE core_agent_run_projections + SET event_json = '{malformed' + WHERE session_id = 'session-1' AND event_type = 'latest_context' + `) + .run(); + } finally { + database.close(); + } + + let scanned = 0; + const counted = countingStore(createSqliteAgentRunStore(root), () => { + scanned += 1; + }); + const first = await readLatestContextDiagnostics(counted, 'session-1'); + assert.equal(first.status, 'available'); + if (first.status !== 'available') return; + assert.deepEqual(first.composition?.tools, [{ name: 'Bash', bytes: 800 }]); + assert.ok(scanned > 0, 'the malformed bytes force a canonical rebuild'); + + scanned = 0; + const second = await readLatestContextDiagnostics(counted, 'session-1'); + assert.equal(second.status, 'available'); + assert.equal(scanned, 0, 'the authority-derived candidate replaced the malformed row'); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test('does not persist a cold answer after canonical authority advances', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-context-diagnostics-')); + try { + const store = createSqliteAgentRunStore(root); + await store.createRun(runHeader('run-1', 1)); + await store.appendEvent( + 'session-1', + 'run-1', + meteringEvent('run-1', 'attempt-1', 10, 'model-1', 40, 200), + { durable: true, latestContext: latestContext('attempt-1', 10, 'model-1') }, + ); + + const database = new DatabaseSync(join(root, 'runtime.sqlite')); + try { + database + .prepare(` + UPDATE core_agent_run_projections + SET event_json = '{malformed' + WHERE session_id = 'session-1' AND event_type = 'latest_context' + `) + .run(); + } finally { + database.close(); + } + + let advanced = false; + const racing: Parameters[0] = { + listSessionRuns: (sessionId) => store.listSessionRuns(sessionId), + readEvents: async (sessionId, runId) => { + const events = await store.readEvents(sessionId, runId); + if (!advanced) { + advanced = true; + await store.appendEvent( + 'session-1', + 'run-1', + meteringEvent('run-1', 'attempt-2', 20, 'model-2', 40, 200), + { durable: true, latestContext: latestContext('attempt-2', 20, 'model-2') }, + ); + } + return events; + }, + readEventProjection: (sessionId, type) => store.readEventProjection(sessionId, type), + readEventLedgerRevision: (sessionId) => store.readEventLedgerRevision(sessionId), + repairEventProjection: (sessionId, type, event, options) => + store.repairEventProjection(sessionId, type, event, options), + }; + + const cold = await readLatestContextDiagnostics(racing, 'session-1'); + assert.equal(cold.status, 'available'); + if (cold.status !== 'available') return; + assert.equal(cold.modelId, 'model-1', 'the in-flight read remains a valid earlier snapshot'); + + const next = await readLatestContextDiagnostics(store, 'session-1'); + assert.equal(next.status, 'available'); + if (next.status !== 'available') return; + assert.equal(next.modelId, 'model-2', 'the stale scan never becomes the warm projection'); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test('rebuilds a nested-malformed v2 projection from the canonical ledger', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-context-diagnostics-')); + try { + const writer = createSqliteAgentRunStore(root); + await writer.createRun(runHeader('run-1', 1)); + await writer.appendEvent( + 'session-1', + 'run-1', + meteringEvent('run-1', 'attempt-1', 10, 'model', 40, 200, { + requestObservation: requestObservation([ + { + kind: 'tool_schema', + index: 0, + cacheable: true, + comparison: 'exact', + digest: `sha256:${'a'.repeat(64)}`, + bytes: 800, + label: 'Bash', + }, + ]), + }), + { durable: true, latestContext: latestContext('attempt-1', 10) }, + ); + await writer.repairEventProjection( + 'session-1', + 'latest_context', + { + type: 'latest_context', + id: 'latest-context-malformed-v2', + runId: 'run-1', + sessionId: 'session-1', + turnId: 'turn-run-1', + ts: 10, + data: { + ...latestContext('attempt-1', 10).snapshot, + composition: { segments: 'not-an-array' }, + }, + }, + { + ifLedgerRevision: await writer.readEventLedgerRevision('session-1'), + replaceEventId: 'latest-context-attempt-1', + }, + ); + + let scanned = 0; + const counted = countingStore(createSqliteAgentRunStore(root), () => { + scanned += 1; + }); + const first = await readLatestContextDiagnostics(counted, 'session-1'); + assert.equal(first.status, 'available'); + if (first.status !== 'available') return; + assert.deepEqual(first.composition?.tools, [{ name: 'Bash', bytes: 800 }]); + assert.ok(scanned > 0, 'the malformed nested value cannot answer the warm read'); + + scanned = 0; + const second = await readLatestContextDiagnostics(counted, 'session-1'); + assert.equal(second.status, 'available'); + assert.equal(scanned, 0, 'the canonical rebuild repaired the rejected projection'); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test('an old readable-order projection is upgraded after one cold rebuild', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-context-diagnostics-')); + try { + const writer = createSqliteAgentRunStore(root); + await writer.createRun(runHeader('run-1', 1)); + await writer.appendEvent( + 'session-1', + 'run-1', + meteringEvent('run-1', 'attempt-1', 10, 'model', 40, 200), + { durable: true, latestContext: latestContext('attempt-1', 10) }, + ); + await writer.repairEventProjection( + 'session-1', + 'latest_context', + { + type: 'latest_context', + id: 'latest-context-attempt-1', + runId: 'run-1', + sessionId: 'session-1', + turnId: 'turn-run-1', + ts: 10, + data: { + ...latestContext('attempt-1', 10).snapshot, + schemaVersion: 1, + composition: { segments: [{ kind: 'tool_definitions', bytes: 999 }] }, + }, + }, + { + ifLedgerRevision: await writer.readEventLedgerRevision('session-1'), + replaceEventId: 'latest-context-attempt-1', + }, + ); + + let scanned = 0; + const counted = countingStore(createSqliteAgentRunStore(root), () => { + scanned += 1; + }); + const first = await readLatestContextDiagnostics(counted, 'session-1'); + assert.equal(first.status, 'available'); + assert.ok(scanned > 0, 'the old schema requires one canonical rebuild'); + + scanned = 0; + const second = await readLatestContextDiagnostics(counted, 'session-1'); + assert.equal(second.status, 'available'); + assert.equal(scanned, 0, 'the rebuilt current schema replaces the old row'); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + function countingStore( reader: ReturnType, onScan: () => void, @@ -543,6 +1007,7 @@ function countingStore( return reader.readEvents(sessionId, runId); }, readEventProjection: (sessionId, type) => reader.readEventProjection(sessionId, type), + readEventLedgerRevision: (sessionId) => reader.readEventLedgerRevision(sessionId), repairEventProjection: (sessionId, type, event, options) => reader.repairEventProjection(sessionId, type, event, options), }; @@ -557,7 +1022,7 @@ function latestContext(attemptId: string, completedAt: number, modelId = 'model' attemptId, orderedAt: completedAt, snapshot: { - schemaVersion: 1, + schemaVersion: 2, attemptId, providerId: 'anthropic', modelId, @@ -640,9 +1105,8 @@ function attemptEvent( } /** - * The DURABLE metering record. It is the anchor now: identity and every - * provider-reported number come from here, and the best-effort capture only - * gets to describe the request this one names (#2323). + * The durable canonical record. Identity, provider-reported numbers, and the + * prepared-request observation all describe this one dispatched attempt. */ function meteringEvent( runId: string, @@ -687,6 +1151,18 @@ function meteringEvent( }; } +function requestObservation(segments: Array>) { + return { + schemaVersion: 1, + digest: `sha256:${'f'.repeat(64)}`, + bytes: segments.reduce( + (total, segment) => total + (typeof segment.bytes === 'number' ? segment.bytes : 0), + 0, + ), + segments, + }; +} + function checkpointEvent( runId: string, ts: number, diff --git a/packages/runtime/src/__tests__/conversation-copy.test.ts b/packages/runtime/src/__tests__/conversation-copy.test.ts index cf7ccbb3a4..82693efa6c 100644 --- a/packages/runtime/src/__tests__/conversation-copy.test.ts +++ b/packages/runtime/src/__tests__/conversation-copy.test.ts @@ -1980,6 +1980,30 @@ test('conversation copy clones one terminal Runtime ledger with new owned identi latencyMs: 0.5, }, }); + await runStore.appendEvent('session-source', 'run-source', { + type: 'provider_request_attempt_recorded', + id: 'attempt-without-capture-source', + runId: 'run-source', + sessionId: 'session-source', + turnId: 'turn-1', + ts: 2.55, + data: { + traceId: 'provider-trace-without-capture-source', + attemptId: 'attempt-without-capture-source', + turnId: 'turn-1', + step: 2, + attempt: 1, + providerId: 'provider-without-capture', + modelId: 'model', + requestHash: 'request-hash-without-capture', + requestBytes: 13, + segments: [], + startedAt: 2.5, + completedAt: 2.55, + status: 'completed', + latencyMs: 0.05, + }, + }); // A legacy event from the retired active-full writer is treated like any // other event this build cannot emit and is therefore not copied. await runStore.appendEvent('session-source', 'run-source', { @@ -2182,6 +2206,7 @@ test('conversation copy clones one terminal Runtime ledger with new owned identi [ 'provider_request_captured', 'provider_request_attempt_recorded', + 'provider_request_attempt_recorded', 'history_compact_checkpoint_recorded', 'run_completed', ], @@ -2190,10 +2215,17 @@ test('conversation copy clones one terminal Runtime ledger with new owned identi (event) => event.type === 'provider_request_captured', ); const targetAttempt = targetOperationalEvents.find( - (event) => event.type === 'provider_request_attempt_recorded', + (event) => + event.type === 'provider_request_attempt_recorded' && event.data?.providerId === 'provider', + ); + const targetAttemptWithoutCapture = targetOperationalEvents.find( + (event) => + event.type === 'provider_request_attempt_recorded' && + event.data?.providerId === 'provider-without-capture', ); assert.ok(targetCapture); assert.ok(targetAttempt); + assert.ok(targetAttemptWithoutCapture); assert.equal(targetCapture.data?.captureId, targetCapture.id); assert.equal(targetCapture.data?.artifactId, 'artifact-target'); assert.notEqual(targetCapture.data?.traceId, 'provider-trace-source'); @@ -2201,6 +2233,8 @@ test('conversation copy clones one terminal Runtime ledger with new owned identi assert.equal(targetAttempt.data?.captureId, targetCapture.id); assert.equal(targetAttempt.data?.captureArtifactId, 'artifact-target'); assert.equal(targetAttempt.data?.traceId, targetCapture.data?.traceId); + assert.equal(targetAttemptWithoutCapture.data?.captureId, undefined); + assert.equal(targetAttemptWithoutCapture.data?.captureArtifactId, undefined); assert.equal(targetEvents[1]?.refs?.providerRequestTraceId, targetCapture.data?.traceId); assert.equal(targetEvents[1]?.refs?.traceEventId, targetCapture.id); assert.doesNotMatch(JSON.stringify(targetOperationalEvents), /OPAQUE_SOURCE_COMPACTION_STATE/); diff --git a/packages/runtime/src/__tests__/history-compact-checkpoint.test.ts b/packages/runtime/src/__tests__/history-compact-checkpoint.test.ts index e36c350178..f9b9e54c52 100644 --- a/packages/runtime/src/__tests__/history-compact-checkpoint.test.ts +++ b/packages/runtime/src/__tests__/history-compact-checkpoint.test.ts @@ -637,12 +637,14 @@ describe('history compact checkpoint', () => { const replacedEventIds: Array = []; const store = { readEventProjection: async () => poisonedProjection, + readEventLedgerRevision: async () => 'ledger-revision', repairEventProjection: async ( _sessionId: string, _type: AgentRunEvent['type'], _event: AgentRunEvent | null, - options?: { replaceEventId?: string }, + options: { ifLedgerRevision: string; replaceEventId?: string }, ) => { + assert.equal(options.ifLedgerRevision, 'ledger-revision'); replacedEventIds.push(options?.replaceEventId); }, listSessionRuns: async () => [run('run-canonical', 10)], @@ -777,11 +779,14 @@ describe('history compact checkpoint', () => { const repaired: Array = []; const store = { readEventProjection: async () => undefined, + readEventLedgerRevision: async () => 'ledger-revision', repairEventProjection: async ( _sessionId: string, _type: AgentRunEvent['type'], repairedEvent: AgentRunEvent | null, + options: { ifLedgerRevision: string }, ) => { + assert.equal(options.ifLedgerRevision, 'ledger-revision'); repaired.push(repairedEvent); }, listSessionRuns: async () => [run('run-recovered', 10)], @@ -794,6 +799,30 @@ describe('history compact checkpoint', () => { assert.deepEqual(repaired, [event]); }); + test('recovers without repairing when the store lacks a ledger revision capability', async () => { + const checkpoint = buildHistoryCompactCheckpoint({ + sessionId: 'session-1', + coveredRuntimeEvents: [textEvent(0)], + summary: 'recovered checkpoint', + summaryFormat: 'legacy_freeform', + }); + const event = checkpointEvent('recovered-event', 'run-recovered', checkpoint, 20); + let repaired = false; + const store = { + readEventProjection: async () => undefined, + repairEventProjection: async () => { + repaired = true; + }, + listSessionRuns: async () => [run('run-recovered', 10)], + readEvents: async () => [event], + }; + + const loaded = await loadLatestHistoryCompactCheckpointFromRunLedger(store, 'session-1'); + + assert.equal(loaded?.checkpointId, checkpoint.checkpointId); + assert.equal(repaired, false); + }); + test('identifies a parseable but invalid projection when repairing from the canonical ledger', async () => { const checkpoint = buildHistoryCompactCheckpoint({ sessionId: 'session-1', @@ -810,12 +839,14 @@ describe('history compact checkpoint', () => { const replacedEventIds: Array = []; const store = { readEventProjection: async () => invalidProjection, + readEventLedgerRevision: async () => 'ledger-revision', repairEventProjection: async ( _sessionId: string, _type: AgentRunEvent['type'], _event: AgentRunEvent | null, - options?: { replaceEventId?: string }, + options: { ifLedgerRevision: string; replaceEventId?: string }, ) => { + assert.equal(options.ifLedgerRevision, 'ledger-revision'); replacedEventIds.push(options?.replaceEventId); }, listSessionRuns: async () => [run('run-canonical', 10)], diff --git a/packages/runtime/src/__tests__/history-compact-summarizer.test.ts b/packages/runtime/src/__tests__/history-compact-summarizer.test.ts index 8379a31c98..0f89fdd084 100644 --- a/packages/runtime/src/__tests__/history-compact-summarizer.test.ts +++ b/packages/runtime/src/__tests__/history-compact-summarizer.test.ts @@ -134,8 +134,7 @@ describe('buildLlmHistorySummarizer', () => { return now; }, newId: () => 'trace-id', - persistCapture: async () => ({ artifactId: 'artifact-1' }), - recordAttempt: () => {}, + persistArtifact: async () => ({ artifactId: 'artifact-1' }), accounting: { sessionId: 'sess-1', resolveRunId: () => 'run-1', @@ -199,8 +198,7 @@ describe('buildLlmHistorySummarizer', () => { turnId: 'turn-1', now: () => 100 + id, newId: () => `request-${++id}`, - persistCapture: async () => ({ artifactId: `artifact-${id}` }), - recordAttempt: () => {}, + persistArtifact: async () => ({ artifactId: `artifact-${id}` }), accounting: { sessionId: 'sess-1', resolveRunId: () => 'run-1', diff --git a/packages/runtime/src/__tests__/latest-context-commit.test.ts b/packages/runtime/src/__tests__/latest-context-commit.test.ts index e1903d94a3..5075d6cbb9 100644 --- a/packages/runtime/src/__tests__/latest-context-commit.test.ts +++ b/packages/runtime/src/__tests__/latest-context-commit.test.ts @@ -36,8 +36,11 @@ import { join } from 'node:path'; import { test } from 'node:test'; import { MockLanguageModelV4, simulateReadableStream } from 'ai/test'; import type { LanguageModelV4StreamPart } from '@ai-sdk/provider'; -import type { ModelCallAttempt } from '@maka/core/model-call-attempt'; -import type { ModelCallCommit } from '@maka/core/agent-run'; +import { + decodeModelCallAttempt, + PREPARED_REQUEST_OBSERVATION_MAX_SEGMENTS, + type ModelCallAttempt, +} from '@maka/core/model-call-attempt'; import { createSqliteAgentRunStore } from '@maka/storage/agent-run-store'; import { createWorkspaceRuntimeStore } from '@maka/storage/runtime-event-persistence'; import { createSessionStore } from '@maka/storage/session-store'; @@ -45,7 +48,7 @@ import { BackendRegistry, SessionManager } from '../session-manager.js'; import { readLatestContextDiagnostics } from '../context-diagnostics.js'; import { createTestAiSdkBackend } from './execution-boundary-test-helpers.js'; -test('a real send seals its row all the way into SQLite, with nothing injected', async () => { +test('a real send seals its observation into SQLite and reconstructs it after restart', async () => { // Tracker → backend → the kernel seam a backend is actually built with → // AgentRun → the storage transaction. Every layer in that list once had a // signature that compiled while dropping the row, and no test crossed all of @@ -133,25 +136,137 @@ test('a real send seals its row all the way into SQLite, with nothing injected', assert.equal(scanned, 0, 'the row was committed by the send, not rebuilt by the read'); await manager.stopSession(session.id, { source: 'stop_button' }); + runStore.close?.(); + + const reopened = createSqliteAgentRunStore(root); + try { + const runs = await reopened.listSessionRuns(session.id); + const canonicalAttempts = ( + await Promise.all( + runs.map(async (run) => { + const events = await reopened.readEvents(session.id, run.runId); + return events + .filter((event) => event.type === 'model_call_attempt_recorded') + .map((event) => decodeModelCallAttempt(event.data)); + }), + ) + ).flat(); + assert.equal(canonicalAttempts.length, 1); + const observation = canonicalAttempts[0]?.requestObservation; + assert.ok(observation); + assert.ok(observation.segments.length <= PREPARED_REQUEST_OBSERVATION_MAX_SEGMENTS); + assert.ok(observation.segments.length > 0); + assert.ok(observation.segments.every((segment) => segment.comparison === 'exact')); + + let coldScans = 0; + const cold = await readLatestContextDiagnostics( + { + listSessionRuns: (sessionId) => reopened.listSessionRuns(sessionId), + readEvents: async (sessionId, runId) => { + coldScans += 1; + return reopened.readEvents(sessionId, runId); + }, + repairEventProjection: (sessionId, type, event, options) => + reopened.repairEventProjection(sessionId, type, event, options), + }, + session.id, + ); + + assert.ok(coldScans > 0, 'omitting the projection reader forces a restart-safe ledger fold'); + assert.equal(cold.status, 'available'); + if (cold.status !== 'available') return; + assert.deepEqual(cold.composition, diagnostics.composition); + } finally { + reopened.close?.(); + } } finally { await rm(root, { recursive: true, force: true }); } }); -test('a layer that forwards only the attempt no longer type-checks', () => { - // The regression this replaces was invisible precisely because it compiled. - // Keeping the shape in a value here means a future narrowing is a build - // failure rather than a silently missing feature. - const forward = (commit: ModelCallCommit) => commit; - const commit = { - attempt: { attemptId: 'a-1' } as ModelCallAttempt, - latestContext: { attemptId: 'a-1', orderedAt: 10, snapshot: { attemptId: 'a-1' } }, - } satisfies ModelCallCommit; +test('an artifact captured before abort does not create a canonical sent attempt', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-aborted-request-chain-')); + try { + const sessionStore = createSessionStore(root); + const runStore = createSqliteAgentRunStore(root); + const runtimeEventStore = createWorkspaceRuntimeStore(root); + const backends = new BackendRegistry(); + let ids = 0; + const newId = () => `abort-chain-${++ids}`; + let providerCalls = 0; + let artifactWrites = 0; + + backends.register('ai-sdk', (ctx) => { + let backend!: ReturnType; + backend = createTestAiSdkBackend({ + sessionId: ctx.sessionId, + header: ctx.header, + appendMessage: async () => {}, + connection: { + slug: 'mock-main', + providerType: 'anthropic', + defaultModel: 'mock-model-id', + models: [{ id: 'mock-model-id', contextWindow: 200_000 }], + }, + apiKey: 'sk-test', + modelId: 'mock-model-id', + modelFactory: () => + new MockLanguageModelV4({ + doStream: async () => { + providerCalls += 1; + return { stream: simulateReadableStream({ chunks: [] }) }; + }, + }), + tools: [], + persistPreparedRequestArtifact: async () => { + artifactWrites += 1; + void backend.stop('user_stop'); + return { artifactId: 'abandoned-artifact' }; + }, + ...(ctx.recordModelCallAttempt + ? { recordModelCallAttempt: ctx.recordModelCallAttempt } + : {}), + newId, + now: () => 1_000 + ids, + }); + return backend; + }); - const forwarded = forward(commit); + const manager = new SessionManager({ + store: sessionStore, + runStore, + runtimeEventStore, + backends, + newId, + now: () => 1_000 + ids, + }); + const session = await manager.createSession({ + cwd: root, + llmConnectionSlug: 'mock-main', + permissionMode: 'bypass', + }); + for await (const _event of manager.sendMessage(session.id, { + turnId: 'turn-aborted-before-dispatch', + text: 'abort after preparing the request', + })) { + // Drain the aborted turn through the real AgentRun store. + } - assert.equal(forwarded.latestContext?.attemptId, 'a-1', 'the derived row survives the hop'); - assert.equal(forwarded.attempt.attemptId, 'a-1'); + const runs = await runStore.listSessionRuns(session.id); + const events = ( + await Promise.all(runs.map((run) => runStore.readEvents(session.id, run.runId))) + ).flat(); + assert.equal(artifactWrites, 1); + assert.equal(providerCalls, 0); + assert.equal(events.filter((event) => event.type === 'model_call_attempt_recorded').length, 0); + assert.deepEqual(await readLatestContextDiagnostics(runStore, session.id), { + status: 'unavailable', + reason: 'no_completed_request', + }); + await manager.stopSession(session.id, { source: 'stop_button' }); + } finally { + await rm(root, { recursive: true, force: true }); + } }); function answeringModel(): MockLanguageModelV4 { diff --git a/packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts b/packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts index 12e9de8ac6..3342589670 100644 --- a/packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts +++ b/packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts @@ -522,7 +522,9 @@ function buildFixture(options: MidTurnFixtureOptions = {}): MidTurnFixture { }, ...(options.meteredSummarizer ? { - recordProviderRequestCapture: async () => ({ artifactId: 'artifact-mid-turn-capture' }), + persistPreparedRequestArtifact: async () => ({ + artifactId: 'artifact-mid-turn-capture', + }), recordModelCallAttempt: (commit: ModelCallCommit) => { commits.push(commit); modelCalls.push(commit.attempt); diff --git a/packages/runtime/src/__tests__/prompt-composition.test.ts b/packages/runtime/src/__tests__/prompt-composition.test.ts index 4c5b911065..6cff37a2ad 100644 --- a/packages/runtime/src/__tests__/prompt-composition.test.ts +++ b/packages/runtime/src/__tests__/prompt-composition.test.ts @@ -30,7 +30,7 @@ import { PROVIDER_REQUEST_ATTEMPT_EVENT_TYPE, } from '../prompt-composition.js'; import type { SizedRequestSegment } from '../prompt-composition.js'; -import { capturePreparedProviderRequest } from '../request-shape.js'; +import { prepareRequestObservation } from '../request-shape.js'; function segment(overrides: Partial = {}): SizedRequestSegment { return { kind: 'message', bytes: 10, ...overrides }; @@ -81,6 +81,17 @@ describe('foldPromptComposition', () => { assert.deepEqual(composition?.segments, [{ kind: 'tool_definitions', bytes: 750 }]); }); + test('preserves the count carried by a bounded tool remainder', () => { + const composition = foldPromptComposition([ + segment({ kind: 'tool_schema', bytes: 500, label: 'Bash' }), + segment({ kind: 'tool_schema', bytes: 9_000, representedSegments: 748 }), + ]); + + assert.deepEqual(composition?.tools, [{ name: 'Bash', bytes: 500 }]); + assert.deepEqual(composition?.remainingTools, { count: 748, bytes: 9_000 }); + assert.equal(composition?.unlabelledToolBytes, undefined); + }); + test('drops a kind nothing contributed to instead of showing it as zero', () => { const composition = foldPromptComposition([ segment({ kind: 'system_prompt', bytes: 400 }), @@ -102,18 +113,18 @@ describe('foldPromptComposition', () => { }); }); -describe('a real capture survives the whole chain into one fold', () => { - test('capture -> JSON -> decode -> fold keeps the same breakdown', () => { +describe('a real observation survives the whole chain into one fold', () => { + test('prepare -> canonical segments -> fold keeps the same breakdown', () => { // Every other test here writes its own segments, so a field renamed on one // side and not the other would pass all of them; and the decode side reads // `label` and `bytes` off an untyped record, so a hand-written fixture // agrees with itself by construction. This is the one test where the // writer, the storage encoding, the reader and the fold all meet. - const capture = capturePreparedProviderRequest({ - providerId: 'anthropic', - modelId: 'claude-test', - instructions: 'you are a helpful assistant', - messages: [{ role: 'user', content: 'hello' }], + const material = prepareRequestObservation({ + prompt: [ + { role: 'system', content: 'you are a helpful assistant' }, + { role: 'user', content: 'hello' }, + ], tools: [ { name: 'Bash', description: 'Run a command', inputSchema: { type: 'object' } }, { name: 'Read', inputSchema: { type: 'object' } }, @@ -121,7 +132,7 @@ describe('a real capture survives the whole chain into one fold', () => { providerOptions: { anthropic: { thinking: { type: 'enabled' } } }, }); - const composition = foldPromptComposition(capture.segments); + const composition = foldPromptComposition(material.observation.segments); assert.deepEqual( composition?.tools?.map((tool) => tool.name), @@ -138,23 +149,27 @@ describe('a real capture survives the whole chain into one fold', () => { composition!.tools!.reduce((carry, tool) => carry + tool.bytes, 0), 'the per-tool rows sum to the tool total above them', ); + }); - const stored = JSON.parse( - JSON.stringify({ - type: PROVIDER_REQUEST_ATTEMPT_EVENT_TYPE, - data: { - attemptId: 'attempt-1', - requestBytes: capture.requestBytes, - segments: capture.segments, - }, - }), - ); + test('a single-tool bounded remainder still counts as one remaining tool', () => { + const material = prepareRequestObservation({ + prompt: [ + { role: 'system', content: 'system' }, + { role: 'user', content: 'hello' }, + { role: 'assistant', content: 'hi' }, + ], + tools: Array.from({ length: 253 }, (_, index) => ({ + name: `tool-${String(index).padStart(3, '0')}`, + inputSchema: { type: 'object' }, + })), + providerOptions: { anthropic: { thinking: { type: 'enabled' } } }, + }); - assert.deepEqual( - readPromptCompositionEvent(stored)?.composition, - composition, - 'and the ledger round-trip changes none of it', - ); + assert.equal(material.observation.segments.length, 256); + const composition = foldPromptComposition(material.observation.segments); + assert.equal(composition?.tools?.length, 64); + assert.equal(composition?.remainingTools?.count, 189); + assert.equal(composition?.unlabelledToolBytes, undefined); }); }); diff --git a/packages/runtime/src/__tests__/provider-request-telemetry.test.ts b/packages/runtime/src/__tests__/provider-request-telemetry.test.ts index 308787e7f0..8aa2970afd 100644 --- a/packages/runtime/src/__tests__/provider-request-telemetry.test.ts +++ b/packages/runtime/src/__tests__/provider-request-telemetry.test.ts @@ -244,94 +244,17 @@ describe('strict provider-request usage', () => { }); }); -describe('provider request capture commit', () => { - test('links body-free metadata and returns the committed artifact reference', async () => { - const ledgerCaptures: Array> = []; - const recordCapture = telemetry.createProviderRequestCaptureRecorder({ - persistArtifact: async () => ({ artifactId: 'artifact-capture-1' }), - recordLedger: async (capture) => { - ledgerCaptures.push(capture as unknown as Record); - }, - }); - - const result = await recordCapture({ - schemaVersion: 2, - traceId: 'trace-1', - captureId: 'capture-1', - turnId: 'turn-1', - step: 0, - providerId: 'openai', - modelId: 'gpt-test', - requestHash: 'sha256:request', - requestPayloadWithoutProviderOptionsHash: 'sha256:shared-request', - requestBytes: 2, - segments: [], - serializedRequest: '{}', - }); - - assert.deepEqual(result, { artifactId: 'artifact-capture-1' }); - assert.equal(ledgerCaptures.length, 1); - assert.equal(ledgerCaptures[0]?.artifactId, 'artifact-capture-1'); - assert.equal(Object.hasOwn(ledgerCaptures[0]!, 'serializedRequest'), false); - }); - - test('retains the request artifact when a failed ledger append may have landed', async () => { - const ledgerError = new Error('capture ledger append failed'); - const ledgerCaptures: Array> = []; - const persistedArtifactIds = new Set(); - const createRecorder = Reflect.get( - telemetry, - 'createProviderRequestCaptureRecorder', - ) as unknown as - | ((input: Record) => (capture: Record) => Promise) - | undefined; - assert.equal(typeof createRecorder, 'function'); - const recordCapture = createRecorder!({ - persistArtifact: async () => { - persistedArtifactIds.add('artifact-capture-1'); - return { artifactId: 'artifact-capture-1' }; - }, - recordLedger: async (capture: Record) => { - ledgerCaptures.push(capture); - throw ledgerError; - }, - }); - - await assert.rejects( - recordCapture({ - schemaVersion: 2, - traceId: 'trace-1', - captureId: 'capture-1', - turnId: 'turn-1', - step: 0, - providerId: 'openai', - modelId: 'gpt-test', - requestHash: 'sha256:request', - requestPayloadWithoutProviderOptionsHash: 'sha256:shared-request', - requestBytes: 2, - segments: [], - serializedRequest: '{}', - }), - (error) => error === ledgerError, - ); - assert.equal(ledgerCaptures.length, 1); - assert.deepEqual([...persistedArtifactIds], ['artifact-capture-1']); - }); -}); - describe('provider request tracker', () => { test('records the request model context window on completed attempts', async () => { - const attempts: telemetry.ProviderRequestAttemptRecord[] = []; + const attempts: ModelCallAttempt[] = []; const tracker = new telemetry.ProviderRequestTracker({ traceId: 'trace-context', turnId: 'turn-context', contextWindow: 200_000, now: () => Date.now(), newId: () => 'id', - persistCapture: async () => ({ artifactId: 'artifact' }), - recordAttempt: async (attempt) => { - attempts.push(attempt); - }, + persistArtifact: async () => ({ artifactId: 'artifact' }), + accounting: canonicalAccounting(attempts), }); const result = await tracker.trackStream({ @@ -346,17 +269,15 @@ describe('provider request tracker', () => { }); test('omits a non-positive request model context window', async () => { - const attempts: telemetry.ProviderRequestAttemptRecord[] = []; + const attempts: ModelCallAttempt[] = []; const tracker = new telemetry.ProviderRequestTracker({ traceId: 'trace-context', turnId: 'turn-context', contextWindow: 0, now: () => Date.now(), newId: () => 'id', - persistCapture: async () => ({ artifactId: 'artifact' }), - recordAttempt: async (attempt) => { - attempts.push(attempt); - }, + persistArtifact: async () => ({ artifactId: 'artifact' }), + accounting: canonicalAccounting(attempts), }); const result = await tracker.trackStream({ @@ -373,44 +294,20 @@ describe('provider request tracker', () => { test('persists a logical capture before each physical attempt and reuses it for retries', async () => { const captures: Array<{ captureId: string; - requestHash: string; serializedRequest: string; }> = []; - const attempts: Array<{ - step: number; - attempt: number; - status: string; - captureId: string; - }> = []; - const Tracker = Reflect.get(telemetry, 'ProviderRequestTracker') as unknown as - | (new ( - input: Record, - ) => { - setStep(step: number): void; - trackStream(input: Record): Promise<{ stream: ReadableStream }>; - }) - | undefined; - assert.equal(typeof Tracker, 'function'); + const attempts: ModelCallAttempt[] = []; let id = 0; - const tracker = new Tracker!({ + const tracker = new telemetry.ProviderRequestTracker({ traceId: 'trace-1', turnId: 'turn-1', now: () => Date.now(), newId: () => `id-${++id}`, - persistCapture: async (capture: { - captureId: string; - requestHash: string; - serializedRequest: string; - }) => { + persistArtifact: async (capture) => { captures.push(capture); return { artifactId: `artifact-${captures.length}` }; }, - recordAttempt: async (attempt: { - step: number; - attempt: number; - status: string; - captureId: string; - }) => attempts.push(attempt), + accounting: canonicalAccounting(attempts), }); tracker.setStep(2); const params = preparedParams('hello'); @@ -461,34 +358,34 @@ describe('provider request tracker', () => { assert.equal(captures.length, 1); assert.deepEqual(JSON.parse(captures[0]!.serializedRequest), params); assert.deepEqual( - attempts.map(({ step, attempt, status, captureId }) => ({ + attempts.map(({ step, attempt, status, captureArtifactId }) => ({ step, attempt, status, - captureId, + captureArtifactId, })), [ { step: 2, - attempt: 1, + attempt: 0, status: 'failed', - captureId: captures[0]!.captureId, + captureArtifactId: 'artifact-1', }, { step: 2, - attempt: 2, + attempt: 1, status: 'completed', - captureId: captures[0]!.captureId, + captureArtifactId: 'artifact-1', }, ], ); - assert.equal((attempts[1] as Record).cacheReadInputSource, 'provider'); - assert.equal((attempts[1] as Record).cacheMissInputSource, 'derived'); + assert.equal(attempts[1]?.cacheReadInputTokens, 4); + assert.equal(attempts[1]?.cacheMissInputTokens, 6); }); test('captures and attributes a non-streaming physical provider call', async () => { const captures: Array<{ captureId: string; serializedRequest: string }> = []; - const attempts: Array> = []; + const attempts: ModelCallAttempt[] = []; let providerCalls = 0; let id = 0; const tracker = new telemetry.ProviderRequestTracker({ @@ -496,13 +393,11 @@ describe('provider request tracker', () => { turnId: 'turn-history', now: () => 1_000 + id, newId: () => `history-${++id}`, - persistCapture: async (capture) => { + persistArtifact: async (capture) => { captures.push(capture); return { artifactId: 'history-artifact' }; }, - recordAttempt: (attempt) => { - attempts.push(attempt as unknown as Record); - }, + accounting: canonicalAccounting(attempts), }); const params = preparedParams('history summary'); const result = await tracker.trackGenerate({ @@ -529,12 +424,12 @@ describe('provider request tracker', () => { assert.equal(captures.length, 1); assert.deepEqual(JSON.parse(captures[0]!.serializedRequest), params); assert.deepEqual( - attempts.map(({ status, finishReason, inputTokens, outputTokens, captureId }) => ({ + attempts.map(({ status, finishReason, inputTokens, outputTokens, captureArtifactId }) => ({ status, finishReason, inputTokens, outputTokens, - captureId, + captureArtifactId, })), [ { @@ -542,24 +437,25 @@ describe('provider request tracker', () => { finishReason: 'stop', inputTokens: 7, outputTokens: 3, - captureId: captures[0]!.captureId, + captureArtifactId: 'history-artifact', }, ], ); }); - test('redacts native compaction state from provider request captures', async () => { - const captures: Array<{ serializedRequest: string }> = []; + test('derives the artifact and canonical opaque observation from one redacted request', async () => { + const captures: telemetry.PreparedRequestArtifactInput[] = []; + const attempts: ModelCallAttempt[] = []; const tracker = new telemetry.ProviderRequestTracker({ traceId: 'compaction-trace', turnId: 'turn-compaction', now: () => 1_000, newId: () => 'compaction-id', - persistCapture: async (capture) => { + persistArtifact: async (capture) => { captures.push(capture); return { artifactId: 'compaction-artifact' }; }, - recordAttempt: () => undefined, + accounting: canonicalAccounting(attempts), }); const params = { image: new URL('https://example.com/provider-image.png'), @@ -607,7 +503,11 @@ describe('provider request tracker', () => { }); assert.equal(captures.length, 1); + assert.equal(attempts.length, 1); + assert.deepEqual(attempts[0]?.requestObservation, captures[0]?.observation); + assert.equal(attempts[0]?.requestObservation?.segments[0]?.comparison, 'opaque'); assert.doesNotMatch(captures[0]!.serializedRequest, /cmp_secret|OPAQUE_ENCRYPTED_STATE/); + assert.doesNotMatch(JSON.stringify(attempts[0]), /cmp_secret|OPAQUE_ENCRYPTED_STATE/); assert.deepEqual(JSON.parse(captures[0]!.serializedRequest), { image: 'https://example.com/provider-image.png', prompt: [ @@ -654,11 +554,10 @@ describe('provider request tracker', () => { beforeDispatch: async () => { throw new Error('Run Composition store unavailable'); }, - persistCapture: async () => { + persistArtifact: async () => { captured = true; return { artifactId: 'unreachable-artifact' }; }, - recordAttempt: () => {}, }); await assert.rejects( @@ -678,26 +577,19 @@ describe('provider request tracker', () => { assert.equal(dispatched, false); }); - test('captures a changed logical body separately and blocks provider calls on capture failure', async () => { + test('dispatches with its observation when private artifact persistence fails', async () => { const captures: string[] = []; - const Tracker = Reflect.get(telemetry, 'ProviderRequestTracker') as unknown as new ( - input: Record, - ) => { - setStep(step: number): void; - trackStream(input: Record): Promise<{ stream: ReadableStream }>; - }; let providerCalls = 0; - const tracker = new Tracker({ + const tracker = new telemetry.ProviderRequestTracker({ traceId: 'trace-2', turnId: 'turn-2', now: () => Date.now(), newId: () => `capture-${captures.length + 1}`, - persistCapture: async (capture: { requestHash: string }) => { - captures.push(capture.requestHash); + persistArtifact: async (capture) => { + captures.push(capture.observation.digest); if (captures.length === 2) throw new Error('capture unavailable'); return { artifactId: 'artifact-1' }; }, - recordAttempt: () => {}, }); tracker.setStep(0); const completed = await tracker.trackStream({ @@ -712,39 +604,73 @@ describe('provider request tracker', () => { }); await drain(completed.stream); - await assert.rejects( - tracker.trackStream({ - providerId: 'anthropic', - modelId: 'claude-test', - params: preparedParams('after'), - abortSignal: new AbortController().signal, - doStream: async () => { - providerCalls += 1; - return { stream: streamOf([finishPart()]) }; - }, - }), - /capture unavailable/, - ); - assert.equal(providerCalls, 1); + const withoutArtifact = await tracker.trackStream({ + providerId: 'anthropic', + modelId: 'claude-test', + params: preparedParams('after'), + abortSignal: new AbortController().signal, + doStream: async () => { + providerCalls += 1; + return { stream: streamOf([finishPart()]) }; + }, + }); + await drain(withoutArtifact.stream); + assert.equal(providerCalls, 2); assert.equal(captures.length, 2); assert.notEqual(captures[0], captures[1]); }); + test('does not wait for private artifact persistence before dispatch or accounting', async () => { + let releaseArtifact!: (value: { artifactId: string }) => void; + const artifactPending = new Promise<{ artifactId: string }>((resolve) => { + releaseArtifact = resolve; + }); + const recorded: ModelCallAttempt[] = []; + let providerCalls = 0; + const tracker = new telemetry.ProviderRequestTracker({ + traceId: 'trace-slow-artifact', + turnId: 'turn-slow-artifact', + now: () => 1_000, + newId: () => 'slow-artifact-id', + persistArtifact: () => artifactPending, + accounting: { + sessionId: 'session-1', + resolveRunId: () => 'run-1', + callKind: 'main', + record: ({ attempt }) => { + recorded.push(attempt); + }, + }, + }); + + const tracked = tracker.trackGenerate({ + providerId: 'anthropic', + modelId: 'claude-test', + params: preparedParams('hello'), + doGenerate: async () => { + providerCalls += 1; + return { finishReason: 'stop' }; + }, + }); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(providerCalls, 1, 'provider dispatch does not wait for artifact persistence'); + assert.equal(recorded.length, 1, 'canonical accounting does not wait for the artifact either'); + releaseArtifact({ artifactId: 'artifact-late' }); + await tracked; + + assert.equal(recorded[0]?.captureArtifactId, undefined); + assert.ok(recorded[0]?.requestObservation); + }); + test('records an errored stream after output as interrupted', async () => { - const attempts: Array<{ status: string }> = []; - const Tracker = Reflect.get(telemetry, 'ProviderRequestTracker') as unknown as new ( - input: Record, - ) => { - setStep(step: number): void; - trackStream(input: Record): Promise<{ stream: ReadableStream }>; - }; - const tracker = new Tracker({ + const attempts: ModelCallAttempt[] = []; + const tracker = new telemetry.ProviderRequestTracker({ traceId: 'trace-3', turnId: 'turn-3', now: () => Date.now(), newId: () => 'id', - persistCapture: async () => ({ artifactId: 'artifact' }), - recordAttempt: async (attempt: { status: string }) => attempts.push(attempt), + persistArtifact: async () => ({ artifactId: 'artifact' }), + accounting: canonicalAccounting(attempts), }); tracker.setStep(0); const result = await tracker.trackStream({ @@ -759,17 +685,15 @@ describe('provider request tracker', () => { }); test('records an in-flight attempt as aborted when its signal is cancelled', async () => { - const attempts: Array<{ status: string }> = []; + const attempts: ModelCallAttempt[] = []; const abort = new AbortController(); const tracker = new telemetry.ProviderRequestTracker({ traceId: 'trace-4', turnId: 'turn-4', now: () => Date.now(), newId: () => 'id', - persistCapture: async () => ({ artifactId: 'artifact' }), - recordAttempt: async (attempt) => { - attempts.push(attempt); - }, + persistArtifact: async () => ({ artifactId: 'artifact' }), + accounting: canonicalAccounting(attempts), }); tracker.setStep(0); await tracker.trackStream({ @@ -788,7 +712,7 @@ describe('provider request tracker', () => { test('does not capture or record an attempt when cancellation predates dispatch', async () => { let captures = 0; - let attempts = 0; + const attempts: ModelCallAttempt[] = []; let providerCalls = 0; const abort = new AbortController(); abort.abort(); @@ -797,13 +721,11 @@ describe('provider request tracker', () => { turnId: 'turn-5', now: () => Date.now(), newId: () => 'id', - persistCapture: async () => { + persistArtifact: async () => { captures += 1; return { artifactId: 'artifact' }; }, - recordAttempt: async () => { - attempts += 1; - }, + accounting: canonicalAccounting(attempts), }); await assert.rejects( @@ -821,13 +743,13 @@ describe('provider request tracker', () => { ); assert.equal(captures, 0); - assert.equal(attempts, 0); + assert.equal(attempts.length, 0); assert.equal(providerCalls, 0); }); test('does not dispatch or record an attempt when cancellation happens during capture', async () => { let captures = 0; - let attempts = 0; + const attempts: ModelCallAttempt[] = []; let providerCalls = 0; const abort = new AbortController(); const tracker = new telemetry.ProviderRequestTracker({ @@ -835,14 +757,12 @@ describe('provider request tracker', () => { turnId: 'turn-6', now: () => Date.now(), newId: () => 'id', - persistCapture: async () => { + persistArtifact: async () => { captures += 1; abort.abort(); return { artifactId: 'artifact' }; }, - recordAttempt: async () => { - attempts += 1; - }, + accounting: canonicalAccounting(attempts), }); await assert.rejects( @@ -860,11 +780,22 @@ describe('provider request tracker', () => { ); assert.equal(captures, 1); - assert.equal(attempts, 0); + assert.equal(attempts.length, 0); assert.equal(providerCalls, 0); }); }); +function canonicalAccounting(attempts: ModelCallAttempt[]): telemetry.ModelCallAccountingInput { + return { + sessionId: 'session-1', + resolveRunId: () => 'run-1', + callKind: 'main', + record: ({ attempt }) => { + attempts.push(attempt); + }, + }; +} + function preparedParams(text: string): Record { return { prompt: [ @@ -934,7 +865,6 @@ describe('canonical model-call accounting', () => { resolveRunId?: () => string | undefined; /** Models a deployment with request capture switched off. */ withoutCapture?: boolean; - recordAttempt?: (attempt: telemetry.ProviderRequestAttemptRecord) => void; callKind?: ModelCallAttempt['callKind']; historyCompactRoute?: ModelCallAttempt['historyCompactRoute']; }): telemetry.ProviderRequestTracker { @@ -946,8 +876,7 @@ describe('canonical model-call accounting', () => { newId: () => `id-${++n}`, ...(overrides.withoutCapture ? {} - : { persistCapture: async () => ({ artifactId: 'artifact-1' }) }), - recordAttempt: overrides.recordAttempt ?? (() => {}), + : { persistArtifact: async () => ({ artifactId: 'artifact-1' }) }), accounting: { sessionId: 'session-1', resolveRunId: overrides.resolveRunId ?? (() => 'run-1'), @@ -962,6 +891,47 @@ describe('canonical model-call accounting', () => { }); } + test('a capture abandoned before dispatch never enters the canonical sent sequence', async () => { + const recorded: ModelCallAttempt[] = []; + let providerCalls = 0; + const abort = new AbortController(); + const tracker = new telemetry.ProviderRequestTracker({ + traceId: 'trace-abandoned-capture', + turnId: 'turn-abandoned-capture', + now: () => 1_000, + newId: () => 'capture-abandoned', + persistArtifact: async () => { + abort.abort(); + return { artifactId: 'artifact-abandoned' }; + }, + accounting: { + sessionId: 'session-1', + resolveRunId: () => 'run-1', + callKind: 'main', + record: ({ attempt }) => { + recorded.push(attempt); + }, + }, + }); + + await assert.rejects( + tracker.trackGenerate({ + providerId: 'anthropic', + modelId: 'claude-test', + params: preparedParams('hello'), + abortSignal: abort.signal, + doGenerate: async () => { + providerCalls += 1; + return { finishReason: 'stop' }; + }, + }), + { name: 'AbortError' }, + ); + + assert.equal(providerCalls, 0); + assert.deepEqual(recorded, []); + }); + test('emits a decodable priced record for a completed call', async () => { const recorded: ModelCallAttempt[] = []; const tracker = accountingTracker({ @@ -995,16 +965,12 @@ describe('canonical model-call accounting', () => { test('persists a structured failure fingerprint and the selected compaction route', async () => { const recorded: ModelCallAttempt[] = []; - const diagnosticAttempts: telemetry.ProviderRequestAttemptRecord[] = []; const tracker = accountingTracker({ callKind: 'history_compact', historyCompactRoute: 'provider_native', record: ({ attempt }) => { recorded.push(attempt); }, - recordAttempt: (attempt) => { - diagnosticAttempts.push(attempt); - }, }); const providerError = Object.assign(new Error('provider payload must not persist'), { name: 'AI_APICallError', @@ -1035,14 +1001,10 @@ describe('canonical model-call accounting', () => { assert.equal(attempt.providerCode, 'rate_limit_exceeded'); assert.equal(attempt.providerRequestId, 'req-compact-1'); assert.equal(attempt.retryable, false); - assert.deepEqual(diagnosticAttempts[0]?.failure, { - errorClass: 'RateLimit', - httpStatus: 429, - providerCode: 'rate_limit_exceeded', - providerRequestId: 'req-compact-1', - retryable: false, - }); - assert.doesNotMatch(JSON.stringify(attempt), /private|prompt|response body/i); + assert.doesNotMatch( + JSON.stringify(attempt), + /private request body|private response body|private prompt/i, + ); }); test('records the physical route when one compaction call falls back', async () => { @@ -1136,15 +1098,11 @@ describe('canonical model-call accounting', () => { // request body is still a record of a call that really was billed, so the // canonical seam must not be gated on the capture sink being configured. const recorded: ModelCallAttempt[] = []; - const attempts: telemetry.ProviderRequestAttemptRecord[] = []; const tracker = accountingTracker({ withoutCapture: true, record: ({ attempt }) => { recorded.push(attempt); }, - recordAttempt: (a) => { - attempts.push(a); - }, }); const result = await tracker.trackStream({ @@ -1158,12 +1116,8 @@ describe('canonical model-call accounting', () => { const attempt = decodeModelCallAttempt(recorded[0]); assert.equal(attempt.usageBasis, 'reported'); assert.equal(attempt.captureArtifactId, undefined, 'there is no artifact to point at'); - // The request shape is computed locally, so it does not need the sink. - assert.equal(attempts.length, 1); - assert.equal(attempts[0]?.captureId, undefined); - assert.equal(attempts[0]?.captureArtifactId, undefined); - assert.ok((attempts[0]?.requestHash?.length ?? 0) > 0); - assert.ok((attempts[0]?.requestBytes ?? 0) > 0); + assert.match(attempt.requestObservation?.digest ?? '', /^sha256:[a-f0-9]{64}$/); + assert.ok((attempt.requestObservation?.segments.length ?? 0) > 0); }); test('an unresolvable price records unpriced rather than zero', async () => { diff --git a/packages/runtime/src/__tests__/request-shape.test.ts b/packages/runtime/src/__tests__/request-shape.test.ts index 0903f2df46..0286eb02e8 100644 --- a/packages/runtime/src/__tests__/request-shape.test.ts +++ b/packages/runtime/src/__tests__/request-shape.test.ts @@ -17,539 +17,181 @@ * under the License. */ -import { describe, test } from 'node:test'; +import { Buffer } from 'node:buffer'; +import { createHash } from 'node:crypto'; import assert from 'node:assert/strict'; +import { describe, test } from 'node:test'; import { canonicalizeToolSet, + prepareRequestObservation, toolSchemaCharsForDiagnostics, - computeRequestShapeDiagnostic, } from '../request-shape.js'; -import * as requestShape from '../request-shape.js'; import type { MakaTool } from '../tool-runtime.js'; function tool(name: string): MakaTool { - return { - name, - description: name, - parameters: {}, - impl: () => ({}), - }; + return { name, description: name, parameters: {}, impl: () => ({}) }; } const invalid = tool('invalid'); describe('canonicalizeToolSet active allow-list', () => { - test('a tool absent from the active set is withheld; the set drives visibility', () => { - const { activeTools } = canonicalizeToolSet( + test('withholds inactive tools without removing them from the dispatch registry', () => { + const { providerTools, activeTools } = canonicalizeToolSet( [tool('Read'), tool('Rive'), tool('tool_search')], invalid, new Set(['Read', 'tool_search']), ); - assert.ok(activeTools.includes('Read'), 'Read is in the active set'); - assert.ok(activeTools.includes('tool_search'), 'tool_search is in the active set'); - assert.ok(!activeTools.includes('Rive'), 'Rive is absent from the active set, so hidden'); - }); - - test('a tool becomes active once it is in the active set', () => { - const { activeTools } = canonicalizeToolSet( - [tool('Read'), tool('Rive')], - invalid, - new Set(['Read', 'Rive']), - ); - assert.ok(activeTools.includes('Rive'), 'Rive is now in the active set'); - }); - test('providerTools keeps the full registry for dispatch; invalid present but not advertised', () => { - const { providerTools, activeTools } = canonicalizeToolSet( - [tool('Read'), tool('Rive')], - invalid, - new Set(['Read']), + assert.deepEqual(activeTools, ['Read', 'tool_search']); + assert.deepEqual( + providerTools.map((candidate) => candidate.name), + ['Read', 'Rive', 'tool_search', 'invalid'], ); - const names = providerTools.map((t) => t.name); - assert.ok(names.includes('Read')); - assert.ok(names.includes('Rive'), 'a hidden tool stays dispatchable in providerTools'); - assert.ok(names.includes('invalid'), 'repair target present in providerTools'); - assert.ok(!activeTools.includes('invalid'), 'invalid is never advertised to the model'); }); -}); -describe('diagnostics measure the provider-visible (active) tool subset', () => { - const connection = { providerType: 'openai', slug: 'c' } as never; - - function rich(name: string, schema: unknown): MakaTool { - return { name, description: name, parameters: schema, impl: () => ({}) }; - } - - function diag( - providerTools: MakaTool[], - activeTools: string[], - prior?: ReturnType, - ) { - return computeRequestShapeDiagnostic( - { - connection, - modelId: 'm', - systemPrompt: 's', - providerOptions: {}, - providerTools, - activeTools, - priorMessages: [], - }, - prior, - ); - } + test('measures only the provider-visible tool schemas', () => { + const tools: MakaTool[] = [ + { ...tool('Read'), parameters: { a: 1 } }, + { ...tool('Rive'), parameters: { big: 'x'.repeat(500) } }, + ]; - test('char count excludes an inactive tool schema', () => { - const tools = [rich('Read', { a: 1 }), rich('Rive', { big: 'x'.repeat(500) })]; - const withoutRive = toolSchemaCharsForDiagnostics(tools, ['Read']); - const withRive = toolSchemaCharsForDiagnostics(tools, ['Read', 'Rive']); assert.ok( - withRive > withoutRive + 400, - 'activating Rive should add its schema chars to the count', + toolSchemaCharsForDiagnostics(tools, ['Read', 'Rive']) > + toolSchemaCharsForDiagnostics(tools, ['Read']) + 400, ); }); - - test('toolSchemaHash ignores an INACTIVE tool schema change', () => { - const a = [rich('Read', { a: 1 }), rich('Rive', { v: 1 })]; - const b = [rich('Read', { a: 1 }), rich('Rive', { v: 2 })]; - assert.equal( - diag(a, ['Read']).componentHashes.toolSchemaHash, - diag(b, ['Read']).componentHashes.toolSchemaHash, - 'a change to an unadvertised schema must not move the hash', - ); - }); - - test('activating a hidden tool moves toolSchemaHash and reports tool_schema_changed', () => { - const tools = [rich('Read', { a: 1 }), rich('Rive', { v: 1 })]; - const before = diag(tools, ['Read']); - const after = diag(tools, ['Read', 'Rive'], before); - assert.notEqual(after.componentHashes.toolSchemaHash, before.componentHashes.toolSchemaHash); - assert.equal(after.prefixChangeReason, 'tool_schema_changed'); - }); }); -describe('prepared provider request capture', () => { - test('records cacheable request segments in provider-prefix order', () => { - const capture = Reflect.get(requestShape, 'capturePreparedProviderRequest') as - | ((input: { - providerId: string; - modelId: string; - instructions: string; - messages: Array<{ role: string; content: string }>; - tools: Array>; - providerOptions: Record; - }) => { - requestHash: string; - requestBytes: number; - serializedRequest: string; - segments: Array<{ - kind: string; - index: number; - cacheable: boolean; - hash: string; - bytes: number; - role?: string; - }>; - }) - | undefined; - - assert.equal(typeof capture, 'function'); - const result = capture!({ - providerId: 'anthropic', - modelId: 'claude-test', - instructions: 'system', - messages: [{ role: 'user', content: 'hello' }], - tools: [{ name: 'Bash', description: 'Run a command', inputSchema: { type: 'object' } }], - providerOptions: { anthropic: { thinking: { type: 'enabled', budgetTokens: 1_024 } } }, +describe('prepared request observation', () => { + test('derives the request digest and bytes from the private serialization', () => { + const material = prepareRequestObservation({ + prompt: [{ role: 'user', content: 'hello' }], + maxOutputTokens: 1_024, }); - assert.deepEqual( - result.segments.map(({ kind, index, cacheable, role }) => ({ - kind, - index, - cacheable, - ...(role ? { role } : {}), - })), - [ - { kind: 'tool_schema', index: 0, cacheable: true }, - { kind: 'system_prompt', index: 0, cacheable: true }, - { kind: 'message', index: 0, cacheable: true, role: 'user' }, - { kind: 'provider_options', index: 0, cacheable: false }, - ], + assert.equal( + material.observation.digest, + `sha256:${createHash('sha256').update(material.serializedRequest).digest('hex')}`, ); - assert.match(result.requestHash, /^sha256:[a-f0-9]{64}$/); - assert.equal(result.requestBytes, Buffer.byteLength(result.serializedRequest, 'utf8')); - assert.ok(result.segments.every((segment) => segment.bytes > 0)); - assert.ok(result.segments.every((segment) => /^sha256:[a-f0-9]{64}$/.test(segment.hash))); + assert.equal(material.observation.bytes, Buffer.byteLength(material.serializedRequest, 'utf8')); }); - test('names a tool schema from the payload, and only that segment kind', () => { - // A size nobody can attribute is not actionable: "tool definitions are 40%" - // names no tool to remove (#2323). - const result = requestShape.capturePreparedProviderRequest({ - providerId: 'anthropic', - modelId: 'claude-test', - instructions: 'system', - messages: [{ role: 'user', content: 'hello' }], - tools: [{ name: 'Bash', inputSchema: { type: 'object' } }, { inputSchema: {} }], - providerOptions: {}, + test('serializes non-JSON values without collapsing their semantic identity', () => { + const observed = prepareRequestObservation({ + bigint: 42n, + missing: undefined, + createdAt: new Date('2026-08-31T00:00:00.000Z'), + headers: new Map([['x-observation', 'present']]), + }); + const plain = prepareRequestObservation({ + bigint: '42', + missing: '[undefined]', + createdAt: '2026-08-31T00:00:00.000Z', + headers: { 'x-observation': 'present' }, }); - const labels = result.segments.map((segment) => [segment.kind, segment.label] as const); - assert.deepEqual(labels, [ - ['tool_schema', 'Bash'], - // A tool the payload does not name gets no invented one. - ['tool_schema', undefined], - ['system_prompt', undefined], - ['message', undefined], - ['provider_options', undefined], - ]); + assert.doesNotThrow(() => JSON.parse(observed.serializedRequest)); + assert.notEqual(observed.observation.digest, plain.observation.digest); }); - test('versions and hashes non-provider-options request parameters for comparison', () => { - const capture = (providerOptions: Record, maxOutputTokens?: number) => - requestShape.capturePreparedProviderRequest({ - providerId: 'provider', - modelId: 'k3', - instructions: 'system', - messages: [{ role: 'user', content: 'hello' }], - tools: [{ name: 'Read', inputSchema: { type: 'object' } }], - providerOptions, - requestPayload: { - prompt: [{ role: 'user', content: 'hello' }], - tools: [{ name: 'Read', inputSchema: { type: 'object' } }], - providerOptions, - ...(maxOutputTokens !== undefined ? { maxOutputTokens } : {}), - }, + test('preserves the semantic identity of binary request content', () => { + const observe = (byte: number) => + prepareRequestObservation({ + prompt: [ + { + role: 'user', + content: [ + { + type: 'file', + data: { type: 'data', data: new Uint8Array([byte]) }, + mediaType: 'application/octet-stream', + }, + ], + }, + ], }); - const anthropic = capture({ anthropic: { effort: 'max' } }, 131_072); - const openai = capture({ kimiCodingPlan: { reasoningEffort: 'max' } }, 131_072); - const changedSharedParameter = capture({ kimiCodingPlan: { reasoningEffort: 'max' } }, 32_768); - - assert.equal(anthropic.schemaVersion, 2); - assert.equal( - anthropic.requestPayloadWithoutProviderOptionsHash, - openai.requestPayloadWithoutProviderOptionsHash, - ); - assert.notEqual( - anthropic.requestPayloadWithoutProviderOptionsHash, - changedSharedParameter.requestPayloadWithoutProviderOptionsHash, - ); + const first = observe(1); + const second = observe(2); + assert.notEqual(first.serializedRequest, second.serializedRequest); + assert.notEqual(first.observation.digest, second.observation.digest); + assert.notEqual(first.observation.segments[0]?.digest, second.observation.segments[0]?.digest); + assert.equal(first.observation.segments[0]?.comparison, 'exact'); }); - test('keeps reasoning effort across provider namespaces in the protocol-independent hash', () => { - const hash = (providerOptions: Record, maxOutputTokens: number) => - requestShape.capturePreparedProviderRequest({ - providerId: 'kimi-coding-plan', - modelId: 'kimi-for-coding', - messages: [{ role: 'user', content: 'hello' }], - tools: [], - providerOptions, - requestPayload: { - prompt: [{ role: 'user', content: 'hello' }], - maxOutputTokens, - providerOptions, - }, - }).requestPayloadWithoutProviderOptionsHash; - - const anthropicMax = hash( - { - anthropic: { - effort: 'max', - thinking: { type: 'enabled', budgetTokens: 1_024 }, + test('marks redacted compaction content comparison-opaque', () => { + const material = prepareRequestObservation({ + prompt: [ + { + role: 'assistant', + content: [ + { + type: 'custom', + kind: 'openai.compaction', + providerOptions: { openai: { redacted: true } }, + }, + ], }, - }, - 31_744, - ); - const openaiMax = hash({ kimiCodingPlan: { reasoningEffort: 'max' } }, 32_768); - const nativeOpenaiMax = hash({ openai: { reasoningEffort: 'max' } }, 32_768); - const nativeOpenaiHigh = hash({ openai: { reasoningEffort: 'high' } }, 32_768); - const zaiHigh = hash({ zaiCodingPlan: { reasoningEffort: 'high' } }, 32_768); - const zaiLow = hash({ zaiCodingPlan: { reasoningEffort: 'low' } }, 32_768); + ], + }); - assert.equal(anthropicMax, openaiMax); - assert.equal(anthropicMax, nativeOpenaiMax); - assert.equal(nativeOpenaiHigh, zaiHigh); - assert.notEqual(zaiHigh, zaiLow); - assert.notEqual(anthropicMax, hash({ kimiCodingPlan: { reasoningEffort: 'low' } }, 32_768)); - assert.notEqual(anthropicMax, hash({ kimiCodingPlan: { reasoningEffort: 'none' } }, 32_768)); + assert.equal(material.observation.segments[0]?.kind, 'message'); + assert.equal(material.observation.segments[0]?.comparison, 'opaque'); }); - test('normalizes Anthropic thinking budget into the protocol-independent output limit', () => { - const capture = (providerOptions: Record, maxOutputTokens: number) => - requestShape.capturePreparedProviderRequest({ - providerId: 'kimi-coding-plan', - modelId: 'kimi-for-coding', - instructions: 'system', - messages: [{ role: 'user', content: 'hello' }], - tools: [], - providerOptions, - requestPayload: { - prompt: [{ role: 'user', content: 'hello' }], - maxOutputTokens, - providerOptions, - }, - }); - - const anthropic = capture( - { anthropic: { thinking: { type: 'enabled', budgetTokens: 1_024 } } }, - 31_744, - ); - const openai = capture({ maka: { kimiReasoningField: 'reasoning_content' } }, 32_768); - - assert.equal( - anthropic.requestPayloadWithoutProviderOptionsHash, - openai.requestPayloadWithoutProviderOptionsHash, + test('bounds ordered segments without dropping their count or bytes', () => { + const prompt = Array.from({ length: 1_000 }, (_, index) => ({ + role: 'user', + content: `message-${index}`, + })); + const material = prepareRequestObservation({ prompt }); + const expectedBytes = prompt.reduce( + (total, message) => + total + prepareRequestObservation({ prompt: [message] }).observation.segments[0]!.bytes, + 0, ); - assert.notEqual(anthropic.requestHash, openai.requestHash); - }); - - test('excludes provider metadata nested in prompt messages and parts', () => { - const capture = (prompt: unknown[], tools: unknown[] = []) => - requestShape.capturePreparedProviderRequest({ - providerId: 'provider', - modelId: 'model', - messages: prompt, - tools, - requestPayload: { prompt, tools }, - }); - const sharedPrompt = [ - { - role: 'assistant', - content: [{ type: 'reasoning', text: 'analysis' }], - }, - ]; - const anthropicPrompt = [ - { - role: 'assistant', - providerOptions: { anthropic: { cacheControl: { type: 'ephemeral' } } }, - content: [ - { - type: 'reasoning', - text: 'analysis', - providerOptions: { anthropic: { signature: 'signed-reasoning' } }, - }, - ], - }, - ]; + assert.ok(material.observation.segments.length <= 256); assert.equal( - capture(anthropicPrompt).requestPayloadWithoutProviderOptionsHash, - capture(sharedPrompt).requestPayloadWithoutProviderOptionsHash, + material.observation.segments.reduce((total, segment) => total + segment.bytes, 0), + expectedBytes, ); - - const sharedToolPrompt = [ - { - role: 'tool', - content: [ - { - type: 'tool-result', - toolCallId: 'call-1', - toolName: 'Inspect', - output: { type: 'content', value: [{ type: 'text', text: 'done' }] }, - }, - ], - }, - ]; - const providerToolPrompt = [ - { - ...sharedToolPrompt[0], - providerOptions: { anthropic: { cacheControl: { type: 'ephemeral' } } }, - content: [ - { - ...sharedToolPrompt[0]!.content[0], - providerOptions: { anthropic: { toolUseId: 'provider-call-1' } }, - output: { - type: 'content', - providerOptions: { anthropic: { resultId: 'provider-result-1' } }, - value: [ - { - type: 'text', - text: 'done', - providerOptions: { anthropic: { blockId: 'provider-block-1' } }, - }, - ], - }, - }, - ], - }, - ]; - const sharedTools = [{ type: 'function', name: 'Inspect', inputSchema: { type: 'object' } }]; - const providerTools = [ - { - ...sharedTools[0], - providerOptions: { anthropic: { deferLoading: true } }, - }, - ]; - assert.equal( - capture(providerToolPrompt, providerTools).requestPayloadWithoutProviderOptionsHash, - capture(sharedToolPrompt, sharedTools).requestPayloadWithoutProviderOptionsHash, + material.observation.segments.reduce( + (total, segment) => total + (segment.representedSegments ?? 1), + 0, + ), + prompt.length, ); + assert.equal(material.observation.segments.at(-1)?.comparison, 'opaque'); }); - test('normalizes provider-local tool and approval bookkeeping', () => { - const hash = (prompt: unknown[]) => - requestShape.capturePreparedProviderRequest({ - providerId: 'provider', - modelId: 'model', - messages: prompt, - tools: [], - requestPayload: { prompt, tools: [] }, - }).requestPayloadWithoutProviderOptionsHash; - const prompt = (suffix: string, approved: boolean) => [ - { - role: 'assistant', - content: [ - { - type: 'tool-call', - toolCallId: `call-${suffix}`, - toolName: 'Inspect', - input: { path: 'README.md' }, - providerExecuted: suffix === 'anthropic', - }, - { - type: 'tool-approval-request', - approvalId: `approval-${suffix}`, - toolCallId: `call-${suffix}`, - isAutomatic: suffix === 'anthropic', - signature: `signature-${suffix}`, - }, - ], - }, - { - role: 'tool', - content: [ - { - type: 'tool-result', - toolCallId: `call-${suffix}`, - toolName: 'Inspect', - output: { type: 'text', value: 'done' }, - }, - { - type: 'tool-approval-response', - approvalId: `approval-${suffix}`, - approved, - providerExecuted: suffix === 'anthropic', - }, - ], - }, - ]; - - assert.equal(hash(prompt('anthropic', true)), hash(prompt('openai', true))); - assert.notEqual(hash(prompt('anthropic', true)), hash(prompt('openai', false))); - }); - - test('preserves same-named fields inside user data and tool schemas', () => { - const hash = (prompt: unknown[], tools: unknown[] = []) => - requestShape.capturePreparedProviderRequest({ - providerId: 'provider', - modelId: 'model', - messages: prompt, - tools, - requestPayload: { prompt, tools }, - }).requestPayloadWithoutProviderOptionsHash; - const toolCall = (value: string) => [ - { - role: 'assistant', - content: [ - { - type: 'tool-call', - toolCallId: 'call-1', - toolName: 'Inspect', - input: { providerOptions: value }, - }, - ], - }, - ]; - const toolResult = (value: string) => [ - { - role: 'tool', - content: [ - { - type: 'tool-result', - toolCallId: 'call-1', - toolName: 'Inspect', - output: { type: 'json', value: { providerOptions: value } }, - }, - ], - }, - ]; - const tool = (description: string) => [ - { - type: 'function', - name: 'Inspect', - inputSchema: { - type: 'object', - properties: { providerOptions: { type: 'string', description } }, - }, - }, - ]; - - assert.notEqual(hash(toolCall('alpha')), hash(toolCall('bravo'))); - assert.notEqual(hash(toolResult('alpha')), hash(toolResult('bravo'))); - assert.notEqual(hash([], tool('alpha')), hash([], tool('bravo'))); - }); - - test('finds the first changed cacheable segment by exact content hash', () => { - const capture = requestShape.capturePreparedProviderRequest; - const findFirstChanged = Reflect.get(requestShape, 'findFirstChangedCacheableSegment') as - | (( - current: ReturnType, - prior: ReturnType, - ) => { kind: string; index: number; role?: string } | undefined) - | undefined; - assert.equal(typeof findFirstChanged, 'function'); - - const prior = capture({ - providerId: 'openai', - modelId: 'gpt-test', - instructions: 'system', - messages: [{ role: 'user', content: 'alpha' }], - tools: [{ name: 'Read', inputSchema: { type: 'object' } }], - providerOptions: { openai: { reasoningEffort: 'low' } }, - }); - const changedMessage = capture({ - providerId: 'openai', - modelId: 'gpt-test', - instructions: 'system', - messages: [{ role: 'user', content: 'bravo' }], - tools: [{ name: 'Read', inputSchema: { type: 'object' } }], - providerOptions: { openai: { reasoningEffort: 'low' } }, - }); - assert.deepEqual(findFirstChanged!(changedMessage, prior), { - kind: 'message', - index: 0, - role: 'user', - }); - - const onlyOptionsChanged = capture({ - providerId: 'openai', - modelId: 'gpt-test', - instructions: 'system', - messages: [{ role: 'user', content: 'alpha' }], - tools: [{ name: 'Read', inputSchema: { type: 'object' } }], - providerOptions: { openai: { reasoningEffort: 'high' } }, + test('records semantic segments in provider-prefix order and labels only tools', () => { + const material = prepareRequestObservation({ + prompt: [ + { role: 'system', content: 'system' }, + { role: 'user', content: 'hello' }, + ], + tools: [{ name: 'Bash', inputSchema: { type: 'object' } }, { inputSchema: {} }], + providerOptions: { anthropic: { thinking: { type: 'enabled' } } }, }); - assert.equal(findFirstChanged!(onlyOptionsChanged, prior), undefined); - const appendedMessage = capture({ - providerId: 'openai', - modelId: 'gpt-test', - instructions: 'system', - messages: [ - { role: 'user', content: 'alpha' }, - { role: 'assistant', content: 'done' }, + assert.deepEqual( + material.observation.segments.map(({ kind, index, cacheable, role, label }) => ({ + kind, + index, + cacheable, + ...(role ? { role } : {}), + ...(label ? { label } : {}), + })), + [ + { kind: 'tool_schema', index: 0, cacheable: true, label: 'Bash' }, + { kind: 'tool_schema', index: 1, cacheable: true }, + { kind: 'system_prompt', index: 0, cacheable: true }, + { kind: 'message', index: 0, cacheable: true, role: 'user' }, + { kind: 'provider_options', index: 0, cacheable: false }, ], - tools: [{ name: 'Read', inputSchema: { type: 'object' } }], - providerOptions: { openai: { reasoningEffort: 'low' } }, - }); - assert.deepEqual(findFirstChanged!(appendedMessage, prior), { - kind: 'message', - index: 1, - role: 'assistant', - }); + ); }); }); diff --git a/packages/runtime/src/__tests__/session-manager.test.ts b/packages/runtime/src/__tests__/session-manager.test.ts index cc04ab9659..6390562560 100644 --- a/packages/runtime/src/__tests__/session-manager.test.ts +++ b/packages/runtime/src/__tests__/session-manager.test.ts @@ -10897,142 +10897,6 @@ describe('SessionManager permission mode updates', () => { assert.strictEqual(JSON.stringify(events).includes('sk-live-secret-token-value'), false); }); - test('required capture and later attempt still write after an attempt append fails', async () => { - const store = new MemorySessionStore(); - const attemptFailureRecorded = makeGate(); - const captureOutcomes: string[] = []; - let failAttemptOnce = true; - const runStore = new MemoryAgentRunStore({ - beforeAgentRunEventAppend: async (_sessionId, _runId, event) => { - if (event.type === 'provider_request_attempt_recorded' && failAttemptOnce) { - failAttemptOnce = false; - throw new Error('diagnostic attempt append failed'); - } - if (event.type === 'trace_write_failed') attemptFailureRecorded.release(); - }, - }); - const backends = new BackendRegistry(); - backends.register( - 'ai-sdk', - (ctx) => - new ProviderCaptureAfterAttemptFailureBackend( - ctx, - attemptFailureRecorded.promise, - captureOutcomes, - ), - ); - const manager = new SessionManager({ - store, - runStore, - runtimeEventStore: runStore, - backends, - newId: nextId(), - now: nextNow(12_762), - }); - const session = await manager.createSession(makeInput()); - - await drain(manager.sendMessage(session.id, { turnId: 'turn-1', text: 'hello' })); - - assert.deepStrictEqual(captureOutcomes, ['fulfilled']); - const [run] = await runStore.listSessionRuns(session.id); - const events = await runStore.readEvents(session.id, run!.runId); - assert.strictEqual( - events.some((event) => event.type === 'provider_request_captured'), - true, - ); - assert.strictEqual( - events.some((event) => event.id === 'attempt-2'), - true, - ); - }); - - test('carries a provider attempt failure latch into the terminal run header', async () => { - const store = new MemorySessionStore(); - let failAttemptAppend = true; - let failFailureLatch = true; - let failFailureSentinel = true; - const runStore = new MemoryAgentRunStore({ - beforeAgentRunUpdate: async (_sessionId, _runId, patch) => { - if (patch.traceWriteError && failFailureLatch) { - failFailureLatch = false; - throw new Error('trace failure latch update failed'); - } - }, - beforeAgentRunEventAppend: async (_sessionId, _runId, event) => { - if (event.type === 'provider_request_attempt_recorded' && failAttemptAppend) { - failAttemptAppend = false; - throw new Error('provider attempt append failed'); - } - if (event.type === 'trace_write_failed' && failFailureSentinel) { - failFailureSentinel = false; - throw new Error('trace failure sentinel append failed'); - } - }, - }); - const backends = new BackendRegistry(); - backends.register('ai-sdk', (ctx) => new ProviderRequestTraceBackend(ctx)); - const manager = new SessionManager({ - store, - runStore, - runtimeEventStore: runStore, - backends, - newId: nextId(), - now: nextNow(12_763), - }); - const session = await manager.createSession(makeInput()); - - await drain(manager.sendMessage(session.id, { turnId: 'turn-1', text: 'hello' })); - - const [run] = await runStore.listSessionRuns(session.id); - assert.strictEqual(run?.status, 'completed'); - assert.match( - String(run?.traceWriteError), - /append provider request attempt: provider attempt append failed/, - ); - const events = await runStore.readEvents(session.id, run!.runId); - assert.strictEqual( - events.some((event) => event.type === 'trace_write_failed'), - false, - ); - }); - - test('finalizes the run when a required provider capture append fails', async () => { - const store = new MemorySessionStore(); - let providerDispatches = 0; - let failCaptureOnce = true; - const runStore = new MemoryAgentRunStore({ - beforeAgentRunEventAppend: async (_sessionId, _runId, event) => { - if (event.type === 'provider_request_captured' && failCaptureOnce) { - failCaptureOnce = false; - throw new Error('required capture append failed'); - } - }, - }); - const backends = new BackendRegistry(); - backends.register( - 'ai-sdk', - (ctx) => new ProviderCaptureGateBackend(ctx, () => (providerDispatches += 1)), - ); - const manager = new SessionManager({ - store, - runStore, - runtimeEventStore: runStore, - backends, - newId: nextId(), - now: nextNow(12_764), - }); - const session = await manager.createSession(makeInput()); - - await drain(manager.sendMessage(session.id, { turnId: 'turn-1', text: 'hello' })).catch( - () => {}, - ); - - assert.strictEqual(providerDispatches, 0); - const [run] = await runStore.listSessionRuns(session.id); - assert.strictEqual(run?.status, 'failed'); - assert.notStrictEqual(run?.completedAt, undefined); - }); - test('history compact cleanup includes continuation events without including child agent events', async () => { const store = new MemorySessionStore(); const runStore = new MemoryAgentRunStore(); @@ -12678,192 +12542,6 @@ class TraceBackend implements AgentBackend { async dispose(): Promise {} } -class ProviderRequestTraceBackend implements AgentBackend { - readonly kind = 'ai-sdk' as const; - readonly sessionId: string; - - constructor(private readonly ctx: BackendFactoryContext) { - this.sessionId = ctx.sessionId; - } - - async *send(input: BackendSendInput): AsyncIterable { - await this.ctx.recordProviderRequestCapture?.({ - schemaVersion: 2, - traceId: 'provider-trace-1', - captureId: 'capture-1', - turnId: input.turnId, - step: 0, - providerId: 'fake', - modelId: 'fake-model', - requestHash: 'sha256:request', - requestPayloadWithoutProviderOptionsHash: 'sha256:shared-request', - requestBytes: 100, - segments: [], - artifactId: 'artifact-capture', - }); - await this.ctx.recordProviderRequestAttempt?.({ - traceId: 'provider-trace-1', - attemptId: 'attempt-1', - turnId: input.turnId, - step: 0, - attempt: 1, - captureId: 'capture-1', - captureArtifactId: 'artifact-capture', - providerId: 'fake', - modelId: 'fake-model', - requestHash: 'sha256:request', - requestBytes: 100, - segments: Array.from({ length: 75 }, (_, index) => ({ - kind: 'message' as const, - index, - cacheable: true, - hash: `sha256:${index}`, - bytes: 1, - })), - startedAt: 1, - completedAt: 2, - status: 'completed', - finishReason: 'stop', - latencyMs: 1, - }); - yield { - type: 'complete', - id: `${input.turnId}-complete`, - turnId: input.turnId, - ts: 3, - stopReason: 'end_turn', - }; - } - - async stop(): Promise {} - async respondToSandboxBoundary(_decision: SandboxBoundaryResponse): Promise {} - async dispose(): Promise {} -} - -class ProviderCaptureGateBackend implements AgentBackend { - readonly kind = 'ai-sdk' as const; - readonly sessionId: string; - - constructor( - private readonly ctx: BackendFactoryContext, - private readonly dispatch: () => void, - ) { - this.sessionId = ctx.sessionId; - } - - async *send(input: BackendSendInput): AsyncIterable { - await this.ctx.recordProviderRequestCapture?.({ - schemaVersion: 2, - traceId: 'provider-trace-gated', - captureId: 'capture-gated', - turnId: input.turnId, - step: 0, - providerId: 'fake', - modelId: 'fake-model', - requestHash: 'sha256:gated', - requestPayloadWithoutProviderOptionsHash: 'sha256:shared-gated', - requestBytes: 100, - segments: [], - artifactId: 'artifact-gated', - }); - this.dispatch(); - yield { - type: 'complete', - id: `${input.turnId}-complete`, - turnId: input.turnId, - ts: 3, - stopReason: 'end_turn', - }; - } - - async stop(): Promise {} - async respondToSandboxBoundary(_decision: SandboxBoundaryResponse): Promise {} - async dispose(): Promise {} -} - -class ProviderCaptureAfterAttemptFailureBackend implements AgentBackend { - readonly kind = 'ai-sdk' as const; - readonly sessionId: string; - - constructor( - private readonly ctx: BackendFactoryContext, - private readonly attemptFailureRecorded: Promise, - private readonly captureOutcomes: string[], - ) { - this.sessionId = ctx.sessionId; - } - - async *send(input: BackendSendInput): AsyncIterable { - await this.ctx.recordProviderRequestAttempt?.({ - traceId: 'provider-trace-1', - attemptId: 'attempt-1', - turnId: input.turnId, - step: 0, - attempt: 1, - captureId: 'capture-1', - captureArtifactId: 'artifact-capture-1', - providerId: 'fake', - modelId: 'fake-model', - requestHash: 'sha256:request-1', - requestBytes: 100, - segments: [], - startedAt: 1, - completedAt: 2, - status: 'completed', - latencyMs: 1, - }); - await this.attemptFailureRecorded; - try { - await this.ctx.recordProviderRequestCapture?.({ - schemaVersion: 2, - traceId: 'provider-trace-1', - captureId: 'capture-2', - turnId: input.turnId, - step: 1, - providerId: 'fake', - modelId: 'fake-model', - requestHash: 'sha256:request-2', - requestPayloadWithoutProviderOptionsHash: 'sha256:shared-request-2', - requestBytes: 120, - segments: [], - artifactId: 'artifact-capture-2', - }); - this.captureOutcomes.push('fulfilled'); - } catch { - this.captureOutcomes.push('rejected'); - } - await this.ctx.recordProviderRequestAttempt?.({ - traceId: 'provider-trace-1', - attemptId: 'attempt-2', - turnId: input.turnId, - step: 1, - attempt: 1, - captureId: 'capture-2', - captureArtifactId: 'artifact-capture-2', - providerId: 'fake', - modelId: 'fake-model', - requestHash: 'sha256:request-2', - requestBytes: 120, - segments: [], - startedAt: 2, - completedAt: 3, - status: 'completed', - latencyMs: 1, - }); - yield { - type: 'complete', - id: `${input.turnId}-complete`, - turnId: input.turnId, - ts: 3, - stopReason: 'end_turn', - }; - } - - async stop(): Promise {} - async respondToSandboxBoundary(_decision: SandboxBoundaryResponse): Promise {} - async dispose(): Promise {} -} - class HistoryCompactCheckpointBackend implements AgentBackend { readonly kind = 'ai-sdk' as const; readonly sessionId: string; @@ -13939,10 +13617,15 @@ class MissingCheckpointProjectionAgentRunStore extends MemoryAgentRunStore { return undefined; } + async readEventLedgerRevision(): Promise { + return 'missing-checkpoint-projection-test-revision'; + } + async repairEventProjection( _sessionId: string, _type: AgentRunEvent['type'], event: AgentRunEvent | null, + _options: { ifLedgerRevision: string; replaceEventId?: string }, ): Promise { this.repairedProjection = event; } diff --git a/packages/runtime/src/agent-run.ts b/packages/runtime/src/agent-run.ts index ba68ad62c5..0b0a08ff8d 100644 --- a/packages/runtime/src/agent-run.ts +++ b/packages/runtime/src/agent-run.ts @@ -85,10 +85,6 @@ import { type RuntimeContinuationStartAdmissionProof, } from './runtime-continuation-admission.js'; import { DEFAULT_TOOL_MODE, isToolMode, type ToolMode } from '@maka/core/tool-mode'; -import type { - ProviderRequestAttemptRecord, - ProviderRequestCaptureLedgerRecord, -} from './provider-request-telemetry.js'; import { materializeRuntimeEventTranscriptProjection } from './runtime-ledger-repair.js'; import { cloneAndFreezeRuntimeSnapshot } from './runtime-snapshot.js'; @@ -441,46 +437,6 @@ export class AgentRun { }); } - recordProviderRequestCapture(capture: ProviderRequestCaptureLedgerRecord): Promise { - if (!this.input.runStore) return Promise.reject(new Error('AgentRun store is not configured')); - return this.enqueueRequiredRunStoreWrite('append provider request capture', async () => { - const { - schemaVersion, - serializedRequest: _serializedRequest, - ...data - } = capture as ProviderRequestCaptureLedgerRecord & { serializedRequest?: string }; - await this.input.runStore?.appendEvent( - this.sessionId, - this.runId, - { - type: 'provider_request_captured', - id: capture.captureId, - runId: this.runId, - sessionId: this.sessionId, - turnId: capture.turnId, - ts: this.input.now(), - data: { schemaVersion, ...data }, - }, - { durable: true }, - ); - }); - } - - recordProviderRequestAttempt(attempt: ProviderRequestAttemptRecord): void { - if (!this.input.runStore) return; - this.enqueueBestEffortProviderAttempt('append provider request attempt', async () => { - await this.input.runStore?.appendEvent(this.sessionId, this.runId, { - type: 'provider_request_attempt_recorded', - id: attempt.attemptId, - runId: this.runId, - sessionId: this.sessionId, - turnId: attempt.turnId, - ts: attempt.completedAt, - data: { ...attempt }, - }); - }); - } - /** * Canonical accounting record for one physical provider request (#1679). * @@ -1577,19 +1533,6 @@ export class AgentRun { return next; } - /** - * Each physical provider request gets its own best-effort diagnostic row. - * One failed attempt append must not suppress later attempts or poison the - * general AgentRun store latch; a required capture independently gates every - * provider dispatch. - */ - private enqueueBestEffortProviderAttempt(label: string, operation: () => Promise): void { - const next = this.traceQueue - .then(operation, operation) - .catch((error) => this.enqueueTraceWriteFailure(error, label)); - this.traceQueue = next.catch(() => {}); - } - /** * Serialize a required Run-store write without consulting the best-effort * latch. A successful required write proves the store is available again; diff --git a/packages/runtime/src/ai-sdk-backend.ts b/packages/runtime/src/ai-sdk-backend.ts index e612551b10..493a969b7b 100644 --- a/packages/runtime/src/ai-sdk-backend.ts +++ b/packages/runtime/src/ai-sdk-backend.ts @@ -106,7 +106,7 @@ import type { PricingConfig, ToolInvocationRecord, } from '@maka/core/usage-stats/types'; -import type { ContextBudgetDiagnostic, PromptSegmentEstimate } from '@maka/core/usage-stats/types'; +import type { ContextBudgetDiagnostic } from '@maka/core/usage-stats/types'; import type { JSONValue, ModelFinishReason, @@ -219,17 +219,12 @@ import { type RuntimeEventModelReplayPlan, type RuntimeEventReplayFallbackGate, } from './model-history.js'; -import { - computeRequestShapeDiagnostic, - toolSchemaCharsForDiagnostics, - type RequestShapeDiagnostic, -} from './request-shape.js'; +import { toolSchemaCharsForDiagnostics } from './request-shape.js'; import type { ModelCallAttempt, ModelCallKind } from '@maka/core/model-call-attempt'; import { ProviderRequestTracker, type ModelCallAccountingInput, - type ProviderRequestAttemptRecord, - type ProviderRequestCaptureRecord, + type PreparedRequestArtifactInput, type ProviderRequestUsage, type ResolvedModelCallCost, } from './provider-request-telemetry.js'; @@ -258,7 +253,6 @@ import { import { applyRuntimeEventContextBudget, buildContextBudgetDiagnosticShell, - buildPromptSegmentEstimates, estimateRuntimeEventsTokens, mergeContextBudgetDiagnostic, mergeContextBudgetDiagnosticPatches, @@ -759,19 +753,10 @@ export interface AiSdkBackendInput extends AiSdkCompactionCapabilities { readChildAgentOutput?: ToolRuntimeInput['readChildAgentOutput']; /** Optional diagnostic trace hook for explaining a runtime turn without changing renderer events. */ recordRunTrace?: RunTraceRecorder; - /** - * Durable prepared-request capture boundary. When configured, rejection - * prevents the corresponding provider request from being dispatched. - */ - recordProviderRequestCapture?: ( - capture: ProviderRequestCaptureRecord, + /** Optional private artifact sink for the secret-free prepared request. */ + persistPreparedRequestArtifact?: ( + input: PreparedRequestArtifactInput, ) => Promise<{ artifactId: string }>; - /** Best-effort durable row for one physical provider request attempt. */ - recordProviderRequestAttempt?: (attempt: ProviderRequestAttemptRecord) => void | Promise; - /** - * Canonical metering sink. Separate from `recordProviderRequestAttempt`, which - * stays a diagnostic trace: this one carries the accounting record. - */ /** * Commits one settled provider request: the canonical attempt and, when it * is the completed main call, the derived latest-context row it authorises. @@ -1089,12 +1074,6 @@ export class AiSdkBackend implements AgentBackend { * read back as absent. */ private readonly activeTurns = new Set(); - /** - * Request-shape baseline for change attribution. Session-scoped on purpose: - * it compares each provider request against whatever this backend sent last, - * across turns. - */ - private priorRequestShape: RequestShapeDiagnostic | undefined; private readonly compaction: AiSdkCompaction; /** Session-scoped running total, deliberately accumulated across turns. */ private cumulativeUsageCheckpoint: NormalizedAiSdkUsage | undefined; @@ -1107,8 +1086,8 @@ export class AiSdkBackend implements AgentBackend { this.maxSteps = input.maxSteps; this.providerRetrySleep = input.providerRetrySleep ?? sleepForProviderRetry; // One resolved options value for every reader: the main call, the - // auxiliary memory-extraction call, and the request-shape diagnostics all - // describe the same request, so they must not disagree on what was sent. + // auxiliary memory-extraction call, and the provider request all use the + // same options value, so they cannot disagree on what was sent. this.resolvedProviderOptions = input.providerOptions ?? buildProviderOptions(input.connection, input.modelId, input.header.thinkingLevel); @@ -1390,7 +1369,7 @@ export class AiSdkBackend implements AgentBackend { // -------------------------------------------------------------------------- async compactHistory(input: BackendCompactHistoryInput): Promise { - return this.compaction.compactHistory(input, this.priorRequestShape?.requestShapeHash); + return this.compaction.compactHistory(input); } // -------------------------------------------------------------------------- @@ -1586,8 +1565,7 @@ export class AiSdkBackend implements AgentBackend { let streamStatus: LlmCallRecord['status'] = 'success'; let streamErrorClass: string | undefined; let runtimeSteps = 0; - let requestShapeForTelemetry: RequestShapeDiagnostic | undefined; - let promptSegmentsForTelemetry: PromptSegmentEstimate[] = []; + let toolAvailabilityForTelemetry: ReturnType = undefined; let contextBudgetForTelemetry: ContextBudgetDiagnostic | undefined; let contextCompactedNoteWritten = false; let contextCompactionFailedOpenNoteWritten = false; @@ -1941,78 +1919,19 @@ export class AiSdkBackend implements AgentBackend { ? currentTurnMessages : [...priorReplay.messages, ...currentTurnMessages]; }; - // Diagnostics describe the provider-visible (active) tool subset. A group - // loaded *this* turn expands that subset on later provider requests, - // so the durable cost record is refined against the final active set once - // the stream is consumed (see below). Both computations classify against - // the same pre-turn baseline. The availability runtime builds the tool - // diagnostic from the same per-step active set + schema-char measurement. + // Tool Availability describes the provider-visible (active) subset. A + // group loaded this turn expands that subset on later requests, so the + // terminal trace is refined against the final active set below. contextBudgetForTelemetry = priorReplay.contextBudget; - const priorShapeBaseline = this.priorRequestShape; - const computeTurnDiagnostics = (active: readonly string[]) => { + const computeToolAvailability = (active: readonly string[]) => { const toolSchemaChars = toolSchemaCharsForDiagnostics(providerTools, active); - const toolAvailabilityDiagnostic = plan.diagnostics(active, toolSchemaChars); - return { - promptSegments: buildPromptSegmentEstimates({ - systemPrompt, - toolSchemaChars, - toolCount: active.length, - priorMessages: priorReplay.messages, - priorRuntimeEventCount: priorReplay.runtimeEventCount, - currentUserContent: input.continuation - ? '' - : formatTextWithInlineRefs(input.text, { - ...(input.attachments !== undefined ? { attachments: input.attachments } : {}), - ...(input.quotes !== undefined ? { quotes: input.quotes } : {}), - }), - }), - requestShape: computeRequestShapeDiagnostic( - { - connection: this.input.connection, - modelId: this.input.modelId, - systemPrompt, - providerOptions: this.resolvedProviderOptions, - providerTools, - activeTools: active, - priorMessages: priorReplay.messages, - ...(toolAvailabilityDiagnostic !== undefined - ? { toolAvailability: toolAvailabilityDiagnostic } - : {}), - }, - priorShapeBaseline, - ), - }; - }; - // Publish a diagnostics snapshot to every telemetry sink at once so the - // cost record and prefix baseline describe the same active tool set. A - // same-turn deferred load re-publishes the final snapshot below. - let turnDiagnostics = computeTurnDiagnostics(activeTools); - const publishTurnDiagnostics = (diag: typeof turnDiagnostics): void => { - turnDiagnostics = diag; - promptSegmentsForTelemetry = diag.promptSegments; - requestShapeForTelemetry = diag.requestShape; - this.priorRequestShape = diag.requestShape; + return plan.diagnostics(active, toolSchemaChars); }; - // Step-0 (turn-start) view: literally what the first request carries, so - // the stream-start trace reports it as the prefix actually sent. - publishTurnDiagnostics(turnDiagnostics); + toolAvailabilityForTelemetry = computeToolAvailability(activeTools); trace.modelStreamStarted(activeTools, { - systemPromptHash: turnDiagnostics.requestShape.componentHashes.systemPromptHash, - prefixHash: turnDiagnostics.requestShape.prefixHash, - prefixChangeReason: turnDiagnostics.requestShape.prefixChangeReason, - requestShapeHash: turnDiagnostics.requestShape.requestShapeHash, - requestShapeChangeReason: turnDiagnostics.requestShape.requestShapeChangeReason, - ...(turnDiagnostics.requestShape.toolSchemaChangeReason !== undefined - ? { - toolSchemaChangeReason: turnDiagnostics.requestShape.toolSchemaChangeReason, - } - : {}), - ...(turnDiagnostics.requestShape.toolAvailability !== undefined - ? { - toolAvailability: turnDiagnostics.requestShape.toolAvailability, - } + ...(toolAvailabilityForTelemetry !== undefined + ? { toolAvailability: toolAvailabilityForTelemetry } : {}), - promptSegments: turnDiagnostics.promptSegments, ...(priorReplay.contextBudget ? { contextBudget: priorReplay.contextBudget } : {}), }); @@ -2778,15 +2697,15 @@ export class AiSdkBackend implements AgentBackend { break agentLoop; } - // Refine the durable cost record + prefix baseline against the final - // active set. Deferred loading may add tools, while boundary convergence - // may remove them; comparing membership avoids missing a same-size swap. + // Refine Tool Availability against the final active set. Deferred + // loading may add tools, while boundary convergence may remove them; + // comparing membership avoids missing a same-size swap. const finalActiveTools = currentRepairToolNames(); if ( finalActiveTools.length !== activeTools.length || finalActiveTools.some((name, index) => name !== activeTools[index]) ) { - publishTurnDiagnostics(computeTurnDiagnostics(finalActiveTools)); + toolAvailabilityForTelemetry = computeToolAvailability(finalActiveTools); } // Final usage event. Each adapter result covers one provider request. @@ -2798,7 +2717,6 @@ export class AiSdkBackend implements AgentBackend { const attemptTotalUsage = providerOutcome.usage; tokenUsage = sawUnusableStepUsage ? undefined : (completedStepUsage ?? attemptTotalUsage); if (tokenUsage) { - const systemPromptHash = turnDiagnostics.requestShape.componentHashes.systemPromptHash; tokenUsageCostUsd = this.computeTokenUsageCostUsd(tokenUsage); const contextBudgetForUsage = contextBudgetWithRequestProjectionDiagnostics( contextBudgetForTelemetry, @@ -2841,12 +2759,6 @@ export class AiSdkBackend implements AgentBackend { ? { cacheCreation: tokenUsage.cacheWriteInputTokens } : {}), ...(tokenUsageCostUsd !== undefined ? { costUsd: tokenUsageCostUsd } : {}), - systemPromptHash, - prefixHash: turnDiagnostics.requestShape.prefixHash, - prefixChangeReason: turnDiagnostics.requestShape.prefixChangeReason, - requestShapeHash: turnDiagnostics.requestShape.requestShapeHash, - requestShapeChangeReason: turnDiagnostics.requestShape.requestShapeChangeReason, - promptSegments: turnDiagnostics.promptSegments, ...(contextBudgetForUsage ? { contextBudget: contextBudgetForUsage } : {}), ...(contextRemainingForUsage !== undefined ? { contextRemaining: contextRemainingForUsage } @@ -3026,27 +2938,8 @@ export class AiSdkBackend implements AgentBackend { ...(contextBudgetForTelemetry !== undefined ? { contextBudget: contextBudgetForTelemetry } : {}), - ...(promptSegmentsForTelemetry.length > 0 - ? { promptSegments: promptSegmentsForTelemetry } - : {}), - ...(requestShapeForTelemetry !== undefined - ? { - systemPromptHash: requestShapeForTelemetry.componentHashes.systemPromptHash, - prefixHash: requestShapeForTelemetry.prefixHash, - prefixChangeReason: requestShapeForTelemetry.prefixChangeReason, - requestShapeHash: requestShapeForTelemetry.requestShapeHash, - requestShapeChangeReason: requestShapeForTelemetry.requestShapeChangeReason, - ...(requestShapeForTelemetry.toolSchemaChangeReason !== undefined - ? { - toolSchemaChangeReason: requestShapeForTelemetry.toolSchemaChangeReason, - } - : {}), - ...(requestShapeForTelemetry.toolAvailability !== undefined - ? { - toolAvailability: requestShapeForTelemetry.toolAvailability, - } - : {}), - } + ...(toolAvailabilityForTelemetry !== undefined + ? { toolAvailability: toolAvailabilityForTelemetry } : {}), }); queue.close(); @@ -3313,14 +3206,14 @@ export class AiSdkBackend implements AgentBackend { /** * One tracker for one physical provider call kind (#1679). * - * Auxiliary calls get the same capture, attempt, and accounting plumbing the + * Auxiliary calls get the same observation, attempt, and accounting plumbing the * main send uses, built here because the sinks and the current run live on * this backend. Callers receive a ready tracker rather than the ingredients: * a half-wired tracker is what produces records nothing can attribute. * - * Absent only when there is nothing to feed: no capture sink *and* no - * canonical sink. Metering deliberately does not depend on capture — capture - * is a diagnostic, and a deployment that turns it off must still be billed. + * Absent only when there is nothing to feed: no artifact sink, canonical + * sink, or dispatch gate. Metering deliberately does not depend on artifact + * persistence: the observation is created in memory for every tracked call. */ private createProviderRequestTracker(input: { turnId: string; @@ -3334,7 +3227,7 @@ export class AiSdkBackend implements AgentBackend { */ runId: string | undefined; }): ProviderRequestTracker | undefined { - const persistCapture = this.input.recordProviderRequestCapture; + const persistArtifact = this.input.persistPreparedRequestArtifact; const accounting = this.modelCallAccounting(input.callKind, { modelId: input.modelId, ...(input.runId ? { runId: input.runId } : {}), @@ -3351,15 +3244,14 @@ export class AiSdkBackend implements AgentBackend { runId, }) : undefined; - if (!persistCapture && !accounting && !beforeDispatch) return undefined; + if (!persistArtifact && !accounting && !beforeDispatch) return undefined; return new ProviderRequestTracker({ traceId: this.newId(), turnId: input.turnId, contextWindow: resolveSelectedModelContextWindow(this.input.connection, input.modelId), now: this.now, newId: this.newId, - ...(persistCapture ? { persistCapture } : {}), - recordAttempt: this.input.recordProviderRequestAttempt ?? (() => {}), + ...(persistArtifact ? { persistArtifact } : {}), ...(beforeDispatch ? { beforeDispatch } : {}), ...(accounting ? { accounting } : {}), }); @@ -3508,7 +3400,6 @@ export class AiSdkBackend implements AgentBackend { runId: scope.runId, runtimeContext: priorRuntimeContext, }, - this.priorRequestShape?.requestShapeHash, automaticMemorySource ? { runId: automaticMemorySource.runId, diff --git a/packages/runtime/src/ai-sdk-compaction-contract.ts b/packages/runtime/src/ai-sdk-compaction-contract.ts index f3a6cbdcdf..a969f03bc4 100644 --- a/packages/runtime/src/ai-sdk-compaction-contract.ts +++ b/packages/runtime/src/ai-sdk-compaction-contract.ts @@ -45,7 +45,6 @@ export interface HistoryCompactSummaryInput { maxEstimatedTokens: number; charsPerToken: number; }; - requestShapeHashBefore?: string; abortSignal?: AbortSignal; /** * Physical-call tracking for this summarization, built by the backend (#1679). diff --git a/packages/runtime/src/ai-sdk-compaction.ts b/packages/runtime/src/ai-sdk-compaction.ts index 3fa12b2ea0..cba17140a0 100644 --- a/packages/runtime/src/ai-sdk-compaction.ts +++ b/packages/runtime/src/ai-sdk-compaction.ts @@ -22,10 +22,10 @@ * from AiSdkBackend (issue #1084, runtime/compaction lane, slice 2). * * Owns the compaction planning and persistence paths that AiSdkBackend's - * Runtime request projection drives. Behavior-neutral collaborator: methods move - * verbatim, turn-scoped state (abortSignal, requestShapeHashBefore) is passed - * per call, and replay/telemetry capabilities that stay on AiSdkBackend are - * injected as host callbacks. + * Runtime request projection drives. Behavior-neutral collaborator: methods + * move verbatim, turn-scoped state (such as abortSignal) is passed per call, + * and replay/telemetry capabilities that stay on AiSdkBackend are injected as + * host callbacks. */ import type { RuntimeEvent } from '@maka/core/runtime-event'; @@ -225,7 +225,6 @@ export class AiSdkCompaction { public async compactHistory( input: Omit & { runId: string | undefined }, - requestShapeHashBefore?: string, automaticMemoryBoundary?: HistoryCompactMemoryExtractionBoundary, ): Promise { const historyCompactAbortController = new AbortController(); @@ -335,7 +334,6 @@ export class AiSdkCompaction { maxEstimatedTokens: policy.maxHistoryEstimatedTokens ?? estimatedTokensBefore, charsPerToken, }, - ...(requestShapeHashBefore ? { requestShapeHashBefore } : {}), abortSignal: historyCompactAbortController.signal, ...(tracker ? { providerRequestTracker: tracker } : {}), }), @@ -451,9 +449,9 @@ export class AiSdkCompaction { historyCompactRoute: this.input.historyCompactRoute, contextBudget: this.input.contextBudget, inputBudget: input.inputBudget, - requestShapeHashBefore: input.requestShapeHashBefore, - previousCheckpointId: input.previousCheckpoint?.checkpointId, + previousCheckpoint: input.previousCheckpoint, foldedRuntimeEvents: input.source.foldedRuntimeEvents, + newlyFoldedRuntimeEvents: input.newlyFoldedRuntimeEvents, }), ); const priorFailure = this.malformedSummaryFailures.get(fingerprint); diff --git a/packages/runtime/src/context-budget.ts b/packages/runtime/src/context-budget.ts index a38ad111c0..20a5ae3062 100644 --- a/packages/runtime/src/context-budget.ts +++ b/packages/runtime/src/context-budget.ts @@ -68,12 +68,10 @@ import { type HistoryCompactionCheckpointReplayFit, } from './history-compaction.js'; -import type { ModelMessage } from './model-protocol.js'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import type { CompactionDecisionDiagnostic, ContextBudgetDiagnostic, - PromptSegmentEstimate, } from '@maka/core/usage-stats/types'; import { compactionDecisionDiagnosticPatch } from './compaction-boundary.js'; import type { HistoryCompactCheckpoint } from './history-compact-checkpoint.js'; @@ -113,16 +111,6 @@ export interface BudgetedRuntimeContext { historyCompactCheckpoint?: HistoryCompactCheckpoint; } -export interface PromptSegmentInput { - systemPrompt?: string; - toolSchemaChars: number; - toolCount: number; - priorMessages: readonly ModelMessage[]; - priorRuntimeEventCount?: number; - currentUserContent: string; - charsPerToken?: number; -} - export function applyRuntimeEventContextBudget( events: readonly RuntimeEvent[], policy: ContextBudgetPolicy | undefined, @@ -191,67 +179,6 @@ export function applyRuntimeEventContextBudget( }; } -export function buildPromptSegmentEstimates(input: PromptSegmentInput): PromptSegmentEstimate[] { - const charsPerToken = input.charsPerToken ?? 4; - return [ - segment('system_prompt', input.systemPrompt?.length ?? 0, charsPerToken), - { - ...segment('tool_schema', input.toolSchemaChars, charsPerToken), - toolCount: input.toolCount, - }, - { - ...segment('prior_history', estimateModelMessagesChars(input.priorMessages), charsPerToken), - messageCount: input.priorMessages.length, - ...(input.priorRuntimeEventCount !== undefined - ? { eventCount: input.priorRuntimeEventCount } - : {}), - }, - segment('current_user', input.currentUserContent.length, charsPerToken), - ]; -} - -export function estimateModelMessagesChars(messages: readonly ModelMessage[]): number { - return messages.reduce((total, message) => total + estimateModelMessageChars(message), 0); -} - -function estimateModelMessageChars(message: ModelMessage): number { - const raw = message as unknown as { content?: unknown }; - return estimateContentChars(raw.content); -} - -function estimateContentChars(content: unknown): number { - if (typeof content === 'string') return content.length; - if (Array.isArray(content)) { - return content.reduce((total, part) => total + estimatePartChars(part), 0); - } - return stableJsonLength(content); -} - -function estimatePartChars(part: unknown): number { - if (!part || typeof part !== 'object') return stableJsonLength(part); - const value = part as Record; - let total = 0; - for (const key of ['text', 'toolName', 'toolCallId'] as const) { - if (typeof value[key] === 'string') total += value[key].length; - } - for (const key of ['input', 'output'] as const) { - if (value[key] !== undefined) total += stableJsonLength(value[key]); - } - return total; -} - -function segment( - kind: PromptSegmentEstimate['kind'], - chars: number, - charsPerToken: number, -): PromptSegmentEstimate { - return { - kind, - chars, - estimatedTokens: estimateTokens(chars, charsPerToken), - }; -} - // ============================================================================ // Replay ordering + context-budget diagnostic merge helpers. // Relocated from ai-sdk-backend.ts: these are pure functions over diff --git a/packages/runtime/src/context-diagnostics.ts b/packages/runtime/src/context-diagnostics.ts index a8d7cfaf0c..bc45466ab0 100644 --- a/packages/runtime/src/context-diagnostics.ts +++ b/packages/runtime/src/context-diagnostics.ts @@ -25,6 +25,7 @@ import { } from '@maka/core/agent-run'; import { decodeModelCallAttempt, type ModelCallAttempt } from '@maka/core/model-call-attempt'; import { + foldPromptComposition, PROVIDER_REQUEST_ATTEMPT_EVENT_TYPE, readPromptCompositionEvent, } from './prompt-composition.js'; @@ -87,15 +88,13 @@ export type ContextDiagnostics = inputTokens?: number; contextWindow?: number; /** - * What the latest request was made of, or absent when the durable - * metering record has no capture to match. + * What the latest request was made of, or absent when its canonical + * attempt has no prepared-request observation. * - * Absence is a state a reader must be able to see. Metering is durable - * because lost spend is unreconstructable; the capture carrying the - * segments is appended best-effort. Reporting the composition of an - * *older* request under a current heading would be the quiet lie this - * separation exists to prevent — so a mismatch reports nothing rather - * than the wrong request (#2323). + * Absence is a state a reader must be able to see. Reporting the + * composition of an *older* request under a current heading would be the + * quiet lie this separation exists to prevent, so readers never join an + * independent capture stream (#2323). */ composition?: ContextDiagnosticsComposition; compaction?: ContextDiagnosticsCompaction; @@ -113,7 +112,11 @@ export interface ContextDiagnosticsComposition { type ContextRunStore = Pick< AgentRunStore, - 'listSessionRuns' | 'readEvents' | 'readEventProjection' | 'repairEventProjection' + | 'listSessionRuns' + | 'readEvents' + | 'readEventProjection' + | 'readEventLedgerRevision' + | 'repairEventProjection' >; /** @@ -122,17 +125,15 @@ type ContextRunStore = Pick< * One sealed row answers this. The `latest_context` projection is written by * the same storage transaction that commits a completed MAIN call's canonical * attempt, freezing that request's identity, its provider-reported numbers, - * the folded composition of its own capture, and the compaction boundary its + * the folded composition of its own observation, and the compaction boundary its * prompt was built under — all at one moment, so no two fields here can * describe different requests. It is a projection, not an event: nothing * appends a record under that name. * - * That sealing is the whole design. The facts come from appends with different - * guarantees (durable metering, best-effort capture) and different owners (the - * compaction boundary belongs to recovery), so reading "the newest of each - * kind" and joining them produces a snapshot whose parts drift apart: a failed - * call replaces the newest metering record, an unmatched capture hides a - * matching one, and the boundary moves on its own. + * That sealing is the whole design. The request facts live on one canonical + * attempt; the compaction boundary remains recovery-owned. Reading "the newest + * of each kind" and joining independent histories would recreate the retired + * second authority and let the snapshot's parts drift apart. * * Warm reads are O(1) — one projection row. The ledger scan below is the cold * path, for a session written before this record existed. @@ -142,6 +143,7 @@ export async function readLatestContextDiagnostics( sessionId: string, ): Promise { try { + let replaceProjectionId: string | undefined; if (runStore.readEventProjection) { // Three states, three answers. `undefined` is an uninitialized // projection — nothing has been decided about this session, so the @@ -157,8 +159,13 @@ export async function readLatestContextDiagnostics( if (projected === null) return { status: 'unavailable', reason: 'no_completed_request' }; const snapshot = readLatestContextSnapshot(projected ?? undefined); if (snapshot) return availableFrom(snapshot); + if (projected) replaceProjectionId = projected.id; } - return await rebuildContextFromLedger(runStore, sessionId); + const ledgerRevision = + runStore.readEventLedgerRevision && runStore.repairEventProjection + ? await runStore.readEventLedgerRevision(sessionId) + : undefined; + return await rebuildContextFromLedger(runStore, sessionId, replaceProjectionId, ledgerRevision); } catch { return { status: 'unavailable', reason: 'trace_unavailable' }; } @@ -167,8 +174,9 @@ export async function readLatestContextDiagnostics( /** * The cold path, and the compatibility path. * - * A ledger written before sealed snapshots existed still has the two records - * they were sealed from, so the reader assembles one — but only here, only + * A ledger written before sealed snapshots existed can still be reconstructed + * from its canonical attempts (or, for legacy sessions only, provider events), + * so the reader assembles one — but only here, only * once, and only when no sealed record is present at all. Nothing is repaired * into another owner's projection: the compaction boundary is read from the * events of this session's own runs, never from recovery's derived row. @@ -176,6 +184,8 @@ export async function readLatestContextDiagnostics( async function rebuildContextFromLedger( runStore: ContextRunStore, sessionId: string, + replaceProjectionId?: string, + ledgerRevision?: string, ): Promise { const runs = (await runStore.listSessionRuns(sessionId)).filter(isSessionInlineRun); let anchor: MeteringAnchor | undefined; @@ -191,7 +201,10 @@ async function rebuildContextFromLedger( // legacy provider rows answer for it would resurrect the very request the // canonical rule declined to report. let sawCanonicalRecord = false; - const captures = new Map(); + // Historical provider rows never select a canonical-era request. They may + // only restore composition for the exact physical attempt selected above + // when that transitional canonical record predates request observations. + const historicalAttempts: LegacyProviderAnchor[] = []; const checkpoints: CheckpointCandidate[] = []; for (const run of runs) { @@ -203,10 +216,11 @@ async function rebuildContextFromLedger( continue; } if (event.type === PROVIDER_REQUEST_ATTEMPT_EVENT_TYPE) { - const attemptId = event.data?.attemptId; - if (typeof attemptId === 'string') captures.set(attemptId, event); const candidate = legacyProviderAnchor(event); - if (candidate && supersedesLatestContext(candidate, legacy)) legacy = candidate; + if (candidate) { + historicalAttempts.push(candidate); + if (supersedesLatestContext(candidate, legacy)) legacy = candidate; + } continue; } if (event.type !== CHECKPOINT_EVENT_TYPE) continue; @@ -222,12 +236,17 @@ async function rebuildContextFromLedger( // authority for data written since canonical metering shipped. const resolved = anchor ?? (sawCanonicalRecord ? undefined : legacy); if (!resolved) { - await repairLatestContext(runStore, sessionId, null); + await repairLatestContext(runStore, sessionId, null, replaceProjectionId, ledgerRevision); return { status: 'unavailable', reason: 'no_completed_request' }; } - const capture = captures.get(resolved.attemptId); - const read = capture ? readPromptCompositionEvent(capture) : undefined; const boundary = latestCheckpointBefore(checkpoints, resolved); + // Selection stays canonical. This is a one-field compatibility join, not a + // fallback for provider/model/status/timing/usage or for a different attempt. + const composition = + resolved.composition ?? + (anchor && !anchor.hasRequestObservation + ? exactHistoricalComposition(anchor, historicalAttempts) + : undefined); const snapshot: LatestContextSnapshot = { schemaVersion: LATEST_CONTEXT_SNAPSHOT_SCHEMA_VERSION, attemptId: resolved.attemptId, @@ -239,13 +258,13 @@ async function rebuildContextFromLedger( ? { cacheReadInputTokens: resolved.cacheReadInputTokens } : {}), ...(resolved.contextWindow !== undefined ? { contextWindow: resolved.contextWindow } : {}), - ...(read?.attemptId === resolved.attemptId ? { composition: read.composition } : {}), + ...(composition ? { composition } : {}), ...(boundary ? { compaction: contextDiagnosticsCompactionOf(boundary.checkpoint) } : {}), }; // Repair on the way out, so this scan happens once per session rather than // on every panel refresh. Best-effort: the caller already has its answer, // and a later cold read can retry the derived write. - await repairLatestContext(runStore, sessionId, snapshot); + await repairLatestContext(runStore, sessionId, snapshot, replaceProjectionId, ledgerRevision); return availableFrom(snapshot); } @@ -260,9 +279,11 @@ async function repairLatestContext( runStore: ContextRunStore, sessionId: string, snapshot: LatestContextSnapshot | null, + replaceProjectionId?: string, + ledgerRevision?: string, ): Promise { const repair = runStore.repairEventProjection; - if (!repair) return; + if (!repair || ledgerRevision === undefined) return; await repair .call( runStore, @@ -279,6 +300,10 @@ async function repairLatestContext( data: snapshot as unknown as Record, } as AgentRunEvent) : null, + { + ifLedgerRevision: ledgerRevision, + ...(replaceProjectionId ? { replaceEventId: replaceProjectionId } : {}), + }, ) .catch(() => {}); } @@ -292,23 +317,28 @@ async function repairLatestContext( function legacyProviderAnchor(event: AgentRunEvent): MeteringAnchor | undefined { const data = event.data; if (!data || data.status !== 'completed') return undefined; - const { attemptId, providerId, modelId, completedAt, startedAt } = data; + const { attemptId, traceId, providerId, modelId, completedAt, startedAt } = data; if ( typeof attemptId !== 'string' || + typeof traceId !== 'string' || typeof providerId !== 'string' || typeof modelId !== 'string' || typeof completedAt !== 'number' ) { return undefined; } + const composition = readPromptCompositionEvent(event)?.composition; return { attemptId, + traceId, providerId, modelId, startedAt: typeof startedAt === 'number' ? startedAt : completedAt, completedAt, + hasRequestObservation: false, ...(typeof data.inputTokens === 'number' ? { inputTokens: data.inputTokens } : {}), ...(typeof data.contextWindow === 'number' ? { contextWindow: data.contextWindow } : {}), + ...(composition ? { composition } : {}), }; } @@ -335,6 +365,7 @@ const CHECKPOINT_EVENT_TYPE = 'history_compact_checkpoint_recorded'; interface MeteringAnchor { attemptId: string; + traceId: string; providerId: string; modelId: string; startedAt: number; @@ -342,6 +373,8 @@ interface MeteringAnchor { inputTokens?: number; cacheReadInputTokens?: number; contextWindow?: number; + composition?: ContextDiagnosticsComposition; + hasRequestObservation: boolean; } interface CheckpointCandidate { @@ -359,20 +392,43 @@ function meteringAnchor(event: AgentRunEvent): MeteringAnchor | undefined { return undefined; } if (attempt.callKind !== 'main' || attempt.status !== 'completed') return undefined; + const composition = attempt.requestObservation + ? foldPromptComposition(attempt.requestObservation.segments) + : undefined; return { attemptId: attempt.attemptId, + traceId: attempt.traceId, providerId: attempt.providerId, modelId: attempt.modelId, startedAt: attempt.startedAt, completedAt: attempt.completedAt, + hasRequestObservation: attempt.requestObservation !== undefined, ...(attempt.inputTokens !== undefined ? { inputTokens: attempt.inputTokens } : {}), ...(attempt.cacheReadInputTokens !== undefined ? { cacheReadInputTokens: attempt.cacheReadInputTokens } : {}), ...(attempt.contextWindow !== undefined ? { contextWindow: attempt.contextWindow } : {}), + ...(composition ? { composition } : {}), }; } +function exactHistoricalComposition( + anchor: MeteringAnchor, + candidates: readonly LegacyProviderAnchor[], +): ContextDiagnosticsComposition | undefined { + const matches = candidates.filter( + (candidate) => + candidate.composition !== undefined && + candidate.attemptId === anchor.attemptId && + candidate.traceId === anchor.traceId && + candidate.providerId === anchor.providerId && + candidate.modelId === anchor.modelId && + candidate.startedAt === anchor.startedAt && + candidate.completedAt === anchor.completedAt, + ); + return matches.length === 1 ? matches[0]!.composition : undefined; +} + function latestCheckpointBefore( candidates: readonly CheckpointCandidate[], anchor: MeteringAnchor, diff --git a/packages/runtime/src/conversation-copy.ts b/packages/runtime/src/conversation-copy.ts index 41d3c72618..9804856fed 100644 --- a/packages/runtime/src/conversation-copy.ts +++ b/packages/runtime/src/conversation-copy.ts @@ -842,8 +842,16 @@ function rewriteProviderRequestAttempt( ...data, traceId: requiredMappedId(providerTraceIds, data.traceId, 'provider trace'), attemptId: eventId, - captureId: requiredMappedId(operationalEventIds, data.captureId, 'provider request capture'), - captureArtifactId: rewriteOwnedArtifactId(data.captureArtifactId, references), + ...(data.captureId !== undefined && data.captureArtifactId !== undefined + ? { + captureId: requiredMappedId( + operationalEventIds, + data.captureId, + 'provider request capture', + ), + captureArtifactId: rewriteOwnedArtifactId(data.captureArtifactId, references), + } + : {}), }; } @@ -915,16 +923,17 @@ function providerRequestCapture(event: AgentRunEvent): Record & function providerRequestAttempt(event: AgentRunEvent): Record & { readonly traceId: string; readonly attemptId: string; - readonly captureId: string; - readonly captureArtifactId: string; + readonly captureId?: string; + readonly captureArtifactId?: string; } { const data = event.data; + const hasCaptureId = typeof data?.captureId === 'string'; + const hasArtifactId = typeof data?.captureArtifactId === 'string'; if ( !data || data.attemptId !== event.id || typeof data.traceId !== 'string' || - typeof data.captureId !== 'string' || - typeof data.captureArtifactId !== 'string' + hasCaptureId !== hasArtifactId ) { throw new Error(`Cannot copy invalid provider request attempt ${event.id}`); } @@ -932,8 +941,9 @@ function providerRequestAttempt(event: AgentRunEvent): Record & ...data, traceId: data.traceId, attemptId: data.attemptId, - captureId: data.captureId, - captureArtifactId: data.captureArtifactId, + ...(hasCaptureId && hasArtifactId + ? { captureId: data.captureId as string, captureArtifactId: data.captureArtifactId as string } + : {}), }; } diff --git a/packages/runtime/src/history-compact-ledger.ts b/packages/runtime/src/history-compact-ledger.ts index 6348877790..9698b92a44 100644 --- a/packages/runtime/src/history-compact-ledger.ts +++ b/packages/runtime/src/history-compact-ledger.ts @@ -76,7 +76,11 @@ export async function loadHistoryCompactCheckpointsFromRunLedger( export async function loadLatestHistoryCompactCheckpointFromRunLedger( runStore: Pick< AgentRunStore, - 'listSessionRuns' | 'readEvents' | 'readEventProjection' | 'repairEventProjection' + | 'listSessionRuns' + | 'readEvents' + | 'readEventProjection' + | 'readEventLedgerRevision' + | 'repairEventProjection' >, sessionId: string, ): Promise { @@ -100,6 +104,10 @@ export async function loadLatestHistoryCompactCheckpointFromRunLedger( // Recover the derived projection from the canonical ledger below. } } + const ledgerRevision = + runStore.readEventLedgerRevision && runStore.repairEventProjection + ? await runStore.readEventLedgerRevision(sessionId) + : undefined; const runs = await runStore.listSessionRuns(sessionId); const candidates: LedgerCheckpointCandidate[] = []; for (let runIndex = runs.length - 1; runIndex >= 0; runIndex -= 1) { @@ -118,12 +126,16 @@ export async function loadLatestHistoryCompactCheckpointFromRunLedger( } } const selected = selectRecoveredCheckpoint(candidates); + if (ledgerRevision === undefined) return selected?.checkpoint; await runStore .repairEventProjection?.( sessionId, 'history_compact_checkpoint_recorded', selected?.event ?? null, - replaceEventId ? { replaceEventId } : undefined, + { + ifLedgerRevision: ledgerRevision, + ...(replaceEventId ? { replaceEventId } : {}), + }, ) .catch(() => { // Recovery succeeded; a later cold read can retry this derived-state repair. diff --git a/packages/runtime/src/latest-context-snapshot.ts b/packages/runtime/src/latest-context-snapshot.ts index 4e070bbb37..d2f53652d6 100644 --- a/packages/runtime/src/latest-context-snapshot.ts +++ b/packages/runtime/src/latest-context-snapshot.ts @@ -33,13 +33,10 @@ import { foldPromptComposition, type SizedRequestSegment } from './prompt-compos /** * One request's context, frozen by the transaction that committed it (#2323). * - * The reason this is a record rather than a read-time join: the facts it holds - * are written by different appends with different guarantees, and reading "the - * newest of each kind" separately produces a snapshot whose parts describe - * different moments. A failed call replaces the newest metering record; a - * capture that never matched replaces the newest capture; the compaction - * boundary moves on its own. Each of those is correct in isolation and wrong - * together. + * The reason this is a record rather than a read-time join: the canonical + * attempt owns the request facts, while the compaction boundary has its own + * recovery lifecycle. Reading independent "latest" records would produce a + * snapshot whose parts describe different moments. * * So the facts are copied into one derived row by the same storage transaction * that commits the canonical completed-main attempt. There is one durable @@ -48,7 +45,7 @@ import { foldPromptComposition, type SizedRequestSegment } from './prompt-compos * request, never authorises a write, so the last good answer stands. */ -export const LATEST_CONTEXT_SNAPSHOT_SCHEMA_VERSION = 1 as const; +export const LATEST_CONTEXT_SNAPSHOT_SCHEMA_VERSION = 2 as const; export interface LatestContextSnapshot { schemaVersion: typeof LATEST_CONTEXT_SNAPSHOT_SCHEMA_VERSION; @@ -63,9 +60,9 @@ export interface LatestContextSnapshot { /** The window this call was metered against, frozen at call time. */ contextWindow?: number; /** - * What the prompt was made of. Absent when the best-effort capture did not - * describe THIS attempt — a request explains itself or says nothing, never - * borrows another request's breakdown. + * What the prompt was made of. Absent when THIS canonical attempt carries no + * prepared-request observation — a request explains itself or says nothing, + * never borrows another request's breakdown. */ composition?: ContextDiagnosticsComposition; /** The boundary that applied when this request was built, if any. */ @@ -120,26 +117,163 @@ export interface LatestContextFacts { /** * Reads a snapshot back off the ledger. * - * Tolerant in the one direction that matters: a record written by a newer - * build may carry fields this one does not know, and dropping the whole - * snapshot for that would lose an answer it could still give. A record missing - * the identity it is anchored on is a different matter, and is rejected. + * Only the current observation-backed schema is trusted. Older projections + * were derived from the retired capture-event path; newer projections may + * change semantics this build cannot safely infer. Either case falls back to the + * canonical ledger and repairs the derived row. */ export function readLatestContextSnapshot( event: Pick | undefined, ): LatestContextSnapshot | undefined { if (!event) return undefined; - const data = event.data; - if (!data || typeof data !== 'object') return undefined; - const record = data as Record; + const record = shapedRecord( + event.data, + ['schemaVersion', 'attemptId', 'providerId', 'modelId', 'completedAt'], + ['inputTokens', 'cacheReadInputTokens', 'contextWindow', 'composition', 'compaction'], + ); + if (!record) return undefined; if ( - typeof record.attemptId !== 'string' || - record.attemptId.length === 0 || - typeof record.providerId !== 'string' || - typeof record.modelId !== 'string' || - typeof record.completedAt !== 'number' + record.schemaVersion !== LATEST_CONTEXT_SNAPSHOT_SCHEMA_VERSION || + !isBoundedString(record.attemptId, 512) || + !isBoundedString(record.providerId, 512) || + !isBoundedString(record.modelId, 512) || + !isCount(record.completedAt) || + !isOptionalCount(record.inputTokens) || + !isOptionalCount(record.cacheReadInputTokens) || + (record.contextWindow !== undefined && + (!isCount(record.contextWindow) || record.contextWindow === 0)) || + (record.composition !== undefined && !isContextDiagnosticsComposition(record.composition)) || + (record.compaction !== undefined && !isContextDiagnosticsCompaction(record.compaction)) ) { return undefined; } return record as unknown as LatestContextSnapshot; } + +function isContextDiagnosticsComposition(value: unknown): value is ContextDiagnosticsComposition { + const composition = shapedRecord( + value, + ['segments'], + ['tools', 'remainingTools', 'unlabelledToolBytes'], + ); + if ( + !composition || + !Array.isArray(composition.segments) || + composition.segments.length === 0 || + composition.segments.length > 4 || + !composition.segments.every(isContextDiagnosticsSegment) || + (composition.tools !== undefined && + (!Array.isArray(composition.tools) || + composition.tools.length === 0 || + composition.tools.length > 64 || + !composition.tools.every(isContextDiagnosticsTool))) || + (composition.remainingTools !== undefined && + !isContextDiagnosticsRemainder(composition.remainingTools)) || + !isOptionalCount(composition.unlabelledToolBytes) || + (composition.unlabelledToolBytes !== undefined && composition.unlabelledToolBytes === 0) + ) { + return false; + } + const valid = composition as unknown as ContextDiagnosticsComposition; + const segmentOrder = ['system_instructions', 'tool_definitions', 'messages', 'other']; + const order = valid.segments.map((segment) => segmentOrder.indexOf(segment.kind)); + if (order.some((value, index) => index > 0 && value <= order[index - 1]!)) return false; + + const toolDefinitions = valid.segments.find((segment) => segment.kind === 'tool_definitions'); + const tools = valid.tools ?? []; + if ( + tools.some( + (tool, index) => + index > 0 && + (tools[index - 1]!.bytes < tool.bytes || + (tools[index - 1]!.bytes === tool.bytes && + tools[index - 1]!.name.localeCompare(tool.name) >= 0)), + ) || + new Set(tools.map((tool) => tool.name)).size !== tools.length + ) { + return false; + } + const describedToolBytes = + tools.reduce((total, tool) => total + tool.bytes, 0) + + (valid.remainingTools?.bytes ?? 0) + + (valid.unlabelledToolBytes ?? 0); + return toolDefinitions + ? describedToolBytes === toolDefinitions.bytes + : describedToolBytes === 0 && valid.remainingTools === undefined; +} + +function isContextDiagnosticsSegment(value: unknown): boolean { + const segment = shapedRecord(value, ['kind', 'bytes'], []); + return Boolean( + segment && + (segment.kind === 'system_instructions' || + segment.kind === 'tool_definitions' || + segment.kind === 'messages' || + segment.kind === 'other') && + isCount(segment.bytes) && + segment.bytes > 0, + ); +} + +function isContextDiagnosticsTool(value: unknown): boolean { + const tool = shapedRecord(value, ['name', 'bytes'], []); + return Boolean(tool && isBoundedString(tool.name, 512) && isCount(tool.bytes) && tool.bytes > 0); +} + +function isContextDiagnosticsRemainder(value: unknown): boolean { + const remainder = shapedRecord(value, ['count', 'bytes'], []); + return Boolean( + remainder && + isCount(remainder.count) && + remainder.count > 0 && + isCount(remainder.bytes) && + remainder.bytes > 0, + ); +} + +function isContextDiagnosticsCompaction(value: unknown): value is ContextDiagnosticsCompaction { + const compaction = shapedRecord( + value, + ['kind', 'phase', 'eventCount', 'turnCount', 'estimatedTokens'], + [], + ); + return Boolean( + compaction && + compaction.kind === 'history' && + (compaction.phase === 'pre_turn' || compaction.phase === 'mid_turn') && + isCount(compaction.eventCount) && + compaction.eventCount > 0 && + isCount(compaction.turnCount) && + compaction.turnCount > 0 && + isCount(compaction.estimatedTokens), + ); +} + +function shapedRecord( + value: unknown, + required: readonly string[], + optional: readonly string[], +): Record | undefined { + if (!value || typeof value !== 'object' || Array.isArray(value)) return undefined; + const record = value as Record; + const allowed = new Set([...required, ...optional]); + if ( + required.some((key) => !Object.hasOwn(record, key)) || + Object.keys(record).some((key) => !allowed.has(key)) + ) { + return undefined; + } + return record; +} + +function isBoundedString(value: unknown, maxLength: number): value is string { + return typeof value === 'string' && value.length > 0 && value.length <= maxLength; +} + +function isCount(value: unknown): value is number { + return Number.isSafeInteger(value) && (value as number) >= 0; +} + +function isOptionalCount(value: unknown): boolean { + return value === undefined || isCount(value); +} diff --git a/packages/runtime/src/prompt-composition.ts b/packages/runtime/src/prompt-composition.ts index 6a2f98cb39..c6863c913e 100644 --- a/packages/runtime/src/prompt-composition.ts +++ b/packages/runtime/src/prompt-composition.ts @@ -21,34 +21,32 @@ import type { ContextDiagnosticsComposition, ContextDiagnosticsSegment, } from './context-diagnostics.js'; -import type { PreparedRequestSegmentKind } from './request-shape.js'; +import type { PreparedRequestObservationSegmentKind } from '@maka/core/model-call-attempt'; /** * The three fields a fold needs, and no more. * - * `PreparedRequestSegment` satisfies this structurally, so a live capture folds - * without conversion — but a decoder reading one back off the ledger does not - * have to invent an `index` or a `hash` it never uses just to produce the wider - * type. A fabricated field is a silent wrong answer waiting for the first - * caller that reads it. + * A current observation segment satisfies this structurally, while the legacy + * event decoder does not have to invent fields the fold never reads. */ export interface SizedRequestSegment { - kind: PreparedRequestSegmentKind; + kind: PreparedRequestObservationSegmentKind; bytes: number; + representedSegments?: number; label?: string; } /** - * Folds one request's captured segments into "what was this prompt made of" + * Folds one request's observed segments into "what was this prompt made of" * (#2323). * * The bar above this in the Inspector answers how full the context is, from * provider-reported tokens. This answers what filled it, and the two are not - * views of one number: composition is measured in **bytes of serialized - * request**, sums to `requestBytes`, and never sums to the reported - * `inputTokens`. Nothing here estimates tokens — a byte count is the fact this - * layer holds, and turning it into a token figure is a display decision that - * has to be labelled as an estimate where it is made (#1679). + * views of one number: composition is measured in **bytes of the observed + * semantic segments** and never sums to the reported `inputTokens`. Nothing + * here estimates tokens — a byte count is the fact this layer holds, and + * turning it into a token figure is a display decision that has to be labelled + * as an estimate where it is made (#1679). * * `tool_schema` folds per tool rather than into one total, because that is the * only breakdown a reader can act on: "tool definitions are 40%" names nothing @@ -61,15 +59,23 @@ export function foldPromptComposition( ): ContextDiagnosticsComposition | undefined { if (segments.length === 0) return undefined; - const byKind = new Map(); + const byKind = new Map(); const byTool = new Map(); let unlabelledToolBytes = 0; + let boundedToolCount = 0; + let boundedToolBytes = 0; for (const segment of segments) { byKind.set(segment.kind, (byKind.get(segment.kind) ?? 0) + segment.bytes); if (segment.kind !== 'tool_schema') continue; - if (segment.label === undefined) unlabelledToolBytes += segment.bytes; - else byTool.set(segment.label, (byTool.get(segment.label) ?? 0) + segment.bytes); + if (segment.label !== undefined) { + byTool.set(segment.label, (byTool.get(segment.label) ?? 0) + segment.bytes); + } else if (segment.representedSegments !== undefined) { + boundedToolCount += segment.representedSegments; + boundedToolBytes += segment.bytes; + } else { + unlabelledToolBytes += segment.bytes; + } } // A zero-byte kind is dropped rather than shown as `≈0`, the same way @@ -93,32 +99,30 @@ export function foldPromptComposition( // remainder, so the rows still account for every tool byte. const tools = ranked.slice(0, MAX_TOOL_ROWS); const remainder = ranked.slice(MAX_TOOL_ROWS); - const remainingToolBytes = remainder.reduce((carry, tool) => carry + tool.bytes, 0); + const remainingToolCount = remainder.length + boundedToolCount; + const remainingToolBytes = + remainder.reduce((carry, tool) => carry + tool.bytes, 0) + boundedToolBytes; return { segments: folded, ...(tools.length > 0 ? { tools } : {}), - ...(remainder.length > 0 - ? { remainingTools: { count: remainder.length, bytes: remainingToolBytes } } + ...(remainingToolCount > 0 + ? { remainingTools: { count: remainingToolCount, bytes: remainingToolBytes } } : {}), ...(unlabelledToolBytes > 0 ? { unlabelledToolBytes } : {}), }; } -/** - * The diagnostic append that carries the segments, alongside the durable - * metering record on the same run stream. - */ +/** Historical provider-attempt event retained only for pre-canonical ledgers. */ export const PROVIDER_REQUEST_ATTEMPT_EVENT_TYPE = 'provider_request_attempt_recorded'; /** * Reads one run event into the composition of the request it describes. * - * Returns undefined for every event that is not a decodable capture, so a - * caller walking the run stream can recognise this alongside the metering - * record without a second read. Absence is the honest outcome: this append is - * best-effort, and a record that will not decode is a composition the reader - * does not have — not a prompt made of nothing. + * Returns undefined for every event that is not a decodable historical + * provider attempt. Current writers put the observation on the canonical + * ModelCallAttempt instead. Absence is the honest outcome: an unreadable legacy + * record is a composition the reader does not have, not a prompt made of nothing. */ export function readPromptCompositionEvent(event: { readonly type: string; @@ -147,12 +151,21 @@ export function readPromptCompositionEvent(event: { function readSegment(value: unknown): SizedRequestSegment | undefined { if (!isRecord(value)) return undefined; const kind = value.kind; - if (!KIND_ORDER.includes(kind as PreparedRequestSegmentKind)) return undefined; + if (!KIND_ORDER.includes(kind as PreparedRequestObservationSegmentKind)) return undefined; if (!isNonNegativeInteger(value.bytes)) return undefined; + if ( + value.representedSegments !== undefined && + (!isNonNegativeInteger(value.representedSegments) || value.representedSegments === 0) + ) { + return undefined; + } if (value.label !== undefined && typeof value.label !== 'string') return undefined; return { - kind: kind as PreparedRequestSegmentKind, + kind: kind as PreparedRequestObservationSegmentKind, bytes: value.bytes, + ...(typeof value.representedSegments === 'number' + ? { representedSegments: value.representedSegments } + : {}), ...(typeof value.label === 'string' ? { label: value.label } : {}), }; } @@ -174,7 +187,7 @@ function isNonNegativeInteger(value: unknown): value is number { */ const MAX_TOOL_ROWS = 64; -const KIND_ORDER: readonly PreparedRequestSegmentKind[] = [ +const KIND_ORDER: readonly PreparedRequestObservationSegmentKind[] = [ 'system_prompt', 'tool_schema', 'message', @@ -186,9 +199,10 @@ const KIND_ORDER: readonly PreparedRequestSegmentKind[] = [ * four buckets already fold the same segments for `readLatestContextDiagnostics` * (#1580), and two names for one fact is how two surfaces start disagreeing. */ -const PART_KINDS: Record = { - system_prompt: 'system_instructions', - tool_schema: 'tool_definitions', - message: 'messages', - provider_options: 'other', -}; +const PART_KINDS: Record = + { + system_prompt: 'system_instructions', + tool_schema: 'tool_definitions', + message: 'messages', + provider_options: 'other', + }; diff --git a/packages/runtime/src/provider-request-telemetry.ts b/packages/runtime/src/provider-request-telemetry.ts index 73d0c99160..49f9809caf 100644 --- a/packages/runtime/src/provider-request-telemetry.ts +++ b/packages/runtime/src/provider-request-telemetry.ts @@ -23,13 +23,10 @@ import { type ModelCallAttempt, type ModelCallKind, type ModelCallUsageBasis, + type PreparedRequestObservation, } from '@maka/core/model-call-attempt'; import type { PricingConfig } from '@maka/core/usage-stats/types'; -import { - capturePreparedProviderRequest, - type PreparedProviderRequestCapture, - type PreparedRequestSegment, -} from './request-shape.js'; +import { prepareRequestObservation, type PreparedRequestMaterial } from './request-shape.js'; import { rawFinishReasonString } from './model-protocol.js'; import { providerFailureDiagnostic, @@ -68,7 +65,7 @@ export interface ProviderRequestUsageLike { export type ProviderRequestAttemptStatus = 'completed' | 'failed' | 'interrupted' | 'aborted'; -export interface ProviderRequestCaptureRecord extends PreparedProviderRequestCapture { +export interface PreparedRequestArtifactInput extends PreparedRequestMaterial { traceId: string; captureId: string; turnId: string; @@ -77,37 +74,20 @@ export interface ProviderRequestCaptureRecord extends PreparedProviderRequestCap modelId: string; } -export interface ProviderRequestCaptureRef { - captureId: string; +export interface PreparedRequestArtifactRef { artifactId: string; } -export type ProviderRequestCaptureLedgerRecord = Omit< - ProviderRequestCaptureRecord, - 'serializedRequest' -> & { - artifactId: string; -}; - -export interface ProviderRequestAttemptRecord extends ProviderRequestUsage { +interface SettledProviderAttempt extends ProviderRequestUsage { traceId: string; attemptId: string; turnId: string; step: number; attempt: number; - /** - * Present only when a capture sink is wired. The request shape below is - * computed locally and always present; these two are the join keys to the - * persisted artifact, so they are absent when there is nothing to join to. - */ - captureId?: string; captureArtifactId?: string; providerId: string; modelId: string; contextWindow?: number; - requestHash: string; - requestBytes: number; - segments: PreparedRequestSegment[]; startedAt: number; completedAt: number; status: ProviderRequestAttemptStatus; @@ -135,14 +115,10 @@ export interface ProviderRequestTrackerInput { now: () => number; newId: () => string; /** - * Request-body capture sink. Optional because capture is a diagnostic, and - * metering must not depend on one: a deployment with capture switched off - * still settles canonical records, it just has no artifact to join them to. + * Optional private artifact sink. Failure leaves the canonical observation + * intact and the attempt explicitly has no artifact join. */ - persistCapture?: ( - capture: ProviderRequestCaptureRecord, - ) => Promise>; - recordAttempt: (attempt: ProviderRequestAttemptRecord) => void | Promise; + persistArtifact?: (input: PreparedRequestArtifactInput) => Promise; /** * Durable run metadata that must exist before any physical provider call. * Kept outside accounting because a dispatch gate is an execution contract, @@ -195,13 +171,6 @@ export interface ModelCallAccountingInput { assertReady?: () => void; } -export interface ProviderRequestCaptureRecorderInput { - persistArtifact: ( - capture: ProviderRequestCaptureRecord, - ) => Promise>; - recordLedger: (capture: ProviderRequestCaptureLedgerRecord) => Promise; -} - export interface TrackProviderStreamInput { providerId: string; modelId: string; @@ -302,19 +271,6 @@ export function withProviderStreamTracking(input: { }); } -export function createProviderRequestCaptureRecorder( - input: ProviderRequestCaptureRecorderInput, -): ( - capture: ProviderRequestCaptureRecord, -) => Promise> { - return async (capture) => { - const artifact = await input.persistArtifact(capture); - const { serializedRequest: _serializedRequest, ...metadata } = capture; - await input.recordLedger({ ...metadata, artifactId: artifact.artifactId }); - return artifact; - }; -} - export interface ProviderStreamResult { stream: ReadableStream; request?: unknown; @@ -330,9 +286,9 @@ export interface ProviderGenerateResult { } interface StoredCapture { - capture: ProviderRequestCaptureRecord; - /** Absent when no capture sink is wired: there is no artifact to point at. */ - ref?: ProviderRequestCaptureRef; + material: PreparedRequestMaterial; + /** Absent when artifact persistence is unavailable or failed. */ + artifactId?: string; } const CANONICAL_USAGE_FIELDS = [ @@ -372,7 +328,7 @@ function modelCallUsageFields( export class ProviderRequestTracker { private step = 0; private readonly attemptsByStep = new Map(); - private readonly captures = new Map>(); + private readonly captures = new Map(); /** * One logical call per step. Retries of the same step are further attempts of * that call, not new calls, so they share this id. @@ -395,7 +351,7 @@ export class ProviderRequestTracker { throwIfAbortedBeforeDispatch(input.abortSignal); this.input.accounting?.assertReady?.(); const step = this.step; - const capture = await this.capture(step, input); + const capture = this.capture(step, input); throwIfAbortedBeforeDispatch(input.abortSignal); let sawOutput = false; const attempt = this.beginAttempt(step, capture, input); @@ -464,7 +420,7 @@ export class ProviderRequestTracker { throwIfAbortedBeforeDispatch(input.abortSignal); this.input.accounting?.assertReady?.(); const step = this.step; - const capture = await this.capture(step, input); + const capture = this.capture(step, input); throwIfAbortedBeforeDispatch(input.abortSignal); const attempt = this.beginAttempt(step, capture, input); try { @@ -531,24 +487,16 @@ export class ProviderRequestTracker { const contextWindow = positiveInteger(this.input.contextWindow); const failure = finish?.error !== undefined ? providerFailureDiagnostic(finish.error) : undefined; - const record: ProviderRequestAttemptRecord = { + const record: SettledProviderAttempt = { traceId: this.input.traceId, attemptId, turnId: this.input.turnId, step, attempt, - ...(capture.ref - ? { - captureId: capture.ref.captureId, - captureArtifactId: capture.ref.artifactId, - } - : {}), + ...(capture.artifactId ? { captureArtifactId: capture.artifactId } : {}), providerId: input.providerId, modelId: input.modelId, ...(contextWindow !== undefined ? { contextWindow } : {}), - requestHash: capture.capture.requestHash, - requestBytes: capture.capture.requestBytes, - segments: capture.capture.segments, startedAt, completedAt, status, @@ -559,15 +507,11 @@ export class ProviderRequestTracker { ...(usage ?? {}), }; accountingSettlement = accountingSettlement.then(async () => { - try { - await this.input.recordAttempt(record); - } catch { - // Attempt telemetry is diagnostic. The provider outcome remains authoritative. - } await this.emitModelCallAttempt(record, { logicalCallId, usage, contextWindow, + requestObservation: capture.material.observation, // Frozen when THIS request was prepared, so a checkpoint published // mid-flight by another turn cannot be sealed into a prompt built // before it existed. @@ -606,11 +550,12 @@ export class ProviderRequestTracker { * reported, not raised. */ private async emitModelCallAttempt( - record: ProviderRequestAttemptRecord, + record: SettledProviderAttempt, context: { logicalCallId: string; usage: ProviderRequestUsage | undefined; contextWindow: number | undefined; + requestObservation: PreparedRequestObservation; historyCompactBoundary: ContextDiagnosticsCompaction | undefined; historyCompactRoute: HistoryCompactRoute | undefined; }, @@ -650,6 +595,7 @@ export class ProviderRequestTracker { ...(record.captureArtifactId !== undefined ? { captureArtifactId: record.captureArtifactId } : {}), + requestObservation: context.requestObservation, startedAt: record.startedAt, completedAt: record.completedAt, latencyMs: record.latencyMs, @@ -678,7 +624,11 @@ export class ProviderRequestTracker { // call commits its metering alone and leaves the last answer standing. const latestContext = attempt.callKind === 'main' && attempt.status === 'completed' - ? latestContextProjectionInput(attempt, record.segments, context.historyCompactBoundary) + ? latestContextProjectionInput( + attempt, + attempt.requestObservation?.segments, + context.historyCompactBoundary, + ) : undefined; try { @@ -689,41 +639,43 @@ export class ProviderRequestTracker { } } - private async capture( + private capture( step: number, input: TrackProviderStreamInput | TrackProviderGenerateInput, - ): Promise { - const prepared = preparedCapture(input.providerId, input.modelId, input.params); - const key = `${step}:${prepared.requestHash}`; + ): StoredCapture { + const material = preparedRequestMaterial(input.params); + const key = `${step}:${input.providerId}:${input.modelId}:${material.observation.digest}`; const existing = this.captures.get(key); - if (existing) return await existing; - - const persistCapture = this.input.persistCapture; - const pending = (async (): Promise => { - const captureId = this.input.newId(); - const capture: ProviderRequestCaptureRecord = { - ...prepared, + if (existing) return existing; + + const persistArtifact = this.input.persistArtifact; + const capture: StoredCapture = { material }; + this.captures.set(key, capture); + if (persistArtifact) { + const artifactInput: PreparedRequestArtifactInput = { + ...material, traceId: this.input.traceId, - captureId, + captureId: this.input.newId(), turnId: this.input.turnId, step, providerId: input.providerId, modelId: input.modelId, }; - // The request shape on `capture` is computed here and needs no sink. Only - // the artifact join keys depend on one, so without it the attempt still - // carries hash, bytes, and segments — it just points at nothing. - if (!persistCapture) return { capture }; - const persisted = await persistCapture(capture); - return { capture, ref: { captureId, artifactId: persisted.artifactId } }; - })(); - this.captures.set(key, pending); - try { - return await pending; - } catch (error) { - this.captures.delete(key); - throw error; + // Persist the private body in parallel. Dispatch and canonical accounting + // are both allowed to finish without it; the bounded observation already + // lives on the canonical attempt. If persistence wins the race, the + // attempt also carries the optional artifact join. + try { + void persistArtifact(artifactInput) + .then((ref) => { + capture.artifactId = ref.artifactId; + }) + .catch(() => undefined); + } catch { + // A synchronous adapter failure is the same optional-artifact miss. + } } + return capture; } } @@ -733,31 +685,9 @@ function throwIfAbortedBeforeDispatch(signal: AbortSignal | undefined): void { } } -function preparedCapture( - providerId: string, - modelId: string, - params: Record, -): PreparedProviderRequestCapture { +function preparedRequestMaterial(params: Record): PreparedRequestMaterial { const safeParams = secretFreeParams(params); - const prompt = Array.isArray(safeParams.prompt) ? safeParams.prompt : []; - const instructions: unknown[] = []; - const messages: unknown[] = []; - for (const item of prompt) { - const record = asRecord(item); - if (record?.role === 'system') instructions.push(record.content); - else messages.push(item); - } - const tools = Array.isArray(safeParams.tools) ? safeParams.tools : []; - const providerOptions = asRecord(safeParams.providerOptions); - return capturePreparedProviderRequest({ - providerId, - modelId, - instructions, - messages, - tools, - ...(providerOptions ? { providerOptions } : {}), - requestPayload: safeParams, - }); + return prepareRequestObservation(safeParams); } function secretFreeParams(params: Record): Record { diff --git a/packages/runtime/src/request-shape.ts b/packages/runtime/src/request-shape.ts index fb76ee45e5..341ba344e4 100644 --- a/packages/runtime/src/request-shape.ts +++ b/packages/runtime/src/request-shape.ts @@ -19,13 +19,14 @@ import { Buffer } from 'node:buffer'; import { createHash } from 'node:crypto'; -import type { RuntimeExecutionConnection } from '@maka/core/llm-connections'; -import type { - PrefixChangeReason, - ToolSchemaChangeReason, - ToolAvailabilityDiagnostic, -} from '@maka/core/usage-stats/types'; -import type { ModelMessage } from './model-protocol.js'; +import { + PREPARED_REQUEST_OBSERVATION_MAX_SEGMENTS, + PREPARED_REQUEST_OBSERVATION_SCHEMA_VERSION, + PREPARED_REQUEST_OBSERVATION_TEXT_MAX_LENGTH, + type PreparedRequestObservation, + type PreparedRequestObservationSegment, + type PreparedRequestObservationSegmentKind, +} from '@maka/core/model-call-attempt'; import { toJSONSchema } from 'zod'; import type { MakaTool } from './tool-runtime.js'; @@ -35,87 +36,13 @@ export interface CanonicalToolSet { activeTools: string[]; } -export interface RequestShapeInput { - connection: RuntimeExecutionConnection; - modelId: string; - systemPrompt?: string; - providerOptions?: Record; - providerTools: readonly MakaTool[]; - activeTools: readonly string[]; - priorMessages: readonly ModelMessage[]; - toolAvailability?: ToolAvailabilityDiagnostic; -} - -export interface RequestShapeComponents { - modelProviderHash: string; - systemPromptHash: string; - providerOptionsHash: string; - toolSchemaHash: string; - historyProjectionHash: string; -} - -export type DurablePrefixComponents = Omit; - -export interface RequestShapeDiagnostic { - /** Durable provider prefix shape, excluding prior-history projection. */ - prefixHash: string; - prefixChangeReason: PrefixChangeReason; - /** Full request shape, including prior-history projection. */ - requestShapeHash: string; - requestShapeChangeReason: PrefixChangeReason; - componentHashes: RequestShapeComponents; - toolSchemaChangeReason?: ToolSchemaChangeReason; - toolAvailability?: ToolAvailabilityDiagnostic; -} - -export type PreparedRequestSegmentKind = - | 'tool_schema' - | 'system_prompt' - | 'message' - | 'provider_options'; - -export interface PreparedRequestSegment { - kind: PreparedRequestSegmentKind; - index: number; - cacheable: boolean; - hash: string; - bytes: number; - role?: string; - /** - * What this segment is, when the seam can name it. Set for `tool_schema` from - * the tool's own name, which the provider payload already carries. - * - * Present so a size can be acted on: "tool definitions are 40% of the prompt" - * names no tool to remove, and every segment kind but this one is already a - * single thing (#2323). Optional because a payload that names nothing is a - * shape this capture still has to describe. - */ - label?: string; -} - -export interface PreparedProviderRequestInput { - providerId: string; - modelId: string; - instructions?: unknown; - messages: readonly unknown[]; - tools?: readonly unknown[]; - providerOptions?: Record; - /** Exact secret-free model-call parameters captured at the provider seam. */ - requestPayload?: unknown; -} - -export interface PreparedProviderRequestCapture { - schemaVersion: 2; - requestHash: string; - /** Hash of protocol-independent model-call semantics for cross-protocol comparison. */ - requestPayloadWithoutProviderOptionsHash: string; - requestBytes: number; +export interface PreparedRequestMaterial { + /** Full secret-free representation for the private request artifact. */ serializedRequest: string; - segments: PreparedRequestSegment[]; + /** Bounded public observation derived from that same representation. */ + observation: PreparedRequestObservation; } -export type PreparedRequestSegmentRef = Pick; - /** * Split the registry into the full dispatch set (`providerTools`) and the * model-visible subset (`activeTools`). @@ -148,53 +75,6 @@ export function canonicalizeToolSet( }; } -export function computeRequestShapeDiagnostic( - input: RequestShapeInput, - prior: RequestShapeDiagnostic | undefined, -): RequestShapeDiagnostic { - const componentHashes: RequestShapeComponents = { - modelProviderHash: stableHash({ - providerId: input.connection.providerType, - connectionSlug: input.connection.slug, - modelId: input.modelId, - }), - systemPromptHash: stableHash(input.systemPrompt ?? ''), - providerOptionsHash: stableHash(input.providerOptions ?? {}), - toolSchemaHash: stableHash({ - activeTools: [...input.activeTools], - // Only the provider-visible (active) subset crosses the wire, so the - // schema hash must reflect that subset — otherwise an inactive deferred - // tool's schema change would falsely fire `tool_schema_changed`, and a - // load would not be distinguishable from churn. - providerTools: providerVisibleTools(input.providerTools, input.activeTools).map( - toolShapeForDiagnostics, - ), - }), - historyProjectionHash: stableHash(input.priorMessages.map(messageShapeForHash)), - }; - const durablePrefixComponents = durableComponents(componentHashes); - const prefixHash = stableHash(durablePrefixComponents); - const requestShapeHash = stableHash(componentHashes); - const toolSchemaChangeReason = classifyToolSchemaChange( - componentHashes, - prior?.componentHashes, - input.toolAvailability, - prior?.toolAvailability, - ); - return { - prefixHash, - prefixChangeReason: classifyDurablePrefixChange( - durablePrefixComponents, - prior ? durableComponents(prior.componentHashes) : undefined, - ), - requestShapeHash, - requestShapeChangeReason: classifyRequestShapeChange(componentHashes, prior?.componentHashes), - componentHashes, - ...(toolSchemaChangeReason !== undefined ? { toolSchemaChangeReason } : {}), - ...(input.toolAvailability !== undefined ? { toolAvailability: input.toolAvailability } : {}), - }; -} - export function toolSchemaCharsForDiagnostics( providerTools: readonly MakaTool[], activeTools: readonly string[], @@ -206,254 +86,121 @@ export function toolSchemaCharsForDiagnostics( } /** - * Capture the standardized request immediately before the provider call. + * Observe the standardized request at the AI SDK model-call seam. * - * Segment order follows the stable Maka request-prefix model used for cache - * diagnostics: tools, system instructions, then conversation messages. - * Provider options are retained for exact replay evidence, but are not claimed - * to be a provider-cacheable prefix segment. + * Segment order follows Maka's semantic request-prefix model: tools, system + * instructions, then conversation messages. Provider options are retained for + * exact request evidence, but are not claimed to be a provider-cacheable prefix + * segment. None of this is presented as the provider's final wire body. */ -export function capturePreparedProviderRequest( - input: PreparedProviderRequestInput, -): PreparedProviderRequestCapture { - const payload = input.requestPayload ?? { - instructions: input.instructions, - messages: input.messages, - tools: input.tools ?? [], - providerOptions: input.providerOptions ?? {}, - }; - // This is the evidence body, not the hash canonicalizer: preserve the exact - // JSON ordering and values presented at the model-call seam. - const serializedRequest = JSON.stringify(payload); - const segments: PreparedRequestSegment[] = []; +export function prepareRequestObservation(payload: unknown): PreparedRequestMaterial { + const normalizedPayload = normalizePreparedValue(payload); + const serializedRequest = JSON.stringify(normalizedPayload.value); + const segments: PreparedRequestObservationSegment[] = []; + const parts = semanticRequestParts(payload); - for (const [index, tool] of (input.tools ?? []).entries()) { + for (const [index, tool] of parts.tools.entries()) { segments.push(preparedSegment('tool_schema', index, tool, true, undefined, toolLabel(tool))); } - if (input.instructions !== undefined) { - const instructions = Array.isArray(input.instructions) - ? input.instructions - : [input.instructions]; + if (parts.instructions !== undefined) { + const instructions = Array.isArray(parts.instructions) + ? parts.instructions + : [parts.instructions]; for (const [index, instruction] of instructions.entries()) { segments.push(preparedSegment('system_prompt', index, instruction, true)); } } - for (const [index, message] of input.messages.entries()) { + for (const [index, message] of parts.messages.entries()) { const role = isObjectLike(message) && typeof message.role === 'string' ? message.role : undefined; segments.push(preparedSegment('message', index, message, true, role)); } - if (input.providerOptions !== undefined) { - segments.push(preparedSegment('provider_options', 0, input.providerOptions, false)); + if (parts.providerOptions !== undefined) { + segments.push(preparedSegment('provider_options', 0, parts.providerOptions, false)); } return { - schemaVersion: 2, - requestHash: stableHash({ - providerId: input.providerId, - modelId: input.modelId, - payload, - }), - requestPayloadWithoutProviderOptionsHash: stableHash( - protocolIndependentRequestPayload(payload), - ), - requestBytes: Buffer.byteLength(serializedRequest, 'utf8'), serializedRequest, - segments, + observation: { + schemaVersion: PREPARED_REQUEST_OBSERVATION_SCHEMA_VERSION, + digest: hashSerialized(serializedRequest), + bytes: Buffer.byteLength(serializedRequest, 'utf8'), + segments: boundPreparedRequestSegments(segments), + }, }; } -function protocolIndependentRequestPayload(payload: unknown): unknown { - if (!isObjectLike(payload)) return payload; - const { providerOptions, ...shared } = payload; - const identities: ProtocolIndependentRequestIdentities = { - approvalIds: new Map(), - toolCallIds: new Map(), - }; - const reasoningEffort = protocolIndependentReasoningEffort(providerOptions); - const protocolIndependent: Record = { - ...shared, - ...(Array.isArray(shared.prompt) - ? { - prompt: shared.prompt.map((message) => withoutPromptProviderOptions(message, identities)), - } - : {}), - ...(Array.isArray(shared.messages) - ? { - messages: shared.messages.map((message) => - withoutPromptProviderOptions(message, identities), - ), - } - : {}), - ...(Array.isArray(shared.tools) - ? { tools: shared.tools.map(withoutObjectProviderOptions) } - : {}), - ...(reasoningEffort !== undefined ? { reasoningEffort } : {}), - }; - const thinkingBudget = anthropicThinkingBudget(providerOptions); - if ( - thinkingBudget === undefined || - !isNonNegativeSafeInteger(protocolIndependent.maxOutputTokens) - ) { - return protocolIndependent; - } - const wireOutputLimit = protocolIndependent.maxOutputTokens + thinkingBudget; - return Number.isSafeInteger(wireOutputLimit) - ? { ...protocolIndependent, maxOutputTokens: wireOutputLimit } - : protocolIndependent; -} +const MAX_PREPARED_REQUEST_REMAINDERS = 4; -function protocolIndependentReasoningEffort( - providerOptions: unknown, -): string | string[] | undefined { - if (!isObjectLike(providerOptions)) return undefined; - const efforts = new Set(); - const anthropic = providerOptions.anthropic; - if (isObjectLike(anthropic) && typeof anthropic.effort === 'string') { - efforts.add(anthropic.effort); - } - for (const namespace of Object.values(providerOptions)) { - if (!isObjectLike(namespace)) continue; - if (typeof namespace.reasoningEffort === 'string') efforts.add(namespace.reasoningEffort); - if (isObjectLike(namespace.thinking) && namespace.thinking.type === 'disabled') { - efforts.add('none'); - } - const thinkingConfig = namespace.thinkingConfig; - if (isObjectLike(thinkingConfig) && typeof thinkingConfig.thinkingLevel === 'string') { - efforts.add(thinkingConfig.thinkingLevel); - } - if (isObjectLike(thinkingConfig) && thinkingConfig.thinkingBudget === 0) { - efforts.add('none'); - } - const chatTemplateKwargs = namespace.chat_template_kwargs; - if (isObjectLike(chatTemplateKwargs) && chatTemplateKwargs.thinking === false) { - efforts.add('none'); +function boundPreparedRequestSegments( + segments: readonly PreparedRequestObservationSegment[], +): PreparedRequestObservationSegment[] { + if (segments.length <= PREPARED_REQUEST_OBSERVATION_MAX_SEGMENTS) return [...segments]; + const kept = segments.slice( + 0, + PREPARED_REQUEST_OBSERVATION_MAX_SEGMENTS - MAX_PREPARED_REQUEST_REMAINDERS, + ); + const remainders: PreparedRequestObservationSegment[] = []; + for (const segment of segments.slice(kept.length)) { + const previous = remainders.at(-1); + if (previous?.kind === segment.kind) { + previous.bytes += segment.bytes; + previous.representedSegments = (previous.representedSegments ?? 1) + 1; + previous.digest = hashSerialized( + JSON.stringify(['prepared-segment-remainder', previous.digest, segment.digest]), + ); + continue; } + remainders.push({ + kind: segment.kind, + index: segment.index, + cacheable: segment.cacheable, + comparison: 'opaque', + digest: hashSerialized(JSON.stringify(['prepared-segment-remainder', segment.digest])), + bytes: segment.bytes, + representedSegments: 1, + }); } - const normalized = [...efforts].sort(); - return normalized.length > 1 ? normalized : normalized[0]; -} - -interface ProtocolIndependentRequestIdentities { - approvalIds: Map; - toolCallIds: Map; -} - -function withoutPromptProviderOptions( - value: unknown, - identities: ProtocolIndependentRequestIdentities, -): unknown { - const message = withoutObjectProviderOptions(value); - if (!isObjectLike(message) || !Array.isArray(message.content)) return message; - return { - ...message, - content: message.content.map((part) => withoutPromptPartProviderOptions(part, identities)), - }; + return [...kept, ...remainders]; } -function withoutPromptPartProviderOptions( - value: unknown, - identities: ProtocolIndependentRequestIdentities, -): unknown { - const part = withoutObjectProviderOptions(value); - if (!isObjectLike(part)) return part; - if (part.type === 'tool-call') { - const { providerExecuted: _providerExecuted, ...shared } = part; - return { - ...shared, - toolCallId: protocolIndependentId(part.toolCallId, identities.toolCallIds, 'tool-call'), - }; - } - if (part.type === 'tool-approval-request') { - const { isAutomatic: _isAutomatic, signature: _signature, ...shared } = part; - return { - ...shared, - approvalId: protocolIndependentId(part.approvalId, identities.approvalIds, 'approval'), - toolCallId: protocolIndependentId(part.toolCallId, identities.toolCallIds, 'tool-call'), - }; - } - if (part.type === 'tool-approval-response') { - const { providerExecuted: _providerExecuted, ...shared } = part; - return { - ...shared, - approvalId: protocolIndependentId(part.approvalId, identities.approvalIds, 'approval'), - }; +function semanticRequestParts(payload: unknown): { + instructions?: unknown; + messages: readonly unknown[]; + tools: readonly unknown[]; + providerOptions?: Record; +} { + if (!isObjectLike(payload)) { + return { messages: [], tools: [] }; } - if (part.type !== 'tool-result') return part; - const output = withoutObjectProviderOptions(part.output); - const normalizedPart = { - ...part, - toolCallId: protocolIndependentId(part.toolCallId, identities.toolCallIds, 'tool-call'), - }; - if (!isObjectLike(output) || output.type !== 'content' || !Array.isArray(output.value)) { - return { ...normalizedPart, output }; + const prompt = Array.isArray(payload.prompt) ? payload.prompt : undefined; + const instructions: unknown[] = []; + const messages: unknown[] = []; + if (prompt) { + for (const item of prompt) { + const record = isObjectLike(item) ? item : undefined; + if (record?.role === 'system') instructions.push(record.content); + else messages.push(item); + } } + const payloadMessages = Array.isArray(payload.messages) ? payload.messages : undefined; + const providerOptions = isPlainObject(payload.providerOptions) + ? payload.providerOptions + : undefined; return { - ...normalizedPart, - output: { ...output, value: output.value.map(withoutObjectProviderOptions) }, + ...(prompt + ? instructions.length > 0 + ? { instructions } + : {} + : payload.instructions !== undefined + ? { instructions: payload.instructions } + : {}), + messages: prompt ? messages : (payloadMessages ?? []), + tools: Array.isArray(payload.tools) ? payload.tools : [], + ...(providerOptions !== undefined ? { providerOptions } : {}), }; } -function protocolIndependentId( - value: unknown, - identities: Map, - prefix: string, -): unknown { - if (typeof value !== 'string') return value; - const existing = identities.get(value); - if (existing) return existing; - const normalized = `${prefix}-${identities.size + 1}`; - identities.set(value, normalized); - return normalized; -} - -function withoutObjectProviderOptions(value: unknown): unknown { - if (!isObjectLike(value)) return value; - const { providerOptions: _providerOptions, ...shared } = value; - return shared; -} - -function anthropicThinkingBudget(providerOptions: unknown): number | undefined { - if (!isObjectLike(providerOptions)) return undefined; - const anthropic = providerOptions.anthropic; - if (!isObjectLike(anthropic)) return undefined; - const thinking = anthropic.thinking; - if (!isObjectLike(thinking) || thinking.type !== 'enabled') return undefined; - return isNonNegativeSafeInteger(thinking.budgetTokens) ? thinking.budgetTokens : undefined; -} - -function isNonNegativeSafeInteger(value: unknown): value is number { - return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0; -} - -export function findFirstChangedCacheableSegment( - current: Pick, - prior: Pick, -): PreparedRequestSegmentRef | undefined { - const currentSegments = current.segments.filter((segment) => segment.cacheable); - const priorSegments = prior.segments.filter((segment) => segment.cacheable); - const segmentCount = Math.max(currentSegments.length, priorSegments.length); - for (let position = 0; position < segmentCount; position += 1) { - const currentSegment = currentSegments[position]; - const priorSegment = priorSegments[position]; - if ( - currentSegment?.kind === priorSegment?.kind && - currentSegment?.index === priorSegment?.index && - currentSegment?.hash === priorSegment?.hash - ) { - continue; - } - const changed = currentSegment ?? priorSegment; - if (!changed) return undefined; - return { - kind: changed.kind, - index: changed.index, - ...(changed.role !== undefined ? { role: changed.role } : {}), - }; - } - return undefined; -} - /** The provider-visible tools — the active subset actually serialized on the wire. */ function providerVisibleTools( providerTools: readonly MakaTool[], @@ -464,31 +211,37 @@ function providerVisibleTools( } function preparedSegment( - kind: PreparedRequestSegmentKind, + kind: PreparedRequestObservationSegmentKind, index: number, value: unknown, cacheable: boolean, role?: string, label?: string, -): PreparedRequestSegment { - const serialized = stableStringify(value); +): PreparedRequestObservationSegment { + const normalized = normalizePreparedValue(value); + const serialized = JSON.stringify(normalized.value); return { kind, index, cacheable, - hash: stableHash(value), + comparison: normalized.opaque || containsComparisonOpaqueRedaction(value) ? 'opaque' : 'exact', + digest: hashSerialized(serialized), bytes: Buffer.byteLength(serialized, 'utf8'), - ...(role !== undefined ? { role } : {}), - ...(label !== undefined ? { label } : {}), + ...(role !== undefined + ? { role: role.slice(0, PREPARED_REQUEST_OBSERVATION_TEXT_MAX_LENGTH) } + : {}), + ...(label !== undefined + ? { label: label.slice(0, PREPARED_REQUEST_OBSERVATION_TEXT_MAX_LENGTH) } + : {}), }; } /** * The tool's own name as the payload carries it. * - * Read off the serialized tool rather than the registry: this capture describes - * what crossed the wire, so a name that is not in the payload is not a name this - * segment can claim. + * Read off the prepared payload rather than the registry: this observation + * describes what Maka handed to the model-call seam, so a name absent there is + * not a name this segment can claim. */ function toolLabel(tool: unknown): string | undefined { if (!isObjectLike(tool)) return undefined; @@ -499,6 +252,10 @@ export function stableHash(value: unknown): `sha256:${string}` { return `sha256:${createHash('sha256').update(stableStringify(value)).digest('hex')}`; } +function hashSerialized(serialized: string): `sha256:${string}` { + return `sha256:${createHash('sha256').update(serialized).digest('hex')}`; +} + export function toolCatalogHash(tools: readonly MakaTool[]): `sha256:${string}` { return stableHash( [...tools] @@ -511,103 +268,180 @@ export function stableStringify(value: unknown): string { return JSON.stringify(canonicalize(value)); } -function classifyDurablePrefixChange( - current: DurablePrefixComponents, - prior: DurablePrefixComponents | undefined, -): PrefixChangeReason { - if (!prior) return 'first_turn'; - if (current.modelProviderHash !== prior.modelProviderHash) return 'model_or_provider_changed'; - if (current.systemPromptHash !== prior.systemPromptHash) return 'system_prompt_changed'; - if (current.toolSchemaHash !== prior.toolSchemaHash) return 'tool_schema_changed'; - if (current.providerOptionsHash !== prior.providerOptionsHash) return 'provider_options_changed'; - return 'stable'; +interface NormalizedPreparedValue { + value: unknown; + opaque: boolean; } -function classifyRequestShapeChange( - current: RequestShapeComponents, - prior: RequestShapeComponents | undefined, -): PrefixChangeReason { - if (!prior) return 'first_turn'; - if (current.modelProviderHash !== prior.modelProviderHash) return 'model_or_provider_changed'; - if (current.systemPromptHash !== prior.systemPromptHash) return 'system_prompt_changed'; - if (current.toolSchemaHash !== prior.toolSchemaHash) return 'tool_schema_changed'; - if (current.providerOptionsHash !== prior.providerOptionsHash) return 'provider_options_changed'; - if (current.historyProjectionHash !== prior.historyProjectionHash) - return 'history_projection_changed'; - return 'stable'; +/** + * Lossless JSON representation for the semantic values accepted by the model + * seam. Every value is tagged, so a bigint cannot collide with a user string + * and an undefined property cannot disappear. Values that cannot be described + * exactly are retained as explicit opaque markers instead of pretending they + * were equal to another request. + */ +function normalizePreparedValue(value: unknown): NormalizedPreparedValue { + const tag = '__makaPreparedValue'; + const ancestors = new Set(); + const visit = (current: unknown, depth: number): NormalizedPreparedValue => { + if (current === null || typeof current === 'string' || typeof current === 'boolean') { + return { value: current, opaque: false }; + } + if (typeof current === 'number') { + if (Number.isFinite(current) && !Object.is(current, -0)) { + return { value: current, opaque: false }; + } + const encoded = Number.isNaN(current) + ? 'NaN' + : current === Infinity + ? 'Infinity' + : current === -Infinity + ? '-Infinity' + : '-0'; + return { value: { [tag]: 'number', value: encoded }, opaque: false }; + } + if (typeof current === 'bigint') { + return { value: { [tag]: 'bigint', value: current.toString() }, opaque: false }; + } + if (typeof current === 'undefined') { + return { value: { [tag]: 'undefined' }, opaque: false }; + } + if (typeof current === 'function' || typeof current === 'symbol') { + return { value: { [tag]: 'opaque', kind: typeof current }, opaque: true }; + } + if (typeof current !== 'object') { + return { value: { [tag]: 'opaque', kind: typeof current }, opaque: true }; + } + if (depth >= 64) { + return { value: { [tag]: 'opaque', kind: 'max-depth' }, opaque: true }; + } + if (ancestors.has(current)) { + return { value: { [tag]: 'opaque', kind: 'cycle' }, opaque: true }; + } + ancestors.add(current); + try { + if (current instanceof ArrayBuffer) { + return { + value: { + [tag]: 'binary', + kind: 'ArrayBuffer', + encoding: 'base64', + value: Buffer.from(current).toString('base64'), + }, + opaque: false, + }; + } + if (ArrayBuffer.isView(current)) { + return { + value: { + [tag]: 'binary', + kind: current.constructor?.name ?? 'ArrayBufferView', + encoding: 'base64', + value: Buffer.from(current.buffer, current.byteOffset, current.byteLength).toString( + 'base64', + ), + }, + opaque: false, + }; + } + if (current instanceof Date) { + const timestamp = current.getTime(); + return { + value: { + [tag]: 'date', + value: Number.isNaN(timestamp) ? 'invalid' : current.toISOString(), + }, + opaque: false, + }; + } + if (current instanceof Map) { + let opaque = false; + const entries = [...current.entries()].map(([key, entry]) => { + const normalizedKey = visit(key, depth + 1); + const normalizedEntry = visit(entry, depth + 1); + opaque ||= normalizedKey.opaque || normalizedEntry.opaque; + return [normalizedKey.value, normalizedEntry.value]; + }); + return { value: { [tag]: 'map', entries }, opaque }; + } + if (current instanceof Set) { + let opaque = false; + const entries = [...current].map((entry) => { + const normalized = visit(entry, depth + 1); + opaque ||= normalized.opaque; + return normalized.value; + }); + return { value: { [tag]: 'set', entries }, opaque }; + } + if (Array.isArray(current)) { + let opaque = false; + const entries = Array.from({ length: current.length }, (_, index) => { + if (!(index in current)) return { [tag]: 'array-hole' }; + const normalized = visit(current[index], depth + 1); + opaque ||= normalized.opaque; + return normalized.value; + }); + return { value: entries, opaque }; + } + if (isPlainObject(current)) { + let opaque = false; + const entries = Object.keys(current).map((key) => { + let normalized: NormalizedPreparedValue; + try { + normalized = visit(current[key], depth + 1); + } catch { + normalized = { + value: { [tag]: 'opaque', kind: 'unreadable-property' }, + opaque: true, + }; + } + opaque ||= normalized.opaque; + return [key, normalized.value]; + }); + if (Object.hasOwn(current, tag)) { + return { value: { [tag]: 'object', entries }, opaque }; + } + return { value: Object.fromEntries(entries), opaque }; + } + const toJSON = (current as { toJSON?: unknown }).toJSON; + if (typeof toJSON === 'function') { + try { + return visit(toJSON.call(current), depth + 1); + } catch { + return { value: { [tag]: 'opaque', kind: 'toJSON-failed' }, opaque: true }; + } + } + return { + value: { + [tag]: 'opaque', + kind: current.constructor?.name ?? 'non-plain-object', + }, + opaque: true, + }; + } finally { + ancestors.delete(current); + } + }; + return visit(value, 0); } -function classifyToolSchemaChange( - current: RequestShapeComponents, - prior: RequestShapeComponents | undefined, - currentAvail: ToolAvailabilityDiagnostic | undefined, - priorAvail: ToolAvailabilityDiagnostic | undefined, -): ToolSchemaChangeReason | undefined { - if (!prior || current.toolSchemaHash === prior.toolSchemaHash) return undefined; - if ( - isEnabledSourceStrictSuperset(currentAvail, priorAvail) && - sourceCatalogStable(currentAvail, priorAvail) - ) { - return 'tool_source_enabled'; - } - if (sourceStateChanged(currentAvail, priorAvail)) { - return 'tool_source_state_changed'; +function containsComparisonOpaqueRedaction(value: unknown, seen = new Set()): boolean { + if (!isObjectLike(value)) return false; + if (seen.has(value)) return false; + seen.add(value); + if (Array.isArray(value)) { + return value.some((entry) => containsComparisonOpaqueRedaction(entry, seen)); } - return 'tool_schema_changed'; -} - -function isEnabledSourceStrictSuperset( - current: ToolAvailabilityDiagnostic | undefined, - prior: ToolAvailabilityDiagnostic | undefined, -): boolean { if ( - !current || - !prior || - current.mode !== prior.mode || - (current.mode !== 'economy' && current.mode !== 'search') - ) - return false; - const currentIds = new Set(current.enabledSourceIds); - const priorIds = new Set(prior.enabledSourceIds); - if (currentIds.size <= priorIds.size) return false; - for (const sourceId of priorIds) { - if (!currentIds.has(sourceId)) return false; + value.type === 'custom' && + value.kind === 'openai.compaction' && + isPlainObject(value.providerOptions) && + isPlainObject(value.providerOptions.openai) && + value.providerOptions.openai.redacted === true + ) { + return true; } - return true; -} - -function sourceCatalogStable( - current: ToolAvailabilityDiagnostic | undefined, - prior: ToolAvailabilityDiagnostic | undefined, -): boolean { - if (!current || !prior) return false; - return ( - stableStringify(sourceCatalogShape(current)) === stableStringify(sourceCatalogShape(prior)) - ); -} - -function sourceStateChanged( - current: ToolAvailabilityDiagnostic | undefined, - prior: ToolAvailabilityDiagnostic | undefined, -): boolean { - return stableStringify(current ?? null) !== stableStringify(prior ?? null); -} - -function sourceCatalogShape(diagnostic: ToolAvailabilityDiagnostic): unknown { - return { - mode: diagnostic.mode, - connectorToolName: diagnostic.connectorToolName, - visibleToolNamesBySource: diagnostic.visibleToolNamesBySource ?? {}, - }; -} - -function durableComponents(components: RequestShapeComponents): DurablePrefixComponents { - return { - modelProviderHash: components.modelProviderHash, - systemPromptHash: components.systemPromptHash, - providerOptionsHash: components.providerOptionsHash, - toolSchemaHash: components.toolSchemaHash, - }; + return Object.values(value).some((entry) => containsComparisonOpaqueRedaction(entry, seen)); } function toolShapeForDiagnostics(tool: MakaTool): unknown { @@ -649,44 +483,6 @@ function stripJsonSchemaRuntimeFields(value: unknown): unknown { return out; } -function messageShapeForHash(message: ModelMessage): unknown { - const raw = message as unknown as { role?: unknown; content?: unknown }; - return { - role: typeof raw.role === 'string' ? raw.role : 'unknown', - content: contentShapeForHash(raw.content), - }; -} - -function contentShapeForHash(content: unknown): unknown { - if (typeof content === 'string') { - return { type: 'text', chars: content.length }; - } - if (Array.isArray(content)) { - return content.map((part) => { - if (!isObjectLike(part)) return { type: typeof part }; - const type = typeof part.type === 'string' ? part.type : 'unknown'; - return { - type, - ...(typeof part.toolName === 'string' ? { toolName: part.toolName } : {}), - ...(typeof part.toolCallId === 'string' ? { toolCallId: part.toolCallId } : {}), - ...(typeof part.text === 'string' ? { chars: part.text.length } : {}), - ...('output' in part ? { output: payloadShapeForHash(part.output) } : {}), - }; - }); - } - return { type: typeof content }; -} - -function payloadShapeForHash(value: unknown): unknown { - const serialized = stableStringify(value); - return { - type: value === null ? 'null' : Array.isArray(value) ? 'array' : typeof value, - chars: serialized.length, - bytes: Buffer.byteLength(serialized, 'utf8'), - hash: stableHash(value), - }; -} - function canonicalize(value: unknown, parentKey?: string): unknown { if (value === null) return null; if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') diff --git a/packages/runtime/src/run-trace.ts b/packages/runtime/src/run-trace.ts index 7eeac0eb1d..2d69a68ebd 100644 --- a/packages/runtime/src/run-trace.ts +++ b/packages/runtime/src/run-trace.ts @@ -22,9 +22,6 @@ import { generalizedErrorMessage, redactSecrets } from '@maka/core/redaction'; import type { CacheMissInputSource, ContextBudgetDiagnostic, - PrefixChangeReason, - PromptSegmentEstimate, - ToolSchemaChangeReason, ToolAvailabilityDiagnostic, } from '@maka/core/usage-stats/types'; @@ -159,21 +156,14 @@ export class RunTrace { modelStreamStarted( activeTools: readonly string[], - prefix?: { - systemPromptHash?: string; - prefixHash: string; - prefixChangeReason: PrefixChangeReason; - requestShapeHash?: string; - requestShapeChangeReason?: PrefixChangeReason; - toolSchemaChangeReason?: ToolSchemaChangeReason; + diagnostics?: { toolAvailability?: ToolAvailabilityDiagnostic; - promptSegments?: PromptSegmentEstimate[]; contextBudget?: ContextBudgetDiagnostic; }, ): void { this.emit('model', 'model_stream_started', 'Model stream started', { activeTools: [...activeTools], - ...(prefix !== undefined ? prefix : {}), + ...(diagnostics !== undefined ? diagnostics : {}), }); } @@ -218,13 +208,6 @@ export class RunTrace { outputTokens?: number; totalTokens?: number; contextBudget?: unknown; - promptSegments?: readonly unknown[]; - systemPromptHash?: string; - prefixHash?: string; - prefixChangeReason?: PrefixChangeReason; - requestShapeHash?: string; - requestShapeChangeReason?: PrefixChangeReason; - toolSchemaChangeReason?: ToolSchemaChangeReason; toolAvailability?: ToolAvailabilityDiagnostic; }): void { this.emit('model', 'send_diagnostics_recorded', 'Send diagnostics recorded', { @@ -236,28 +219,6 @@ export class RunTrace { ...(diagnostics.contextBudget !== undefined ? { contextBudget: diagnostics.contextBudget } : {}), - ...(diagnostics.promptSegments !== undefined && diagnostics.promptSegments.length > 0 - ? { promptSegments: diagnostics.promptSegments } - : {}), - // The FINAL request shape, not step 0's: a same-turn tool load changes it - // mid-send, and `model_stream_started` reports only what the first - // request carried. - ...(diagnostics.systemPromptHash !== undefined - ? { systemPromptHash: diagnostics.systemPromptHash } - : {}), - ...(diagnostics.prefixHash !== undefined ? { prefixHash: diagnostics.prefixHash } : {}), - ...(diagnostics.prefixChangeReason !== undefined - ? { prefixChangeReason: diagnostics.prefixChangeReason } - : {}), - ...(diagnostics.requestShapeHash !== undefined - ? { requestShapeHash: diagnostics.requestShapeHash } - : {}), - ...(diagnostics.requestShapeChangeReason !== undefined - ? { requestShapeChangeReason: diagnostics.requestShapeChangeReason } - : {}), - ...(diagnostics.toolSchemaChangeReason !== undefined - ? { toolSchemaChangeReason: diagnostics.toolSchemaChangeReason } - : {}), ...(diagnostics.toolAvailability !== undefined ? { toolAvailability: diagnostics.toolAvailability } : {}), diff --git a/packages/runtime/src/runtime-kernel.ts b/packages/runtime/src/runtime-kernel.ts index ccfe7f86c2..f2f048ecc8 100644 --- a/packages/runtime/src/runtime-kernel.ts +++ b/packages/runtime/src/runtime-kernel.ts @@ -2223,8 +2223,6 @@ export class RuntimeKernel implements RuntimeKernelLike { }): Pick< BackendFactoryContext, | 'recordRunTrace' - | 'recordProviderRequestCapture' - | 'recordProviderRequestAttempt' | 'recordModelCallAttempt' | 'recordRunComposition' | 'loadHistoryCompactCheckpoint' @@ -2243,15 +2241,6 @@ export class RuntimeKernel implements RuntimeKernelLike { }, ...(this.deps.runStore ? { - recordProviderRequestCapture: (capture) => { - const run = runFor(capture.turnId); - if (!run) - return Promise.reject(new Error('No active AgentRun for provider request capture')); - return run.recordProviderRequestCapture(capture); - }, - recordProviderRequestAttempt: (attempt) => { - runFor(attempt.turnId)?.recordProviderRequestAttempt(attempt); - }, // Resolved by runId rather than turnId: the canonical record names // the run it belongs to, so it needs no turn-to-run indirection. recordModelCallAttempt: (commit) => { diff --git a/packages/runtime/src/session-manager.ts b/packages/runtime/src/session-manager.ts index b8497a9df0..cf2a233f80 100644 --- a/packages/runtime/src/session-manager.ts +++ b/packages/runtime/src/session-manager.ts @@ -155,10 +155,6 @@ import type { MakaTool } from './tool-runtime.js'; import type { TurnShellPlan } from './shell-detect.js'; import type { RunTraceRecorder } from './run-trace.js'; import type { ModelCallAttempt } from '@maka/core/model-call-attempt'; -import type { - ProviderRequestAttemptRecord, - ProviderRequestCaptureLedgerRecord, -} from './provider-request-telemetry.js'; import { readLatestContextDiagnostics, type ContextDiagnostics } from './context-diagnostics.js'; import type { ModelCallCommit } from '@maka/core/agent-run'; import type { ShellRunProcessManager } from './shell-run-manager.js'; @@ -669,14 +665,9 @@ export interface BackendFactoryContext { /** Turn-scoped shell plan captured with a bound child tool ceiling. */ turnShellPlan?: TurnShellPlan; recordRunTrace?: RunTraceRecorder; - /** Durable AgentRun metadata row written after the private capture artifact. */ - recordProviderRequestCapture?: (capture: ProviderRequestCaptureLedgerRecord) => Promise; - /** Best-effort AgentRun row for one physical provider call. */ - recordProviderRequestAttempt?: (attempt: ProviderRequestAttemptRecord) => void; /** - * Durable AgentRun row carrying the canonical accounting record for one - * physical provider call. Distinct from the diagnostic row above: this one is - * the metering source of truth (#1679). + * Durable AgentRun row carrying the canonical record for one physical + * provider call, including metering and its prepared-request observation. */ recordModelCallAttempt?: (commit: ModelCallCommit) => Promise; /** Immutable Run policy snapshot; provider dispatch waits for this durable commit. */ diff --git a/packages/runtime/src/system-prompt/main-session-prompt.ts b/packages/runtime/src/system-prompt/main-session-prompt.ts index 72467be668..5a46c19f5b 100644 --- a/packages/runtime/src/system-prompt/main-session-prompt.ts +++ b/packages/runtime/src/system-prompt/main-session-prompt.ts @@ -34,7 +34,7 @@ * fragment order. * * The identity and response-format guidance are pure static text: constant - * across turns, so they never churn the systemPromptHash / prefix-cache (see + * across turns, so they never churn the provider-stable request prefix (see * request-shape.ts). They are intentionally NOT injected into sub-agent * (childInstruction) paths, which already carry their own role identities. */ diff --git a/packages/storage/src/__tests__/sqlite-core-execution-store.test.ts b/packages/storage/src/__tests__/sqlite-core-execution-store.test.ts index 8ce0f081a4..8ef650997b 100644 --- a/packages/storage/src/__tests__/sqlite-core-execution-store.test.ts +++ b/packages/storage/src/__tests__/sqlite-core-execution-store.test.ts @@ -140,6 +140,213 @@ describe('SQLite core execution stores', () => { }); }); + test('commits canonical authority without guessing a malformed projection order', async () => { + await withRoot(async (root) => { + const store = createSqliteAgentRunStore(root); + await store.createRun(runHeader()); + await store.appendEvent( + 'session-1', + 'run-1', + { + ...runEvent(), + id: 'model-call-newer', + type: 'model_call_attempt_recorded', + ts: 100, + data: { + ...modelCallAttempt({ + attemptId: 'attempt-newer', + completedAt: 100, + latencyMs: 99, + }), + }, + }, + { + latestContext: { + attemptId: 'attempt-newer', + orderedAt: 100, + snapshot: { + schemaVersion: 2, + attemptId: 'attempt-newer', + providerId: 'openai', + modelId: 'gpt-5', + completedAt: 100, + }, + }, + }, + ); + store.close?.(); + + const database = new DatabaseSync(join(root, 'runtime.sqlite')); + try { + database + .prepare(` + UPDATE core_agent_run_projections + SET event_json = '{malformed' + WHERE session_id = 'session-1' AND event_type = 'latest_context' + `) + .run(); + } finally { + database.close(); + } + + const reopened = createSqliteAgentRunStore(root); + try { + await reopened.appendEvent( + 'session-1', + 'run-1', + { + ...runEvent(), + id: 'model-call-older', + type: 'model_call_attempt_recorded', + ts: 50, + data: { + ...modelCallAttempt({ + logicalCallId: 'call-older', + attemptId: 'attempt-older', + traceId: 'trace-older', + completedAt: 50, + latencyMs: 49, + }), + }, + }, + { + latestContext: { + attemptId: 'attempt-older', + orderedAt: 50, + snapshot: { + schemaVersion: 2, + attemptId: 'attempt-older', + providerId: 'openai', + modelId: 'gpt-5', + completedAt: 50, + }, + }, + }, + ); + + assert.ok( + (await reopened.readEvents('session-1', 'run-1')).some( + (event) => event.id === 'model-call-older', + ), + ); + } finally { + reopened.close?.(); + } + + const inspected = new DatabaseSync(join(root, 'runtime.sqlite'), { readOnly: true }); + try { + assert.equal( + inspected + .prepare(` + SELECT event_json AS eventJson + FROM core_agent_run_projections + WHERE session_id = 'session-1' AND event_type = 'latest_context' + `) + .get()?.eventJson, + '{malformed', + 'unknown incumbent ordering stays untouched until a ledger rebuild can repair it', + ); + } finally { + inspected.close(); + } + }); + }); + + test('does not repair a malformed projection from a stale ledger revision', async () => { + await withRoot(async (root) => { + const store = createSqliteAgentRunStore(root); + await store.createRun(runHeader()); + await store.appendEvent('session-1', 'run-1', runEvent()); + await store.repairEventProjection( + 'session-1', + 'history_compact_checkpoint_recorded', + { + ...runEvent(), + id: 'checkpoint-a', + type: 'history_compact_checkpoint_recorded', + }, + { ifLedgerRevision: await store.readEventLedgerRevision('session-1') }, + ); + + const database = new DatabaseSync(join(root, 'runtime.sqlite')); + try { + database + .prepare(` + UPDATE core_agent_run_projections + SET event_json = '{malformed' + WHERE session_id = 'session-1' + AND event_type = 'history_compact_checkpoint_recorded' + `) + .run(); + } finally { + database.close(); + } + + const staleRevision = await store.readEventLedgerRevision('session-1'); + await store.appendEvent('session-1', 'run-1', { + ...runEvent(), + id: 'event-2', + ts: 2, + }); + await store.repairEventProjection( + 'session-1', + 'history_compact_checkpoint_recorded', + { + ...runEvent(), + id: 'checkpoint-a', + type: 'history_compact_checkpoint_recorded', + }, + { ifLedgerRevision: staleRevision }, + ); + + const inspected = new DatabaseSync(join(root, 'runtime.sqlite'), { readOnly: true }); + try { + assert.equal( + inspected + .prepare(` + SELECT event_json AS eventJson + FROM core_agent_run_projections + WHERE session_id = 'session-1' + AND event_type = 'history_compact_checkpoint_recorded' + `) + .get()?.eventJson, + '{malformed', + ); + } finally { + inspected.close(); + store.close?.(); + } + }); + }); + + test('rejects a projection repair without a canonical ledger revision', async () => { + await withRoot(async (root) => { + const store = createSqliteAgentRunStore(root); + await store.createRun(runHeader()); + await store.appendEvent('session-1', 'run-1', runEvent()); + const before = await store.readEventProjection( + 'session-1', + 'history_compact_checkpoint_recorded', + ); + + await assert.rejects( + // @ts-expect-error A repair must prove which canonical ledger revision it rebuilt. + store.repairEventProjection('session-1', 'history_compact_checkpoint_recorded', { + ...runEvent(), + id: 'checkpoint-a', + type: 'history_compact_checkpoint_recorded', + }), + /ledger revision/i, + ); + + assert.equal( + await store.readEventProjection('session-1', 'history_compact_checkpoint_recorded'), + before, + ); + store.close?.(); + }); + }); + test('backfills the model-call high-water when upgrading existing AgentRun rows', async () => { await withRoot(async (root) => { const store = createSqliteAgentRunStore(root); diff --git a/packages/storage/src/agent-run-store.ts b/packages/storage/src/agent-run-store.ts index 6ab33b96cf..c861f0284f 100644 --- a/packages/storage/src/agent-run-store.ts +++ b/packages/storage/src/agent-run-store.ts @@ -236,11 +236,12 @@ export interface DurableAgentRunStore sessionId: string, type: AgentRunProjectionKey, ): Promise; + readEventLedgerRevision(sessionId: string): Promise; repairEventProjection( sessionId: string, type: AgentRunProjectionKey, event: AgentRunEvent | null, - options?: { replaceEventId?: string }, + options: { ifLedgerRevision: string; replaceEventId?: string }, ): Promise; ready?(): Promise; close?(): void; @@ -560,12 +561,13 @@ class SqliteAgentRunStore implements DurableAgentRunStore { const type = normalized.type as AgentRunEventType; const projectsCheckpoint = type === 'history_compact_checkpoint_recorded'; const projection = projectsCheckpoint - ? readSqliteAgentRunProjection(this.#lease.database, sessionId, type) + ? inspectSqliteAgentRunProjection(this.#lease.database, sessionId, type) : undefined; insertAgentRunEvent(this.#lease.database, normalized); - if (projectsCheckpoint) { - const row = shouldPreserveCheckpointProjectionDuringAppend(projection, normalized) - ? projection! + if (projection && projection.state !== 'malformed') { + const current = projectionValue(projection); + const row = shouldPreserveCheckpointProjectionDuringAppend(current, normalized) + ? current! : normalized; writeSqliteAgentRunProjection(this.#lease.database, sessionId, type, row); } @@ -597,11 +599,16 @@ class SqliteAgentRunStore implements DurableAgentRunStore { event: AgentRunEvent, latest: LatestContextProjectionInput, ): void { - const existing = readSqliteAgentRunProjection( + const inspected = inspectSqliteAgentRunProjection( this.#lease.database, sessionId, LATEST_CONTEXT_PROJECTION_TYPE, ); + // The canonical append must survive a damaged derived row, but the row's + // ordering is unknowable. Leave it untouched until a ledger rebuild can + // select the real latest attempt and repair it without guessing. + if (inspected.state === 'malformed') return; + const existing = projectionValue(inspected); // Compared against the stored row's own completion, which the snapshot // carries — not against an ordering field the row does not have, which is // how the first version of this guard silently never fired. The rule @@ -671,19 +678,35 @@ class SqliteAgentRunStore implements DurableAgentRunStore { return readSqliteAgentRunProjection(this.#lease.database, sessionId, type); } + async readEventLedgerRevision(sessionId: string): Promise { + assertSafeId(sessionId, 'Invalid session id'); + return readSqliteAgentRunLedgerRevision(this.#lease.database, sessionId); + } + async repairEventProjection( sessionId: string, type: AgentRunProjectionKey, event: AgentRunEvent | null, - options: { replaceEventId?: string } = {}, + options: { ifLedgerRevision: string; replaceEventId?: string }, ): Promise { assertSafeId(sessionId, 'Invalid session id'); + if (!options || typeof options.ifLedgerRevision !== 'string') { + throw new Error('AgentRun projection repair requires a canonical ledger revision'); + } if (event !== null && !isProjectedAgentRunEvent(event, sessionId, type)) { throw new Error(`Invalid AgentRun event projection repair for ${type}`); } this.#lease.transaction('write', () => { - const current = readSqliteAgentRunProjection(this.#lease.database, sessionId, type); if ( + readSqliteAgentRunLedgerRevision(this.#lease.database, sessionId) !== + options.ifLedgerRevision + ) { + return; + } + const inspected = inspectSqliteAgentRunProjection(this.#lease.database, sessionId, type); + const current = projectionValue(inspected); + if ( + inspected.state !== 'malformed' && current?.id !== options.replaceEventId && shouldPreserveProjectionDuringRepair(current, event, type) ) { @@ -866,6 +889,39 @@ class SqliteAgentRunStore implements DurableAgentRunStore { } } +function readSqliteAgentRunLedgerRevision(db: DatabaseSync, sessionId: string): string { + const rows = db + .prepare(` + SELECT run.run_id, COUNT(event.sequence) AS event_count, + COALESCE(MAX(event.sequence), -1) AS high_water + FROM core_agent_runs AS run + LEFT JOIN core_agent_run_events AS event + ON event.session_id = run.session_id AND event.run_id = run.run_id + WHERE run.session_id = ? + GROUP BY run.run_id + ORDER BY run.run_id + `) + .all(sessionId) as Array<{ + run_id?: unknown; + event_count?: unknown; + high_water?: unknown; + }>; + return JSON.stringify( + rows.map((row) => { + if ( + typeof row.run_id !== 'string' || + typeof row.event_count !== 'number' || + !Number.isSafeInteger(row.event_count) || + typeof row.high_water !== 'number' || + !Number.isSafeInteger(row.high_water) + ) { + throw new Error('Invalid SQLite AgentRun ledger revision'); + } + return [row.run_id, row.event_count, row.high_water]; + }), + ); +} + function normalizeCurrentAgentRunHeader( value: unknown, sessionId: string, @@ -1072,6 +1128,24 @@ function readSqliteAgentRunProjection( // derived row nothing ever appends under (#2323). type: string, ): AgentRunEvent | null | undefined { + const inspected = inspectSqliteAgentRunProjection(db, sessionId, type); + if (inspected.state === 'malformed') { + throw new Error(`Invalid AgentRun event projection for ${type}`); + } + return projectionValue(inspected); +} + +type SqliteAgentRunProjectionInspection = + | { state: 'missing' } + | { state: 'empty' } + | { state: 'malformed' } + | { state: 'valid'; event: AgentRunEvent }; + +function inspectSqliteAgentRunProjection( + db: DatabaseSync, + sessionId: string, + type: string, +): SqliteAgentRunProjectionInspection { const row = db .prepare(` SELECT event_json @@ -1079,16 +1153,26 @@ function readSqliteAgentRunProjection( WHERE session_id = ? AND event_type = ? `) .get(sessionId, type) as { event_json?: unknown } | undefined; - if (!row) return undefined; - if (row.event_json === null) return null; - if (typeof row.event_json !== 'string') { - throw new Error(`Invalid AgentRun event projection for ${type}`); + if (!row) return { state: 'missing' }; + if (row.event_json === null) return { state: 'empty' }; + if (typeof row.event_json !== 'string') return { state: 'malformed' }; + let event: unknown; + try { + event = JSON.parse(row.event_json); + } catch { + return { state: 'malformed' }; } - const event = JSON.parse(row.event_json); if (!isProjectedAgentRunEvent(event, sessionId, type)) { - throw new Error(`Invalid AgentRun event projection for ${type}`); + return { state: 'malformed' }; } - return event; + return { state: 'valid', event }; +} + +function projectionValue( + inspected: SqliteAgentRunProjectionInspection, +): AgentRunEvent | null | undefined { + if (inspected.state === 'valid') return inspected.event; + return inspected.state === 'empty' ? null : undefined; } function writeSqliteAgentRunProjection( diff --git a/packages/storage/src/execution-stores.ts b/packages/storage/src/execution-stores.ts index 27507b5f9b..5a9c18c4b7 100644 --- a/packages/storage/src/execution-stores.ts +++ b/packages/storage/src/execution-stores.ts @@ -501,6 +501,8 @@ async function createExecutionStoresForWrite agentRunStore.readEventsForEvidence(sessionId, runId)), readEventProjection: (sessionId, type) => run(() => agentRunStore.readEventProjection(sessionId, type)), + readEventLedgerRevision: (sessionId) => + run(() => agentRunStore.readEventLedgerRevision(sessionId)), repairEventProjection: (sessionId, type, event, options) => run(() => agentRunStore.repairEventProjection(sessionId, type, event, options)), admitRootTurn: (input: AdmitRootTurnInput): Promise =>