From efbfcf24d86825c50ec85a208cf4095dd9aa58b9 Mon Sep 17 00:00:00 2001 From: Xiao Liu Date: Mon, 31 Aug 2026 09:21:21 +0800 Subject: [PATCH] fix(runtime): don't count image bytes as raw chars in the context-budget verdict The mid-turn final-request capacity verdict sized the outgoing payload with `JSON.stringify(messages).length`, which serializes an inline image at the size of its transported bytes: a Uint8Array/Buffer expands to a digit map (`{"0":..}` or `{"type":"Buffer","data":[...]}` once toJSON runs) and a base64 string is counted at full length. Divided by charsPerToken that reads as hundreds of thousands of tokens, so a single attached image could trip `context_budget_exhausted` before any request was sent -- most visibly on an unknown-window model whose `policy_fallback` capacity runs the verdict on step 0 (Steps 0, ~263ms, no provider round-trip). The reported Desktop upload path reads a session-file as a Node Buffer. Walk the message content and discount only genuine IMAGES to a bounded per-image estimate (~1.6K tokens, the vision per-image ceiling) scaled by the policy's charsPerToken. Image classification mirrors the AI SDK's own normalization: an `image` part, an `image` top-level mediaType (bare `image` or `image/*`, incl. a data: URL's own type), or inline bytes whose signature is a known image (which the SDK uses to override the declared type). Each image is charged once regardless of how it is carried (inline bytes, a data: URL, or a remote reference); only inline bytes are stripped, and a remote URL stays verbatim. Non-image files (PDF, text, audio) and look-alike content (e.g. a `{type:'data'}` tool-call input) stay fully measured, so the budget never silently under-counts them, and the image=0 under-count of #3372 is avoided. Covered by estimator unit tests (Buffer / Uint8Array / base64 / data: URL / remote URL string / bare `image` type / generic-MIME + magic bytes / PDF / tool-call input / charsPerToken) and an end-to-end backend test: an unknown-window first turn with a 200 KB Buffer image now reaches the provider instead of a pre-send context_budget_exhausted. Fixes #4290 Generated-by: Claude Code --- .../mid-turn-capacity-backend.test.ts | 40 +- .../mid-turn-image-payload-chars.test.ts | 434 ++++++++++++++++++ packages/runtime/src/ai-sdk-compaction.ts | 241 +++++++++- 3 files changed, 712 insertions(+), 3 deletions(-) create mode 100644 packages/runtime/src/__tests__/mid-turn-image-payload-chars.test.ts diff --git a/packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts b/packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts index 12e9de8ac6..4822a2825c 100644 --- a/packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts +++ b/packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts @@ -19,6 +19,7 @@ import type { ModelCallCommit } from '@maka/core/agent-run'; import assert from 'node:assert/strict'; +import { Buffer } from 'node:buffer'; import { describe, test } from 'node:test'; import { setImmediate as flushMacrotask } from 'node:timers/promises'; import { MockLanguageModelV4, simulateReadableStream } from 'ai/test'; @@ -137,6 +138,12 @@ interface MidTurnFixtureOptions { priorShape?: 'text' | 'tool_heavy' | 'image_tool'; /** Put one image attachment on the durable current-turn user anchor. */ currentImage?: boolean; + /** + * Return attachment bytes as a Node `Buffer` (the production session-file + * reader) rather than a `Uint8Array`, so the estimate sees what ships + * (apache/maka#4290). + */ + attachmentBytesAsBuffer?: boolean; /** First tool result is huge (finding C: prune must be able to rescue it). */ hugeFirstResult?: boolean; /** Exact first Read result for capacity-ordering regressions. */ @@ -428,7 +435,9 @@ function buildFixture(options: MidTurnFixtureOptions = {}): MidTurnFixture { supportsVision: true, readAttachmentBytes: async () => ({ ok: true as const, - bytes: new Uint8Array(imageBytes), + bytes: options.attachmentBytesAsBuffer + ? Buffer.from(new Uint8Array(imageBytes)) + : new Uint8Array(imageBytes), }), } : {}), @@ -1474,6 +1483,35 @@ describe('mid-turn capacity default-on safety guards (issue #882 PR 3)', () => { assert.equal(promptJson(fixture, 2).includes('RAW_SPAN_ONE_'), true); }); + test('a first-turn image upload reaches the provider on an unknown-window model (apache/maka#4290)', async () => { + // The reported failure end to end: an unknown-window model runs the + // capacity verdict on step 0 against the 48,384-token policy_fallback + // bound. The current turn's only message is a ~200 KB image upload, read as + // a Node Buffer. Before the fix its serialized bytes alone exceeded the + // bound and the turn died with context_budget_exhausted before any request. + const fixture = buildFixture({ + withoutContextWindow: true, + withoutPriorTurns: true, + currentImage: true, + imageBytes: 200_000, + attachmentBytesAsBuffer: true, + }); + await runFixtureTurn(fixture); + + // The provider actually received the request — the turn was not killed on + // step 0 — and the image was materialized into it. + assert.ok(fixture.model.doStreamCalls.length >= 1, 'the first request must reach the provider'); + assert.match(promptJson(fixture, 0), /"mediaType":"image\/png"/); + const complete = fixture.events.find((event) => event.type === 'complete'); + assert.notEqual( + complete?.type === 'complete' ? complete.stopReason : undefined, + 'context_budget_exhausted', + ); + // Nothing was foldable (first turn), so no compaction should have run. + assert.equal(fixture.summarizerCalls, 0); + assert.equal(fixture.recorded.length, 0); + }); + test('compacts one oversized prior turn before an unknown-model request', async () => { const fixture = buildFixture({ useRuntimeDefaultPolicy: true, diff --git a/packages/runtime/src/__tests__/mid-turn-image-payload-chars.test.ts b/packages/runtime/src/__tests__/mid-turn-image-payload-chars.test.ts new file mode 100644 index 0000000000..d165e453e7 --- /dev/null +++ b/packages/runtime/src/__tests__/mid-turn-image-payload-chars.test.ts @@ -0,0 +1,434 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from 'node:assert/strict'; +import { Buffer } from 'node:buffer'; +import { describe, test } from 'node:test'; +import { midTurnRequestPayloadChars } from '../ai-sdk-compaction.js'; +import { estimateNextRequestTokens } from '../history-compaction.js'; +import { toolSchemaCharsForDiagnostics } from '../request-shape.js'; +import type { ModelMessage } from '../model-protocol.js'; + +/** + * The unknown-model `policy_fallback` capacity: `maxHistoryEstimatedTokens` + * (32_000) + mid-turn `reserveTokens` (16_384). See context-budget-policy.ts + * and context-budget-mid-turn-policy.test.ts. This is the capacity the step-0 + * verdict measures the first request against when the selected model has no + * known context window — the exact scenario in apache/maka#4290. + */ +const POLICY_FALLBACK_CAPACITY_TOKENS = 48_384; +const CHARS_PER_TOKEN = 4; +/** Mirrors IMAGE_PART_ESTIMATED_TOKENS in ai-sdk-compaction.ts. */ +const IMAGE_PART_ESTIMATED_TOKENS = 1_600; + +function bytes(size: number): Uint8Array { + // 0x89 alone is NOT a full image signature (PNG needs 0x89 0x50 0x4e 0x47), + // so a `bytes()`-filled part is classified as an image only via its declared + // mediaType — never by magic-byte sniffing. Content scales with `size`; the + // estimate must not. + return new Uint8Array(size).fill(0x89); +} + +/** Bytes that begin with the real PNG signature, so magic-byte sniffing fires. */ +function pngBytes(size: number): Uint8Array { + const out = new Uint8Array(Math.max(8, size)).fill(0); + out.set([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + return out; +} + +/** Bytes that begin with the `%PDF` signature — a genuine non-image document. */ +function pdfBytes(size: number): Uint8Array { + const out = new Uint8Array(Math.max(4, size)).fill(0); + out.set([0x25, 0x50, 0x44, 0x46]); + return out; +} + +/** + * A first-turn user message: short prompt + one inline image attachment. The + * `data` payload type varies by source — a session-file upload arrives as a + * Node Buffer, a workspace/computer-use image as a Uint8Array (ai-sdk-backend). + */ +function userImageMessage(imageBytes: Uint8Array | Buffer): ModelMessage { + return { + role: 'user', + content: [ + { type: 'text', text: '看看这张图包含什么内容' }, + { + type: 'file', + data: { type: 'data', data: imageBytes }, + mediaType: 'image/png', + }, + ], + } as unknown as ModelMessage; +} + +/** A tool-result image, materialized as an inline base64 string (ai-sdk-backend). */ +function toolResultImageMessage(imageBase64: string): ModelMessage { + return { + role: 'tool', + content: [ + { + type: 'tool-result', + toolCallId: 'call_1', + toolName: 'read_image', + output: { + type: 'content', + value: [ + { type: 'text', text: 'Image read successfully.' }, + { + type: 'file', + data: { type: 'data', data: imageBase64 }, + mediaType: 'image/png', + }, + ], + }, + }, + ], + } as unknown as ModelMessage; +} + +/** A single-part user message carrying a `file` part of the given media type. */ +function fileMessage(mediaType: string, data: Uint8Array | Buffer | string): ModelMessage { + return { + role: 'user', + content: [{ type: 'file', data: { type: 'data', data }, mediaType }], + } as unknown as ModelMessage; +} + +/** + * A `file` part whose `data` carries a `data:` URL — as a bare `URL` instance + * (`data: new URL(...)`) or a tagged `{ type: 'url', url }`, the two shapes the + * AI SDK unpacks and re-sniffs. Declared mediaType stays generic so only the + * decoded bytes can prove it is an image. + */ +function fileDataUrlMessage( + mediaType: string, + data: URL | { type: 'url'; url: string | URL }, +): ModelMessage { + return { + role: 'user', + content: [{ type: 'file', data, mediaType }], + } as unknown as ModelMessage; +} + +/** A single-part user message carrying an `image` part with the given payload. */ +function imagePartMessage(image: unknown): ModelMessage { + return { role: 'user', content: [{ type: 'image', image }] } as unknown as ModelMessage; +} + +function coldStartTokens(payloadChars: number, charsPerToken = CHARS_PER_TOKEN): number { + return estimateNextRequestTokens({ + appendedChars: 0, + coldStartChars: payloadChars, + charsPerToken, + }); +} + +describe('midTurnRequestPayloadChars image accounting (apache/maka#4290)', () => { + test('a first-turn user upload (session_file Buffer) estimates under the fallback capacity', () => { + // The reported production path: a Desktop upload is read as a Node Buffer + // (artifact-attachments.ts) and inlined into the request. JSON.stringify + // turns a Buffer into a {"type":"Buffer","data":[...]} digit map via its + // toJSON, so a raw estimate blows past the window on step 0. The fix must + // discount it structurally (Buffer is a Uint8Array before serialization). + const messages = [userImageMessage(Buffer.from(bytes(200_000)))]; + const payloadChars = midTurnRequestPayloadChars(messages, [], [], 4_000); + assert.ok( + coldStartTokens(payloadChars) < POLICY_FALLBACK_CAPACITY_TOKENS, + `a Buffer upload + short prompt must fit the fallback capacity, got ~${coldStartTokens(payloadChars)} tokens`, + ); + // The pre-fix accounting (raw serialization of the same messages) would + // have exceeded the window — this is what killed the turn on step 0. + assert.ok( + Math.ceil(JSON.stringify(messages).length / CHARS_PER_TOKEN) > + POLICY_FALLBACK_CAPACITY_TOKENS, + ); + }); + + test('a Uint8Array image (workspace / computer-use path) also fits the fallback capacity', () => { + const payloadChars = midTurnRequestPayloadChars( + [userImageMessage(bytes(200_000))], + [], + [], + 4_000, + ); + assert.ok(coldStartTokens(payloadChars) < POLICY_FALLBACK_CAPACITY_TOKENS); + }); + + test('the estimate does not scale with image byte size (Buffer, Uint8Array, base64)', () => { + const buf = (n: number) => + midTurnRequestPayloadChars([userImageMessage(Buffer.from(bytes(n)))], [], [], 0); + const u8 = (n: number) => midTurnRequestPayloadChars([userImageMessage(bytes(n))], [], [], 0); + const b64 = (n: number) => + midTurnRequestPayloadChars( + [toolResultImageMessage(Buffer.from(bytes(n)).toString('base64'))], + [], + [], + 0, + ); + // A 100x larger image must not change the estimate on any inline-binary + // representation: the provider bills the rendered image, not the bytes. + assert.equal(buf(20_000), buf(2_000_000)); + assert.equal(u8(20_000), u8(2_000_000)); + assert.equal(b64(20_000), b64(2_000_000)); + }); + + test('a non-image file (e.g. PDF) is NOT discounted — it stays fully measured', () => { + // The per-image cost applies only to images; a PDF/text/audio file part + // carries content the provider really bills, so it must keep full + // measurement rather than collapse to the ~1.6K-token image estimate. + const pdf = (n: number) => + midTurnRequestPayloadChars( + [fileMessage('application/pdf', Buffer.from(pdfBytes(n)))], + [], + [], + 0, + ); + assert.notEqual(pdf(1_000), pdf(200_000)); + assert.ok( + pdf(200_000) > 200_000, + `an inline non-image file must be counted in full, got ${pdf(200_000)} chars`, + ); + }); + + test('a bare "image" top-level mediaType is discounted (AI SDK deprecates image parts for it)', () => { + // The AI SDK's recommended replacement for an image part is a file part + // with mediaType `image` (top-level, no subtype). `startsWith('image/')` + // misses it; the estimate must still treat it as an image. + const img = (n: number) => + midTurnRequestPayloadChars([fileMessage('image', Buffer.from(pngBytes(n)))], [], [], 0); + assert.equal(img(20_000), img(2_000_000)); + assert.ok( + coldStartTokens( + midTurnRequestPayloadChars( + [fileMessage('image', Buffer.from(pngBytes(200_000)))], + [], + [], + 4_000, + ), + ) < POLICY_FALLBACK_CAPACITY_TOKENS, + ); + }); + + test('a generic-MIME file with image bytes is discounted (magic-byte parity with the SDK)', () => { + // The AI SDK detects the image from the byte signature and overrides the + // declared mediaType, so the provider bills it as an image. The estimate + // must match, or a mislabelled screenshot trips the pre-send verdict. + const img = (n: number) => + midTurnRequestPayloadChars( + [fileMessage('application/octet-stream', Buffer.from(pngBytes(n)))], + [], + [], + 0, + ); + assert.equal(img(20_000), img(2_000_000)); + assert.ok( + coldStartTokens( + midTurnRequestPayloadChars( + [fileMessage('application/octet-stream', Buffer.from(pngBytes(200_000)))], + [], + [], + 4_000, + ), + ) < POLICY_FALLBACK_CAPACITY_TOKENS, + ); + }); + + test('a generic-MIME file whose data: URL decodes to image bytes is discounted', () => { + // The reachable production combination the Buffer test above misses: a + // `file` part with a generic declared mediaType whose `data` is a `data:` + // URL carrying image bytes. The AI SDK splits the data URL, decodes its + // base64 body, and re-sniffs — reclassifying it to image/png and billing it + // as an image. The estimate must decode+sniff the SAME way, across both + // carrier shapes (bare `URL` and tagged `{ type: 'url', url }`), or a + // ~200 KB screenshot passed this way trips the step-0 verdict + // (apache/maka#4290). + const dataUrl = (n: number) => + `data:application/octet-stream;base64,${Buffer.from(pngBytes(n)).toString('base64')}`; + const asBareUrl = (n: number) => + midTurnRequestPayloadChars( + [fileDataUrlMessage('application/octet-stream', new URL(dataUrl(n)))], + [], + [], + 0, + ); + const asTaggedUrl = (n: number) => + midTurnRequestPayloadChars( + [fileDataUrlMessage('application/octet-stream', { type: 'url', url: dataUrl(n) })], + [], + [], + 0, + ); + // The estimate must not scale with byte size on either carrier shape. + assert.equal(asBareUrl(20_000), asBareUrl(2_000_000)); + assert.equal(asTaggedUrl(20_000), asTaggedUrl(2_000_000)); + // And a 200 KB image passed this way must fit the fallback capacity, not + // trip the pre-send verdict as it did before the data-URL sniff. + assert.ok( + coldStartTokens( + midTurnRequestPayloadChars( + [fileDataUrlMessage('application/octet-stream', new URL(dataUrl(200_000)))], + [], + [], + 4_000, + ), + ) < POLICY_FALLBACK_CAPACITY_TOKENS, + ); + assert.ok( + coldStartTokens( + midTurnRequestPayloadChars( + [fileDataUrlMessage('application/octet-stream', { type: 'url', url: dataUrl(200_000) })], + [], + [], + 4_000, + ), + ) < POLICY_FALLBACK_CAPACITY_TOKENS, + ); + }); + + test('a generic-MIME file whose data: URL is NOT an image stays fully measured', () => { + // Precision guard, matching the SDK: the byte override fires only for image + // signatures. A PDF carried as a generic-MIME data: URL must keep full + // measurement, never collapse to the per-image estimate. + const pdfDataUrl = (n: number) => + new URL( + `data:application/octet-stream;base64,${Buffer.from(pdfBytes(n)).toString('base64')}`, + ); + const pdf = (n: number) => + midTurnRequestPayloadChars( + [fileDataUrlMessage('application/octet-stream', pdfDataUrl(n))], + [], + [], + 0, + ); + assert.notEqual(pdf(1_000), pdf(200_000)); + }); + + test('a remote-URL image still costs the per-image budget (no under-count)', () => { + // An http(s) URL / provider reference has no inline bytes to strip, but it + // is still an image the provider fetches and bills — it must carry the + // per-image token cost, or a URL image would be estimated at ~zero. + const at = (cpt: number) => + midTurnRequestPayloadChars( + [imagePartMessage({ type: 'url', url: new URL('https://example.com/chart.png') })], + [], + [], + 0, + cpt, + ); + assert.equal(at(4) - at(1), IMAGE_PART_ESTIMATED_TOKENS * (4 - 1)); + }); + + test('a bare remote-URL string image is left verbatim, not blanked', () => { + // A parseable bare string is a URL to the SDK; blanking it would drop a + // long signed URL from the count. It stays verbatim (and still costs the + // per-image budget). A data: URL string, by contrast, is inline bytes. + const url = `https://example.com/${'a'.repeat(2_000)}.png`; + const payloadChars = midTurnRequestPayloadChars([imagePartMessage(url)], [], [], 0); + assert.ok( + payloadChars >= url.length, + `a remote URL string must be counted, not blanked, got ${payloadChars} chars`, + ); + }); + + test('a data: URL image is stripped, not counted at full base64', () => { + // AI SDK treats a data: URL as inline data. Whether it arrives as a string + // or a URL instance, its bytes must not drive the estimate — otherwise a + // ~200 KB data: URL would still trip the pre-send verdict (apache/maka#4290). + const dataUrl = (n: number) => + `data:image/png;base64,${Buffer.from(bytes(n)).toString('base64')}`; + const asString = (n: number) => + midTurnRequestPayloadChars([imagePartMessage(dataUrl(n))], [], [], 0); + const asUrl = (n: number) => + midTurnRequestPayloadChars([imagePartMessage(new URL(dataUrl(n)))], [], [], 0); + assert.equal(asString(20_000), asString(2_000_000)); + assert.equal(asUrl(20_000), asUrl(2_000_000)); + assert.ok( + coldStartTokens( + midTurnRequestPayloadChars([imagePartMessage(dataUrl(200_000))], [], [], 4_000), + ) < POLICY_FALLBACK_CAPACITY_TOKENS, + ); + }); + + test('a non-image {type:"data"} payload (e.g. a tool-call input) is NOT discounted', () => { + // Precision guard: the discount must apply only to genuine image/file + // parts, never to arbitrary content that happens to look like a data + // wrapper — else a large tool-call input would silently skip the budget. + const bigInput = 'x'.repeat(300_000); + const messages = [ + { + role: 'assistant', + content: [ + { + type: 'tool-call', + toolCallId: 'c1', + toolName: 'run', + input: { type: 'data', data: bigInput }, + }, + ], + }, + ] as unknown as ModelMessage[]; + const payloadChars = midTurnRequestPayloadChars(messages, [], [], 0); + assert.ok( + payloadChars >= bigInput.length, + `a non-image data payload must be counted in full, got ${payloadChars} chars`, + ); + }); + + test('the per-image cost scales with the configured charsPerToken', () => { + // The image is billed at a fixed TOKEN budget, so its CHAR cost must track + // charsPerToken (the caller divides the payload back down by it). With one + // image the only charsPerToken-dependent term is that per-image cost. + const at = (cpt: number) => + midTurnRequestPayloadChars([userImageMessage(bytes(50_000))], [], [], 0, cpt); + assert.equal(at(4) - at(1), IMAGE_PART_ESTIMATED_TOKENS * (4 - 1)); + assert.equal(at(8) - at(4), IMAGE_PART_ESTIMATED_TOKENS * (8 - 4)); + }); + + test('images add a bounded, linear per-part cost', () => { + const one = midTurnRequestPayloadChars([userImageMessage(bytes(50_000))], [], [], 0); + const two = midTurnRequestPayloadChars( + [userImageMessage(bytes(50_000)), userImageMessage(bytes(50_000))], + [], + [], + 0, + ); + const perImageDelta = two - one; + assert.ok(perImageDelta > 0, 'a second image must add cost'); + assert.ok( + perImageDelta < IMAGE_PART_ESTIMATED_TOKENS * CHARS_PER_TOKEN + 2_000, + `per-image marginal cost must stay bounded, got ${perImageDelta} chars`, + ); + }); + + test('text-only messages are unchanged (no image discount, no regression)', () => { + const messages = [ + { role: 'user', content: 'plain text, no attachments' }, + { role: 'assistant', content: [{ type: 'text', text: 'a reply' }] }, + ] as unknown as ModelMessage[]; + const systemPromptChars = 1_234; + // With no images the messages term must equal the original definition + // (`JSON.stringify(messages).length`); system-prompt and tool-schema terms + // are unchanged by this fix. + assert.equal( + midTurnRequestPayloadChars(messages, [], [], systemPromptChars), + systemPromptChars + JSON.stringify(messages).length + toolSchemaCharsForDiagnostics([], []), + ); + }); +}); diff --git a/packages/runtime/src/ai-sdk-compaction.ts b/packages/runtime/src/ai-sdk-compaction.ts index 3fa12b2ea0..05b16975c4 100644 --- a/packages/runtime/src/ai-sdk-compaction.ts +++ b/packages/runtime/src/ai-sdk-compaction.ts @@ -70,6 +70,7 @@ import { type MalformedHistoryCompactSummaryReason, } from './history-compact-error.js'; import { createHash } from 'node:crypto'; +import { Buffer } from 'node:buffer'; import type { ModelMessage } from './model-protocol.js'; import type { ModelAdapter } from './model-adapter.js'; import type { @@ -778,6 +779,7 @@ export class AiSdkCompaction { providerTools, activeToolsForStep, systemPromptChars, + charsPerToken, ); const forcedEstimate = state.forcedTriggerEstimate; state.forcedTriggerEstimate = undefined; @@ -1051,6 +1053,7 @@ export class AiSdkCompaction { providerTools, activeToolsForStep, systemPromptChars, + charsPerToken, ); if (replacedPayloadChars >= input.referencePayloadChars) { return { @@ -1171,6 +1174,7 @@ export class AiSdkCompaction { input.providerTools, input.activeTools, input.systemPromptChars, + this.input.contextBudget?.charsPerToken ?? 4, ); const phase = input.stepNumber === 0 ? 'pre_turn' : 'mid_turn'; const outcome = await this.compactActiveRequestHistory({ @@ -1294,6 +1298,7 @@ export class AiSdkCompaction { providerTools, result?.activeTools ?? options.activeTools ?? fallbackActiveTools(), systemPromptChars, + charsPerToken, ); let payloadChars = finalPayloadChars(); if ( @@ -1581,15 +1586,247 @@ export class MidTurnCapacityCompactState { * usage sample) is the whole payload, so omitting it would under-estimate by * exactly the system prompt and let an over-window request stream. */ -function midTurnRequestPayloadChars( +/** + * Vision models bill an image by its rendered dimensions, not its transported + * bytes: one image is bounded at roughly this many tokens (the Anthropic + * per-image ceiling; other providers are lower). Counted linearly per part. + */ +const IMAGE_PART_ESTIMATED_TOKENS = 1_600; + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null; +} + +/** The inline binary of an image payload slot, elided before serialization. */ +const ELIDED_IMAGE_BINARY = ''; + +/** + * Image magic-byte prefixes, mirroring `@ai-sdk/provider-utils`' + * `detectMediaType({ topLevelType: 'image' })`. The AI SDK runs this over every + * inline file payload and OVERRIDES the declared mediaType when the bytes are a + * known image, so the provider bills such a part as an image whatever its + * declared type. We classify the same way, or a generic-MIME image (or the + * SDK's recommended bare `image` type) would be measured at full byte size and + * trip the pre-send verdict (apache/maka#4290). `null` matches any byte. + */ +const IMAGE_BYTE_SIGNATURES: readonly (readonly (number | null)[])[] = [ + [0x47, 0x49, 0x46], // GIF + [0x89, 0x50, 0x4e, 0x47], // PNG + [0xff, 0xd8], // JPEG + [0x52, 0x49, 0x46, 0x46, null, null, null, null, 0x57, 0x45, 0x42, 0x50], // WEBP (RIFF…WEBP) + [0x42, 0x4d], // BMP + [0x49, 0x49, 0x2a, 0x00], // TIFF (little-endian) + [0x4d, 0x4d, 0x00, 0x2a], // TIFF (big-endian) + [0x00, 0x00, 0x00, 0x20, 0x66, 0x74, 0x79, 0x70, 0x61, 0x76, 0x69, 0x66], // AVIF + [0x00, 0x00, 0x00, 0x20, 0x66, 0x74, 0x79, 0x70, 0x68, 0x65, 0x69, 0x63], // HEIC +]; + +function bytesAreKnownImage(bytes: Uint8Array): boolean { + return IMAGE_BYTE_SIGNATURES.some( + (signature) => + bytes.length >= signature.length && + signature.every((byte, index) => byte === null || bytes[index] === byte), + ); +} + +function topLevelMediaType(mediaType: unknown): string | undefined { + if (typeof mediaType !== 'string') return undefined; + const slash = mediaType.indexOf('/'); + return (slash === -1 ? mediaType : mediaType.slice(0, slash)).toLowerCase(); +} + +function isDataUrl(value: unknown): boolean { + if (value instanceof URL) return value.protocol === 'data:'; + return typeof value === 'string' && value.slice(0, 5).toLowerCase() === 'data:'; +} + +function isRemoteUrlString(value: unknown): boolean { + return typeof value === 'string' && /^https?:\/\//i.test(value); +} + +/** The mediaType named by a `data:;base64,…` payload, if any. */ +function dataUrlMediaType(value: unknown): string | undefined { + const text = value instanceof URL ? value.toString() : typeof value === 'string' ? value : ''; + if (!isDataUrl(text)) return undefined; + const comma = text.indexOf(','); + const header = text.slice('data:'.length, comma === -1 ? text.length : comma); + const mediaType = header.split(';')[0]?.trim(); + return mediaType && mediaType.length > 0 ? mediaType : undefined; +} + +/** + * The body of a `data:[;base64],` payload (string or URL + * instance), if any. The AI SDK treats this body as base64 and re-sniffs its + * bytes to override the declared mediaType, so we decode it the same way. + */ +function dataUrlBody(value: unknown): string | undefined { + const text = value instanceof URL ? value.toString() : typeof value === 'string' ? value : ''; + if (!isDataUrl(text)) return undefined; + const comma = text.indexOf(','); + return comma === -1 ? undefined : text.slice(comma + 1); +} + +/** Decode a leading window of a base64 string into raw bytes for sniffing. */ +function base64BytePrefix(base64: string): Uint8Array | undefined { + try { + // 24 base64 chars → 18 bytes, past the longest image signature (12 bytes). + return new Uint8Array(Buffer.from(base64.slice(0, 24), 'base64')); + } catch { + return undefined; + } +} + +/** + * A small leading byte window of an inline payload, for image-signature + * sniffing: raw bytes, a bare base64 string, or the (base64) body of a `data:` + * URL carried as a string or `URL` instance. The AI SDK decodes a `data:` URL + * and re-sniffs its bytes to override the declared type, so we must sniff it + * too — otherwise a generic-MIME image transported as a `data:` URL measures + * at full byte size and trips the pre-send verdict (apache/maka#4290). A remote + * `http(s)` URL carries no inline bytes. + */ +function inlineBytePrefix(payload: unknown): Uint8Array | undefined { + if (payload instanceof Uint8Array) return payload.subarray(0, 16); + if (payload instanceof ArrayBuffer) return new Uint8Array(payload).subarray(0, 16); + const body = dataUrlBody(payload); + if (body !== undefined) return base64BytePrefix(body); + if (typeof payload === 'string' && !isRemoteUrlString(payload)) { + return base64BytePrefix(payload); + } + return undefined; +} + +/** The inline payload of a `FilePart.data`, unwrapping a `{ type: 'data'|'url' }`. */ +function fileDataPayload(data: unknown): unknown { + if (isRecord(data)) { + if (data.type === 'data') return data.data; + if (data.type === 'url') return data.url; + } + return data; +} + +/** + * Whether the provider will bill a `file` part as an image, mirroring the AI + * SDK's normalization: an `image` top-level mediaType (bare `image` or + * `image/*`, incl. a `data:` URL's own type), or inline bytes whose signature + * is a known image (which the SDK uses to override the declared type). + */ +function fileBillsAsImage(part: Record): boolean { + const payload = fileDataPayload(part.data); + const declared = dataUrlMediaType(payload) ?? part.mediaType; + if (topLevelMediaType(declared) === 'image') return true; + const prefix = inlineBytePrefix(payload); + return prefix !== undefined && bytesAreKnownImage(prefix); +} + +/** + * Return an image payload with its INLINE bytes elided: raw bytes (Uint8Array — + * a Node Buffer is one, before `JSON.stringify` applies its toJSON — or an + * ArrayBuffer), a base64 string, and the `{ type: 'data' }` / `data:`-URL + * wrappers all carry bytes and are elided. A remote `http(s)` URL or provider + * reference carries none and stays verbatim; the caller still adds the + * per-image token cost. + */ +function elideInlineImageBytes(payload: unknown): unknown { + if (payload instanceof Uint8Array || payload instanceof ArrayBuffer) return ELIDED_IMAGE_BINARY; + if (payload instanceof URL) return isDataUrl(payload) ? ELIDED_IMAGE_BINARY : payload; + if (typeof payload === 'string') + return isRemoteUrlString(payload) ? payload : ELIDED_IMAGE_BINARY; + if (isRecord(payload)) { + if (payload.type === 'data') return { ...payload, data: ELIDED_IMAGE_BINARY }; + if (payload.type === 'url') { + return isDataUrl(payload.url) ? { ...payload, url: ELIDED_IMAGE_BINARY } : payload; + } + } + return payload; +} + +/** + * Discount a single content part when it is an IMAGE — an `image` part, or a + * `file` part the provider bills as an image (see fileBillsAsImage) — including + * one nested in a tool-result's content output. Every image contributes the + * per-image token cost regardless of how it is carried (inline bytes, a `data:` + * URL, or a remote reference); only inline bytes are stripped from the + * serialized size. Non-image files (PDF, text, audio, …) and look-alike content + * (e.g. a `{ type: 'data' }` tool-call input) are left to serialize at full + * size — the budget must not silently under-count them (apache/maka#4290). + */ +function elideImageBinaryInPart(part: unknown): { part: unknown; imageParts: number } { + if (!isRecord(part)) return { part, imageParts: 0 }; + if (part.type === 'image') { + return { part: { ...part, image: elideInlineImageBytes(part.image) }, imageParts: 1 }; + } + if (part.type === 'file') { + if (!fileBillsAsImage(part)) return { part, imageParts: 0 }; + return { part: { ...part, data: elideInlineImageBytes(part.data) }, imageParts: 1 }; + } + if ( + part.type === 'tool-result' && + isRecord(part.output) && + part.output.type === 'content' && + Array.isArray(part.output.value) + ) { + let imageParts = 0; + let changed = false; + const value = part.output.value.map((inner) => { + const result = elideImageBinaryInPart(inner); + if (result.part !== inner) changed = true; + imageParts += result.imageParts; + return result.part; + }); + return changed + ? { part: { ...part, output: { ...part.output, value } }, imageParts } + : { part, imageParts }; + } + return { part, imageParts: 0 }; +} + +/** + * Serialized char size of the outgoing messages, with inline image/file BINARY + * payloads discounted to a bounded per-image estimate instead of their raw + * transported size. + * + * `JSON.stringify(messages).length` counts an image at the size of its bytes: a + * `Uint8Array`/`Buffer` serializes to a `{"0":..}` (or `{"type":"Buffer",…}`) + * digit map (~10 chars/byte) and an inline base64 string at its full length — + * hundreds of KB to megabytes for one screenshot. Divided by `charsPerToken` + * that reads as hundreds of thousands of tokens, so a single attached image can + * trip `context_budget_exhausted` before any request is sent (apache/maka#4290), + * most visibly on an unknown-window model whose `policy_fallback` capacity puts + * the verdict on step 0. We walk the message content and discount only genuine + * image/file payloads: every other byte of structural overhead is preserved, + * and the opposite (image = 0) under-count of #3372 is avoided. + */ +function imageAwareMessagesPayloadChars( + messages: readonly ModelMessage[], + charsPerToken: number, +): number { + let imageParts = 0; + const sanitized = messages.map((message) => { + if (!isRecord(message) || !Array.isArray(message.content)) return message; + let changed = false; + const content = message.content.map((part) => { + const result = elideImageBinaryInPart(part); + if (result.part !== part) changed = true; + imageParts += result.imageParts; + return result.part; + }); + return changed ? { ...message, content } : message; + }); + const perImageChars = IMAGE_PART_ESTIMATED_TOKENS * Math.max(1, charsPerToken); + return JSON.stringify(sanitized).length + imageParts * perImageChars; +} + +export function midTurnRequestPayloadChars( messages: readonly ModelMessage[], providerTools: readonly MakaTool[], activeTools: readonly string[], systemPromptChars: number, + charsPerToken = 4, ): number { return ( Math.max(0, Math.floor(systemPromptChars)) + - JSON.stringify(messages).length + + imageAwareMessagesPayloadChars(messages, charsPerToken) + toolSchemaCharsForDiagnostics(providerTools, activeTools) ); }