From b43c309c4c65717367918056eff3be5030f1eed3 Mon Sep 17 00:00:00 2001 From: Daniel Gil Sancho Date: Sun, 2 Aug 2026 14:42:34 +0300 Subject: [PATCH] fix(server): transcode HEIC attachments instead of failing the turn Attaching or pasting a photo straight from an Apple device sends `image/heic`, which the Claude adapter rejects with "Unsupported Claude image attachment type". HEIC is the default capture format on iOS, so this fires constantly from the mobile composer and from pasting an iPhone screenshot on macOS. The attachment itself is stored fine (`imageMime.ts` already lists HEIC as a safe type) and the mobile composer already labels it `image/heic`, so the only thing missing was converting it before handing it to the provider. `transcodeImageToJpeg` shells out to a tool that ships with the host rather than adding a HEIC decoder dependency: `sips` on macOS, which is always present, and `heif-convert` from libheif elsewhere. If neither is available the conversion fails and the caller reports the attachment as unsupported exactly like before, so the change is strictly additive. The helper lives in its own module with no orchestration imports, so it survives the orchestration-v2 move; the adapter call site is a single `yield*`. Fixes #4579 Co-Authored-By: Claude Opus 5 (1M context) --- apps/server/src/imageTranscode.test.ts | 91 ++++++++++++++ apps/server/src/imageTranscode.ts | 116 ++++++++++++++++++ .../src/provider/Layers/ClaudeAdapter.ts | 34 ++++- 3 files changed, 240 insertions(+), 1 deletion(-) create mode 100644 apps/server/src/imageTranscode.test.ts create mode 100644 apps/server/src/imageTranscode.ts diff --git a/apps/server/src/imageTranscode.test.ts b/apps/server/src/imageTranscode.test.ts new file mode 100644 index 00000000000..c365260f25e --- /dev/null +++ b/apps/server/src/imageTranscode.test.ts @@ -0,0 +1,91 @@ +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeChildProcess from "node:child_process"; +import * as NodeFSP from "node:fs/promises"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; + +import { describe, expect, it } from "vite-plus/test"; + +import { + isTranscodableImageMimeType, + TRANSCODED_IMAGE_MIME_TYPE, + transcodeImageToJpeg, +} from "./imageTranscode.ts"; + +// 8x8 RGB PNG, used as the source for the HEIC fixture below. +const SAMPLE_PNG_BASE64 = + "iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAIAAABLbSncAAAAbElEQVR4nA3JQQEAMAgDMZRUCUqqpEpQgoh7o2jLN1WFii5cpJhiiyuqhEQLi4gRK04/GjXduEkzzTbXP4xMG5uYMWvOP4JCB4eECRsuPwYNPXjIMMMONz8WLb14yTLLLrc/Dh19+Mgxxx53PKaVZoFj4h8/AAAAAElFTkSuQmCC"; + +const JPEG_START_OF_IMAGE = [0xff, 0xd8, 0xff]; + +function runSips(args: Array): Promise { + return new Promise((resolve, reject) => { + NodeChildProcess.execFile("sips", args, (error) => + error === null ? resolve() : reject(error), + ); + }); +} + +// `sips` is the macOS transcoder, and also the only way to build a real HEIC +// fixture without shipping a binary in the repo. Probing for it keeps this +// suite green on hosts that do not have it. +const SIPS_AVAILABLE = await runSips(["--version"]).then( + () => true, + () => false, +); + +async function makeHeicFixture(): Promise { + const workingDir = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-heic-fixture-")); + const pngPath = NodePath.join(workingDir, "source.png"); + const heicPath = NodePath.join(workingDir, "source.heic"); + + try { + await NodeFSP.writeFile(pngPath, Buffer.from(SAMPLE_PNG_BASE64, "base64")); + await runSips(["-s", "format", "heic", pngPath, "--out", heicPath]); + return new Uint8Array(await NodeFSP.readFile(heicPath)); + } finally { + await NodeFSP.rm(workingDir, { recursive: true, force: true }).catch(() => {}); + } +} + +describe("imageTranscode", () => { + it("recognizes the mime types Claude cannot ingest", () => { + expect(isTranscodableImageMimeType("image/heic")).toBe(true); + expect(isTranscodableImageMimeType("image/heif")).toBe(true); + expect(isTranscodableImageMimeType("IMAGE/HEIC")).toBe(true); + expect(isTranscodableImageMimeType(" image/heic ")).toBe(true); + }); + + it("leaves natively supported mime types alone", () => { + for (const mimeType of ["image/jpeg", "image/png", "image/gif", "image/webp"]) { + expect(isTranscodableImageMimeType(mimeType)).toBe(false); + } + }); + + it("targets jpeg, which every provider accepts", () => { + expect(TRANSCODED_IMAGE_MIME_TYPE).toBe("image/jpeg"); + }); + + it.skipIf(!SIPS_AVAILABLE)("converts HEIC bytes into JPEG bytes", async () => { + const heic = await makeHeicFixture(); + // Guards against the fixture silently degrading into a non-HEIC file. + expect(Buffer.from(heic.subarray(4, 12)).toString("ascii")).toBe("ftypheic"); + + const jpeg = await transcodeImageToJpeg({ + bytes: heic, + platform: "darwin", + }); + + expect(jpeg.byteLength).toBeGreaterThan(0); + expect([...jpeg.subarray(0, 3)]).toEqual(JPEG_START_OF_IMAGE); + }); + + it.skipIf(!SIPS_AVAILABLE)("rejects bytes that are not a decodable image", async () => { + await expect( + transcodeImageToJpeg({ + bytes: new TextEncoder().encode("not an image"), + platform: "darwin", + }), + ).rejects.toThrow(); + }); +}); diff --git a/apps/server/src/imageTranscode.ts b/apps/server/src/imageTranscode.ts new file mode 100644 index 00000000000..aef943a10e0 --- /dev/null +++ b/apps/server/src/imageTranscode.ts @@ -0,0 +1,116 @@ +// @effect-diagnostics nodeBuiltinImport:off +/** + * Transcodes image formats the model providers cannot ingest (HEIC/HEIF) into + * JPEG, which every provider accepts. + * + * iPhones capture HEIC by default, so pasting or attaching a photo straight + * from an Apple device otherwise fails at the provider boundary even though the + * attachment itself was stored just fine. + * + * Transcoding shells out to a tool that ships with the host rather than pulling + * in a HEIC decoder dependency: `sips` on macOS (always present) and + * `heif-convert` from libheif elsewhere (packaged on most desktop Linux). When + * neither is available the caller surfaces the original "unsupported type" + * failure, so this is strictly additive. + * + * @module imageTranscode + */ +import * as NodeChildProcess from "node:child_process"; +import * as NodeFSP from "node:fs/promises"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; + +export const TRANSCODABLE_IMAGE_MIME_TYPES: ReadonlySet = new Set([ + "image/heic", + "image/heic-sequence", + "image/heif", + "image/heif-sequence", +]); + +export const TRANSCODED_IMAGE_MIME_TYPE = "image/jpeg"; + +/** Whether `mimeType` is one we can convert into a provider-supported format. */ +export function isTranscodableImageMimeType(mimeType: string): boolean { + return TRANSCODABLE_IMAGE_MIME_TYPES.has(mimeType.trim().toLowerCase()); +} + +interface Transcoder { + readonly command: string; + readonly args: (input: { + readonly inputPath: string; + readonly outputPath: string; + }) => Array; +} + +const SIPS_TRANSCODER: Transcoder = { + command: "sips", + args: ({ inputPath, outputPath }) => ["-s", "format", "jpeg", inputPath, "--out", outputPath], +}; + +const HEIF_CONVERT_TRANSCODER: Transcoder = { + command: "heif-convert", + args: ({ inputPath, outputPath }) => [inputPath, outputPath], +}; + +function transcodersFor(platform: NodeJS.Platform): ReadonlyArray { + return platform === "darwin" ? [SIPS_TRANSCODER] : [HEIF_CONVERT_TRANSCODER]; +} + +function runTranscoder(input: { + readonly transcoder: Transcoder; + readonly inputPath: string; + readonly outputPath: string; +}): Promise { + return new Promise((resolve, reject) => { + NodeChildProcess.execFile( + input.transcoder.command, + input.transcoder.args({ + inputPath: input.inputPath, + outputPath: input.outputPath, + }), + // A photo is a bounded workload; the cap only guards against a wedged + // helper process holding the turn open forever. + { timeout: 30_000, maxBuffer: 1024 * 1024 }, + (error) => (error === null ? resolve() : reject(error)), + ); + }); +} + +/** + * Converts HEIC/HEIF bytes to JPEG bytes. + * + * Rejects when no transcoder is available on the host or the conversion fails, + * so callers can fall back to reporting the attachment as unsupported. + */ +export async function transcodeImageToJpeg(input: { + readonly bytes: Uint8Array; + readonly platform: NodeJS.Platform; +}): Promise { + const workingDir = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-image-transcode-")); + const inputPath = NodePath.join(workingDir, "input"); + const outputPath = NodePath.join(workingDir, "output.jpg"); + + try { + await NodeFSP.writeFile(inputPath, input.bytes); + + let lastError: unknown = new Error("No image transcoder is available on this host."); + for (const transcoder of transcodersFor(input.platform)) { + try { + await runTranscoder({ transcoder, inputPath, outputPath }); + const converted = await NodeFSP.readFile(outputPath); + if (converted.byteLength === 0) { + // Some builds of `sips` exit 0 after writing nothing when the input + // is not decodable, so an empty result has to count as a failure. + throw new Error(`${transcoder.command} produced an empty image.`); + } + return new Uint8Array(converted); + } catch (error) { + lastError = error; + } + } + + throw lastError; + } finally { + await NodeFSP.rm(workingDir, { recursive: true, force: true }).catch(() => {}); + } +} diff --git a/apps/server/src/provider/Layers/ClaudeAdapter.ts b/apps/server/src/provider/Layers/ClaudeAdapter.ts index f87d5be7446..1aace2c4fa7 100644 --- a/apps/server/src/provider/Layers/ClaudeAdapter.ts +++ b/apps/server/src/provider/Layers/ClaudeAdapter.ts @@ -21,6 +21,7 @@ import { type ModelUsage, } from "@anthropic-ai/claude-agent-sdk"; import { parseCliArgs } from "@t3tools/shared/cliArgs"; +import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; import { ApprovalRequestId, type CanonicalItemType, @@ -69,6 +70,11 @@ import * as Stream from "effect/Stream"; import { resolveAttachmentPath } from "../../attachmentStore.ts"; import { ServerConfig } from "../../config.ts"; +import { + isTranscodableImageMimeType, + TRANSCODED_IMAGE_MIME_TYPE, + transcodeImageToJpeg, +} from "../../imageTranscode.ts"; import * as McpProviderSession from "../../mcp/McpProviderSession.ts"; import { resolveClaudeSdkExecutablePath } from "../Drivers/ClaudeExecutable.ts"; import { makeClaudeEnvironment } from "../Drivers/ClaudeHome.ts"; @@ -951,7 +957,11 @@ const buildUserMessageEffect = Effect.fn("buildUserMessageEffect")(function* ( continue; } - if (!SUPPORTED_CLAUDE_IMAGE_MIME_TYPES.has(attachment.mimeType)) { + // HEIC/HEIF is the default capture format on Apple devices, so it arrives + // constantly via paste and the mobile composer. Convert it instead of + // failing the turn. + const needsTranscode = isTranscodableImageMimeType(attachment.mimeType); + if (!needsTranscode && !SUPPORTED_CLAUDE_IMAGE_MIME_TYPES.has(attachment.mimeType)) { return yield* new ProviderAdapterRequestError({ provider: PROVIDER, method: "turn/start", @@ -983,6 +993,28 @@ const buildUserMessageEffect = Effect.fn("buildUserMessageEffect")(function* ( ), ); + if (needsTranscode) { + const hostPlatform = yield* HostProcessPlatform; + const converted = yield* Effect.tryPromise({ + try: () => transcodeImageToJpeg({ bytes, platform: hostPlatform }), + catch: (cause) => + new ProviderAdapterRequestError({ + provider: PROVIDER, + method: "turn/start", + detail: `Unsupported Claude image attachment type '${attachment.mimeType}' and converting it to JPEG failed.`, + cause, + }), + }); + + sdkContent.push( + buildClaudeImageContentBlock({ + mimeType: TRANSCODED_IMAGE_MIME_TYPE, + bytes: converted, + }), + ); + continue; + } + sdkContent.push( buildClaudeImageContentBlock({ mimeType: attachment.mimeType,