diff --git a/apps/server/src/imageTranscode.test.ts b/apps/server/src/imageTranscode.test.ts new file mode 100644 index 00000000000..c365260f25e --- /dev/null +++ b/apps/server/src/imageTranscode.test.ts @@ -0,0 +1,91 @@ +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeChildProcess from "node:child_process"; +import * as NodeFSP from "node:fs/promises"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; + +import { describe, expect, it } from "vite-plus/test"; + +import { + isTranscodableImageMimeType, + TRANSCODED_IMAGE_MIME_TYPE, + transcodeImageToJpeg, +} from "./imageTranscode.ts"; + +// 8x8 RGB PNG, used as the source for the HEIC fixture below. +const SAMPLE_PNG_BASE64 = + "iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAIAAABLbSncAAAAbElEQVR4nA3JQQEAMAgDMZRUCUqqpEpQgoh7o2jLN1WFii5cpJhiiyuqhEQLi4gRK04/GjXduEkzzTbXP4xMG5uYMWvOP4JCB4eECRsuPwYNPXjIMMMONz8WLb14yTLLLrc/Dh19+Mgxxx53PKaVZoFj4h8/AAAAAElFTkSuQmCC"; + +const JPEG_START_OF_IMAGE = [0xff, 0xd8, 0xff]; + +function runSips(args: Array): Promise { + return new Promise((resolve, reject) => { + NodeChildProcess.execFile("sips", args, (error) => + error === null ? resolve() : reject(error), + ); + }); +} + +// `sips` is the macOS transcoder, and also the only way to build a real HEIC +// fixture without shipping a binary in the repo. Probing for it keeps this +// suite green on hosts that do not have it. +const SIPS_AVAILABLE = await runSips(["--version"]).then( + () => true, + () => false, +); + +async function makeHeicFixture(): Promise { + const workingDir = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-heic-fixture-")); + const pngPath = NodePath.join(workingDir, "source.png"); + const heicPath = NodePath.join(workingDir, "source.heic"); + + try { + await NodeFSP.writeFile(pngPath, Buffer.from(SAMPLE_PNG_BASE64, "base64")); + await runSips(["-s", "format", "heic", pngPath, "--out", heicPath]); + return new Uint8Array(await NodeFSP.readFile(heicPath)); + } finally { + await NodeFSP.rm(workingDir, { recursive: true, force: true }).catch(() => {}); + } +} + +describe("imageTranscode", () => { + it("recognizes the mime types Claude cannot ingest", () => { + expect(isTranscodableImageMimeType("image/heic")).toBe(true); + expect(isTranscodableImageMimeType("image/heif")).toBe(true); + expect(isTranscodableImageMimeType("IMAGE/HEIC")).toBe(true); + expect(isTranscodableImageMimeType(" image/heic ")).toBe(true); + }); + + it("leaves natively supported mime types alone", () => { + for (const mimeType of ["image/jpeg", "image/png", "image/gif", "image/webp"]) { + expect(isTranscodableImageMimeType(mimeType)).toBe(false); + } + }); + + it("targets jpeg, which every provider accepts", () => { + expect(TRANSCODED_IMAGE_MIME_TYPE).toBe("image/jpeg"); + }); + + it.skipIf(!SIPS_AVAILABLE)("converts HEIC bytes into JPEG bytes", async () => { + const heic = await makeHeicFixture(); + // Guards against the fixture silently degrading into a non-HEIC file. + expect(Buffer.from(heic.subarray(4, 12)).toString("ascii")).toBe("ftypheic"); + + const jpeg = await transcodeImageToJpeg({ + bytes: heic, + platform: "darwin", + }); + + expect(jpeg.byteLength).toBeGreaterThan(0); + expect([...jpeg.subarray(0, 3)]).toEqual(JPEG_START_OF_IMAGE); + }); + + it.skipIf(!SIPS_AVAILABLE)("rejects bytes that are not a decodable image", async () => { + await expect( + transcodeImageToJpeg({ + bytes: new TextEncoder().encode("not an image"), + platform: "darwin", + }), + ).rejects.toThrow(); + }); +}); diff --git a/apps/server/src/imageTranscode.ts b/apps/server/src/imageTranscode.ts new file mode 100644 index 00000000000..aef943a10e0 --- /dev/null +++ b/apps/server/src/imageTranscode.ts @@ -0,0 +1,116 @@ +// @effect-diagnostics nodeBuiltinImport:off +/** + * Transcodes image formats the model providers cannot ingest (HEIC/HEIF) into + * JPEG, which every provider accepts. + * + * iPhones capture HEIC by default, so pasting or attaching a photo straight + * from an Apple device otherwise fails at the provider boundary even though the + * attachment itself was stored just fine. + * + * Transcoding shells out to a tool that ships with the host rather than pulling + * in a HEIC decoder dependency: `sips` on macOS (always present) and + * `heif-convert` from libheif elsewhere (packaged on most desktop Linux). When + * neither is available the caller surfaces the original "unsupported type" + * failure, so this is strictly additive. + * + * @module imageTranscode + */ +import * as NodeChildProcess from "node:child_process"; +import * as NodeFSP from "node:fs/promises"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; + +export const TRANSCODABLE_IMAGE_MIME_TYPES: ReadonlySet = new Set([ + "image/heic", + "image/heic-sequence", + "image/heif", + "image/heif-sequence", +]); + +export const TRANSCODED_IMAGE_MIME_TYPE = "image/jpeg"; + +/** Whether `mimeType` is one we can convert into a provider-supported format. */ +export function isTranscodableImageMimeType(mimeType: string): boolean { + return TRANSCODABLE_IMAGE_MIME_TYPES.has(mimeType.trim().toLowerCase()); +} + +interface Transcoder { + readonly command: string; + readonly args: (input: { + readonly inputPath: string; + readonly outputPath: string; + }) => Array; +} + +const SIPS_TRANSCODER: Transcoder = { + command: "sips", + args: ({ inputPath, outputPath }) => ["-s", "format", "jpeg", inputPath, "--out", outputPath], +}; + +const HEIF_CONVERT_TRANSCODER: Transcoder = { + command: "heif-convert", + args: ({ inputPath, outputPath }) => [inputPath, outputPath], +}; + +function transcodersFor(platform: NodeJS.Platform): ReadonlyArray { + return platform === "darwin" ? [SIPS_TRANSCODER] : [HEIF_CONVERT_TRANSCODER]; +} + +function runTranscoder(input: { + readonly transcoder: Transcoder; + readonly inputPath: string; + readonly outputPath: string; +}): Promise { + return new Promise((resolve, reject) => { + NodeChildProcess.execFile( + input.transcoder.command, + input.transcoder.args({ + inputPath: input.inputPath, + outputPath: input.outputPath, + }), + // A photo is a bounded workload; the cap only guards against a wedged + // helper process holding the turn open forever. + { timeout: 30_000, maxBuffer: 1024 * 1024 }, + (error) => (error === null ? resolve() : reject(error)), + ); + }); +} + +/** + * Converts HEIC/HEIF bytes to JPEG bytes. + * + * Rejects when no transcoder is available on the host or the conversion fails, + * so callers can fall back to reporting the attachment as unsupported. + */ +export async function transcodeImageToJpeg(input: { + readonly bytes: Uint8Array; + readonly platform: NodeJS.Platform; +}): Promise { + const workingDir = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-image-transcode-")); + const inputPath = NodePath.join(workingDir, "input"); + const outputPath = NodePath.join(workingDir, "output.jpg"); + + try { + await NodeFSP.writeFile(inputPath, input.bytes); + + let lastError: unknown = new Error("No image transcoder is available on this host."); + for (const transcoder of transcodersFor(input.platform)) { + try { + await runTranscoder({ transcoder, inputPath, outputPath }); + const converted = await NodeFSP.readFile(outputPath); + if (converted.byteLength === 0) { + // Some builds of `sips` exit 0 after writing nothing when the input + // is not decodable, so an empty result has to count as a failure. + throw new Error(`${transcoder.command} produced an empty image.`); + } + return new Uint8Array(converted); + } catch (error) { + lastError = error; + } + } + + throw lastError; + } finally { + await NodeFSP.rm(workingDir, { recursive: true, force: true }).catch(() => {}); + } +} diff --git a/apps/server/src/provider/Layers/ClaudeAdapter.ts b/apps/server/src/provider/Layers/ClaudeAdapter.ts index f87d5be7446..1aace2c4fa7 100644 --- a/apps/server/src/provider/Layers/ClaudeAdapter.ts +++ b/apps/server/src/provider/Layers/ClaudeAdapter.ts @@ -21,6 +21,7 @@ import { type ModelUsage, } from "@anthropic-ai/claude-agent-sdk"; import { parseCliArgs } from "@t3tools/shared/cliArgs"; +import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; import { ApprovalRequestId, type CanonicalItemType, @@ -69,6 +70,11 @@ import * as Stream from "effect/Stream"; import { resolveAttachmentPath } from "../../attachmentStore.ts"; import { ServerConfig } from "../../config.ts"; +import { + isTranscodableImageMimeType, + TRANSCODED_IMAGE_MIME_TYPE, + transcodeImageToJpeg, +} from "../../imageTranscode.ts"; import * as McpProviderSession from "../../mcp/McpProviderSession.ts"; import { resolveClaudeSdkExecutablePath } from "../Drivers/ClaudeExecutable.ts"; import { makeClaudeEnvironment } from "../Drivers/ClaudeHome.ts"; @@ -951,7 +957,11 @@ const buildUserMessageEffect = Effect.fn("buildUserMessageEffect")(function* ( continue; } - if (!SUPPORTED_CLAUDE_IMAGE_MIME_TYPES.has(attachment.mimeType)) { + // HEIC/HEIF is the default capture format on Apple devices, so it arrives + // constantly via paste and the mobile composer. Convert it instead of + // failing the turn. + const needsTranscode = isTranscodableImageMimeType(attachment.mimeType); + if (!needsTranscode && !SUPPORTED_CLAUDE_IMAGE_MIME_TYPES.has(attachment.mimeType)) { return yield* new ProviderAdapterRequestError({ provider: PROVIDER, method: "turn/start", @@ -983,6 +993,28 @@ const buildUserMessageEffect = Effect.fn("buildUserMessageEffect")(function* ( ), ); + if (needsTranscode) { + const hostPlatform = yield* HostProcessPlatform; + const converted = yield* Effect.tryPromise({ + try: () => transcodeImageToJpeg({ bytes, platform: hostPlatform }), + catch: (cause) => + new ProviderAdapterRequestError({ + provider: PROVIDER, + method: "turn/start", + detail: `Unsupported Claude image attachment type '${attachment.mimeType}' and converting it to JPEG failed.`, + cause, + }), + }); + + sdkContent.push( + buildClaudeImageContentBlock({ + mimeType: TRANSCODED_IMAGE_MIME_TYPE, + bytes: converted, + }), + ); + continue; + } + sdkContent.push( buildClaudeImageContentBlock({ mimeType: attachment.mimeType,