diff --git a/.changeset/local-process-exec-utf8.md b/.changeset/local-process-exec-utf8.md new file mode 100644 index 0000000000..51e3460619 --- /dev/null +++ b/.changeset/local-process-exec-utf8.md @@ -0,0 +1,5 @@ +--- +'@tanstack/ai-sandbox-local-process': patch +--- + +`exec()` no longer corrupts multi-byte UTF-8 characters in `stdout` and `stderr`. A character that a pipe split across two reads used to decode as U+FFFD. Output is now decoded across chunks, like `spawn()` already does. diff --git a/packages/ai-sandbox-local-process/src/handle.ts b/packages/ai-sandbox-local-process/src/handle.ts index 9c556d5d6f..b8bfe453fa 100644 --- a/packages/ai-sandbox-local-process/src/handle.ts +++ b/packages/ai-sandbox-local-process/src/handle.ts @@ -886,8 +886,10 @@ export class LocalProcessHandle implements SandboxHandle { this.track(child) let stdout = '' let stderr = '' - child.stdout.on('data', (d: Buffer) => (stdout += d.toString('utf8'))) - child.stderr.on('data', (d: Buffer) => (stderr += d.toString('utf8'))) + // setEncoding decodes across chunks, so a multi-byte character split + // between two `data` events is not turned into U+FFFD (see decodeStream). + child.stdout.setEncoding('utf8').on('data', (d: string) => (stdout += d)) + child.stderr.setEncoding('utf8').on('data', (d: string) => (stderr += d)) const onAbort = (): void => { killTree(child, undefined, this.options.logger) } diff --git a/packages/ai-sandbox-local-process/tests/local-process.test.ts b/packages/ai-sandbox-local-process/tests/local-process.test.ts index 9b59b04fff..0bf845a96d 100644 --- a/packages/ai-sandbox-local-process/tests/local-process.test.ts +++ b/packages/ai-sandbox-local-process/tests/local-process.test.ts @@ -393,6 +393,28 @@ describe('local-process spawn stdout — UTF-8 decoding', () => { await sbx.destroy() }) + it('exec() reassembles a multi-byte character split across chunk boundaries', async () => { + const sbx = await fresh() + await sbx.fs.write( + '/workspace/split-utf8.mjs', + [ + `const euro = Buffer.from('€', 'utf8')`, + `process.stdout.write(euro.subarray(0, 1))`, + `process.stderr.write(euro.subarray(0, 1))`, + `setTimeout(() => {`, + ` process.stdout.write(euro.subarray(1))`, + ` process.stderr.write(euro.subarray(1))`, + `}, 50)`, + ].join('\n'), + ) + const result = await sbx.process.exec('node split-utf8.mjs', { + cwd: '/workspace', + }) + expect(result.stdout).toBe('€') + expect(result.stderr).toBe('€') + await sbx.destroy() + }) + it('flushes a genuinely truncated trailing sequence at end of stream (as U+FFFD, not silently dropped)', async () => { const sbx = await fresh() await sbx.fs.write( diff --git a/testing/e2e/src/routeTree.gen.ts b/testing/e2e/src/routeTree.gen.ts index 4ef2f9f08f..98f5a564c1 100644 --- a/testing/e2e/src/routeTree.gen.ts +++ b/testing/e2e/src/routeTree.gen.ts @@ -210,6 +210,7 @@ import { Route as ApiResponsesAnswerItemsWireRouteImport } from './routes/api.re import { Route as ApiRetryAfterRouteImport } from './routes/api.retry-after' import { Route as ApiSandboxDurabilityRouteImport } from './routes/api.sandbox-durability' import { Route as ApiSandboxFilePersistenceRouteImport } from './routes/api.sandbox-file-persistence' +import { Route as ApiSandboxLocalProcessExecRouteImport } from './routes/api.sandbox-local-process-exec' import { Route as ApiSandboxToolHistoryRouteImport } from './routes/api.sandbox-tool-history' import { Route as ApiServerToolStoreWireRouteImport } from './routes/api.server-tool-store-wire' import { Route as ApiStructuredOutputLengthWireRouteImport } from './routes/api.structured-output-length-wire' @@ -1298,6 +1299,12 @@ const ApiSandboxFilePersistenceRoute = path: '/api/sandbox-file-persistence', getParentRoute: () => rootRouteImport, } as any) +const ApiSandboxLocalProcessExecRoute = + ApiSandboxLocalProcessExecRouteImport.update({ + id: '/api/sandbox-local-process-exec', + path: '/api/sandbox-local-process-exec', + getParentRoute: () => rootRouteImport, + } as any) const ApiSandboxToolHistoryRoute = ApiSandboxToolHistoryRouteImport.update({ id: '/api/sandbox-tool-history', path: '/api/sandbox-tool-history', @@ -1665,6 +1672,7 @@ export interface FileRoutesByFullPath { '/api/retry-after': typeof ApiRetryAfterRoute '/api/sandbox-durability': typeof ApiSandboxDurabilityRoute '/api/sandbox-file-persistence': typeof ApiSandboxFilePersistenceRoute + '/api/sandbox-local-process-exec': typeof ApiSandboxLocalProcessExecRoute '/api/sandbox-tool-history': typeof ApiSandboxToolHistoryRoute '/api/server-tool-store-wire': typeof ApiServerToolStoreWireRoute '/api/structured-output-length-wire': typeof ApiStructuredOutputLengthWireRoute @@ -1900,6 +1908,7 @@ export interface FileRoutesByTo { '/api/retry-after': typeof ApiRetryAfterRoute '/api/sandbox-durability': typeof ApiSandboxDurabilityRoute '/api/sandbox-file-persistence': typeof ApiSandboxFilePersistenceRoute + '/api/sandbox-local-process-exec': typeof ApiSandboxLocalProcessExecRoute '/api/sandbox-tool-history': typeof ApiSandboxToolHistoryRoute '/api/server-tool-store-wire': typeof ApiServerToolStoreWireRoute '/api/structured-output-length-wire': typeof ApiStructuredOutputLengthWireRoute @@ -2136,6 +2145,7 @@ export interface FileRoutesById { '/api/retry-after': typeof ApiRetryAfterRoute '/api/sandbox-durability': typeof ApiSandboxDurabilityRoute '/api/sandbox-file-persistence': typeof ApiSandboxFilePersistenceRoute + '/api/sandbox-local-process-exec': typeof ApiSandboxLocalProcessExecRoute '/api/sandbox-tool-history': typeof ApiSandboxToolHistoryRoute '/api/server-tool-store-wire': typeof ApiServerToolStoreWireRoute '/api/structured-output-length-wire': typeof ApiStructuredOutputLengthWireRoute @@ -2373,6 +2383,7 @@ export interface FileRouteTypes { | '/api/retry-after' | '/api/sandbox-durability' | '/api/sandbox-file-persistence' + | '/api/sandbox-local-process-exec' | '/api/sandbox-tool-history' | '/api/server-tool-store-wire' | '/api/structured-output-length-wire' @@ -2608,6 +2619,7 @@ export interface FileRouteTypes { | '/api/retry-after' | '/api/sandbox-durability' | '/api/sandbox-file-persistence' + | '/api/sandbox-local-process-exec' | '/api/sandbox-tool-history' | '/api/server-tool-store-wire' | '/api/structured-output-length-wire' @@ -2843,6 +2855,7 @@ export interface FileRouteTypes { | '/api/retry-after' | '/api/sandbox-durability' | '/api/sandbox-file-persistence' + | '/api/sandbox-local-process-exec' | '/api/sandbox-tool-history' | '/api/server-tool-store-wire' | '/api/structured-output-length-wire' @@ -3079,6 +3092,7 @@ export interface RootRouteChildren { ApiRetryAfterRoute: typeof ApiRetryAfterRoute ApiSandboxDurabilityRoute: typeof ApiSandboxDurabilityRoute ApiSandboxFilePersistenceRoute: typeof ApiSandboxFilePersistenceRoute + ApiSandboxLocalProcessExecRoute: typeof ApiSandboxLocalProcessExecRoute ApiSandboxToolHistoryRoute: typeof ApiSandboxToolHistoryRoute ApiServerToolStoreWireRoute: typeof ApiServerToolStoreWireRoute ApiStructuredOutputLengthWireRoute: typeof ApiStructuredOutputLengthWireRoute @@ -4518,6 +4532,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof ApiSandboxFilePersistenceRouteImport parentRoute: typeof rootRouteImport } + '/api/sandbox-local-process-exec': { + id: '/api/sandbox-local-process-exec' + path: '/api/sandbox-local-process-exec' + fullPath: '/api/sandbox-local-process-exec' + preLoaderRoute: typeof ApiSandboxLocalProcessExecRouteImport + parentRoute: typeof rootRouteImport + } '/api/sandbox-tool-history': { id: '/api/sandbox-tool-history' path: '/api/sandbox-tool-history' @@ -5010,6 +5031,7 @@ const rootRouteChildren: RootRouteChildren = { ApiRetryAfterRoute: ApiRetryAfterRoute, ApiSandboxDurabilityRoute: ApiSandboxDurabilityRoute, ApiSandboxFilePersistenceRoute: ApiSandboxFilePersistenceRoute, + ApiSandboxLocalProcessExecRoute: ApiSandboxLocalProcessExecRoute, ApiSandboxToolHistoryRoute: ApiSandboxToolHistoryRoute, ApiServerToolStoreWireRoute: ApiServerToolStoreWireRoute, ApiStructuredOutputLengthWireRoute: ApiStructuredOutputLengthWireRoute, diff --git a/testing/e2e/src/routes/api.sandbox-local-process-exec.ts b/testing/e2e/src/routes/api.sandbox-local-process-exec.ts new file mode 100644 index 0000000000..a05a113e9e --- /dev/null +++ b/testing/e2e/src/routes/api.sandbox-local-process-exec.ts @@ -0,0 +1,62 @@ +import { tmpdir } from 'node:os' +import { resolve, sep } from 'node:path' +import { createFileRoute } from '@tanstack/react-router' +import { localProcessSandbox } from '@tanstack/ai-sandbox-local-process' + +/** + * `exec()` of the local-process sandbox, with output that has multi-byte + * characters. Body: `{ root }`, a folder in the temp folder that the test + * made. The answer is the JSON of the exec result. + * + * The script writes each byte of `TEXT` by itself, with a pause between the + * writes, to stdout and to stderr. So each multi-byte character comes to the + * sandbox in more than one `data` chunk, and `exec()` must decode across the + * chunks to give back `TEXT`, not U+FFFD. + */ + +/** Two, three and four byte UTF-8 characters, and Persian text. */ +export const TEXT = 'é € 😀 سلام' + +const SCRIPT = [ + `const bytes = Buffer.from(${JSON.stringify(TEXT)}, 'utf8')`, + `let index = 0`, + `const next = () => {`, + ` if (index >= bytes.length) return`, + ` const byte = bytes.subarray(index, index + 1)`, + ` process.stdout.write(byte)`, + ` process.stderr.write(byte)`, + ` index += 1`, + ` setTimeout(next, 5)`, + `}`, + `next()`, +].join('\n') + +export const Route = createFileRoute('/api/sandbox-local-process-exec')({ + server: { + handlers: { + POST: async ({ request }) => { + const body: unknown = await request.json() + const rootField = + typeof body === 'object' && body !== null && 'root' in body + ? body.root + : undefined + const root = typeof rootField === 'string' ? resolve(rootField) : '' + if (!root.startsWith(resolve(tmpdir()) + sep)) { + return new Response('root must be in the temp folder', { + status: 400, + }) + } + const sandbox = await localProcessSandbox({ dir: root }).create({}) + try { + await sandbox.fs.write('/workspace/split-utf8.mjs', SCRIPT) + const result = await sandbox.process.exec('node split-utf8.mjs', { + cwd: '/workspace', + }) + return Response.json(result) + } finally { + await sandbox.destroy() + } + }, + }, + }, +}) diff --git a/testing/e2e/tests/sandbox-local-process-exec.spec.ts b/testing/e2e/tests/sandbox-local-process-exec.spec.ts new file mode 100644 index 0000000000..b65ce75f13 --- /dev/null +++ b/testing/e2e/tests/sandbox-local-process-exec.spec.ts @@ -0,0 +1,42 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { expect, test } from '@playwright/test' +import { TEXT } from '../src/routes/api.sandbox-local-process-exec' + +/** + * `exec()` of the local-process sandbox decodes UTF-8 across output chunks. + * The route runs a command that writes one byte at a time, so each + * multi-byte character is split between `data` chunks. + */ +test.describe('local-process sandbox exec() output', () => { + let root = '' + + test.beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'e2e-local-process-exec-')) + }) + + test.afterEach(async () => { + await rm(root, { recursive: true, force: true }) + }) + + test('keeps multi-byte characters split across chunks', async ({ + request, + }) => { + const response = await request.post('/api/sandbox-local-process-exec', { + data: { root }, + }) + const body = await response.text() + expect(response.ok(), body).toBe(true) + + const result = JSON.parse(body) + expect(result).toEqual({ stdout: TEXT, stderr: TEXT, exitCode: 0 }) + }) + + test('rejects a root outside the temp folder', async ({ request }) => { + const response = await request.post('/api/sandbox-local-process-exec', { + data: { root: '/' }, + }) + expect(response.status()).toBe(400) + }) +})