Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/local-process-exec-utf8.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'@tanstack/ai-sandbox-local-process': patch
---

`exec()` no longer corrupts multi-byte UTF-8 characters in `stdout` and `stderr`. A character that a pipe split across two reads used to decode as U+FFFD. Output is now decoded across chunks, like `spawn()` already does.
6 changes: 4 additions & 2 deletions packages/ai-sandbox-local-process/src/handle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -886,8 +886,10 @@ export class LocalProcessHandle implements SandboxHandle {
this.track(child)
let stdout = ''
let stderr = ''
child.stdout.on('data', (d: Buffer) => (stdout += d.toString('utf8')))
child.stderr.on('data', (d: Buffer) => (stderr += d.toString('utf8')))
// setEncoding decodes across chunks, so a multi-byte character split
// between two `data` events is not turned into U+FFFD (see decodeStream).
child.stdout.setEncoding('utf8').on('data', (d: string) => (stdout += d))
child.stderr.setEncoding('utf8').on('data', (d: string) => (stderr += d))
const onAbort = (): void => {
killTree(child, undefined, this.options.logger)
}
Expand Down
22 changes: 22 additions & 0 deletions packages/ai-sandbox-local-process/tests/local-process.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -393,6 +393,28 @@ describe('local-process spawn stdout — UTF-8 decoding', () => {
await sbx.destroy()
})

it('exec() reassembles a multi-byte character split across chunk boundaries', async () => {
const sbx = await fresh()
await sbx.fs.write(
'/workspace/split-utf8.mjs',
[
`const euro = Buffer.from('€', 'utf8')`,
`process.stdout.write(euro.subarray(0, 1))`,
`process.stderr.write(euro.subarray(0, 1))`,
`setTimeout(() => {`,
` process.stdout.write(euro.subarray(1))`,
` process.stderr.write(euro.subarray(1))`,
`}, 50)`,
].join('\n'),
)
const result = await sbx.process.exec('node split-utf8.mjs', {
cwd: '/workspace',
})
expect(result.stdout).toBe('€')
expect(result.stderr).toBe('€')
await sbx.destroy()
})

it('flushes a genuinely truncated trailing sequence at end of stream (as U+FFFD, not silently dropped)', async () => {
const sbx = await fresh()
await sbx.fs.write(
Expand Down
22 changes: 22 additions & 0 deletions testing/e2e/src/routeTree.gen.ts
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,7 @@ import { Route as ApiResponsesAnswerItemsWireRouteImport } from './routes/api.re
import { Route as ApiRetryAfterRouteImport } from './routes/api.retry-after'
import { Route as ApiSandboxDurabilityRouteImport } from './routes/api.sandbox-durability'
import { Route as ApiSandboxFilePersistenceRouteImport } from './routes/api.sandbox-file-persistence'
import { Route as ApiSandboxLocalProcessExecRouteImport } from './routes/api.sandbox-local-process-exec'
import { Route as ApiSandboxToolHistoryRouteImport } from './routes/api.sandbox-tool-history'
import { Route as ApiServerToolStoreWireRouteImport } from './routes/api.server-tool-store-wire'
import { Route as ApiStructuredOutputLengthWireRouteImport } from './routes/api.structured-output-length-wire'
Expand Down Expand Up @@ -1298,6 +1299,12 @@ const ApiSandboxFilePersistenceRoute =
path: '/api/sandbox-file-persistence',
getParentRoute: () => rootRouteImport,
} as any)
const ApiSandboxLocalProcessExecRoute =
ApiSandboxLocalProcessExecRouteImport.update({
id: '/api/sandbox-local-process-exec',
path: '/api/sandbox-local-process-exec',
getParentRoute: () => rootRouteImport,
} as any)
const ApiSandboxToolHistoryRoute = ApiSandboxToolHistoryRouteImport.update({
id: '/api/sandbox-tool-history',
path: '/api/sandbox-tool-history',
Expand Down Expand Up @@ -1665,6 +1672,7 @@ export interface FileRoutesByFullPath {
'/api/retry-after': typeof ApiRetryAfterRoute
'/api/sandbox-durability': typeof ApiSandboxDurabilityRoute
'/api/sandbox-file-persistence': typeof ApiSandboxFilePersistenceRoute
'/api/sandbox-local-process-exec': typeof ApiSandboxLocalProcessExecRoute
'/api/sandbox-tool-history': typeof ApiSandboxToolHistoryRoute
'/api/server-tool-store-wire': typeof ApiServerToolStoreWireRoute
'/api/structured-output-length-wire': typeof ApiStructuredOutputLengthWireRoute
Expand Down Expand Up @@ -1900,6 +1908,7 @@ export interface FileRoutesByTo {
'/api/retry-after': typeof ApiRetryAfterRoute
'/api/sandbox-durability': typeof ApiSandboxDurabilityRoute
'/api/sandbox-file-persistence': typeof ApiSandboxFilePersistenceRoute
'/api/sandbox-local-process-exec': typeof ApiSandboxLocalProcessExecRoute
'/api/sandbox-tool-history': typeof ApiSandboxToolHistoryRoute
'/api/server-tool-store-wire': typeof ApiServerToolStoreWireRoute
'/api/structured-output-length-wire': typeof ApiStructuredOutputLengthWireRoute
Expand Down Expand Up @@ -2136,6 +2145,7 @@ export interface FileRoutesById {
'/api/retry-after': typeof ApiRetryAfterRoute
'/api/sandbox-durability': typeof ApiSandboxDurabilityRoute
'/api/sandbox-file-persistence': typeof ApiSandboxFilePersistenceRoute
'/api/sandbox-local-process-exec': typeof ApiSandboxLocalProcessExecRoute
'/api/sandbox-tool-history': typeof ApiSandboxToolHistoryRoute
'/api/server-tool-store-wire': typeof ApiServerToolStoreWireRoute
'/api/structured-output-length-wire': typeof ApiStructuredOutputLengthWireRoute
Expand Down Expand Up @@ -2373,6 +2383,7 @@ export interface FileRouteTypes {
| '/api/retry-after'
| '/api/sandbox-durability'
| '/api/sandbox-file-persistence'
| '/api/sandbox-local-process-exec'
| '/api/sandbox-tool-history'
| '/api/server-tool-store-wire'
| '/api/structured-output-length-wire'
Expand Down Expand Up @@ -2608,6 +2619,7 @@ export interface FileRouteTypes {
| '/api/retry-after'
| '/api/sandbox-durability'
| '/api/sandbox-file-persistence'
| '/api/sandbox-local-process-exec'
| '/api/sandbox-tool-history'
| '/api/server-tool-store-wire'
| '/api/structured-output-length-wire'
Expand Down Expand Up @@ -2843,6 +2855,7 @@ export interface FileRouteTypes {
| '/api/retry-after'
| '/api/sandbox-durability'
| '/api/sandbox-file-persistence'
| '/api/sandbox-local-process-exec'
| '/api/sandbox-tool-history'
| '/api/server-tool-store-wire'
| '/api/structured-output-length-wire'
Expand Down Expand Up @@ -3079,6 +3092,7 @@ export interface RootRouteChildren {
ApiRetryAfterRoute: typeof ApiRetryAfterRoute
ApiSandboxDurabilityRoute: typeof ApiSandboxDurabilityRoute
ApiSandboxFilePersistenceRoute: typeof ApiSandboxFilePersistenceRoute
ApiSandboxLocalProcessExecRoute: typeof ApiSandboxLocalProcessExecRoute
ApiSandboxToolHistoryRoute: typeof ApiSandboxToolHistoryRoute
ApiServerToolStoreWireRoute: typeof ApiServerToolStoreWireRoute
ApiStructuredOutputLengthWireRoute: typeof ApiStructuredOutputLengthWireRoute
Expand Down Expand Up @@ -4518,6 +4532,13 @@ declare module '@tanstack/react-router' {
preLoaderRoute: typeof ApiSandboxFilePersistenceRouteImport
parentRoute: typeof rootRouteImport
}
'/api/sandbox-local-process-exec': {
id: '/api/sandbox-local-process-exec'
path: '/api/sandbox-local-process-exec'
fullPath: '/api/sandbox-local-process-exec'
preLoaderRoute: typeof ApiSandboxLocalProcessExecRouteImport
parentRoute: typeof rootRouteImport
}
'/api/sandbox-tool-history': {
id: '/api/sandbox-tool-history'
path: '/api/sandbox-tool-history'
Expand Down Expand Up @@ -5010,6 +5031,7 @@ const rootRouteChildren: RootRouteChildren = {
ApiRetryAfterRoute: ApiRetryAfterRoute,
ApiSandboxDurabilityRoute: ApiSandboxDurabilityRoute,
ApiSandboxFilePersistenceRoute: ApiSandboxFilePersistenceRoute,
ApiSandboxLocalProcessExecRoute: ApiSandboxLocalProcessExecRoute,
ApiSandboxToolHistoryRoute: ApiSandboxToolHistoryRoute,
ApiServerToolStoreWireRoute: ApiServerToolStoreWireRoute,
ApiStructuredOutputLengthWireRoute: ApiStructuredOutputLengthWireRoute,
Expand Down
62 changes: 62 additions & 0 deletions testing/e2e/src/routes/api.sandbox-local-process-exec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
import { tmpdir } from 'node:os'
import { resolve, sep } from 'node:path'
import { createFileRoute } from '@tanstack/react-router'
import { localProcessSandbox } from '@tanstack/ai-sandbox-local-process'

/**
* `exec()` of the local-process sandbox, with output that has multi-byte
* characters. Body: `{ root }`, a folder in the temp folder that the test
* made. The answer is the JSON of the exec result.
*
* The script writes each byte of `TEXT` by itself, with a pause between the
* writes, to stdout and to stderr. So each multi-byte character comes to the
* sandbox in more than one `data` chunk, and `exec()` must decode across the
* chunks to give back `TEXT`, not U+FFFD.
*/

/** Two, three and four byte UTF-8 characters, and Persian text. */
export const TEXT = 'é € 😀 سلام'

const SCRIPT = [
`const bytes = Buffer.from(${JSON.stringify(TEXT)}, 'utf8')`,
`let index = 0`,
`const next = () => {`,
` if (index >= bytes.length) return`,
` const byte = bytes.subarray(index, index + 1)`,
` process.stdout.write(byte)`,
` process.stderr.write(byte)`,
` index += 1`,
` setTimeout(next, 5)`,
`}`,
`next()`,
].join('\n')

export const Route = createFileRoute('/api/sandbox-local-process-exec')({
server: {
handlers: {
POST: async ({ request }) => {
const body: unknown = await request.json()
const rootField =
typeof body === 'object' && body !== null && 'root' in body
? body.root
: undefined
const root = typeof rootField === 'string' ? resolve(rootField) : ''
if (!root.startsWith(resolve(tmpdir()) + sep)) {
return new Response('root must be in the temp folder', {
status: 400,
})
}
const sandbox = await localProcessSandbox({ dir: root }).create({})
try {
await sandbox.fs.write('/workspace/split-utf8.mjs', SCRIPT)
const result = await sandbox.process.exec('node split-utf8.mjs', {
cwd: '/workspace',
})
return Response.json(result)
} finally {
await sandbox.destroy()
}
},
},
},
})
42 changes: 42 additions & 0 deletions testing/e2e/tests/sandbox-local-process-exec.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
import { mkdtemp, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { expect, test } from '@playwright/test'
import { TEXT } from '../src/routes/api.sandbox-local-process-exec'

/**
* `exec()` of the local-process sandbox decodes UTF-8 across output chunks.
* The route runs a command that writes one byte at a time, so each
* multi-byte character is split between `data` chunks.
*/
test.describe('local-process sandbox exec() output', () => {
let root = ''

test.beforeEach(async () => {
root = await mkdtemp(join(tmpdir(), 'e2e-local-process-exec-'))
})

test.afterEach(async () => {
await rm(root, { recursive: true, force: true })
})

test('keeps multi-byte characters split across chunks', async ({
request,
}) => {
const response = await request.post('/api/sandbox-local-process-exec', {
data: { root },
})
const body = await response.text()
expect(response.ok(), body).toBe(true)

const result = JSON.parse(body)
expect(result).toEqual({ stdout: TEXT, stderr: TEXT, exitCode: 0 })
})

test('rejects a root outside the temp folder', async ({ request }) => {
const response = await request.post('/api/sandbox-local-process-exec', {
data: { root: '/' },
})
expect(response.status()).toBe(400)
})
})
Loading