From 86a8e2982541b8d0e00f2e1a3091a89eddad8710 Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Tue, 21 Jul 2026 12:39:45 -0400 Subject: [PATCH 1/8] Execute bounded PHP commands in clean Playground processes --- .../runtime-playground/src/php-bootstrap.ts | 10 ++-- .../src/phpunit-command-handlers.ts | 5 ++ .../src/playground-cli-runner.ts | 54 +++---------------- .../src/playground-runtime.ts | 49 +++-------------- .../runtime-playground/src/preview-server.ts | 2 +- ...isposable-mysql-mysqli.integration.test.ts | 2 +- tests/phpunit-project-autoload.test.ts | 1 + ...layground-cli-runner-bootstrap-ini.test.ts | 20 ++----- 8 files changed, 34 insertions(+), 109 deletions(-) diff --git a/packages/runtime-playground/src/php-bootstrap.ts b/packages/runtime-playground/src/php-bootstrap.ts index 3995da35..9c46fafe 100644 --- a/packages/runtime-playground/src/php-bootstrap.ts +++ b/packages/runtime-playground/src/php-bootstrap.ts @@ -21,11 +21,15 @@ ${phpBody(code)}` } export function bootstrapPhpCode(spec: RuntimeCreateSpec, code: string, args: string[], wpCliBridge?: PhpBootstrapBridge, failureDiagnosticFile?: string): string { + const command = splitLeadingStrictTypesDeclare(code) if (argValue(args, "bootstrap") === "none") { - return code + return ` or declare phpunit bootstrap'); diff --git a/packages/runtime-playground/src/playground-cli-runner.ts b/packages/runtime-playground/src/playground-cli-runner.ts index 2f3266e3..cbd8bd8f 100644 --- a/packages/runtime-playground/src/playground-cli-runner.ts +++ b/packages/runtime-playground/src/playground-cli-runner.ts @@ -3,7 +3,7 @@ import { PlaygroundCliExitError, type PlaygroundCliBufferedOutput } from "./play import { PlaygroundPreviewPortUnavailableError, assertPreviewPortAvailable, errorHasCode, withPreviewProxy, type PlaygroundCliServer } from "./preview-server.js" import { startProgrammaticPlaygroundServer } from "./programmatic-playground-runner.js" import { normalizeLiveProgressEvent, previewLease, type BrowserStartupProgressEvent, type BrowserStartupProgressPhase, type BrowserStartupProgressStatus, type MountSpec, type PreviewLease, type RuntimeCreateSpec, type RuntimePreviewLeaseProvider } from "@automattic/wp-codebox-core" -import { randomBytes, randomInt } from "node:crypto" +import { randomInt } from "node:crypto" import { existsSync } from "node:fs" import { createServer as createHttpServer, type Server as HttpServer } from "node:http" import { mkdir, readFile, rename, rm, stat, unlink, utimes, writeFile } from "node:fs/promises" @@ -80,11 +80,6 @@ export async function startPlaygroundCliServer(spec: RuntimeCreateSpec, mounts: const wordpressInstallMode = spec.environment.wordpressInstallMode ?? "install-from-existing-files" const bootstrapIniEntries = runtimeBootstrapPhpIniEntries(spec) const useProgrammaticRunner = shouldUseProgrammaticPlaygroundRunner(spec, options) - const requestWorkerEndpoint = useProgrammaticRunner ? undefined : { - route: `/wp-codebox-execute-${randomBytes(12).toString("hex")}.php`, - token: randomBytes(32).toString("base64url"), - payloadDirectory: join(spec.artifactsDirectory ?? "artifacts", "playground-internal-shared"), - } usesArchiveCache = !wordpressDirectory && !spec.environment.assets?.wordpressZip readonlyMountStaging = await stageReadonlyPlaygroundMounts(mounts) const stagedMounts = readonlyMountStaging.mounts @@ -137,7 +132,7 @@ export async function startPlaygroundCliServer(spec: RuntimeCreateSpec, mounts: }, Boolean(spec.preview?.port)) : await startPlaygroundCliWithDynamicPortRetry(async (port) => { const { runCLI } = options.cliModule ?? (await import("@wp-playground/cli")) as unknown as PlaygroundCliModule const localAssetServer = wordpressStartupAsset?.localPath ? await serveLocalStartupAsset(wordpressStartupAsset.localPath) : undefined - const bootstrapSharedMounts = await pluginRuntimeBootstrapSharedMounts(spec, requestWorkerEndpoint) + const bootstrapSharedMounts = await pluginRuntimeBootstrapSharedMounts(spec) try { return await runCLI({ command: "server", @@ -179,7 +174,7 @@ export async function startPlaygroundCliServer(spec: RuntimeCreateSpec, mounts: fixedPreviewPort: spec.preview?.port ?? null, }) - const proxiedServer = await withPreviewLeaseProvider(await withPreviewProxy({ ...server, ...(requestWorkerEndpoint ? { requestWorkerEndpoint } : {}) }, spec.preview?.port ?? 0, spec.preview?.bind), spec) + const proxiedServer = await withPreviewLeaseProvider(await withPreviewProxy(server, spec.preview?.port ?? 0, spec.preview?.bind), spec) emitProgress("preview:ready", "complete", "Preview ready", { localUrl: proxiedServer.serverUrl, upstreamUrl: server.serverUrl, @@ -303,9 +298,10 @@ export function shouldUseProgrammaticPlaygroundRunner(spec: RuntimeCreateSpec, o && (Boolean(runtimeBootstrapPhpIniEntries(spec)) || Boolean(spec.environment.extensions?.length)) } -async function pluginRuntimeBootstrapSharedMounts(spec: RuntimeCreateSpec, requestWorkerEndpoint?: { route: string; token: string; payloadDirectory: string }): Promise> { +async function pluginRuntimeBootstrapSharedMounts(spec: RuntimeCreateSpec): Promise> { const iniEntries = runtimeBootstrapPhpIniEntries(spec) - if (!iniEntries && !requestWorkerEndpoint) { + const externalWpConfig = externalDatabaseWpConfig(spec) + if (!iniEntries && !externalWpConfig) { return [] } @@ -315,8 +311,6 @@ async function pluginRuntimeBootstrapSharedMounts(spec: RuntimeCreateSpec, reque await writeFile(join(directory, "php.ini"), phpIniContent(iniEntries), "utf8") await writeFile(join(directory, "wp-codebox-auto-prepend.php"), runtimeAutoPrependPhp(spec), "utf8") } - if (requestWorkerEndpoint) await writeFile(join(directory, "request-worker.php"), requestWorkerPhp(requestWorkerEndpoint.token), "utf8") - const externalWpConfig = externalDatabaseWpConfig(spec) if (externalWpConfig) await writeFile(join(directory, "wp-config.php"), externalWpConfig, "utf8") return [ @@ -324,46 +318,10 @@ async function pluginRuntimeBootstrapSharedMounts(spec: RuntimeCreateSpec, reque { hostPath: join(directory, "php.ini"), vfsPath: "/internal/shared/php.ini" }, { hostPath: join(directory, "wp-codebox-auto-prepend.php"), vfsPath: "/internal/shared/wp-codebox-auto-prepend.php" }, ] : []), - ...(requestWorkerEndpoint ? [ - { hostPath: directory, vfsPath: "/internal/wp-codebox" }, - { hostPath: join(directory, "request-worker.php"), vfsPath: `/wordpress${requestWorkerEndpoint.route}` }, - ] : []), ...(externalWpConfig ? [{ hostPath: join(directory, "wp-config.php"), vfsPath: "/wordpress/wp-config.php" }] : []), ] } -function requestWorkerPhp(token: string): string { - return ` $wp_codebox_value) { - if (!is_string($wp_codebox_name) || !is_string($wp_codebox_value)) { - http_response_code(400); - exit; - } - putenv($wp_codebox_name . '=' . $wp_codebox_value); - $_ENV[$wp_codebox_name] = $wp_codebox_value; - $_SERVER[$wp_codebox_name] = $wp_codebox_value; -} -eval('?>' . $wp_codebox_code); -` -} - function runtimeBootstrapPhpIniEntries(spec: RuntimeCreateSpec): Record | undefined { const entries = pluginRuntimeBootstrapPhpIniEntries(spec) ?? {} if (Object.keys(entries).length === 0 && !runtimeAutoPrependPhpBody(spec)) { diff --git a/packages/runtime-playground/src/playground-runtime.ts b/packages/runtime-playground/src/playground-runtime.ts index f0598ad5..076092c8 100644 --- a/packages/runtime-playground/src/playground-runtime.ts +++ b/packages/runtime-playground/src/playground-runtime.ts @@ -1,6 +1,6 @@ import { randomBytes } from "node:crypto" import { AsyncLocalStorage } from "node:async_hooks" -import { mkdir, readFile, realpath, unlink, writeFile } from "node:fs/promises" +import { mkdir, readFile, realpath, writeFile } from "node:fs/promises" import type { IncomingMessage, ServerResponse } from "node:http" import { dirname, join, resolve } from "node:path" import { HostToolRegistry, PREVIEW_LEASE_SCHEMA, RUNTIME_EPISODE_OBSERVATION_SCHEMA, RUNTIME_EPISODE_SNAPSHOT_SCHEMA, RuntimeActionExecutionError, assertRuntimeCommandAllowed, commandAgentRunResultJson, createCommandAgentRunResult, createHostToolRegistry, createRuntimeCommandResultEnvelope, parseCommandAgentRunRequest, previewLease, resolveArtifactPath, resolveCommandPath, runtimeCommandResultEnvelopeFromOutput, runtimeEpisodeDigest } from "@automattic/wp-codebox-core" @@ -251,8 +251,7 @@ class PlaygroundRuntime implements Runtime { private cliServerPromise?: Promise private readonly activeExecutionAbortControllers = new Set() private readonly executionSignals = new AsyncLocalStorage() - private readonly requestWorkerExecutions = new AsyncLocalStorage>() - private requestWorkerReady?: Promise + private readonly isolatedProcessExecutions = new AsyncLocalStorage() private reviewerAuthBootstrapRouteRegistered = false private readonly reviewerAuthBootstraps = new Map() @@ -365,9 +364,7 @@ class PlaygroundRuntime implements Runtime { this.activeExecutionAbortControllers.add(abortController) try { const executionSpec = executionSpecWithEnvironment(spec) - const output = await this.executionSignals.run(abortController.signal, async () => spec.processIdentity - ? await this.requestWorkerExecutions.run(spec.environment ?? {}, async () => await timeoutPlaygroundCommand(executePlaygroundCommand(this, executionSpec, this.hostTools), spec, abortController)) - : await timeoutPlaygroundCommand(executePlaygroundCommand(this, executionSpec, this.hostTools), spec, abortController)) + const output = await this.executionSignals.run(abortController.signal, async () => await this.isolatedProcessExecutions.run(Boolean(spec.processIdentity), async () => await timeoutPlaygroundCommand(executePlaygroundCommand(this, executionSpec, this.hostTools), spec, abortController))) const finishedAt = now() const envelope = typeof output === "string" ? runtimeCommandResultEnvelopeFromOutput({ @@ -1769,11 +1766,11 @@ class PlaygroundRuntime implements Runtime { private async runPlaygroundCommand(command: string, server: PlaygroundCliServer, options: { code: string } | { scriptPath: string }): Promise { try { - const requestWorkerEnvironment = this.requestWorkerExecutions.getStore() - if (requestWorkerEnvironment && "code" in options && server.requestWorkerEndpoint) { - await this.prepareRequestWorker(server) - const response = await this.executeRequestWorker(server, options.code, requestWorkerEnvironment, this.executionSignals.getStore()) - return { text: response.text, exitCode: response.ok ? 0 : 1, ...(!response.ok ? { errors: response.text } : {}) } + if (this.isolatedProcessExecutions.getStore() && "code" in options) { + if (!server.playground.runInFreshProcess) { + throw new Error("The Playground runtime does not support clean PHP process execution.") + } + return await abortable(server.playground.runInFreshProcess(options), this.executionSignals.getStore()) } return await abortable(server.playground.run(options), this.executionSignals.getStore()) } catch (error) { @@ -1786,36 +1783,6 @@ class PlaygroundRuntime implements Runtime { } } - private async prepareRequestWorker(server: PlaygroundCliServer): Promise { - if (!server.requestWorkerEndpoint) return - this.requestWorkerReady ??= (async () => { - const response = await this.executeRequestWorker(server, ", signal?: AbortSignal): Promise<{ ok: boolean; status: number; text: string }> { - const endpoint = server.requestWorkerEndpoint - if (!endpoint) throw new Error("Playground request worker endpoint is unavailable.") - const payloadId = randomBytes(16).toString("hex") - const payloadPath = join(endpoint.payloadDirectory, `execution-${payloadId}.json`) - await writeFile(payloadPath, JSON.stringify({ code, environment }), "utf8") - try { - const response = await fetch(new URL(endpoint.route, server.serverUrl), { - method: "POST", - headers: { - "X-WP-Codebox-Execution-Token": endpoint.token, - "X-WP-Codebox-Execution-Payload": payloadId, - }, - signal, - }) - return { ok: response.ok, status: response.status, text: await response.text() } - } finally { - await unlink(payloadPath).catch(() => undefined) - } - } - async inspectMountedInputs(): Promise { const server = await this.bootPlayground() const response = await server.playground.run({ diff --git a/packages/runtime-playground/src/preview-server.ts b/packages/runtime-playground/src/preview-server.ts index 79e3b454..8601d400 100644 --- a/packages/runtime-playground/src/preview-server.ts +++ b/packages/runtime-playground/src/preview-server.ts @@ -11,12 +11,12 @@ export interface PlaygroundServerRunResponse { export interface PlaygroundCliServer { playground: { run(options: { code: string } | { scriptPath: string }): Promise + runInFreshProcess?(options: { code: string }): Promise onMessage?(listener: (data: string) => Promise | string | void): Promise<(() => Promise | void) | void> | (() => Promise | void) | void readFileAsText?(path: string): string | Promise writeFile?(path: string, contents: string): Promise } serverUrl: string - requestWorkerEndpoint?: { route: string; token: string; payloadDirectory: string } previewLease?: PreviewLease previewRoutes?: PlaygroundPreviewRouteRegistry previewProxyDiagnostics?: PlaygroundPreviewProxyDiagnostics diff --git a/tests/disposable-mysql-mysqli.integration.test.ts b/tests/disposable-mysql-mysqli.integration.test.ts index 0a404e0a..92c4b861 100644 --- a/tests/disposable-mysql-mysqli.integration.test.ts +++ b/tests/disposable-mysql-mysqli.integration.test.ts @@ -77,7 +77,7 @@ if (!await dockerAvailable()) { await mkdir(join(plugin, "tests"), { recursive: true }) await writeFile(join(plugin, "bounded-phpunit-fixture.php"), "tests\n") - await writeFile(join(plugin, "tests", "bootstrap.php"), "xpath('//testsuite/file') ?: array() as $file)")) assert.ok(projectModeCode.includes("list($directories, $suffixes, $prefixes, $excludes, $configured_files) = wp_codebox_phpunit_parse_config")) assert.ok(projectModeCode.includes("$test_files = wp_codebox_phpunit_discover($directories, $suffixes, $prefixes, $excludes, $configured_files);")) diff --git a/tests/playground-cli-runner-bootstrap-ini.test.ts b/tests/playground-cli-runner-bootstrap-ini.test.ts index 918491de..9e94d29e 100644 --- a/tests/playground-cli-runner-bootstrap-ini.test.ts +++ b/tests/playground-cli-runner-bootstrap-ini.test.ts @@ -64,14 +64,12 @@ try { await server[Symbol.asyncDispose]() assert.equal(calls.length, 1) - assert.equal(calls[0]["mount-before-install"]?.length, 6) + assert.equal(calls[0]["mount-before-install"]?.length, 4) assert.equal(calls[0]["mount-before-install"]?.[0]?.vfsPath, "/internal/shared/php.ini") assert.equal(calls[0]["mount-before-install"]?.[1]?.vfsPath, "/internal/shared/wp-codebox-auto-prepend.php") - assert.equal(calls[0]["mount-before-install"]?.[2]?.vfsPath, "/internal/wp-codebox") - assert.match(calls[0]["mount-before-install"]?.[3]?.vfsPath ?? "", /^\/wordpress\/wp-codebox-execute-[a-f0-9]{24}\.php$/) // A wordpress-develop checkout is the runtime root, not an ordinary post-startup mount. - assert.equal(calls[0]["mount-before-install"]?.[4]?.vfsPath, "/wordpress/wp-config.php") - assert.deepEqual(calls[0]["mount-before-install"]?.[5], { hostPath: wordpressDevelopDirectory, vfsPath: "/wordpress" }) + assert.equal(calls[0]["mount-before-install"]?.[2]?.vfsPath, "/wordpress/wp-config.php") + assert.deepEqual(calls[0]["mount-before-install"]?.[3], { hostPath: wordpressDevelopDirectory, vfsPath: "/wordpress" }) assert.deepEqual(calls[0].mount, []) assert.equal(calls[0].workers, 6) assert.equal(calls[0].wordpressInstallMode, "do-not-attempt-installing") @@ -92,13 +90,7 @@ try { const sharedAutoPrepend = await readFile(sharedAutoPrependPath as string, "utf8") assert.match(sharedAutoPrepend, /require_once '\/internal\/shared\/auto_prepend_file\.php'/) assert.match(sharedAutoPrepend, /putenv\("TC_MYSQL_PORT=33060"\);/) - const requestWorkerPath = calls[0]["mount-before-install"]?.[3]?.hostPath - assert.equal(typeof requestWorkerPath, "string") - const requestWorker = await readFile(requestWorkerPath as string, "utf8") - assert.match(requestWorker, /HTTP_X_WP_CODEBOX_EXECUTION_TOKEN/) - assert.match(requestWorker, /hash_equals/) - assert.match(requestWorker, /\$_ENV\[\$wp_codebox_name\] = \$wp_codebox_value/) - const externalWpConfigPath = calls[0]["mount-before-install"]?.[4]?.hostPath + const externalWpConfigPath = calls[0]["mount-before-install"]?.[2]?.hostPath assert.equal(typeof externalWpConfigPath, "string") const externalWpConfig = await readFile(externalWpConfigPath as string, "utf8") assert.match(externalWpConfig, /define\('DB_HOST', "127\.0\.0\.1:33061"\)/) @@ -134,9 +126,7 @@ try { await downloadedWordPressServer[Symbol.asyncDispose]() assert.equal(calls.length, 1) - assert.equal(calls[0]["mount-before-install"]?.length, 2) - assert.equal(calls[0]["mount-before-install"]?.[0]?.vfsPath, "/internal/wp-codebox") - assert.match(calls[0]["mount-before-install"]?.[1]?.vfsPath ?? "", /^\/wordpress\/wp-codebox-execute-[a-f0-9]{24}\.php$/) + assert.equal(calls[0]["mount-before-install"], undefined) assert.equal(calls[0].wordpressInstallMode, undefined) assert.equal(shouldUseProgrammaticPlaygroundRunner(downloadedWordPressSpec), false) From 8662de9974e0002b7c89f8b3d17ed2820b6dc008 Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Tue, 21 Jul 2026 13:10:40 -0400 Subject: [PATCH 2/8] Decode wrapped PHPUnit bootstrap assertions --- tests/phpunit-project-autoload.test.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/phpunit-project-autoload.test.ts b/tests/phpunit-project-autoload.test.ts index 8a248812..63ab6fea 100644 --- a/tests/phpunit-project-autoload.test.ts +++ b/tests/phpunit-project-autoload.test.ts @@ -540,12 +540,17 @@ echo json_encode(array($legacy_project_autoload_file, $harness_autoload_file)); `) assert.deepEqual(JSON.parse(execFileSync("php", [canonicalHarnessProbe], { encoding: "utf8" })), ["", "/tmp/wp-codebox-inputs/0-wp-codebox-vendor-73845ca47d2f/autoload.php"], "a canonical staged harness path remains the harness in project mode") +function decodeSubmittedBootstrap(code: string): string { + const encodedBootstrap = code.match(/base64_decode\("([A-Za-z0-9+/=]+)"\)/)?.[1] + return encodedBootstrap ? Buffer.from(encodedBootstrap, "base64").toString("utf8") : code +} + let capturedCanonicalHarnessCode = "" await runPhpunitCommand({ artifactRoot: mkdtempSync(join(tmpdir(), "wp-codebox-phpunit-artifacts-")), mounts: [], runPlaygroundCommand: async (_command, _server, input) => { - capturedCanonicalHarnessCode = input.code + capturedCanonicalHarnessCode = decodeSubmittedBootstrap(input.code) return { text: "ok", exitCode: 0 } }, runtimeSpec: phpunitRuntimeSpec, @@ -573,7 +578,7 @@ await runPhpunitCommand({ artifactRoot: mkdtempSync(join(tmpdir(), "wp-codebox-phpunit-artifacts-")), mounts: [], runPlaygroundCommand: async (_command, _server, input) => { - capturedExplicitCode = input.code + capturedExplicitCode = decodeSubmittedBootstrap(input.code) return { text: "ok", exitCode: 0 } }, runtimeSpec: phpunitRuntimeSpec, From 72e2acd3cfda5ff7a7a16b164e3d6d49aad502b3 Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Tue, 21 Jul 2026 18:09:53 -0400 Subject: [PATCH 3/8] Preserve strict PHPUnit config handling after rebase --- .../src/phpunit-command-handlers.ts | 3 +++ tests/phpunit-project-autoload.test.ts | 11 +++-------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/packages/runtime-playground/src/phpunit-command-handlers.ts b/packages/runtime-playground/src/phpunit-command-handlers.ts index 0a92b984..78013154 100644 --- a/packages/runtime-playground/src/phpunit-command-handlers.ts +++ b/packages/runtime-playground/src/phpunit-command-handlers.ts @@ -837,6 +837,9 @@ function pg_run_project_bootstrap_stage(array $cfg): void { $from_config = $bootstrap !== ''; } if ($bootstrap === '') { + if (!$phpunit_xml_is_default && !is_readable($phpunit_xml)) { + throw new RuntimeException('explicit PHPUnit config is not readable: ' . $phpunit_xml); + } pg_log('NOTICE:project bootstrap not declared; continuing without one'); pg_stage_ok('project_bootstrap'); return; diff --git a/tests/phpunit-project-autoload.test.ts b/tests/phpunit-project-autoload.test.ts index 63ab6fea..1f325e89 100644 --- a/tests/phpunit-project-autoload.test.ts +++ b/tests/phpunit-project-autoload.test.ts @@ -540,17 +540,12 @@ echo json_encode(array($legacy_project_autoload_file, $harness_autoload_file)); `) assert.deepEqual(JSON.parse(execFileSync("php", [canonicalHarnessProbe], { encoding: "utf8" })), ["", "/tmp/wp-codebox-inputs/0-wp-codebox-vendor-73845ca47d2f/autoload.php"], "a canonical staged harness path remains the harness in project mode") -function decodeSubmittedBootstrap(code: string): string { - const encodedBootstrap = code.match(/base64_decode\("([A-Za-z0-9+/=]+)"\)/)?.[1] - return encodedBootstrap ? Buffer.from(encodedBootstrap, "base64").toString("utf8") : code -} - let capturedCanonicalHarnessCode = "" await runPhpunitCommand({ artifactRoot: mkdtempSync(join(tmpdir(), "wp-codebox-phpunit-artifacts-")), mounts: [], runPlaygroundCommand: async (_command, _server, input) => { - capturedCanonicalHarnessCode = decodeSubmittedBootstrap(input.code) + capturedCanonicalHarnessCode = input.code return { text: "ok", exitCode: 0 } }, runtimeSpec: phpunitRuntimeSpec, @@ -571,14 +566,14 @@ const decodedCanonicalHarnessCode = decodedBootstrapWrapper(capturedCanonicalHar assert.ok(decodedCanonicalHarnessCode.includes('$autoload_file = "/tmp/wp-codebox-inputs/0-wp-codebox-vendor-73845ca47d2f/autoload.php";')) assert.ok(decodedCanonicalHarnessCode.includes('$autoload_file_role = "harness";')) assert.ok(decodedCanonicalHarnessCode.includes('putenv("TC_MYSQL_PORT=3306");'), "runtime service environment is passed to the PHP executed by wordpress.phpunit") -assert.ok(decodedCanonicalHarnessCode.indexOf('putenv("TC_MYSQL_PORT=3306");') < decodedCanonicalHarnessCode.indexOf("require_once '/wordpress/wp-load.php';"), "runtime environment is available to project bootstrap code") +assert.ok(!decodedCanonicalHarnessCode.includes("require_once '/wordpress/wp-load.php';"), "project bootstrap mode does not load the managed WordPress runtime first") let capturedExplicitCode = "" await runPhpunitCommand({ artifactRoot: mkdtempSync(join(tmpdir(), "wp-codebox-phpunit-artifacts-")), mounts: [], runPlaygroundCommand: async (_command, _server, input) => { - capturedExplicitCode = decodeSubmittedBootstrap(input.code) + capturedExplicitCode = input.code return { text: "ok", exitCode: 0 } }, runtimeSpec: phpunitRuntimeSpec, From 9025ac7109346e0eceb1dde644718dd89c485fce Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Tue, 21 Jul 2026 20:58:51 -0400 Subject: [PATCH 4/8] Reject incomplete PHPUnit bootstrap runs --- packages/runtime-playground/src/playground-runtime.ts | 4 +++- packages/runtime-playground/src/wordpress-command-runners.ts | 3 +++ tests/phpunit-runtime-failure-diagnostics.test.ts | 4 ++-- .../playground-phpunit-bootstrap-failure.integration.test.ts | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/runtime-playground/src/playground-runtime.ts b/packages/runtime-playground/src/playground-runtime.ts index 076092c8..ad38f06e 100644 --- a/packages/runtime-playground/src/playground-runtime.ts +++ b/packages/runtime-playground/src/playground-runtime.ts @@ -1898,9 +1898,11 @@ echo json_encode(array('command' => 'inspect-mounted-inputs', 'mounts' => $inspe function executionSpecWithEnvironment(spec: ExecutionSpec): ExecutionSpec { if (!spec.environment || Object.keys(spec.environment).length === 0) return spec + const args = spec.args ?? [] + const runtimeEnvironment = JSON.parse(argValue(args, "runtime-env-json") ?? "{}") as Record return { ...spec, - args: [...(spec.args ?? []).filter((argument) => !argument.startsWith("runtime-env-json=")), `runtime-env-json=${JSON.stringify(spec.environment)}`], + args: [...args.filter((argument) => !argument.startsWith("runtime-env-json=")), `runtime-env-json=${JSON.stringify({ ...runtimeEnvironment, ...spec.environment })}`], } } diff --git a/packages/runtime-playground/src/wordpress-command-runners.ts b/packages/runtime-playground/src/wordpress-command-runners.ts index a85fccf1..6036bbd1 100644 --- a/packages/runtime-playground/src/wordpress-command-runners.ts +++ b/packages/runtime-playground/src/wordpress-command-runners.ts @@ -966,6 +966,9 @@ export async function runPhpunitCommand({ } throw error } + if (structured) { + throw attachPlaygroundDiagnostics(new Error("wordpress.phpunit terminated before completing bootstrap"), "wordpress.phpunit structured diagnostics", structured) + } return response.text } diff --git a/tests/phpunit-runtime-failure-diagnostics.test.ts b/tests/phpunit-runtime-failure-diagnostics.test.ts index 93ec3c23..4b3d6516 100644 --- a/tests/phpunit-runtime-failure-diagnostics.test.ts +++ b/tests/phpunit-runtime-failure-diagnostics.test.ts @@ -15,7 +15,7 @@ await assert.rejects( mounts: [], runPlaygroundCommand: async (_command, _server, input) => { submittedCode = input.code - return { exitCode: 1, errors: "", text: "" } + return { exitCode: 0, errors: "", text: "" } }, runtimeSpec: { environment: { kind: "wordpress", name: "test", version: "latest" }, policy: { commands: ["wordpress.phpunit"] } } as never, server: { @@ -29,7 +29,7 @@ await assert.rejects( spec: { command: "wordpress.phpunit", args: ["plugin-slug=demo-plugin"] }, }), (error: Error) => { - assert.match(error.message, /wordpress\.phpunit failed with exit code 1/) + assert.match(error.message, /wordpress\.phpunit terminated before completing bootstrap/) assert.match(error.message, /wordpress\.phpunit structured diagnostics/) assert.match(error.message, /Bootstrap failed with token: \[redacted\]/) assert.match(error.message, /\[diagnostic truncated\]/) diff --git a/tests/playground-phpunit-bootstrap-failure.integration.test.ts b/tests/playground-phpunit-bootstrap-failure.integration.test.ts index d1a3d140..a4c7031f 100644 --- a/tests/playground-phpunit-bootstrap-failure.integration.test.ts +++ b/tests/playground-phpunit-bootstrap-failure.integration.test.ts @@ -52,7 +52,7 @@ try { assert.match(commandLog, /PHPUnit bootstrap fixture token: \[redacted\]/) assert.doesNotMatch(commandLog, new RegExp(secret)) - await writeFile(fatalMuPlugin, ` Date: Tue, 21 Jul 2026 22:56:53 -0400 Subject: [PATCH 5/8] Require real scoped PHPUnit execution --- packages/cli/src/recipe-validation.ts | 10 +++-- packages/cli/src/runtime-services.ts | 4 +- packages/runtime-core/src/recipe-schema.ts | 7 +++- .../runtime-core/src/runtime-contracts.ts | 4 +- .../src/phpunit-command-handlers.ts | 42 ++++++++++++++++--- .../src/wordpress-command-runners.ts | 6 ++- tests/phpunit-project-autoload.test.ts | 28 +++++++++++-- ...hpunit-runtime-failure-diagnostics.test.ts | 13 ++++++ tests/runtime-services.test.ts | 9 +++- 9 files changed, 104 insertions(+), 19 deletions(-) diff --git a/packages/cli/src/recipe-validation.ts b/packages/cli/src/recipe-validation.ts index 74411ff4..ad10c3cd 100644 --- a/packages/cli/src/recipe-validation.ts +++ b/packages/cli/src/recipe-validation.ts @@ -743,11 +743,13 @@ function validateRecipeRuntimeServices(recipe: WorkspaceRecipe, addIssue: (code: if (ids.has(service.id)) addIssue("duplicate-runtime-service-id", `${path}.id`, `Runtime service ids must be unique: ${service.id}`) ids.add(service.id) if (service.kind !== "mysql") addIssue("unsupported-runtime-service-kind", `${path}.kind`, `Unsupported managed runtime service kind: ${service.kind}`) - for (const [output, name] of Object.entries(service.outputs)) { + for (const [output, names] of Object.entries(service.outputs)) { if (!/^(host|port|username|password|database)$/.test(output)) addIssue("unknown-runtime-service-output", `${path}.outputs.${output}`, `Unsupported ${service.kind} service output: ${output}`) - if (!/^[A-Z_][A-Z0-9_]*$/.test(name)) addIssue("invalid-runtime-service-env", `${path}.outputs.${output}`, "Runtime service environment variable names must match /^[A-Z_][A-Z0-9_]*$/.") - if (environment.has(name)) addIssue("duplicate-runtime-service-env", `${path}.outputs.${output}`, `Runtime service output environment variable is already declared: ${name}`) - environment.add(name) + for (const name of Array.isArray(names) ? names : [names]) { + if (!/^[A-Z_][A-Z0-9_]*$/.test(name)) addIssue("invalid-runtime-service-env", `${path}.outputs.${output}`, "Runtime service environment variable names must match /^[A-Z_][A-Z0-9_]*$/.") + if (environment.has(name)) addIssue("duplicate-runtime-service-env", `${path}.outputs.${output}`, `Runtime service output environment variable is already declared: ${name}`) + environment.add(name) + } } } } diff --git a/packages/cli/src/runtime-services.ts b/packages/cli/src/runtime-services.ts index 90ed2758..fc955371 100644 --- a/packages/cli/src/runtime-services.ts +++ b/packages/cli/src/runtime-services.ts @@ -61,7 +61,7 @@ const defaultDependencies: RuntimeServiceDependencies = { randomBytes, } -export function runtimeServicePlan(services: WorkspaceRecipeRuntimeService[]): Array<{ id: string; kind: string; provider: string; version: string; bind: "loopback"; port: "ephemeral"; persistentVolume: false; configuration?: WorkspaceRecipeRuntimeService["configuration"]; outputs: Record }> { +export function runtimeServicePlan(services: WorkspaceRecipeRuntimeService[]): Array<{ id: string; kind: string; provider: string; version: string; bind: "loopback"; port: "ephemeral"; persistentVolume: false; configuration?: WorkspaceRecipeRuntimeService["configuration"]; outputs: Record }> { return services.map((service) => { const provider = runtimeServiceProvider(service.kind) return { id: service.id, kind: service.kind, provider: provider.name, version: provider.version(service), bind: "loopback", port: "ephemeral", persistentVolume: false, ...(service.configuration ? { configuration: service.configuration } : {}), outputs: service.outputs } @@ -177,7 +177,7 @@ async function provisionMysqlDockerService(service: WorkspaceRecipeRuntimeServic evidence.readiness = "ready" evidence.lifecycle = "provisioned" const values: Record = { host: "127.0.0.1", port: String(port), username: "runtime", password, database: "runtime" } - return { env: Object.fromEntries(Object.entries(service.outputs).map(([output, name]) => [name, values[output] ?? ""])), evidence, async release() { await releaseService(container, evidence, dependencies) } } + return { env: Object.fromEntries(Object.entries(service.outputs).flatMap(([output, names]) => (Array.isArray(names) ? names : [names]).map((name) => [name, values[output] ?? ""]))), evidence, async release() { await releaseService(container, evidence, dependencies) } } } catch (error) { evidence.readiness = "failed" evidence.lifecycle = "failed" diff --git a/packages/runtime-core/src/recipe-schema.ts b/packages/runtime-core/src/recipe-schema.ts index 083b7d3a..ecce763a 100644 --- a/packages/runtime-core/src/recipe-schema.ts +++ b/packages/runtime-core/src/recipe-schema.ts @@ -930,7 +930,12 @@ export function createWorkspaceRecipeJsonSchema(options: WorkspaceRecipeJsonSche outputs: { type: "object", minProperties: 1, - additionalProperties: { type: "string", pattern: "^[A-Z_][A-Z0-9_]*$" }, + additionalProperties: { + anyOf: [ + { type: "string", pattern: "^[A-Z_][A-Z0-9_]*$" }, + { type: "array", minItems: 1, uniqueItems: true, items: { type: "string", pattern: "^[A-Z_][A-Z0-9_]*$" } }, + ], + }, }, }, }, diff --git a/packages/runtime-core/src/runtime-contracts.ts b/packages/runtime-core/src/runtime-contracts.ts index 3e018a64..6d40a0e3 100644 --- a/packages/runtime-core/src/runtime-contracts.ts +++ b/packages/runtime-core/src/runtime-contracts.ts @@ -142,8 +142,8 @@ export interface WorkspaceRecipeRuntimeService { rootAuthentication?: "generated-password" | "empty-password" foreignKeyTargetPolicy?: "unique-only" | "indexed" } - /** Explicit map from a provider output (for example `port`) to a runtime env name. */ - outputs: Record + /** Explicit map from a provider output (for example `port`) to one or more runtime env names. */ + outputs: Record } export interface WorkspaceRecipeRuntimeStack { diff --git a/packages/runtime-playground/src/phpunit-command-handlers.ts b/packages/runtime-playground/src/phpunit-command-handlers.ts index 78013154..30c2c05c 100644 --- a/packages/runtime-playground/src/phpunit-command-handlers.ts +++ b/packages/runtime-playground/src/phpunit-command-handlers.ts @@ -97,7 +97,7 @@ function phpunitConfigDiscoveryPhp(options: PhpunitConfigDiscoveryPhpOptions): s const suffixAssignment = options.replaceDefaultMatchers ? "$suffixes = $config_suffixes;" : "$suffixes = array_merge($suffixes, $config_suffixes);" const prefixAssignment = options.replaceDefaultMatchers ? "$prefixes = $config_prefixes;" : "$prefixes = array_merge($prefixes, $config_prefixes);" - return `function ${options.functionName}($xml_path, $test_dir_default) { + return `function ${options.functionName}($xml_path, $test_dir_default, $requested_suites = array()) { $directories = array($test_dir_default); $suffixes = array('Test.php'); $prefixes = array('test-'); @@ -118,11 +118,23 @@ function phpunitConfigDiscoveryPhp(options: PhpunitConfigDiscoveryPhpOptions): s $first = $errors ? trim($errors[0]->message) : 'unknown'; throw new RuntimeException('PHPUnit config could not be parsed at ' . $xml_path . ': ' . $first); } + $suite_nodes = $xml->xpath('//testsuite') ?: array(); + if (!empty($requested_suites)) { + $requested_lookup = array_fill_keys(array_map('strval', $requested_suites), true); + $suite_nodes = array_values(array_filter($suite_nodes, static function($suite) use ($requested_lookup) { + return isset($requested_lookup[(string) ($suite['name'] ?? '')]); + })); + if (empty($suite_nodes)) { + throw new RuntimeException('Requested PHPUnit testsuite was not found in config: ' . implode(',', array_keys($requested_lookup))); + } + } + $directories = array(); $base = ${options.basePathExpression}; $config_dirs = array(); $config_suffixes = array(); $config_prefixes = array(); - foreach ($xml->xpath('//testsuite/directory') ?: array() as $dir) { + foreach ($suite_nodes as $suite) { + foreach ($suite->xpath('./directory') ?: array() as $dir) { $raw = trim((string) $dir);${directoryRestriction} $config_dirs[] = $raw[0] === '/' ? rtrim($raw, '/') : rtrim($base . '/' . $raw, '/'); foreach (explode(',', (string) ($dir['suffix'] ?? '')) as $suffix) { @@ -137,18 +149,23 @@ function phpunitConfigDiscoveryPhp(options: PhpunitConfigDiscoveryPhpOptions): s $config_prefixes[] = $prefix; } } + } } - foreach ($xml->xpath('//testsuite/exclude') ?: array() as $exclude) { + foreach ($suite_nodes as $suite) { + foreach ($suite->xpath('./exclude') ?: array() as $exclude) { $raw = trim((string) $exclude); if ($raw !== '') { $excludes[] = $raw[0] === '/' ? rtrim($raw, '/') : rtrim($base . '/' . $raw, '/'); } + } } - foreach ($xml->xpath('//testsuite/file') ?: array() as $file) { + foreach ($suite_nodes as $suite) { + foreach ($suite->xpath('./file') ?: array() as $file) { $raw = trim((string) $file); if ($raw !== '') { $files[] = $raw[0] === '/' ? $raw : $base . '/' . $raw; } + } } if (!empty($config_dirs)) { $directories = $config_dirs; @@ -1227,13 +1244,28 @@ ${phpunitConfigDiscoveryPhp({ ${phpunitDiscoveryPhp("wp_codebox_phpunit_discover", "pg_log")} +function wp_codebox_requested_phpunit_suites($args): array { + $suites = array(); + if (!is_array($args)) { + return $suites; + } + foreach ($args as $index => $arg) { + if ($arg === '--testsuite' && isset($args[$index + 1])) { + $suites = array_merge($suites, explode(',', (string) $args[$index + 1])); + } elseif (is_string($arg) && strpos($arg, '--testsuite=') === 0) { + $suites = array_merge($suites, explode(',', substr($arg, strlen('--testsuite=')))); + } + } + return array_values(array_filter(array_map('trim', $suites), static function($suite) { return $suite !== ''; })); +} + pg_stage_begin('discover_tests'); try { $test_dir = $test_root; if (!is_dir($test_dir)) { throw new RuntimeException('configured PHPUnit test root is not a readable directory: ' . $test_dir); } - list($directories, $suffixes, $prefixes, $excludes, $configured_files) = wp_codebox_phpunit_parse_config(${JSON.stringify(options.phpunitXml)}, $test_dir); + list($directories, $suffixes, $prefixes, $excludes, $configured_files) = wp_codebox_phpunit_parse_config(${JSON.stringify(options.phpunitXml)}, $test_dir, wp_codebox_requested_phpunit_suites($phpunit_args_raw)); $test_files = wp_codebox_phpunit_discover($directories, $suffixes, $prefixes, $excludes, $configured_files); $test_files = pg_filter_changed_test_files($test_files, $changed_test_files_raw, $test_dir); if ($selected_test_file !== '') { diff --git a/packages/runtime-playground/src/wordpress-command-runners.ts b/packages/runtime-playground/src/wordpress-command-runners.ts index 6036bbd1..ee9c80f7 100644 --- a/packages/runtime-playground/src/wordpress-command-runners.ts +++ b/packages/runtime-playground/src/wordpress-command-runners.ts @@ -913,6 +913,7 @@ export async function runPhpunitCommand({ const bootstrapMode = argValue(args, "bootstrap-mode")?.trim() || "managed" const autoloadFile = argValue(args, "autoload-file")?.trim() || (bootstrapMode === "project" ? "" : "/wp-codebox-vendor/autoload.php") const autoloadFileRole = argValue(args, "autoload-file-role")?.trim() === "harness" ? "harness" : undefined + const phpunitArgs = jsonArrayArg(args, "phpunit-args-json").filter((value): value is string => typeof value === "string") const processIdentity = boundedProcessIdentity(spec.processIdentity) const resultFile = processIdentity ? `/tmp/wp-codebox-phpunit-result-${processIdentity}.txt` : PLUGIN_PHPUNIT_RESULT_FILE const diagnosticHostFile = `/wordpress/wp-content/plugins/${pluginSlug}/.pg-test-result${processIdentity ? `-${processIdentity}` : ""}.txt` @@ -928,7 +929,7 @@ export async function runPhpunitCommand({ phpunitXmlIsDefault: phpunitXmlArg === undefined || booleanArg(args, "phpunit-xml-default"), selectedTestFile: argValue(args, "test-file")?.trim() || "", changedTestFiles: changedTestFilesArg(args), - phpunitArgs: jsonArrayArg(args, "phpunit-args-json").filter((value): value is string => typeof value === "string"), + phpunitArgs, env: jsonObjectArg(args, "env-json"), wpConfigDefines: jsonObjectArg(args, "wp-config-defines-json"), dependencyMounts: commaListArg(args, "dependency-mounts"), @@ -969,6 +970,9 @@ export async function runPhpunitCommand({ if (structured) { throw attachPlaygroundDiagnostics(new Error("wordpress.phpunit terminated before completing bootstrap"), "wordpress.phpunit structured diagnostics", structured) } + if (!phpunitArgs.includes("--list-tests") && !/\bOK(?:, but there were issues!)? \([1-9]\d* tests?,/m.test(response.text)) { + throw new Error("wordpress.phpunit exited successfully without a non-zero PHPUnit test summary") + } return response.text } diff --git a/tests/phpunit-project-autoload.test.ts b/tests/phpunit-project-autoload.test.ts index 1f325e89..efa8e27c 100644 --- a/tests/phpunit-project-autoload.test.ts +++ b/tests/phpunit-project-autoload.test.ts @@ -136,6 +136,27 @@ echo "ok\n"; assert.equal(execFileSync("php", [scriptPath], { encoding: "utf8" }), "ok\n") } +function assertNamedTestsuiteScopesDiscovery(source: string): void { + const tempDir = mkdtempSync(join(tmpdir(), "wp-codebox-named-testsuite-")) + const selected = join(tempDir, "SelectedTest.php") + const unrelated = join(tempDir, "UnrelatedTest.php") + const config = join(tempDir, "phpunit.xml") + const scriptPath = join(tempDir, "assert-named-testsuite.php") + writeFileSync(selected, "SelectedTest.phpUnrelatedTest.php`) + writeFileSync(scriptPath, `xpath('//testsuite/file') ?: array() as $file)")) +assert.ok(projectModeCode.includes("foreach ($suite->xpath('./file') ?: array() as $file)")) assert.ok(projectModeCode.includes("list($directories, $suffixes, $prefixes, $excludes, $configured_files) = wp_codebox_phpunit_parse_config")) assert.ok(projectModeCode.includes("$test_files = wp_codebox_phpunit_discover($directories, $suffixes, $prefixes, $excludes, $configured_files);")) assert.ok(projectModeCode.includes("' files=' . count($configured_files)")) @@ -492,6 +513,7 @@ assert.equal(projectModeCode.match(/return array\(\$directories, \$suffixes, \$p assert.equal(projectModeCode.match(/return \$return_values\(\);/g)?.length, 1) assert.match(projectModeCode, /configured PHPUnit test root is not a readable directory/) assertPhpunitConfigurationAndDiscoveryFailures(projectModeCode, "wp_codebox_phpunit_parse_config", "wp_codebox_phpunit_discover", "pg_log", false) +assertNamedTestsuiteScopesDiscovery(projectModeCode) assertChangedScopeNoOp(projectModeCode, "pg_filter_changed_test_files", "pg_component_relative_path") assertSelectedTestFileResolution(projectModeCode) @@ -546,7 +568,7 @@ await runPhpunitCommand({ mounts: [], runPlaygroundCommand: async (_command, _server, input) => { capturedCanonicalHarnessCode = input.code - return { text: "ok", exitCode: 0 } + return { text: "OK (1 test, 1 assertion)", exitCode: 0 } }, runtimeSpec: phpunitRuntimeSpec, server: { playground: {} } as never, @@ -574,7 +596,7 @@ await runPhpunitCommand({ mounts: [], runPlaygroundCommand: async (_command, _server, input) => { capturedExplicitCode = input.code - return { text: "ok", exitCode: 0 } + return { text: "OK (1 test, 1 assertion)", exitCode: 0 } }, runtimeSpec: phpunitRuntimeSpec, server: { playground: {} } as never, diff --git a/tests/phpunit-runtime-failure-diagnostics.test.ts b/tests/phpunit-runtime-failure-diagnostics.test.ts index 4b3d6516..f6fd7373 100644 --- a/tests/phpunit-runtime-failure-diagnostics.test.ts +++ b/tests/phpunit-runtime-failure-diagnostics.test.ts @@ -49,4 +49,17 @@ const preBootstrapRecorder = decodedBootstrap.indexOf("STAGE_FATAL:bootstrap:") const wordpressBootstrap = decodedBootstrap.indexOf("require_once '/wordpress/wp-load.php';") assert.ok(preBootstrapRecorder >= 0 && preBootstrapRecorder < wordpressBootstrap, "fatal diagnostics must be recorded before the WordPress bootstrap boundary") +const emptySuccessArtifactRoot = await mkdtemp(join(tmpdir(), "wp-codebox-phpunit-empty-success-")) +await assert.rejects( + () => runPhpunitCommand({ + artifactRoot: emptySuccessArtifactRoot, + mounts: [], + runPlaygroundCommand: async () => ({ exitCode: 0, errors: "", text: "WPCOM Codebox PHPUnit shutdown: mysql_port=unset" }), + runtimeSpec: { environment: { kind: "wordpress", name: "test", version: "latest" }, policy: { commands: ["wordpress.phpunit"] } } as never, + server: { playground: { readFileAsText: async () => { throw new Error("missing") } } } as never, + spec: { command: "wordpress.phpunit", args: ["plugin-slug=demo-plugin"] }, + }), + /exited successfully without a non-zero PHPUnit test summary/, +) + console.log("phpunit runtime failure diagnostics ok") diff --git a/tests/runtime-services.test.ts b/tests/runtime-services.test.ts index 27552adc..23af5540 100644 --- a/tests/runtime-services.test.ts +++ b/tests/runtime-services.test.ts @@ -18,6 +18,9 @@ assert.throws(() => parseLoopbackPort("0.0.0.0:3306"), /loopback/) const valid = validateWorkspaceRecipeJsonSchema({ schema: "wp-codebox/workspace-recipe/v1", inputs: { services: [service] }, workflow: { steps: [{ command: "wordpress.run-php" }] } }) assert.equal(valid.valid, true) +const aliasedPortService = { ...service, outputs: { ...service.outputs, port: ["DB_PORT", "TC_MYSQL_PORT"] } } +assert.equal(validateWorkspaceRecipeJsonSchema({ schema: "wp-codebox/workspace-recipe/v1", inputs: { services: [aliasedPortService] }, workflow: { steps: [{ command: "wordpress.run-php" }] } }).valid, true) +assert.equal(validateWorkspaceRecipeJsonSchema({ schema: "wp-codebox/workspace-recipe/v1", inputs: { services: [{ ...service, outputs: { port: [] } }] }, workflow: { steps: [{ command: "wordpress.run-php" }] } }).valid, false) const unsafe = validateWorkspaceRecipeJsonSchema({ schema: "wp-codebox/workspace-recipe/v1", inputs: { services: [{ ...service, outputs: { port: "bad-name" } }] }, workflow: { steps: [{ command: "wordpress.run-php" }] } }) assert.equal(unsafe.valid, false) const emptyRootService = { ...service, configuration: { rootAuthentication: "empty-password" as const } } @@ -103,6 +106,10 @@ const dependencies: RuntimeServiceDependencies = { const provisioned = await provisionRuntimeServices([service], { dependencies }) assert.equal(provisioned.env.DB_PORT, "41001") assert.equal(provisioned.env.DB_PASSWORD, Buffer.alloc(24, 7).toString("base64url")) +const aliasedPort = await provisionRuntimeServices([aliasedPortService], { dependencies }) +assert.equal(aliasedPort.env.DB_PORT, "41001") +assert.equal(aliasedPort.env.TC_MYSQL_PORT, "41001") +await aliasedPort.release() const runCall = calls.find((call) => call.args[0] === "run") assert.ok(runCall?.args.includes("MYSQL_PASSWORD")) assert.ok(runCall?.args.includes("127.0.0.1::3306"), "Docker publishes MySQL on a loopback ephemeral port") @@ -118,7 +125,7 @@ assert.equal(runCall?.env?.DOCKER_HOST, process.env.DOCKER_HOST, "Docker provide assert.equal(calls[0]?.args[0], "image", "the provider checks the image before starting the service") await provisioned.release() await provisioned.release() -assert.equal(calls.filter((call) => call.args[0] === "rm").length, 1, "release is idempotent") +assert.equal(calls.filter((call) => call.args[0] === "rm").length, 2, "each service is released exactly once") const emptyRootCalls: Array<{ args: string[]; env?: NodeJS.ProcessEnv }> = [] const emptyRootDependencies: RuntimeServiceDependencies = { From e7b41708937199fdbda73728de2cea06ec5ec921 Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Wed, 22 Jul 2026 18:33:07 -0400 Subject: [PATCH 6/8] Preserve PHPUnit zero-summary diagnostics --- .../src/wordpress-command-runners.ts | 6 +++++- ...hpunit-runtime-failure-diagnostics.test.ts | 20 +++++++++++++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/packages/runtime-playground/src/wordpress-command-runners.ts b/packages/runtime-playground/src/wordpress-command-runners.ts index ee9c80f7..0b87aed7 100644 --- a/packages/runtime-playground/src/wordpress-command-runners.ts +++ b/packages/runtime-playground/src/wordpress-command-runners.ts @@ -971,7 +971,11 @@ export async function runPhpunitCommand({ throw attachPlaygroundDiagnostics(new Error("wordpress.phpunit terminated before completing bootstrap"), "wordpress.phpunit structured diagnostics", structured) } if (!phpunitArgs.includes("--list-tests") && !/\bOK(?:, but there were issues!)? \([1-9]\d* tests?,/m.test(response.text)) { - throw new Error("wordpress.phpunit exited successfully without a non-zero PHPUnit test summary") + const diagnostics = [ + response.errors?.trim() ? `--- stderr ---\n${response.errors.trim()}` : "", + response.text.trim() ? `--- stdout ---\n${response.text.trim()}` : "", + ].filter(Boolean).join("\n") + throw attachPlaygroundDiagnostics(new Error("wordpress.phpunit exited successfully without a non-zero PHPUnit test summary"), "wordpress.phpunit successful response diagnostics", diagnostics) } return response.text diff --git a/tests/phpunit-runtime-failure-diagnostics.test.ts b/tests/phpunit-runtime-failure-diagnostics.test.ts index f6fd7373..32b68ee0 100644 --- a/tests/phpunit-runtime-failure-diagnostics.test.ts +++ b/tests/phpunit-runtime-failure-diagnostics.test.ts @@ -50,16 +50,32 @@ const wordpressBootstrap = decodedBootstrap.indexOf("require_once '/wordpress/wp assert.ok(preBootstrapRecorder >= 0 && preBootstrapRecorder < wordpressBootstrap, "fatal diagnostics must be recorded before the WordPress bootstrap boundary") const emptySuccessArtifactRoot = await mkdtemp(join(tmpdir(), "wp-codebox-phpunit-empty-success-")) +const successfulResponseSecret = "ghp_abcdefghijklmnopqrstuvwxyz1234567890" await assert.rejects( () => runPhpunitCommand({ artifactRoot: emptySuccessArtifactRoot, mounts: [], - runPlaygroundCommand: async () => ({ exitCode: 0, errors: "", text: "WPCOM Codebox PHPUnit shutdown: mysql_port=unset" }), + runPlaygroundCommand: async () => ({ + exitCode: 0, + errors: `PHPUnit stderr token=${successfulResponseSecret}`, + text: `WPCOM Codebox PHPUnit shutdown: mysql_port=unset\n${"x".repeat(25_000)}`, + }), runtimeSpec: { environment: { kind: "wordpress", name: "test", version: "latest" }, policy: { commands: ["wordpress.phpunit"] } } as never, server: { playground: { readFileAsText: async () => { throw new Error("missing") } } } as never, spec: { command: "wordpress.phpunit", args: ["plugin-slug=demo-plugin"] }, }), - /exited successfully without a non-zero PHPUnit test summary/, + (error: Error) => { + assert.match(error.message, /exited successfully without a non-zero PHPUnit test summary/) + assert.match(error.message, /wordpress\.phpunit successful response diagnostics/) + assert.match(error.message, /--- stderr ---/) + assert.match(error.message, /--- stdout ---/) + assert.match(error.message, /WPCOM Codebox \[redacted\] shutdown: mysql_port=unset/) + assert.match(error.message, /PHPUnit stderr token=\[redacted\]/) + assert.match(error.message, /\[diagnostic truncated\]/) + assert.doesNotMatch(error.message, new RegExp(successfulResponseSecret)) + assert.ok(error.message.length < 21_000, "successful response diagnostics must remain bounded") + return true + }, ) console.log("phpunit runtime failure diagnostics ok") From ddae9e8513ec81c3227b21694ef816bda4691967 Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Wed, 22 Jul 2026 18:44:34 -0400 Subject: [PATCH 7/8] Bound PHPUnit response streams independently --- .../src/wordpress-command-runners.ts | 13 +++++++++---- tests/phpunit-runtime-failure-diagnostics.test.ts | 6 +++--- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/packages/runtime-playground/src/wordpress-command-runners.ts b/packages/runtime-playground/src/wordpress-command-runners.ts index 0b87aed7..b26952ad 100644 --- a/packages/runtime-playground/src/wordpress-command-runners.ts +++ b/packages/runtime-playground/src/wordpress-command-runners.ts @@ -971,16 +971,21 @@ export async function runPhpunitCommand({ throw attachPlaygroundDiagnostics(new Error("wordpress.phpunit terminated before completing bootstrap"), "wordpress.phpunit structured diagnostics", structured) } if (!phpunitArgs.includes("--list-tests") && !/\bOK(?:, but there were issues!)? \([1-9]\d* tests?,/m.test(response.text)) { - const diagnostics = [ - response.errors?.trim() ? `--- stderr ---\n${response.errors.trim()}` : "", - response.text.trim() ? `--- stdout ---\n${response.text.trim()}` : "", - ].filter(Boolean).join("\n") + const diagnostics = successfulPhpunitResponseDiagnostics(response) throw attachPlaygroundDiagnostics(new Error("wordpress.phpunit exited successfully without a non-zero PHPUnit test summary"), "wordpress.phpunit successful response diagnostics", diagnostics) } return response.text } +function successfulPhpunitResponseDiagnostics(response: PlaygroundRunResponse): string { + const boundedStream = (value: string): string => value.length > 9_000 ? `${value.slice(0, 9_000)}\n[stream truncated]` : value + return [ + response.errors?.trim() ? `--- stderr ---\n${boundedStream(response.errors.trim())}` : "", + response.text.trim() ? `--- stdout ---\n${boundedStream(response.text.trim())}` : "", + ].filter(Boolean).join("\n") +} + function boundedProcessIdentity(value: string | undefined): string { if (!value) return "" if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(value)) throw new Error(`Invalid PHPUnit process identity: ${value}`) diff --git a/tests/phpunit-runtime-failure-diagnostics.test.ts b/tests/phpunit-runtime-failure-diagnostics.test.ts index 32b68ee0..ff21cb22 100644 --- a/tests/phpunit-runtime-failure-diagnostics.test.ts +++ b/tests/phpunit-runtime-failure-diagnostics.test.ts @@ -57,7 +57,7 @@ await assert.rejects( mounts: [], runPlaygroundCommand: async () => ({ exitCode: 0, - errors: `PHPUnit stderr token=${successfulResponseSecret}`, + errors: `PHPUnit stderr token=${successfulResponseSecret}\n${"e".repeat(25_000)}`, text: `WPCOM Codebox PHPUnit shutdown: mysql_port=unset\n${"x".repeat(25_000)}`, }), runtimeSpec: { environment: { kind: "wordpress", name: "test", version: "latest" }, policy: { commands: ["wordpress.phpunit"] } } as never, @@ -71,9 +71,9 @@ await assert.rejects( assert.match(error.message, /--- stdout ---/) assert.match(error.message, /WPCOM Codebox \[redacted\] shutdown: mysql_port=unset/) assert.match(error.message, /PHPUnit stderr token=\[redacted\]/) - assert.match(error.message, /\[diagnostic truncated\]/) + assert.match(error.message, /\[stream truncated\]/) assert.doesNotMatch(error.message, new RegExp(successfulResponseSecret)) - assert.ok(error.message.length < 21_000, "successful response diagnostics must remain bounded") + assert.ok(error.message.length < 19_000, "successful response diagnostics must remain bounded") return true }, ) From 66ae688bebaac1f04d74a6d54bd7b1fd0c69c780 Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Wed, 22 Jul 2026 23:08:41 -0400 Subject: [PATCH 8/8] Support TeamCity PHPUnit runtime contracts --- packages/cli/src/commands/recipe-build.ts | 4 +++- packages/runtime-core/src/recipe-builders.ts | 4 +++- .../runtime-playground/src/wordpress-command-runners.ts | 6 +++++- tests/phpunit-runtime-failure-diagnostics.test.ts | 5 ++++- tests/runtime-services.test.ts | 1 + 5 files changed, 16 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/commands/recipe-build.ts b/packages/cli/src/commands/recipe-build.ts index 40dee3eb..9e091aba 100644 --- a/packages/cli/src/commands/recipe-build.ts +++ b/packages/cli/src/commands/recipe-build.ts @@ -1,5 +1,5 @@ import { readFile, writeFile } from "node:fs/promises" -import { buildGenericAbilityRuntimeRunRecipe, buildRuntimePackageRunRecipe, buildWordPressBenchRecipe, buildWordPressPhpunitRecipe, compileRecipeTemplate, type GenericAbilityRuntimeRunOptions, type RecipeTemplateInput, type RuntimePackageRunRecipeOptions, type RuntimeWordPressInstallMode, type WorkspaceRecipe, type WorkspaceRecipeExtraPlugin, type WorkspaceRecipeMount, type WorkspaceRecipePHPWasmExtensionManifest, type WorkspaceRecipeRuntimeBackendPackage, type WorkspaceRecipeRuntimeService, type WorkspaceRecipeStep } from "@automattic/wp-codebox-core" +import { buildGenericAbilityRuntimeRunRecipe, buildRuntimePackageRunRecipe, buildWordPressBenchRecipe, buildWordPressPhpunitRecipe, compileRecipeTemplate, type GenericAbilityRuntimeRunOptions, type RecipeTemplateInput, type RuntimePackageRunRecipeOptions, type RuntimeWordPressInstallMode, type WorkspaceRecipe, type WorkspaceRecipeExtraPlugin, type WorkspaceRecipeMount, type WorkspaceRecipePHPWasmExtensionManifest, type WorkspaceRecipePluginRuntime, type WorkspaceRecipeRuntimeBackendPackage, type WorkspaceRecipeRuntimeService, type WorkspaceRecipeStep } from "@automattic/wp-codebox-core" interface RecipeBuildOptions { recipeType: "phpunit" | "bench" | "template" | "generic-ability-runtime-run" | "runtime-package-run" @@ -16,6 +16,7 @@ interface WordPressPhpunitBuilderOptions { backendPackage?: WorkspaceRecipeRuntimeBackendPackage mounts?: WorkspaceRecipeMount[] services?: WorkspaceRecipeRuntimeService[] + pluginRuntime?: WorkspaceRecipePluginRuntime extra_plugins?: WorkspaceRecipeExtraPlugin[] pluginSource?: string pluginSlug: string @@ -86,6 +87,7 @@ function buildRecipe(recipeType: RecipeBuildOptions["recipeType"], options: Word backendPackage: phpunitOptions.backendPackage, mounts: Array.isArray(phpunitOptions.mounts) ? phpunitOptions.mounts : [], services: Array.isArray(phpunitOptions.services) ? phpunitOptions.services : [], + pluginRuntime: phpunitOptions.pluginRuntime ? plainObject(phpunitOptions.pluginRuntime) as WorkspaceRecipePluginRuntime : undefined, extra_plugins: Array.isArray(phpunitOptions.extra_plugins) ? phpunitOptions.extra_plugins : [], pluginSource: stringOrUndefined(phpunitOptions.pluginSource), pluginSlug: requiredString(phpunitOptions.pluginSlug, "pluginSlug"), diff --git a/packages/runtime-core/src/recipe-builders.ts b/packages/runtime-core/src/recipe-builders.ts index da5b18e1..fe6b890c 100644 --- a/packages/runtime-core/src/recipe-builders.ts +++ b/packages/runtime-core/src/recipe-builders.ts @@ -1,4 +1,4 @@ -import type { RuntimePreviewSpec, RuntimeWordPressInstallMode, WorkspaceRecipe, WorkspaceRecipeExtraPlugin, WorkspaceRecipeMount, WorkspaceRecipePHPWasmExtensionManifest, WorkspaceRecipeRuntimeBackendPackage, WorkspaceRecipeRuntimeService, WorkspaceRecipeStep } from "./runtime-contracts.js" +import type { RuntimePreviewSpec, RuntimeWordPressInstallMode, WorkspaceRecipe, WorkspaceRecipeExtraPlugin, WorkspaceRecipeMount, WorkspaceRecipePHPWasmExtensionManifest, WorkspaceRecipePluginRuntime, WorkspaceRecipeRuntimeBackendPackage, WorkspaceRecipeRuntimeService, WorkspaceRecipeStep } from "./runtime-contracts.js" import { commandArg, commandJsonArg, commandStringListArg } from "./command-codecs.js" export { buildRuntimePackageRunRecipe, CODEBOX_RUN_RUNTIME_PACKAGE_ABILITY, RUNTIME_PACKAGE_ARTIFACT_DECLARATION_SCHEMA, RUNTIME_PACKAGE_EXECUTION_INPUT_SCHEMA, RUNTIME_PACKAGE_EXECUTION_RESULT_SCHEMA, RUNTIME_PACKAGE_OUTPUT_PROJECTION_SCHEMA, runtimePackageExecutionInput, type RuntimePackageArtifactDeclaration, type RuntimePackageExecutionInput, type RuntimePackageOutputProjection, type RuntimePackageRunRecipeOptions } from "./runtime-package-execution.js" export { RUNTIME_PACKAGE_DIAGNOSTIC_SCHEMA, RUNTIME_PACKAGE_RESULT_SCHEMA, RUNTIME_PACKAGE_TASK_SCHEMA, normalizeRuntimePackageResult, normalizeRuntimePackageTask, validateRuntimePackageTask, type RuntimePackageDiagnostic, type RuntimePackageResult, type RuntimePackageTask } from "./runtime-package-contracts.js" @@ -21,6 +21,7 @@ export interface WordPressPhpunitRecipeOptions { preview?: RuntimePreviewSpec mounts?: WorkspaceRecipeMount[] services?: WorkspaceRecipeRuntimeService[] + pluginRuntime?: WorkspaceRecipePluginRuntime extra_plugins?: WorkspaceRecipeExtraPlugin[] pluginSource?: string pluginSlug: string @@ -88,6 +89,7 @@ export function buildWordPressPhpunitRecipe(options: WordPressPhpunitRecipeOptio inputs: { extra_plugins: normalizeExtraPlugins(options.extra_plugins), ...(options.services && options.services.length > 0 ? { services: options.services } : {}), + ...(options.pluginRuntime ? { pluginRuntime: options.pluginRuntime } : {}), mounts: normalizeRecipeMounts([ ...(options.pluginSource ? [{ source: options.pluginSource, target: pluginTarget } satisfies WorkspaceRecipeMount] : []), ...(options.mounts ?? []), diff --git a/packages/runtime-playground/src/wordpress-command-runners.ts b/packages/runtime-playground/src/wordpress-command-runners.ts index b26952ad..4cdbba94 100644 --- a/packages/runtime-playground/src/wordpress-command-runners.ts +++ b/packages/runtime-playground/src/wordpress-command-runners.ts @@ -970,7 +970,7 @@ export async function runPhpunitCommand({ if (structured) { throw attachPlaygroundDiagnostics(new Error("wordpress.phpunit terminated before completing bootstrap"), "wordpress.phpunit structured diagnostics", structured) } - if (!phpunitArgs.includes("--list-tests") && !/\bOK(?:, but there were issues!)? \([1-9]\d* tests?,/m.test(response.text)) { + if (!phpunitArgs.includes("--list-tests") && !hasSuccessfulPhpunitSummary(response.text)) { const diagnostics = successfulPhpunitResponseDiagnostics(response) throw attachPlaygroundDiagnostics(new Error("wordpress.phpunit exited successfully without a non-zero PHPUnit test summary"), "wordpress.phpunit successful response diagnostics", diagnostics) } @@ -978,6 +978,10 @@ export async function runPhpunitCommand({ return response.text } +export function hasSuccessfulPhpunitSummary(output: string): boolean { + return /\bOK(?:, but (?:there were issues!|incomplete, skipped, or risky tests!))?(?: \([1-9]\d* tests?,|\s+Tests:\s*[1-9]\d*,\s*Assertions:)/m.test(output) +} + function successfulPhpunitResponseDiagnostics(response: PlaygroundRunResponse): string { const boundedStream = (value: string): string => value.length > 9_000 ? `${value.slice(0, 9_000)}\n[stream truncated]` : value return [ diff --git a/tests/phpunit-runtime-failure-diagnostics.test.ts b/tests/phpunit-runtime-failure-diagnostics.test.ts index ff21cb22..dd59b3c1 100644 --- a/tests/phpunit-runtime-failure-diagnostics.test.ts +++ b/tests/phpunit-runtime-failure-diagnostics.test.ts @@ -3,7 +3,7 @@ import { mkdtemp, readFile } from "node:fs/promises" import { tmpdir } from "node:os" import { join } from "node:path" import { PLUGIN_PHPUNIT_RESULT_FILE } from "../packages/runtime-playground/src/phpunit-command-handlers.js" -import { runPhpunitCommand } from "../packages/runtime-playground/src/wordpress-command-runners.js" +import { hasSuccessfulPhpunitSummary, runPhpunitCommand } from "../packages/runtime-playground/src/wordpress-command-runners.js" const artifactRoot = await mkdtemp(join(tmpdir(), "wp-codebox-phpunit-runtime-diagnostics-")) const secret = "sk-abcdefghijklmnopqrstuvwxyz" @@ -51,6 +51,9 @@ assert.ok(preBootstrapRecorder >= 0 && preBootstrapRecorder < wordpressBootstrap const emptySuccessArtifactRoot = await mkdtemp(join(tmpdir(), "wp-codebox-phpunit-empty-success-")) const successfulResponseSecret = "ghp_abcdefghijklmnopqrstuvwxyz1234567890" +assert.equal(hasSuccessfulPhpunitSummary("OK (1 test, 1 assertion)"), true) +assert.equal(hasSuccessfulPhpunitSummary("OK, but incomplete, skipped, or risky tests!\nTests: 21, Assertions: 27398, Skipped: 1."), true) +assert.equal(hasSuccessfulPhpunitSummary("Tests: 21, Assertions: 27398, Failures: 1."), false) await assert.rejects( () => runPhpunitCommand({ artifactRoot: emptySuccessArtifactRoot, diff --git a/tests/runtime-services.test.ts b/tests/runtime-services.test.ts index 23af5540..1ee14f0e 100644 --- a/tests/runtime-services.test.ts +++ b/tests/runtime-services.test.ts @@ -33,6 +33,7 @@ assert.equal(runtimeServicePlan([mariaDbService])[0]?.version, "mariadb:11.4") assert.equal(validateWorkspaceRecipeJsonSchema({ schema: "wp-codebox/workspace-recipe/v1", inputs: { services: [{ ...service, configuration: { foreignKeyTargetPolicy: "anything" } }] }, workflow: { steps: [{ command: "wordpress.run-php" }] } }).valid, false) assert.deepEqual(buildWordPressPhpunitRecipe({ pluginSlug: "example", services: [emptyRootService] }).inputs?.services, [emptyRootService]) assert.equal(buildWordPressPhpunitRecipe({ pluginSlug: "example", wordpressInstallMode: "do-not-attempt-installing" }).runtime?.wordpressInstallMode, "do-not-attempt-installing") +assert.deepEqual(buildWordPressPhpunitRecipe({ pluginSlug: "example", pluginRuntime: { php: { memoryLimit: "2G" } } }).inputs?.pluginRuntime, { php: { memoryLimit: "2G" } }) const builderDirectory = await mkdtemp(join(tmpdir(), "wp-codebox-phpunit-builder-")) try { const optionsPath = join(builderDirectory, "options.json")