Skip to content

Fail closed on PHPUnit discovery errors #1940

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
chubes4 merged 4 commits into from
Jul 21, 2026
Merged

Fail closed on PHPUnit discovery errors #1940

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5ac3898
Capture malformed PHPUnit bootstrap payloads
chubes4 Jul 21, 2026
ac31ce0
Treat empty PHPUnit suites as successful
chubes4 Jul 21, 2026
df5f4d7
Merge remote-tracking branch 'origin/main' into fix/1916-malformed-bo…
chubes4 Jul 21, 2026
f6321d1
Fail closed on PHPUnit discovery errors
chubes4 Jul 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions packages/runtime-core/src/command-registry.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -944,8 +944,9 @@ export const commandRegistry = [
{ name: "project-autoload-file", description: "Project/plugin autoload path loaded after the project bootstrap has provided WordPress functions.", format: "sandbox path" },
{ name: "tests-dir", description: "WP PHPUnit tests directory inside the sandbox.", format: "sandbox path" },
{ name: "phpunit-xml", description: "phpunit.xml path inside the plugin.", format: "path" },
{ name: "phpunit-xml-default", description: "Marks phpunit-xml as the standard default path, allowing an adjacent phpunit.xml fallback only when phpunit.xml.dist is absent.", format: "boolean" },
{ name: "test-file", description: "Single test file to run.", format: "path" },
{ name: "changed-tests-json", description: "Changed test files for diagnostics.", format: "JSON array" },
{ name: "changed-tests-json", description: "Optional changed-scope selection. A non-empty JSON array of test file paths runs only matching files and may legitimately select zero tests.", format: "JSON array of non-empty paths" },
{ name: "env-json", description: "PHPUnit environment values.", format: "JSON object" },
{ name: "wp-config-defines-json", description: "wp-config.php constants for the run.", format: "JSON object" },
{ name: "dependency-mounts", description: "Comma-separated mounted dependency paths loaded and activated after managed PHPUnit installation, before tests execute.", format: "comma-separated sandbox paths" },
Expand Down Expand Up @@ -1062,8 +1063,9 @@ export const commandRegistry = [
{ name: "core-root", description: "WordPress develop checkout root inside the sandbox. Must contain vendor/ with Composer dev dependencies (PHPUnit + yoast/phpunit-polyfills) installed before mounting.", format: "sandbox path" },
{ name: "tests-dir", description: "Core tests directory inside the sandbox (expects includes/bootstrap.php under it).", format: "sandbox path" },
{ name: "phpunit-xml", description: "phpunit.xml path.", format: "path" },
{ name: "phpunit-xml-default", description: "Marks phpunit-xml as the standard default path, allowing an adjacent phpunit.xml fallback only when phpunit.xml.dist is absent.", format: "boolean" },
{ name: "test-file", description: "Single test file to run.", format: "path" },
{ name: "changed-tests-json", description: "Changed test files for diagnostics.", format: "JSON array" },
{ name: "changed-tests-json", description: "Optional changed-scope selection. A non-empty JSON array of test file paths runs only matching files and may legitimately select zero tests.", format: "JSON array of non-empty paths" },
{ name: "autoload-file", description: "Autoload path inside the sandbox (typically <core-root>/vendor/autoload.php from a completed composer install).", format: "sandbox path" },
{ name: "wp-config-defines-json", description: "wp-config.php constants for the run.", format: "JSON object" },
{ name: "multisite", description: "Run as multisite.", format: "boolean" },
Expand Down
1 change: 1 addition & 0 deletions packages/runtime-core/src/recipe-builders.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ export function buildWordPressPhpunitRecipe(options: WordPressPhpunitRecipeOptio
commandArg("tests-dir", options.testsDir ?? "/wp-codebox-vendor/wp-phpunit/wp-phpunit"),
commandArg("test-root", options.testRoot ?? `${pluginTarget}/tests`),
commandArg("phpunit-xml", options.phpunitXml ?? `${pluginTarget}/phpunit.xml.dist`),
commandArg("phpunit-xml-default", options.phpunitXml === undefined ? "1" : ""),
commandStringListArg("dependency-mounts", options.dependencyMounts ?? []),
commandJsonArg("bootstrap-files-json", options.bootstrapFiles ?? []),
commandJsonArg("preload-files-json", options.preloadFiles ?? []),
Expand Down
77 changes: 26 additions & 51 deletions packages/runtime-playground/src/phpunit-command-handlers.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,9 @@ export interface PhpunitRunCodeOptions {
testsDir: string
testRoot?: string
phpunitXml: string
phpunitXmlIsDefault: boolean
selectedTestFile: string
changedTestFiles: unknown[]
changedTestFiles: string[]
phpunitArgs: string[]
env: Record<string, unknown>
wpConfigDefines: Record<string, unknown>
Expand All @@ -32,8 +33,9 @@ export interface CorePhpunitRunCodeOptions {
coreRoot: string
testsDir: string
phpunitXml: string
phpunitXmlIsDefault: boolean
selectedTestFile: string
changedTestFiles: unknown[]
changedTestFiles: string[]
autoloadFile: string
wpConfigDefines: Record<string, unknown>
multisite: boolean
Expand All @@ -51,9 +53,6 @@ export const PLUGIN_PHPUNIT_RESULT_FILE = "/tmp/wp-codebox-phpunit-result.txt"
interface PhpunitConfigDiscoveryPhpOptions {
functionName: string
logFunction: string
missingConfigMessage: string
parseFailureMessage: string
includeParseFailureDetail: boolean
loadedConfigMessage: string
fallbackXmlDist: boolean
restrictDirectoriesToTests: boolean
Expand All @@ -68,7 +67,6 @@ interface PhpunitChangedTestFilterPhpOptions {
logFunction: string
rootParameterName: string
testsPathFallback: boolean
emptyWantedNotice: boolean
}

function phpunitConfigDiscoveryPhp(options: PhpunitConfigDiscoveryPhpOptions): string {
Expand All @@ -90,9 +88,6 @@ function phpunitConfigDiscoveryPhp(options: PhpunitConfigDiscoveryPhpOptions): s
const returnValues = options.uniqueReturnValues
? "array(array_values(array_unique($directories)), array_values(array_unique($suffixes)), array_values(array_unique($prefixes)), $excludes, array_values(array_unique($files)))"
: "array($directories, $suffixes, $prefixes, $excludes, $files)"
const parseFailureLog = options.includeParseFailureDetail
? `${options.logFunction}('${options.parseFailureMessage}' . $first . '); using defaults');`
: `${options.logFunction}('${options.parseFailureMessage}');`
const suffixAssignment = options.replaceDefaultMatchers ? "$suffixes = $config_suffixes;" : "$suffixes = array_merge($suffixes, $config_suffixes);"
const prefixAssignment = options.replaceDefaultMatchers ? "$prefixes = $config_prefixes;" : "$prefixes = array_merge($prefixes, $config_prefixes);"

Expand All @@ -106,8 +101,7 @@ function phpunitConfigDiscoveryPhp(options: PhpunitConfigDiscoveryPhpOptions): s
return ${returnValues};
};${fallbackXmlDist}
if (!is_readable($xml_path)) {
${options.logFunction}('${options.missingConfigMessage}' . $xml_path . '; using defaults');
return $return_values();
throw new RuntimeException('PHPUnit config is not readable: ' . $xml_path);
}
$prev = libxml_use_internal_errors(true);
$xml = @simplexml_load_file($xml_path);
Expand All @@ -116,8 +110,7 @@ function phpunitConfigDiscoveryPhp(options: PhpunitConfigDiscoveryPhpOptions): s
libxml_use_internal_errors($prev);
if ($xml === false) {
$first = $errors ? trim($errors[0]->message) : 'unknown';
${parseFailureLog}
return $return_values();
throw new RuntimeException('PHPUnit config could not be parsed at ' . $xml_path . ': ' . $first);
}
$base = ${options.basePathExpression};
$config_dirs = array();
Expand Down Expand Up @@ -170,11 +163,10 @@ function phpunitDiscoveryPhp(functionName: string, logFunction: string): string
$found = array();
foreach ($files as $file) {
if (!is_string($file) || $file === '') {
continue;
throw new RuntimeException('configured PHPUnit test file is invalid');
}
if (!is_file($file) || pathinfo($file, PATHINFO_EXTENSION) !== 'php') {
${logFunction}('NOTICE:test file does not exist: ' . $file);
continue;
if (!is_file($file) || !is_readable($file) || pathinfo($file, PATHINFO_EXTENSION) !== 'php') {
throw new RuntimeException('configured PHPUnit test file is not a readable PHP file: ' . $file);
}
foreach ($excludes as $exclude) {
if (strpos($file, $exclude) === 0) {
Expand All @@ -184,9 +176,8 @@ function phpunitDiscoveryPhp(functionName: string, logFunction: string): string
$found[] = $file;
}
foreach ($directories as $dir) {
if (!is_dir($dir)) {
${logFunction}('NOTICE:test directory does not exist: ' . $dir);
continue;
if (!is_dir($dir) || !is_readable($dir)) {
throw new RuntimeException('configured PHPUnit test directory is not a readable directory: ' . $dir);
}
$iterator = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($dir, RecursiveDirectoryIterator::SKIP_DOTS), RecursiveIteratorIterator::LEAVES_ONLY);
foreach ($iterator as $file) {
Expand Down Expand Up @@ -229,12 +220,6 @@ function phpunitChangedTestFilterPhp(options: PhpunitChangedTestFilterPhpOptions
const testsPathFallback = options.testsPathFallback ? ` elseif (strpos($path, '/tests/') !== false) {
$path = substr($path, strpos($path, '/tests/') + 1);
}` : ""
const emptyWantedNotice = options.emptyWantedNotice ? `
if (empty($wanted)) {
${options.logFunction}('NOTICE:changed tests did not contain usable test paths');
return array();
}` : ""

return `function ${options.filterFunctionName}(array $test_files, string $changed_files_json, string $${options.rootParameterName}): array {
$decoded = json_decode($changed_files_json, true);
if (!is_array($decoded) || empty($decoded)) {
Expand All @@ -249,7 +234,7 @@ function phpunitChangedTestFilterPhp(options: PhpunitChangedTestFilterPhpOptions
if ($normalized !== '') {
$wanted[$normalized] = true;
}
}${emptyWantedNotice}
}
$filtered = array();
foreach ($test_files as $file) {
if (isset($wanted[${options.relativeFunctionName}((string) $file, $${options.rootParameterName})])) {
Expand Down Expand Up @@ -360,6 +345,7 @@ $tests_dir = ${JSON.stringify(options.testsDir)};
$test_root = ${JSON.stringify(options.testRoot || `/wordpress/wp-content/plugins/${options.pluginSlug}/tests`)};
$selected_test_file = ${JSON.stringify(options.selectedTestFile)};
$changed_test_files_raw = ${JSON.stringify(JSON.stringify(options.changedTestFiles))};
$changed_test_scope = ${JSON.stringify(options.changedTestFiles.length > 0)};
$phpunit_args_raw = json_decode(${JSON.stringify(JSON.stringify(options.phpunitArgs))}, true);
$bench_env = json_decode(${JSON.stringify(JSON.stringify(options.env))}, true);
$wp_config_defines = json_decode(${JSON.stringify(JSON.stringify(options.wpConfigDefines))}, true);
Expand Down Expand Up @@ -765,8 +751,8 @@ function pg_plugin_real_path(string $relative_path, string $kind): ?string {
return $real;
}

function pg_project_bootstrap_from_config(string $xml_path): string {
if (!is_readable($xml_path) && basename($xml_path) === 'phpunit.xml.dist') {
function pg_project_bootstrap_from_config(string &$xml_path, bool $xml_is_default): string {
if ($xml_is_default && !is_readable($xml_path) && basename($xml_path) === 'phpunit.xml.dist') {
$alternate = dirname($xml_path) . '/phpunit.xml';
if (is_readable($alternate)) {
$xml_path = $alternate;
Expand Down Expand Up @@ -838,9 +824,10 @@ function pg_run_project_bootstrap_stage(array $cfg): void {
try {
$bootstrap = trim((string) ($cfg['project_bootstrap'] ?? ''));
$phpunit_xml = (string) ($cfg['phpunit_xml'] ?? '');
$phpunit_xml_is_default = !empty($cfg['phpunit_xml_is_default']);
$from_config = false;
if ($bootstrap === '') {
$bootstrap = pg_project_bootstrap_from_config($phpunit_xml);
$bootstrap = pg_project_bootstrap_from_config($phpunit_xml, $phpunit_xml_is_default);
$from_config = $bootstrap !== '';
}
$bootstrap_real = pg_project_bootstrap_real_path($bootstrap, $phpunit_xml, $from_config);
Expand Down Expand Up @@ -1038,7 +1025,6 @@ ${phpunitChangedTestFilterPhp({
logFunction: "pg_log",
rootParameterName: "plugin_path",
testsPathFallback: true,
emptyWantedNotice: true,
})}

pg_install_diagnostics_handlers();
Expand Down Expand Up @@ -1089,7 +1075,7 @@ $config_path = pg_run_boot_stage(array('extra_defines' => $wp_config_defines, 't
if ($bootstrap_mode === 'project') {
pg_prepare_project_bootstrap_environment($config_path);
pg_skip_project_bootstrap_shell_install();
pg_run_project_bootstrap_stage(array('project_bootstrap' => $project_bootstrap, 'phpunit_xml' => ${JSON.stringify(options.phpunitXml)}));
pg_run_project_bootstrap_stage(array('project_bootstrap' => $project_bootstrap, 'phpunit_xml' => ${JSON.stringify(options.phpunitXml)}, 'phpunit_xml_is_default' => ${JSON.stringify(options.phpunitXmlIsDefault)}));
pg_run_project_autoload_stage($project_autoload_file !== '' ? $project_autoload_file : $legacy_project_autoload_file);
} else {
if ($bootstrap_mode !== 'managed') {
Expand Down Expand Up @@ -1216,11 +1202,8 @@ try {
${phpunitConfigDiscoveryPhp({
functionName: "wp_codebox_phpunit_parse_config",
logFunction: "pg_log",
missingConfigMessage: "NOTICE:phpunit.xml.dist not readable at ",
parseFailureMessage: "NOTICE:phpunit.xml.dist parse failed (",
includeParseFailureDetail: true,
loadedConfigMessage: "NOTICE:phpunit.xml.dist loaded from ",
fallbackXmlDist: true,
fallbackXmlDist: options.phpunitXmlIsDefault,
restrictDirectoriesToTests: !options.testRoot,
basePathExpression: "dirname($xml_path)",
uniqueReturnValues: false,
Expand All @@ -1233,10 +1216,7 @@ pg_stage_begin('discover_tests');
try {
$test_dir = $test_root;
if (!is_dir($test_dir)) {
pg_log('NO_TEST_FILES');
pg_log('NOTICE:tests directory not found at ' . $test_dir);
pg_stage_ok('discover_tests');
exit(0);
throw new RuntimeException('configured PHPUnit test root is not a readable directory: ' . $test_dir);
}
list($directories, $suffixes, $prefixes, $excludes, $configured_files) = wp_codebox_phpunit_parse_config(${JSON.stringify(options.phpunitXml)}, $test_dir);
$test_files = wp_codebox_phpunit_discover($directories, $suffixes, $prefixes, $excludes, $configured_files);
Expand All @@ -1258,10 +1238,9 @@ try {
$test_files = array($selected_abs);
}
pg_log('DISCOVERY: dirs=' . implode(',', $directories) . ' files=' . count($configured_files) . ' suffixes=' . implode(',', $suffixes) . ' prefixes=' . implode(',', $prefixes) . ' excludes=' . count($excludes) . ' found=' . count($test_files));
if (empty($test_files)) {
if (empty($test_files) && !$changed_test_scope) {
pg_log('NO_TEST_FILES');
pg_stage_ok('discover_tests');
exit(0);
throw new RuntimeException('PHPUnit discovery found no test files');
}
pg_stage_ok('discover_tests');
} catch (Throwable $e) {
Expand Down Expand Up @@ -1347,6 +1326,7 @@ $tests_dir = rtrim(${JSON.stringify(options.testsDir)}, '/');
$phpunit_xml = ${JSON.stringify(options.phpunitXml)};
$selected_test_file = ${JSON.stringify(options.selectedTestFile)};
$changed_test_files_raw = ${JSON.stringify(JSON.stringify(options.changedTestFiles))};
$changed_test_scope = ${JSON.stringify(options.changedTestFiles.length > 0)};
$autoload_file = ${JSON.stringify(options.autoloadFile)};
$wp_config_defines = json_decode(${JSON.stringify(JSON.stringify(options.wpConfigDefines))}, true);
$multisite = ${JSON.stringify(options.multisite)};
Expand Down Expand Up @@ -1482,11 +1462,8 @@ function core_pg_write_tests_config(string $core_root, array $extra_defines): st
${phpunitConfigDiscoveryPhp({
functionName: "core_pg_parse_phpunit_config",
logFunction: "core_pg_log",
missingConfigMessage: "NOTICE:phpunit config not readable at ",
parseFailureMessage: "NOTICE:phpunit config parse failed; using defaults",
includeParseFailureDetail: false,
loadedConfigMessage: "NOTICE:phpunit config loaded from ",
fallbackXmlDist: false,
fallbackXmlDist: options.phpunitXmlIsDefault,
restrictDirectoriesToTests: false,
basePathExpression: "dirname($xml_path)",
uniqueReturnValues: true,
Expand All @@ -1501,7 +1478,6 @@ ${phpunitChangedTestFilterPhp({
logFunction: "core_pg_log",
rootParameterName: "core_root",
testsPathFallback: false,
emptyWantedNotice: false,
})}

${phpunitArgsPhp("core_pg_phpunit_args", "core_pg_log")}
Expand Down Expand Up @@ -1584,10 +1560,9 @@ try {
$test_files = array($selected_abs);
}
core_pg_log('DISCOVERY: dirs=' . implode(',', $directories) . ' files=' . count($configured_files) . ' suffixes=' . implode(',', $suffixes) . ' prefixes=' . implode(',', $prefixes) . ' excludes=' . count($excludes) . ' found=' . count($test_files));
if (empty($test_files)) {
if (empty($test_files) && !$changed_test_scope) {
core_pg_log('NO_TEST_FILES');
core_pg_stage_ok('discover_tests');
exit(0);
throw new RuntimeException('PHPUnit discovery found no test files');
}
core_pg_stage_ok('discover_tests');
} catch (Throwable $e) {
Expand Down
21 changes: 17 additions & 4 deletions packages/runtime-playground/src/wordpress-command-runners.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -907,6 +907,7 @@ export async function runPhpunitCommand({
spec: ExecutionSpec
}): Promise<string> {
const args = spec.args ?? []
const phpunitXmlArg = argValue(args, "phpunit-xml")
const explicitCode = argValue(args, "code") || argValue(args, "code-file")
const pluginSlug = argValue(args, "plugin-slug")?.trim() || ""
const bootstrapMode = argValue(args, "bootstrap-mode")?.trim() || "managed"
Expand All @@ -923,9 +924,10 @@ export async function runPhpunitCommand({
projectAutoloadFile: argValue(args, "project-autoload-file")?.trim() || "",
testsDir: argValue(args, "tests-dir")?.trim() || "/wp-codebox-vendor/wp-phpunit/wp-phpunit",
testRoot: argValue(args, "test-root")?.trim() || `/wordpress/wp-content/plugins/${pluginSlug}/tests`,
phpunitXml: argValue(args, "phpunit-xml")?.trim() || `/wordpress/wp-content/plugins/${pluginSlug}/phpunit.xml.dist`,
phpunitXml: phpunitXmlArg?.trim() || `/wordpress/wp-content/plugins/${pluginSlug}/phpunit.xml.dist`,
phpunitXmlIsDefault: phpunitXmlArg === undefined || booleanArg(args, "phpunit-xml-default"),
selectedTestFile: argValue(args, "test-file")?.trim() || "",
changedTestFiles: jsonArrayArg(args, "changed-tests-json"),
changedTestFiles: changedTestFilesArg(args),
phpunitArgs: jsonArrayArg(args, "phpunit-args-json").filter((value): value is string => typeof value === "string"),
env: jsonObjectArg(args, "env-json"),
wpConfigDefines: jsonObjectArg(args, "wp-config-defines-json"),
Expand Down Expand Up @@ -974,6 +976,15 @@ function boundedProcessIdentity(value: string | undefined): string {
return value
}

function changedTestFilesArg(args: string[]): string[] {
return jsonArrayArg(args, "changed-tests-json").map((value) => {
if (typeof value !== "string" || value.trim() === "") {
throw new Error("changed-tests-json must be a JSON array of non-empty test file paths")
}
return value.trim()
})
}

export async function runCorePhpunitCommand({
artifactRoot,
runPlaygroundCommand,
Expand All @@ -986,6 +997,7 @@ export async function runCorePhpunitCommand({
spec: ExecutionSpec
}): Promise<string> {
const args = spec.args ?? []
const phpunitXmlArg = argValue(args, "phpunit-xml")
const explicitCode = argValue(args, "code") || argValue(args, "code-file")
// Write structured diagnostics to a sandbox-internal /tmp path rather than inside
// the (often read-only) core mount, so the result survives read-only mounts and a
Expand All @@ -994,9 +1006,10 @@ export async function runCorePhpunitCommand({
const code = explicitCode ? await phpCodeFromArgs(args, "wordpress.core-phpunit") : normalizePhpCode(corePhpunitRunCode({
coreRoot: argValue(args, "core-root")?.trim() || "/wordpress",
testsDir: argValue(args, "tests-dir")?.trim() || "/wordpress/tests/phpunit",
phpunitXml: argValue(args, "phpunit-xml")?.trim() || "/wordpress/tests/phpunit/phpunit.xml.dist",
phpunitXml: phpunitXmlArg?.trim() || "/wordpress/tests/phpunit/phpunit.xml.dist",
phpunitXmlIsDefault: phpunitXmlArg === undefined || booleanArg(args, "phpunit-xml-default"),
selectedTestFile: argValue(args, "test-file")?.trim() || "",
changedTestFiles: jsonArrayArg(args, "changed-tests-json"),
changedTestFiles: changedTestFilesArg(args),
autoloadFile: argValue(args, "autoload-file")?.trim() || "/wordpress/vendor/autoload.php",
wpConfigDefines: jsonObjectArg(args, "wp-config-defines-json"),
multisite: booleanArg(args, "multisite"),
Expand Down
Loading
Loading