diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 2fefb7e..3ea95f4 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -53,6 +53,32 @@ jobs: - name: Run root policy tests run: php vendor/bin/phpunit --configuration phpunit.xml.dist tests/Architecture tests/Documentation + benchmark-policy: + name: Benchmark policy (PHP 8.4) + runs-on: ubuntu-24.04 + + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up PHP + uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 # 2.37.2 + with: + php-version: '8.4' + tools: composer:v2 + coverage: none + + - name: Validate benchmark Composer manifest and lockfile + run: composer validate --working-dir=benchmarks --strict --check-lock + + - name: Install benchmark dependencies + run: composer install --working-dir=benchmarks --no-interaction --no-progress --prefer-dist + + - name: Run CI-safe benchmark policy checks + run: composer --working-dir=benchmarks ci:policy + workspace-quality: name: Workspace quality (PHP ${{ matrix.php }}) runs-on: ubuntu-24.04 diff --git a/benchmarks/README.md b/benchmarks/README.md index 6755043..7c409d6 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -1,6 +1,6 @@ # EvolvePHP 2 Benchmarks -EvolvePHP performance work starts from measurements instead of guesses. This benchmark harness provides correctness checks, environment fingerprinting, local baseline output, and measured evidence for later optimization work. +EvolvePHP performance work starts from measurements instead of guesses. This benchmark harness provides correctness checks, environment fingerprinting, local baseline output, measured evidence for later optimization work, and benchmark-only budget evaluation for controlled EvolvePHP evidence. This harness is benchmark infrastructure only. It does not optimize production framework code and it does not make a fastest-framework claim. There is no current fastest-framework claim and no current top-three performance claim. @@ -18,7 +18,9 @@ Benchmark-only dependencies are installed inside `benchmarks/vendor/` and locked The primary comparison lane is PHP 8.4 with OPcache enabled, JIT disabled, production-like framework configuration, and no debugging or profiling extension altering timings. PHP 8.5 results may be useful, but they must not be combined with PHP 8.4 results as if they came from the same environment. -one-off stopwatch results are not performance evidence. Warmup is required. Multiple iterations are required. The environment fingerprint must match before two baselines are treated as comparable. Shared GitHub-hosted runners are useful for smoke validation, but they are not authoritative sources for absolute wall-clock regression budgets. Future blocking budgets must come from a controlled or proven low-noise environment. +one-off stopwatch results are not performance evidence. Warmup is required. Multiple iterations are required. The environment fingerprint must match before two baselines are treated as comparable. Shared GitHub-hosted runners are useful for smoke validation and benchmark policy checks, but they are not authoritative sources for absolute wall-clock regression budgets. + +The initial performance budget uses p50 as the blocking metric. p95, p99, mean, relative standard deviation, throughput, and memory remain diagnostic evidence. No blocking memory budget is active because the accepted calibration did not establish a cross-run memory noise floor. ## Commands @@ -52,6 +54,27 @@ Run the fast benchmark smoke: php benchmarks\bin\benchmark-smoke.php ``` +Validate the tracked performance budget and compact reference summary without running a benchmark: + +```powershell +php benchmarks\bin\performance-budget.php --budget benchmarks\budgets\performance-budget.json --validate-reference +``` + +Evaluate a controlled EvolvePHP comparator candidate directory: + +```powershell +php benchmarks\bin\performance-budget.php --budget benchmarks\budgets\performance-budget.json --candidate benchmarks\results\local\comparator-candidate +``` + +Budget evaluation states: + +- `pass`: p50 is within the accepted observed calibration envelope. +- `warn`: p50 is outside the observed calibration envelope but not a blocking regression under the scenario policy. +- `fail`: a blocking warm HTTP p50 threshold was exceeded. +- `incomparable`: identity, protocol, availability, source-cleanliness, sample-count, or normalized-result requirements do not allow timing comparison. + +Exit code `0` means pass or non-blocking warning. Exit code `1` means a blocking regression. Exit code `2` means incomparable evidence or invalid policy/reference data. + Run internal Core scenarios: ```powershell @@ -112,7 +135,9 @@ A result qualifies as a reference baseline only when the documented PHP 8.4 prot Baseline comparison must reject casual comparisons when the environment fingerprint differs. The normalized result schema includes scenario identifiers, sample counts, timing statistics, percentiles when enough samples exist, relative standard deviation, throughput where derivable, memory fields, environment fingerprint, source SHA, and schema version. -Cross-framework comparison, optimization work, and regression budgets are intentionally outside this initial harness. They should be introduced only after baseline measurements are reproducible and the relevant comparison methodology is defined. +Cross-framework comparison, optimization work, and regression budgets require reproducible controlled evidence and matching comparison identity. The benchmark-only budget policy lives under `benchmarks/budgets/performance-budget.json`, and the compact reference artifacts live under `benchmarks/results/reference/`. + +Warm HTTP p50 scenarios have blocking thresholds derived from the accepted EvolvePHP-only controlled calibration. `application_boot` is currently monitor-only because its accepted calibration showed high within-run relative standard deviation and volatile tail timing. It can warn when it exceeds the observed envelope or the provisional observation boundary, but it does not produce a blocking timing failure in this policy version. ## Cross-Framework Comparator Infrastructure @@ -130,7 +155,7 @@ Install the benchmark harness explicitly: composer install --working-dir=benchmarks --no-interaction ``` -Install comparator fixtures only when doing framework-maintainer comparator work: +Install comparator fixtures only when doing controlled comparator work: ```powershell composer install --working-dir=benchmarks/comparators/evolvephp --no-interaction @@ -253,4 +278,13 @@ Candidate evidence is useful while developing or reviewing the harness. Store it Canonical reference evidence requires PHP exactly 8.4.25, OPcache enabled for CLI, JIT disabled, the same php.ini/configuration and extension set for all comparator processes, and ext-phalcon 5.20.3 loaded when the five-framework lane is claimed. Shared GitHub-hosted runner wall-clock timing is not authoritative comparator evidence. +The tracked reference directory contains compact, intentional artifacts: + +- `benchmarks/results/reference/performance-summary.json` +- `benchmarks/results/reference/performance-report.md` + +Raw 100-sample process records, command streams, and disposable candidate directories remain local or externally archived. They should not be committed to the repository. + +The budget evaluator rejects blind timing comparisons. It requires matching execution-environment fingerprint, comparator identity, scenario identity, matrix hash, EvolvePHP comparator lock hash, fixture identity hash, sample protocol, repeated-warm operations protocol, availability state, clean candidate evidence, valid source SHA, and well-formed normalized results. + The public reporting policy is a non-ranking policy. Per-scenario evidence and limitations may be published, but broad claims such as fastest framework, top-three placement, composite rankings, or one framework generally beating another belong to later accepted performance-budget work. diff --git a/benchmarks/bin/performance-budget.php b/benchmarks/bin/performance-budget.php new file mode 100644 index 0000000..e3bbf1b --- /dev/null +++ b/benchmarks/bin/performance-budget.php @@ -0,0 +1,215 @@ +assertValidBudget($budget); + validateReferenceSummary($budget); + + echo json_encode([ + 'schema_version' => PerformanceBudgetEvaluator::EVALUATION_SCHEMA_VERSION, + 'status' => 'pass', + 'validation' => 'reference policy is valid', + ], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR) . PHP_EOL; + exit(0); + } + + if (!isset($options['candidate']) || !is_string($options['candidate'])) { + fwrite(STDERR, "Missing required --candidate option unless --validate-reference is used.\n"); + exit(2); + } + + $evaluation = $evaluator->evaluate($budget, readCandidateEvidence($options['candidate'])); + + echo json_encode($evaluation, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR) . PHP_EOL; + + exit(match ($evaluation['status']) { + 'pass', 'warn' => 0, + 'fail' => 1, + default => 2, + }); +} catch (Throwable $exception) { + fwrite(STDERR, $exception->getMessage() . PHP_EOL); + exit(2); +} + +/** + * @return array + */ +function readJsonObject(string $path): array +{ + if (!is_file($path)) { + throw new RuntimeException("JSON file not found: {$path}"); + } + + $decoded = json_decode((string) file_get_contents($path), true, flags: JSON_THROW_ON_ERROR); + + if (!is_array($decoded)) { + throw new RuntimeException("JSON file must decode to an object: {$path}"); + } + + return $decoded; +} + +/** + * @return array{manifest: array, results: list>} + */ +function readCandidateEvidence(string $candidateDir): array +{ + if (!is_dir($candidateDir)) { + throw new RuntimeException("Candidate directory not found: {$candidateDir}"); + } + + $candidateDir = realpath($candidateDir); + if ($candidateDir === false) { + throw new RuntimeException('Candidate directory could not be resolved.'); + } + + $manifest = readJsonObject($candidateDir . DIRECTORY_SEPARATOR . 'manifest.json'); + $results = []; + + foreach ($manifest['results'] ?? [] as $result) { + if (!is_array($result) || !isset($result['normalized_result']['path']) || !is_string($result['normalized_result']['path'])) { + continue; + } + + $hash = $result['normalized_result']['sha256'] ?? null; + if (!is_string($hash) || preg_match('/\A[a-f0-9]{64}\z/', $hash) !== 1) { + throw new RuntimeException('Manifest normalized_result.sha256 must be a 64-character lowercase hexadecimal hash.'); + } + + $normalizedPath = resolveCandidateFile($candidateDir, $result['normalized_result']['path']); + $actualHash = hash_file('sha256', $normalizedPath); + + if ($actualHash !== $hash) { + throw new RuntimeException('Normalized result hash does not match manifest evidence.'); + } + + $results[] = readJsonObject($normalizedPath); + } + + return [ + 'manifest' => $manifest, + 'results' => $results, + ]; +} + +function resolveCandidateFile(string $candidateDir, string $relativePath): string +{ + if (preg_match('/\A(?:[A-Za-z]:[\\\\\/]|[\\\\\/])/', $relativePath) === 1) { + throw new RuntimeException('Normalized result path must be relative to the candidate directory.'); + } + + $path = realpath($candidateDir . DIRECTORY_SEPARATOR . str_replace(['/', '\\'], DIRECTORY_SEPARATOR, $relativePath)); + + if ($path === false || !is_file($path)) { + throw new RuntimeException('Normalized result file does not exist.'); + } + + $root = str_replace('\\', '/', rtrim($candidateDir, DIRECTORY_SEPARATOR)); + $resolved = str_replace('\\', '/', $path); + + if (!str_starts_with($resolved, $root . '/')) { + throw new RuntimeException('Normalized result path escapes the candidate directory.'); + } + + return $path; +} + +/** + * @param array $budget + */ +function validateReferenceSummary(array $budget): void +{ + $referencePath = dirname(__DIR__) . DIRECTORY_SEPARATOR . 'results' . DIRECTORY_SEPARATOR . 'reference' . DIRECTORY_SEPARATOR . 'performance-summary.json'; + $summary = readJsonObject($referencePath); + + if (($summary['schema_version'] ?? null) !== 'evolvephp.performance-reference-summary.v1') { + throw new RuntimeException('Reference summary schema_version is not supported.'); + } + + foreach ([ + 'regression_baseline_source_sha' => $budget['baseline_source_sha'], + 'canonical_environment_fingerprint' => $budget['comparison_identity']['execution_environment_fingerprint'], + ] as $field => $expected) { + if (($summary[$field] ?? null) !== $expected) { + throw new RuntimeException("Reference summary {$field} does not match the performance budget."); + } + } + + $protocol = is_array($summary['calibration_protocol'] ?? null) ? $summary['calibration_protocol'] : []; + foreach ([ + 'run_count' => $budget['calibration']['run_count'], + 'sample_count' => $budget['calibration']['sample_count'], + 'php_version' => $budget['canonical_runtime_policy']['php_version'], + 'opcache_cli_enabled' => $budget['canonical_runtime_policy']['opcache_cli_enabled'], + 'jit_enabled' => $budget['canonical_runtime_policy']['jit_enabled'], + 'primary_metric' => $budget['primary_metric'], + 'repeated_warm_request_count' => $budget['sample_protocol']['request_count'], + ] as $field => $expected) { + if (($protocol[$field] ?? null) !== $expected) { + throw new RuntimeException("Reference summary calibration_protocol.{$field} does not match the performance budget."); + } + } + + foreach ($budget['scenarios'] as $scenarioId => $policy) { + if (!isset($summary['scenarios'][$scenarioId]) || !is_array($summary['scenarios'][$scenarioId])) { + throw new RuntimeException("Reference summary is missing scenario {$scenarioId}."); + } + + foreach ([ + 'p50_microseconds_by_run' => $policy['observed_p50_microseconds'], + 'reference_median_p50_microseconds' => $policy['reference_p50_microseconds'], + 'observed_maximum_p50_microseconds' => $policy['observed_maximum_p50_microseconds'], + 'observed_range_percent' => $policy['cross_run_range_percent'], + 'budget_classification' => $policy['mode'], + ] as $field => $expected) { + if (($summary['scenarios'][$scenarioId][$field] ?? null) !== $expected) { + throw new RuntimeException("Reference summary {$scenarioId}.{$field} does not match the performance budget."); + } + } + + if (($policy['mode'] ?? null) === 'blocking') { + if (($summary['scenarios'][$scenarioId]['blocking_threshold_p50_microseconds'] ?? null) !== $policy['blocking_threshold_p50_microseconds']) { + throw new RuntimeException("Reference summary {$scenarioId}.blocking_threshold_p50_microseconds does not match the performance budget."); + } + } elseif (($summary['scenarios'][$scenarioId]['observation_threshold_p50_microseconds'] ?? null) !== $policy['observation_threshold_p50_microseconds']) { + throw new RuntimeException("Reference summary {$scenarioId}.observation_threshold_p50_microseconds does not match the performance budget."); + } + + if ( + $scenarioId === 'application_boot' + && (($summary['scenarios'][$scenarioId]['diagnostic_rsd_percent_by_run'] ?? null) !== ($policy['diagnostic_rsd_percent'] ?? null)) + ) { + throw new RuntimeException('Reference summary application_boot.diagnostic_rsd_percent_by_run does not match the performance budget.'); + } + + if ( + $scenarioId === 'http_repeated_warm' + && (($summary['scenarios'][$scenarioId]['operations_per_sample'] ?? null) !== $budget['sample_protocol']['repeated_warm_operations_per_sample']) + ) { + throw new RuntimeException('Reference summary http_repeated_warm.operations_per_sample does not match the performance budget.'); + } + } +} diff --git a/benchmarks/budgets/performance-budget.json b/benchmarks/budgets/performance-budget.json new file mode 100644 index 0000000..2fe3136 --- /dev/null +++ b/benchmarks/budgets/performance-budget.json @@ -0,0 +1,142 @@ +{ + "schema_version": "evolvephp.performance-budget.v1", + "baseline_source_sha": "c62cecb16cb4fcdc93bfbb0188a4b63d8cf704ce", + "calibration": { + "source": "accepted controlled EvolvePHP-only regression calibration", + "run_count": 3, + "sample_count": 100, + "primary_metric": "p50", + "raw_evidence_policy": "full raw controlled evidence is retained outside this repository" + }, + "canonical_runtime_policy": { + "php_version": "8.4.25", + "opcache_cli_enabled": true, + "jit_enabled": false + }, + "warmup_policy": { + "warm_http_subject_warmups": 5, + "application_boot_measured_worker_in_process_warmups": 0, + "application_boot_discarded_worker_processes_per_measured_sample": 1 + }, + "sample_protocol": { + "sample_count": 100, + "warmups": 5, + "request_count": 25, + "process_isolation_model": "subprocess_per_measured_sample", + "repeated_warm_operations_per_sample": 25, + "boot_protocol": { + "discarded_worker_processes_per_measured_sample": 1, + "sample_order": "rotating_round_robin", + "measured_worker_in_process_warmups": 0, + "outlier_policy": "retain_all_measured_samples", + "primary_central_statistic": "p50" + } + }, + "comparison_identity": { + "manifest_schema_version": "evolvephp.comparator.evidence-manifest.v1", + "comparator_result_schema_version": "evolvephp.comparator.result.v1", + "comparator_id": "evolvephp", + "execution_environment_fingerprint": "9c06a992d7f01cb7096a60b893f33a34aff7b2a86fba157c8b879d9ac55457a2", + "matrix_sha256": "e2219a90c660657249e7a2aac225dcf55d618a61cb7bcdd227786f4532146778", + "comparator_lock_sha256": "f792575ec5491c8d3aa171ba5f7de3b38558bfbd82b977beea45e603fd79e491", + "fixture_identity_hash": "b331add71f193a353e866d1123dd2d654a677aed87a731c1526230d098b86386" + }, + "primary_metric": "p50", + "diagnostic_metrics": [ + "p95", + "p99", + "mean", + "relative_standard_deviation_percent", + "throughput_per_second", + "memory" + ], + "memory_policy": { + "blocking_budget": false, + "reason": "accepted calibration did not establish a cross-run memory noise floor" + }, + "scenarios": { + "application_boot": { + "mode": "monitor", + "observed_p50_microseconds": [ + 1068.25, + 1042.4, + 1044.05 + ], + "reference_p50_microseconds": 1044.05, + "observed_maximum_p50_microseconds": 1068.25, + "cross_run_range_percent": 2.4798541826554, + "observation_threshold_p50_microseconds": 1148.455, + "diagnostic_rsd_percent": [ + 29.76929761631, + 18.666326692293, + 25.872643772215 + ], + "policy_note": "monitor-only because accepted calibration showed high within-run RSD and volatile p99" + }, + "http_static": { + "mode": "blocking", + "observed_p50_microseconds": [ + 30.9, + 30.8, + 30.7 + ], + "reference_p50_microseconds": 30.8, + "observed_maximum_p50_microseconds": 30.9, + "cross_run_range_percent": 0.6514657980456, + "blocking_threshold_p50_microseconds": 32.34, + "policy_note": "5 percent actionable regression budget above accepted reference median p50" + }, + "http_parameterized": { + "mode": "blocking", + "observed_p50_microseconds": [ + 34.4, + 34.4, + 34.5 + ], + "reference_p50_microseconds": 34.4, + "observed_maximum_p50_microseconds": 34.5, + "cross_run_range_percent": 0.29069767441861, + "blocking_threshold_p50_microseconds": 36.12, + "policy_note": "5 percent actionable regression budget above accepted reference median p50" + }, + "http_middleware": { + "mode": "blocking", + "observed_p50_microseconds": [ + 37.2, + 37.2, + 37.3 + ], + "reference_p50_microseconds": 37.2, + "observed_maximum_p50_microseconds": 37.3, + "cross_run_range_percent": 0.26881720430106, + "blocking_threshold_p50_microseconds": 39.06, + "policy_note": "5 percent actionable regression budget above accepted reference median p50" + }, + "http_not_found": { + "mode": "blocking", + "observed_p50_microseconds": [ + 23.9, + 23.8, + 23.8 + ], + "reference_p50_microseconds": 23.8, + "observed_maximum_p50_microseconds": 23.9, + "cross_run_range_percent": 0.42016806722688, + "blocking_threshold_p50_microseconds": 24.99, + "policy_note": "5 percent actionable regression budget above accepted reference median p50" + }, + "http_repeated_warm": { + "mode": "blocking", + "observed_p50_microseconds": [ + 26.662, + 26.652, + 26.54 + ], + "reference_p50_microseconds": 26.652, + "observed_maximum_p50_microseconds": 26.662, + "cross_run_range_percent": 0.45968349660889, + "blocking_threshold_p50_microseconds": 27.9846, + "policy_note": "5 percent actionable regression budget above accepted reference median p50" + } + } +} diff --git a/benchmarks/composer.json b/benchmarks/composer.json index 7ff25d8..e01f56c 100644 --- a/benchmarks/composer.json +++ b/benchmarks/composer.json @@ -49,12 +49,20 @@ }, "scripts": { "bench:local": "@php vendor/bin/phpbench run --config=phpbench.json --report=aggregate", + "budget:validate": "@php bin/performance-budget.php --budget budgets/performance-budget.json --validate-reference", + "ci:policy": [ + "@syntax", + "@test:budget", + "@smoke", + "@budget:validate" + ], "comparator:preflight": "@php bin/comparator-preflight.php", "comparator:run": "@php bin/comparator-run.php", "capture-environment": "@php bin/capture-environment.php", "smoke": "@php bin/benchmark-smoke.php", "syntax": "@php bin/check-syntax.php", "test": "@php vendor/bin/phpunit --configuration phpunit.xml.dist", + "test:budget": "@php vendor/bin/phpunit --configuration phpunit.xml.dist tests/PerformanceBudgetEvaluatorTest.php", "quality": [ "@syntax", "@test", diff --git a/benchmarks/results/README.md b/benchmarks/results/README.md index 17dfe17..32b1d88 100644 --- a/benchmarks/results/README.md +++ b/benchmarks/results/README.md @@ -1,6 +1,6 @@ # Benchmark Results -Committed files in this directory describe the result protocol only. +Committed files in this directory describe the result protocol and compact reference artifacts only. Local and disposable output belongs under ignored paths: @@ -11,6 +11,15 @@ Local and disposable output belongs under ignored paths: A reference baseline should be recorded only after the documented protocol is run in a controlled, low-noise PHP 8.4 environment with the matching OPcache/JIT policy and environment fingerprint. +The tracked reference directory contains compact evidence summaries: + +```text +results/reference/performance-summary.json +results/reference/performance-report.md +``` + +These files intentionally summarize the accepted policy and public engineering report. They do not contain raw 100-sample process records, raw command streams, or disposable candidate directories. + Controlled comparator execution writes this local structure when `benchmarks/bin/comparator-run.php` is used: ```text @@ -35,3 +44,21 @@ For `application_boot`, the manifest records one separate discarded application_ The `application_boot` normalized output keeps p50 available while mean, p95, p99 and relative standard deviation remain visible as tail and noise evidence. No measured sample is removed as an outlier. The measured process is fresh and performs its first framework construction inside the measured subject; only host-level paths may have been primed by the separate discarded process. Candidate output from a dirty or uncommitted worktree is not canonical reference evidence. Canonical reference evidence must be regenerated from the exact committed implementation ref in the controlled PHP 8.4.25 lane described in the benchmark README. + +Performance budget validation uses `benchmarks/bin/performance-budget.php`: + +```powershell +php benchmarks\bin\performance-budget.php --budget benchmarks\budgets\performance-budget.json --validate-reference +php benchmarks\bin\performance-budget.php --budget benchmarks\budgets\performance-budget.json --candidate benchmarks\results\local\comparator-candidate +``` + +Budget states are: + +- `pass`: the scenario is within the accepted observed p50 envelope. +- `warn`: the scenario exceeds the observed p50 envelope but is non-blocking under the current policy. +- `fail`: a blocking warm HTTP p50 threshold was exceeded. +- `incomparable`: required identity, protocol, availability, cleanliness, sample-count, or normalized-result evidence is absent or mismatched. + +Warm HTTP p50 thresholds are blocking only for controlled evidence that matches the accepted comparison identity. `application_boot` is monitor-only in this policy version because accepted calibration showed high within-run relative standard deviation and volatile tail timing. + +Ordinary shared GitHub-hosted CI may validate benchmark syntax, smoke behaviour, evaluator tests, and reference-policy schema. It must not run the canonical 100-sample comparator timing suite or fail on absolute wall-clock timing from the shared runner. diff --git a/benchmarks/results/reference/performance-report.md b/benchmarks/results/reference/performance-report.md new file mode 100644 index 0000000..2452079 --- /dev/null +++ b/benchmarks/results/reference/performance-report.md @@ -0,0 +1,77 @@ +# EvolvePHP 2 Performance Reference + +This report records the initial benchmark reference used for EvolvePHP 2 performance-budget evaluation. It separates regression policy from cross-framework comparison evidence: the regression budget is calibrated from EvolvePHP-only controlled runs, while the comparison table is a per-scenario reference from the accepted controlled comparator matrix. + +## Methodology + +All accepted timing evidence uses PHP 8.4.25 with OPcache enabled for CLI and JIT disabled. Each scenario contains 100 measured samples. Warm HTTP scenarios use prepared framework instances and configured in-process subject warmups. The repeated-warm scenario reports per-request latency from batches of 25 requests. Application boot is intentionally cold inside each measured worker, with one separate discarded boot worker before every measured worker to prime host-level paths without warming the measured process in-process. + +The canonical execution-environment fingerprint for the EvolvePHP regression calibration is: + +```text +9c06a992d7f01cb7096a60b893f33a34aff7b2a86fba157c8b879d9ac55457a2 +``` + +The regression calibration source SHA is: + +```text +c62cecb16cb4fcdc93bfbb0188a4b63d8cf704ce +``` + +The accepted cross-framework comparator source ref is: + +```text +debfb4228c4d652a5f6d0bdc4ff0f3a9c0a6c1c2 +``` + +These are separate provenance identities and should not be merged. + +## Regression Calibration + +| Scenario | Run 1 p50 us | Run 2 p50 us | Run 3 p50 us | Reference p50 us | Cross-run range | +| --- | ---: | ---: | ---: | ---: | ---: | +| application_boot | 1068.25 | 1042.40 | 1044.05 | 1044.05 | 2.4799% | +| http_static | 30.90 | 30.80 | 30.70 | 30.80 | 0.6515% | +| http_parameterized | 34.40 | 34.40 | 34.50 | 34.40 | 0.2907% | +| http_middleware | 37.20 | 37.20 | 37.30 | 37.20 | 0.2688% | +| http_not_found | 23.90 | 23.80 | 23.80 | 23.80 | 0.4202% | +| http_repeated_warm | 26.662 | 26.652 | 26.540 | 26.652 | 0.4597% | + +Warm HTTP scenarios receive a 5 percent actionable regression budget above the accepted reference median p50. That margin is intentionally larger than the measured cross-run noise of the warm scenarios. It is an engineering policy margin informed by controlled evidence, not a statistical-certainty claim. + +| Scenario | Reference p50 us | Observed maximum p50 us | Blocking threshold p50 us | +| --- | ---: | ---: | ---: | +| http_static | 30.80 | 30.90 | 32.34 | +| http_parameterized | 34.40 | 34.50 | 36.12 | +| http_middleware | 37.20 | 37.30 | 39.06 | +| http_not_found | 23.80 | 23.90 | 24.99 | +| http_repeated_warm | 26.652 | 26.662 | 27.9846 | + +`application_boot` remains monitor-only in this first policy. Its p50 and p99 showed higher volatility, and its within-run relative standard deviation was 29.7693%, 18.6663%, and 25.8726% across the three calibration runs. The current reference p50 is 1044.05 us, the observed maximum p50 is 1068.25 us, and the provisional 10 percent observation boundary is 1148.455 us. Exceeding that boundary asks for repeat investigation, but it is not a blocking timing failure in this policy. + +## Cross-Framework Reference + +The table below is comparison evidence from the accepted controlled comparator matrix. It is not the regression threshold source for EvolvePHP. + +| Scenario | EvolvePHP p50 us | Laravel p50 us | Phalcon p50 us | Slim p50 us | Symfony p50 us | +| --- | ---: | ---: | ---: | ---: | ---: | +| application_boot | 1163.65 | 1144.90 | 264.05 | 699.05 | 579.60 | +| http_static | 30.80 | 117.40 | 29.60 | 25.40 | 93.60 | +| http_parameterized | 34.40 | 126.50 | 33.45 | 29.20 | 103.45 | +| http_middleware | 37.40 | 189.25 | 31.50 | 27.85 | 98.80 | +| http_not_found | 24.00 | 47.60 | 30.10 | 40.30 | 71.20 | +| http_repeated_warm | 26.526 | 110.660 | 25.992 | 21.554 | 87.372 | + +The warm HTTP p50 results are competitive in this controlled matrix. Cold boot is the visible comparative gap and should be profiled before remediation decisions are made. The current data does not prove that framework architecture is the definitive cause of cold-boot cost. + +## CI And Regression Policy + +Ordinary GitHub-hosted CI validates benchmark tooling and policy shape. It should not run the canonical 100-sample timing suite, install every comparator fixture root merely to generate timing, or fail on absolute p50, p95, p99, or wall-clock values from shared runners. + +Controlled candidate evidence can be evaluated with `benchmarks/bin/performance-budget.php` when it was produced under the canonical environment and protocol. Evidence with mismatched environment fingerprint, matrix identity, EvolvePHP comparator lock identity, fixture identity, sample protocol, availability state, source cleanliness, or normalized result shape is classified as incomparable rather than as a timing regression. + +## Limitations + +This reference does not commit raw 100-sample process records, command streams, or disposable candidate directories. It does not define a blocking memory budget because the accepted calibration bundle did not establish a cross-run memory noise floor. It does not certify FrankenPHP, RoadRunner, or another persistent-runtime adapter. + +This is a non-ranking report. It does not make general cross-framework speed claims or provide an overall ranking. diff --git a/benchmarks/results/reference/performance-summary.json b/benchmarks/results/reference/performance-summary.json new file mode 100644 index 0000000..d130ce1 --- /dev/null +++ b/benchmarks/results/reference/performance-summary.json @@ -0,0 +1,99 @@ +{ + "schema_version": "evolvephp.performance-reference-summary.v1", + "regression_baseline_source_sha": "c62cecb16cb4fcdc93bfbb0188a4b63d8cf704ce", + "canonical_environment_fingerprint": "9c06a992d7f01cb7096a60b893f33a34aff7b2a86fba157c8b879d9ac55457a2", + "calibration_protocol": { + "run_count": 3, + "sample_count": 100, + "php_version": "8.4.25", + "opcache_cli_enabled": true, + "jit_enabled": false, + "primary_metric": "p50", + "repeated_warm_request_count": 25, + "raw_evidence_note": "Full raw controlled evidence is intentionally not committed here." + }, + "scenarios": { + "application_boot": { + "p50_microseconds_by_run": [ + 1068.25, + 1042.4, + 1044.05 + ], + "reference_median_p50_microseconds": 1044.05, + "observed_maximum_p50_microseconds": 1068.25, + "observed_range_percent": 2.4798541826554, + "budget_classification": "monitor", + "observation_threshold_p50_microseconds": 1148.455, + "diagnostic_rsd_percent_by_run": [ + 29.76929761631, + 18.666326692293, + 25.872643772215 + ] + }, + "http_static": { + "p50_microseconds_by_run": [ + 30.9, + 30.8, + 30.7 + ], + "reference_median_p50_microseconds": 30.8, + "observed_maximum_p50_microseconds": 30.9, + "observed_range_percent": 0.6514657980456, + "budget_classification": "blocking", + "blocking_threshold_p50_microseconds": 32.34 + }, + "http_parameterized": { + "p50_microseconds_by_run": [ + 34.4, + 34.4, + 34.5 + ], + "reference_median_p50_microseconds": 34.4, + "observed_maximum_p50_microseconds": 34.5, + "observed_range_percent": 0.29069767441861, + "budget_classification": "blocking", + "blocking_threshold_p50_microseconds": 36.12 + }, + "http_middleware": { + "p50_microseconds_by_run": [ + 37.2, + 37.2, + 37.3 + ], + "reference_median_p50_microseconds": 37.2, + "observed_maximum_p50_microseconds": 37.3, + "observed_range_percent": 0.26881720430106, + "budget_classification": "blocking", + "blocking_threshold_p50_microseconds": 39.06 + }, + "http_not_found": { + "p50_microseconds_by_run": [ + 23.9, + 23.8, + 23.8 + ], + "reference_median_p50_microseconds": 23.8, + "observed_maximum_p50_microseconds": 23.9, + "observed_range_percent": 0.42016806722688, + "budget_classification": "blocking", + "blocking_threshold_p50_microseconds": 24.99 + }, + "http_repeated_warm": { + "p50_microseconds_by_run": [ + 26.662, + 26.652, + 26.54 + ], + "reference_median_p50_microseconds": 26.652, + "observed_maximum_p50_microseconds": 26.662, + "observed_range_percent": 0.45968349660889, + "budget_classification": "blocking", + "blocking_threshold_p50_microseconds": 27.9846, + "operations_per_sample": 25 + } + }, + "threshold_policy": { + "warm_http": "pass at or below observed maximum p50, warn above observed maximum through the 5 percent threshold, fail above threshold", + "application_boot": "monitor-only; warn above observed maximum and continue to warn above the provisional observation boundary" + } +} diff --git a/benchmarks/src/Support/PerformanceBudgetEvaluator.php b/benchmarks/src/Support/PerformanceBudgetEvaluator.php new file mode 100644 index 0000000..661c5c7 --- /dev/null +++ b/benchmarks/src/Support/PerformanceBudgetEvaluator.php @@ -0,0 +1,684 @@ + + */ + private const REQUIRED_SCENARIOS = [ + 'application_boot', + 'http_static', + 'http_parameterized', + 'http_middleware', + 'http_not_found', + 'http_repeated_warm', + ]; + + /** + * @param array $budget + * @param array $candidateEvidence + * @return array + */ + public function evaluate(array $budget, array $candidateEvidence): array + { + $this->assertValidBudget($budget); + + $globalReasons = $this->globalIncomparabilityReasons($budget, $candidateEvidence); + $manifestSourceSha = $this->manifestSourceSha($candidateEvidence['manifest'] ?? null); + $scenarioResults = []; + + foreach ($budget['scenarios'] as $scenarioId => $policy) { + $candidate = $this->candidateResultForScenario($candidateEvidence, (string) $scenarioId); + $scenarioResults[(string) $scenarioId] = $this->evaluateScenario( + (string) $scenarioId, + $policy, + $budget, + $candidate, + $manifestSourceSha, + ); + } + + foreach ($scenarioResults as $scenarioResult) { + if (($scenarioResult['status'] ?? null) === 'incomparable') { + foreach ($scenarioResult['reasons'] as $reason) { + $globalReasons[] = (string) $reason; + } + } + } + + $status = $globalReasons !== [] + ? 'incomparable' + : $this->aggregateStatus(array_column($scenarioResults, 'status')); + + return [ + 'schema_version' => self::EVALUATION_SCHEMA_VERSION, + 'status' => $status, + 'blocking' => $status === 'fail', + 'primary_metric' => $budget['primary_metric'], + 'scenarios' => $scenarioResults, + 'reasons' => array_values(array_unique($globalReasons)), + ]; + } + + /** + * @param array $budget + */ + public function assertValidBudget(array $budget): void + { + if (($budget['schema_version'] ?? null) !== self::BUDGET_SCHEMA_VERSION) { + throw new InvalidArgumentException('Invalid performance budget: unsupported schema_version.'); + } + + $this->requireSha($budget, 'baseline_source_sha', 40); + $this->requireExactString($budget, 'primary_metric', 'p50'); + + $calibration = $this->requireArray($budget, 'calibration'); + $this->requireString($calibration, 'source'); + $this->requireString($calibration, 'raw_evidence_policy'); + $this->requireExactPositiveInteger($calibration, 'run_count', 3); + $this->requireExactString($calibration, 'primary_metric', 'p50'); + + $runtime = $this->requireArray($budget, 'canonical_runtime_policy'); + $this->requireString($runtime, 'php_version'); + $this->requireBoolean($runtime, 'opcache_cli_enabled'); + $this->requireBoolean($runtime, 'jit_enabled'); + + $warmup = $this->requireArray($budget, 'warmup_policy'); + $warmHttpWarmups = $this->requireNonNegativeInteger($warmup, 'warm_http_subject_warmups'); + $bootWarmups = $this->requireNonNegativeInteger($warmup, 'application_boot_measured_worker_in_process_warmups'); + $discardedBootWorkers = $this->requirePositiveInteger($warmup, 'application_boot_discarded_worker_processes_per_measured_sample'); + + $identity = $this->requireArray($budget, 'comparison_identity'); + $this->requireExactString($identity, 'manifest_schema_version', 'evolvephp.comparator.evidence-manifest.v1'); + $this->requireExactString($identity, 'comparator_result_schema_version', 'evolvephp.comparator.result.v1'); + $this->requireExactString($identity, 'comparator_id', 'evolvephp'); + + foreach ([ + 'execution_environment_fingerprint', + 'matrix_sha256', + 'comparator_lock_sha256', + 'fixture_identity_hash', + ] as $key) { + $this->requireSha($identity, $key, 64); + } + + $protocol = $this->requireArray($budget, 'sample_protocol'); + $sampleCount = $this->requirePositiveInteger($protocol, 'sample_count'); + $this->requireExactPositiveInteger($calibration, 'sample_count', $sampleCount); + $warmups = $this->requireNonNegativeInteger($protocol, 'warmups'); + $requestCount = $this->requirePositiveInteger($protocol, 'request_count'); + $repeatedWarmOperations = $this->requirePositiveInteger($protocol, 'repeated_warm_operations_per_sample'); + $this->requireExactString($protocol, 'process_isolation_model', 'subprocess_per_measured_sample'); + + if ($warmups !== $warmHttpWarmups) { + throw new InvalidArgumentException('Invalid performance budget: warmup policy does not match sample protocol.'); + } + + if ($requestCount !== $repeatedWarmOperations) { + throw new InvalidArgumentException('Invalid performance budget: repeated-warm operation count must match request count.'); + } + + $bootProtocol = $this->requireArray($protocol, 'boot_protocol'); + $this->requireExactPositiveInteger($bootProtocol, 'discarded_worker_processes_per_measured_sample', $discardedBootWorkers); + $this->requireExactString($bootProtocol, 'sample_order', 'rotating_round_robin'); + $this->requireExactNonNegativeInteger($bootProtocol, 'measured_worker_in_process_warmups', $bootWarmups); + $this->requireExactString($bootProtocol, 'outlier_policy', 'retain_all_measured_samples'); + $this->requireExactString($bootProtocol, 'primary_central_statistic', 'p50'); + + $scenarios = $this->requireArray($budget, 'scenarios'); + if ($scenarios === []) { + throw new InvalidArgumentException('Invalid performance budget: scenarios must not be empty.'); + } + + $scenarioIds = array_keys($scenarios); + sort($scenarioIds, SORT_STRING); + $requiredScenarioIds = self::REQUIRED_SCENARIOS; + sort($requiredScenarioIds, SORT_STRING); + + if ($scenarioIds !== $requiredScenarioIds) { + throw new InvalidArgumentException('Invalid performance budget: scenarios must contain exactly the supported scenario set.'); + } + + foreach ($scenarios as $scenarioId => $scenario) { + $this->assertValidScenarioPolicy((string) $scenarioId, $scenario, (int) $calibration['run_count']); + } + } + + /** + * @param array $budget + * @param array $candidateEvidence + * @return list + */ + private function globalIncomparabilityReasons(array $budget, array $candidateEvidence): array + { + $manifest = $candidateEvidence['manifest'] ?? null; + + if (!is_array($manifest)) { + return ['candidate evidence is missing manifest']; + } + + $identity = $budget['comparison_identity']; + $protocol = $budget['sample_protocol']; + $reasons = []; + + $this->appendMismatch( + $reasons, + 'manifest schema version', + $identity['manifest_schema_version'], + $manifest['schema_version'] ?? null, + ); + $this->appendMismatch($reasons, 'manifest status', 'completed', $manifest['status'] ?? null); + + if (!is_array($manifest['source'] ?? null)) { + $reasons[] = 'candidate source evidence is missing'; + } elseif (!$this->validSha($manifest['source']['git_sha'] ?? null, 40)) { + $reasons[] = 'candidate source SHA is missing or invalid'; + } + + if (!is_array($manifest['source'] ?? null) || !array_key_exists('dirty', $manifest['source']) || !is_bool($manifest['source']['dirty']) || $manifest['source']['dirty'] !== false) { + $reasons[] = 'candidate canonical evidence dirty state is missing, invalid, or dirty'; + } + + $this->appendMismatch( + $reasons, + 'execution environment fingerprint', + $identity['execution_environment_fingerprint'], + $manifest['execution_environment_fingerprint'] ?? null, + ); + $this->appendMismatch($reasons, 'matrix hash', $identity['matrix_sha256'], $manifest['matrix']['sha256'] ?? null); + $this->appendMismatch( + $reasons, + 'process isolation model', + $protocol['process_isolation_model'], + $manifest['process_isolation']['model'] ?? null, + ); + $this->appendMismatch($reasons, 'sample count protocol', $protocol['sample_count'], $manifest['samples'] ?? null); + $this->appendMismatch($reasons, 'warmup protocol', $protocol['warmups'], $manifest['warmups'] ?? null); + $this->appendMismatch($reasons, 'request count protocol', $protocol['request_count'], $manifest['request_count'] ?? null); + + if (($manifest['boot_protocol'] ?? null) !== $protocol['boot_protocol']) { + $reasons[] = 'application boot protocol identity mismatch'; + } + + return $reasons; + } + + /** + * @param array $policy + * @param array $budget + * @param array|null $candidate + * @return array + */ + private function evaluateScenario( + string $scenarioId, + array $policy, + array $budget, + ?array $candidate, + ?string $manifestSourceSha, + ): array { + if ($candidate === null) { + return $this->scenarioResult($policy, 'incomparable', null, ['missing required scenario ' . $scenarioId]); + } + + $reasons = $this->scenarioIncomparabilityReasons($scenarioId, $candidate, $budget, $manifestSourceSha); + + if ($reasons !== []) { + return $this->scenarioResult($policy, 'incomparable', null, $reasons); + } + + $scenario = $candidate['baseline_result']['scenarios'][0]; + $p50 = (float) $scenario['p50']; + $observedMaximum = (float) $policy['observed_maximum_p50_microseconds']; + $mode = (string) $policy['mode']; + + if ($p50 <= $observedMaximum) { + return $this->scenarioResult($policy, 'pass', $p50); + } + + if ($mode === 'blocking' && $p50 > (float) $policy['blocking_threshold_p50_microseconds']) { + return $this->scenarioResult($policy, 'fail', $p50, ['p50 exceeds blocking threshold']); + } + + $reasons = ['p50 exceeds observed calibration envelope']; + + if ( + $mode === 'monitor' + && $p50 > (float) $policy['observation_threshold_p50_microseconds'] + ) { + $reasons[] = 'p50 exceeds observation boundary; repeat investigation is required'; + } + + return $this->scenarioResult($policy, 'warn', $p50, $reasons); + } + + /** + * @param array $candidate + * @param array $budget + * @return list + */ + private function scenarioIncomparabilityReasons( + string $scenarioId, + array $candidate, + array $budget, + ?string $manifestSourceSha, + ): array { + $identity = $budget['comparison_identity']; + $protocol = $budget['sample_protocol']; + $reasons = []; + + $this->appendMismatch( + $reasons, + 'comparator result schema version', + $identity['comparator_result_schema_version'], + $candidate['comparator_schema_version'] ?? null, + ); + $this->appendMismatch($reasons, 'comparator identity', $identity['comparator_id'], $candidate['comparator_id'] ?? null); + $this->appendMismatch($reasons, 'scenario identity', $scenarioId, $candidate['scenario_id'] ?? null); + $this->appendMismatch( + $reasons, + 'execution environment fingerprint', + $identity['execution_environment_fingerprint'], + $candidate['execution_environment_fingerprint'] ?? null, + ); + $this->appendMismatch($reasons, 'matrix hash', $identity['matrix_sha256'], $candidate['matrix_sha256'] ?? null); + $this->appendMismatch($reasons, 'EvolvePHP comparator lock hash', $identity['comparator_lock_sha256'], $candidate['comparator_lock_sha256'] ?? null); + $this->appendMismatch($reasons, 'fixture identity hash', $identity['fixture_identity_hash'], $candidate['fixture_identity_hash'] ?? null); + + if (($candidate['availability'] ?? null) !== 'available') { + $reasons[] = 'comparator result is not available'; + } + + if (!$this->validSha($candidate['source_evolvephp_sha'] ?? null, 40)) { + $reasons[] = 'candidate source SHA is missing or invalid'; + } elseif ($manifestSourceSha !== null && $candidate['source_evolvephp_sha'] !== $manifestSourceSha) { + $reasons[] = 'candidate source SHA does not match manifest source SHA'; + } + + if (!array_key_exists('source_dirty', $candidate) || !is_bool($candidate['source_dirty']) || $candidate['source_dirty'] !== false) { + $reasons[] = 'candidate normalized evidence dirty state is missing, invalid, or dirty'; + } + + $baseline = $candidate['baseline_result'] ?? null; + if (!is_array($baseline) || !is_array($baseline['scenarios'] ?? null) || !isset($baseline['scenarios'][0]) || !is_array($baseline['scenarios'][0])) { + $reasons[] = 'malformed or missing normalized result'; + + return $reasons; + } + + if (array_key_exists('source_sha', $baseline) && $baseline['source_sha'] !== $manifestSourceSha) { + $reasons[] = 'baseline result source SHA does not match manifest source SHA'; + } + + $scenario = $baseline['scenarios'][0]; + $this->appendMismatch($reasons, 'normalized scenario identity', $scenarioId, $scenario['id'] ?? null); + $this->appendMismatch($reasons, 'normalized timing unit', $this->expectedUnit($scenarioId), $scenario['unit'] ?? null); + + if (!$this->positiveFiniteNumber($scenario['p50'] ?? null)) { + $reasons[] = 'normalized p50 is missing, non-finite, or not greater than zero'; + } + + if (($scenario['p50_status'] ?? 'available') !== 'available') { + $reasons[] = 'normalized p50 status is not available'; + } + + if (($scenario['sample_count'] ?? null) !== $protocol['sample_count']) { + $reasons[] = 'normalized sample count protocol mismatch'; + } + + if ( + $scenarioId === 'http_repeated_warm' + && ($scenario['operations_per_sample'] ?? null) !== $protocol['repeated_warm_operations_per_sample'] + ) { + $reasons[] = 'repeated-warm operations per sample protocol mismatch'; + } + + return $reasons; + } + + /** + * @param array $policy + * @param list $reasons + * @return array + */ + private function scenarioResult(array $policy, string $status, ?float $p50, array $reasons = []): array + { + return [ + 'status' => $status, + 'blocking' => $status === 'fail', + 'mode' => $policy['mode'], + 'p50_microseconds' => $p50, + 'reference_p50_microseconds' => $policy['reference_p50_microseconds'], + 'observed_maximum_p50_microseconds' => $policy['observed_maximum_p50_microseconds'], + 'threshold_p50_microseconds' => $policy['blocking_threshold_p50_microseconds'] + ?? $policy['observation_threshold_p50_microseconds'], + 'reasons' => $reasons, + ]; + } + + /** + * @param mixed $scenario + */ + private function assertValidScenarioPolicy(string $scenarioId, mixed $scenario, int $calibrationRunCount): void + { + if ($scenarioId === '') { + throw new InvalidArgumentException('Invalid performance budget: scenario ids must be non-empty strings.'); + } + + if (!is_array($scenario)) { + throw new InvalidArgumentException("Invalid performance budget: scenario {$scenarioId} must be an object."); + } + + $mode = $this->requireString($scenario, 'mode'); + if ($scenarioId === 'application_boot') { + $this->requireExactString($scenario, 'mode', 'monitor'); + } else { + $this->requireExactString($scenario, 'mode', 'blocking'); + } + + $observed = $this->requirePositiveFiniteNumberList($scenario, 'observed_p50_microseconds', $calibrationRunCount); + $reference = $this->requirePositiveFiniteNumber($scenario, 'reference_p50_microseconds'); + $observedMaximum = $this->requirePositiveFiniteNumber($scenario, 'observed_maximum_p50_microseconds'); + $rangePercent = $this->requireNonNegativeFiniteNumber($scenario, 'cross_run_range_percent'); + + if ($reference > $observedMaximum) { + throw new InvalidArgumentException("Invalid performance budget: scenario {$scenarioId} reference p50 exceeds observed maximum."); + } + + if (!$this->floatEquals($reference, $this->median($observed))) { + throw new InvalidArgumentException("Invalid performance budget: scenario {$scenarioId} reference p50 does not match observed median."); + } + + if (!$this->floatEquals($observedMaximum, max($observed))) { + throw new InvalidArgumentException("Invalid performance budget: scenario {$scenarioId} observed maximum does not match observed p50 values."); + } + + if (!$this->floatEquals($rangePercent, $this->rangePercent($observed))) { + throw new InvalidArgumentException("Invalid performance budget: scenario {$scenarioId} cross-run range does not match observed p50 values."); + } + + if ($mode === 'blocking') { + $threshold = $this->requirePositiveFiniteNumber($scenario, 'blocking_threshold_p50_microseconds'); + if ($threshold <= $observedMaximum) { + throw new InvalidArgumentException("Invalid performance budget: scenario {$scenarioId} blocking threshold must exceed observed maximum."); + } + + return; + } + + $threshold = $this->requirePositiveFiniteNumber($scenario, 'observation_threshold_p50_microseconds'); + if ($threshold <= $observedMaximum) { + throw new InvalidArgumentException("Invalid performance budget: scenario {$scenarioId} observation threshold must exceed observed maximum."); + } + } + + /** + * @param array $candidateEvidence + * @return array|null + */ + private function candidateResultForScenario(array $candidateEvidence, string $scenarioId): ?array + { + if (!is_array($candidateEvidence['results'] ?? null)) { + return null; + } + + foreach ($candidateEvidence['results'] as $result) { + if (is_array($result) && ($result['scenario_id'] ?? null) === $scenarioId) { + return $result; + } + } + + return null; + } + + /** + * @param list $statuses + */ + private function aggregateStatus(array $statuses): string + { + foreach (['incomparable', 'fail', 'warn'] as $state) { + if (in_array($state, $statuses, true)) { + return $state; + } + } + + return 'pass'; + } + + /** + * @param list $reasons + */ + private function appendMismatch(array &$reasons, string $label, mixed $expected, mixed $actual): void + { + if ($actual !== $expected) { + $reasons[] = $label . ' mismatch'; + } + } + + private function expectedUnit(string $scenarioId): string + { + return $scenarioId === 'http_repeated_warm' ? 'per_operation_microseconds' : 'microseconds'; + } + + private function manifestSourceSha(mixed $manifest): ?string + { + if (!is_array($manifest) || !is_array($manifest['source'] ?? null)) { + return null; + } + + $sha = $manifest['source']['git_sha'] ?? null; + + return $this->validSha($sha, 40) ? $sha : null; + } + + /** + * @param array $data + */ + private function requireString(array $data, string $key): string + { + if (!isset($data[$key]) || !is_string($data[$key]) || $data[$key] === '') { + throw new InvalidArgumentException("Invalid performance budget: {$key} is required."); + } + + return $data[$key]; + } + + /** + * @param array $data + */ + private function requireExactString(array $data, string $key, string $expected): string + { + $actual = $this->requireString($data, $key); + + if ($actual !== $expected) { + throw new InvalidArgumentException("Invalid performance budget: {$key} must be {$expected}."); + } + + return $actual; + } + + /** + * @param array $data + * @return array + */ + private function requireArray(array $data, string $key): array + { + if (!isset($data[$key]) || !is_array($data[$key])) { + throw new InvalidArgumentException("Invalid performance budget: {$key} is required."); + } + + return $data[$key]; + } + + /** + * @param array $data + */ + private function requireSha(array $data, string $key, int $length): void + { + if (!$this->validSha($data[$key] ?? null, $length)) { + throw new InvalidArgumentException("Invalid performance budget: {$key} must be a {$length}-character hexadecimal hash."); + } + } + + /** + * @param array $data + */ + private function requireBoolean(array $data, string $key): bool + { + if (!array_key_exists($key, $data) || !is_bool($data[$key])) { + throw new InvalidArgumentException("Invalid performance budget: {$key} must be boolean."); + } + + return $data[$key]; + } + + /** + * @param array $data + */ + private function requirePositiveInteger(array $data, string $key): int + { + if (!isset($data[$key]) || !is_int($data[$key]) || $data[$key] < 1) { + throw new InvalidArgumentException("Invalid performance budget: {$key} must be a positive integer."); + } + + return $data[$key]; + } + + /** + * @param array $data + */ + private function requireNonNegativeInteger(array $data, string $key): int + { + if (!array_key_exists($key, $data) || !is_int($data[$key]) || $data[$key] < 0) { + throw new InvalidArgumentException("Invalid performance budget: {$key} must be a non-negative integer."); + } + + return $data[$key]; + } + + /** + * @param array $data + */ + private function requireExactPositiveInteger(array $data, string $key, int $expected): int + { + $actual = $this->requirePositiveInteger($data, $key); + + if ($actual !== $expected) { + throw new InvalidArgumentException("Invalid performance budget: {$key} must be {$expected}."); + } + + return $actual; + } + + /** + * @param array $data + */ + private function requireExactNonNegativeInteger(array $data, string $key, int $expected): int + { + $actual = $this->requireNonNegativeInteger($data, $key); + + if ($actual !== $expected) { + throw new InvalidArgumentException("Invalid performance budget: {$key} must be {$expected}."); + } + + return $actual; + } + + /** + * @param array $data + */ + private function requirePositiveFiniteNumber(array $data, string $key): float + { + if (!$this->positiveFiniteNumber($data[$key] ?? null)) { + throw new InvalidArgumentException("Invalid performance budget: {$key} must be a positive finite number."); + } + + return (float) $data[$key]; + } + + /** + * @param array $data + */ + private function requireNonNegativeFiniteNumber(array $data, string $key): float + { + if (!isset($data[$key]) || !is_numeric($data[$key]) || !is_finite((float) $data[$key]) || (float) $data[$key] < 0.0) { + throw new InvalidArgumentException("Invalid performance budget: {$key} must be a non-negative finite number."); + } + + return (float) $data[$key]; + } + + /** + * @param array $data + * @return list + */ + private function requirePositiveFiniteNumberList(array $data, string $key, int $expectedCount): array + { + if (!isset($data[$key]) || !is_array($data[$key]) || count($data[$key]) !== $expectedCount) { + throw new InvalidArgumentException("Invalid performance budget: {$key} must contain {$expectedCount} values."); + } + + $values = []; + foreach ($data[$key] as $value) { + if (!$this->positiveFiniteNumber($value)) { + throw new InvalidArgumentException("Invalid performance budget: {$key} contains a non-positive or non-finite value."); + } + + $values[] = (float) $value; + } + + return $values; + } + + private function positiveFiniteNumber(mixed $value): bool + { + return is_numeric($value) && is_finite((float) $value) && (float) $value > 0.0; + } + + /** + * @param list $values + */ + private function median(array $values): float + { + sort($values, SORT_NUMERIC); + $count = count($values); + $middle = intdiv($count, 2); + + if ($count % 2 === 1) { + return $values[$middle]; + } + + return ($values[$middle - 1] + $values[$middle]) / 2; + } + + /** + * @param list $values + */ + private function rangePercent(array $values): float + { + $minimum = min($values); + + return ((max($values) - $minimum) / $minimum) * 100; + } + + private function floatEquals(float $left, float $right): bool + { + return abs($left - $right) <= self::FLOAT_TOLERANCE; + } + + private function validSha(mixed $value, int $length): bool + { + return is_string($value) && preg_match('/\A[a-f0-9]{' . $length . '}\z/', $value) === 1; + } +} diff --git a/benchmarks/tests/PerformanceBudgetEvaluatorTest.php b/benchmarks/tests/PerformanceBudgetEvaluatorTest.php new file mode 100644 index 0000000..f8d5197 --- /dev/null +++ b/benchmarks/tests/PerformanceBudgetEvaluatorTest.php @@ -0,0 +1,720 @@ + + */ + private array $temporaryDirectories = []; + + protected function tearDown(): void + { + foreach ($this->temporaryDirectories as $directory) { + $this->removeDirectory($directory); + } + + $this->temporaryDirectories = []; + } + + public function testWarmScenarioPassesWithinObservedEnvelope(): void + { + $result = $this->evaluateWithScenarioP50('http_static', 30.90); + + self::assertSame('pass', $result['status']); + self::assertSame('pass', $result['scenarios']['http_static']['status']); + self::assertFalse($result['blocking']); + } + + public function testWarmScenarioWarnsBetweenObservedMaximumAndBlockingThreshold(): void + { + $result = $this->evaluateWithScenarioP50('http_static', 31.00); + + self::assertSame('warn', $result['status']); + self::assertSame('warn', $result['scenarios']['http_static']['status']); + self::assertFalse($result['blocking']); + } + + public function testWarmScenarioFailsBeyondBlockingThreshold(): void + { + $result = $this->evaluateWithScenarioP50('http_static', 32.35); + + self::assertSame('fail', $result['status']); + self::assertSame('fail', $result['scenarios']['http_static']['status']); + self::assertTrue($result['blocking']); + } + + public function testApplicationBootPassesWithinObservedEnvelope(): void + { + $result = $this->evaluateWithScenarioP50('application_boot', 1068.25); + + self::assertSame('pass', $result['status']); + self::assertSame('pass', $result['scenarios']['application_boot']['status']); + self::assertFalse($result['blocking']); + } + + public function testApplicationBootWarnsBeyondObservedEnvelope(): void + { + $result = $this->evaluateWithScenarioP50('application_boot', 1068.26); + + self::assertSame('warn', $result['status']); + self::assertSame('warn', $result['scenarios']['application_boot']['status']); + self::assertFalse($result['blocking']); + } + + public function testApplicationBootRemainsNonBlockingBeyondObservationBoundary(): void + { + $result = $this->evaluateWithScenarioP50('application_boot', 1148.456); + + self::assertSame('warn', $result['status']); + self::assertSame('warn', $result['scenarios']['application_boot']['status']); + self::assertFalse($result['blocking']); + self::assertStringContainsString('observation boundary', implode(' ', $result['scenarios']['application_boot']['reasons'])); + } + + public function testEnvironmentFingerprintMismatchIsIncomparable(): void + { + $candidate = $this->candidateEvidence(); + $candidate['manifest']['execution_environment_fingerprint'] = str_repeat('0', 64); + + $this->assertIncomparable($candidate, 'execution environment fingerprint'); + } + + public function testMatrixLockAndProtocolMismatchesAreIncomparable(): void + { + $matrixCandidate = $this->candidateEvidence(); + $matrixCandidate['manifest']['matrix']['sha256'] = str_repeat('1', 64); + $this->assertIncomparable($matrixCandidate, 'matrix'); + + $lockCandidate = $this->candidateEvidence(); + $lockCandidate['results'][0]['comparator_lock_sha256'] = str_repeat('2', 64); + $this->assertIncomparable($lockCandidate, 'lock'); + + $protocolCandidate = $this->candidateEvidence(); + $protocolCandidate['manifest']['warmups'] = 4; + $this->assertIncomparable($protocolCandidate, 'warmup'); + } + + public function testUnavailableFailedDirtyAndInsufficientEvidenceAreIncomparable(): void + { + $unavailable = $this->candidateEvidence(); + $unavailable['results'][0]['availability'] = 'unavailable'; + $this->assertIncomparable($unavailable, 'available'); + + $failed = $this->candidateEvidence(); + $failed['results'][0]['availability'] = 'failed'; + $this->assertIncomparable($failed, 'available'); + + $dirty = $this->candidateEvidence(); + $dirty['results'][0]['source_dirty'] = true; + $this->assertIncomparable($dirty, 'dirty'); + + $insufficient = $this->candidateEvidence(); + $insufficient['results'][0]['baseline_result']['scenarios'][0]['sample_count'] = 99; + $this->assertIncomparable($insufficient, 'sample count'); + } + + public function testCleanlinessEvidenceMustBePositivelyProven(): void + { + $manifestDirty = $this->candidateEvidence(); + $manifestDirty['manifest']['source']['dirty'] = true; + $this->assertIncomparable($manifestDirty, 'dirty'); + + $manifestMissing = $this->candidateEvidence(); + unset($manifestMissing['manifest']['source']['dirty']); + $this->assertIncomparable($manifestMissing, 'dirty'); + + $manifestWrongType = $this->candidateEvidence(); + $manifestWrongType['manifest']['source']['dirty'] = 'false'; + $this->assertIncomparable($manifestWrongType, 'dirty'); + + $normalizedDirty = $this->candidateEvidence(); + $normalizedDirty['results'][0]['source_dirty'] = true; + $this->assertIncomparable($normalizedDirty, 'dirty'); + + $normalizedMissing = $this->candidateEvidence(); + unset($normalizedMissing['results'][0]['source_dirty']); + $this->assertIncomparable($normalizedMissing, 'dirty'); + + $normalizedWrongType = $this->candidateEvidence(); + $normalizedWrongType['results'][0]['source_dirty'] = 'false'; + $this->assertIncomparable($normalizedWrongType, 'dirty'); + } + + public function testCandidateSourceShaMustBeConsistentAcrossManifestAndNormalizedResults(): void + { + $normalizedMismatch = $this->candidateEvidence(); + $normalizedMismatch['results'][0]['source_evolvephp_sha'] = str_repeat('1', 40); + $this->assertIncomparable($normalizedMismatch, 'source SHA'); + + $baselineMismatch = $this->candidateEvidence(); + $baselineMismatch['results'][0]['baseline_result']['source_sha'] = str_repeat('2', 40); + $this->assertIncomparable($baselineMismatch, 'source SHA'); + + $missingNormalized = $this->candidateEvidence(); + unset($missingNormalized['results'][0]['source_evolvephp_sha']); + $this->assertIncomparable($missingNormalized, 'source SHA'); + + $invalidNormalized = $this->candidateEvidence(); + $invalidNormalized['results'][0]['source_evolvephp_sha'] = 'not-a-sha'; + $this->assertIncomparable($invalidNormalized, 'source SHA'); + } + + public function testMissingRequiredScenarioAndMalformedResultAreIncomparable(): void + { + $missing = $this->candidateEvidence(); + $missing['results'] = array_values(array_filter( + $missing['results'], + static fn(array $result): bool => $result['scenario_id'] !== 'http_static', + )); + $this->assertIncomparable($missing, 'missing required scenario'); + + $malformed = $this->candidateEvidence(); + $malformed['results'][0]['baseline_result'] = null; + $this->assertIncomparable($malformed, 'normalized result'); + } + + public function testRepeatedWarmOperationsProtocolMismatchIsIncomparable(): void + { + $candidate = $this->candidateEvidence(); + + foreach ($candidate['results'] as &$result) { + if ($result['scenario_id'] === 'http_repeated_warm') { + $result['baseline_result']['scenarios'][0]['operations_per_sample'] = 24; + } + } + + $this->assertIncomparable($candidate, 'operations per sample'); + } + + public function testNormalizedUnitSampleCountAndP50ProtocolMustMatchExactly(): void + { + $wrongUnit = $this->candidateEvidence(); + $wrongUnit['results'][0]['baseline_result']['scenarios'][0]['unit'] = 'nanoseconds'; + $this->assertIncomparable($wrongUnit, 'unit'); + + $tooFewSamples = $this->candidateEvidence(); + $tooFewSamples['results'][0]['baseline_result']['scenarios'][0]['sample_count'] = 99; + $this->assertIncomparable($tooFewSamples, 'sample count'); + + $tooManySamples = $this->candidateEvidence(); + $tooManySamples['results'][0]['baseline_result']['scenarios'][0]['sample_count'] = 101; + $this->assertIncomparable($tooManySamples, 'sample count'); + + $zeroP50 = $this->candidateEvidence(); + $zeroP50['results'][0]['baseline_result']['scenarios'][0]['p50'] = 0.0; + $this->assertIncomparable($zeroP50, 'p50'); + + $negativeP50 = $this->candidateEvidence(); + $negativeP50['results'][0]['baseline_result']['scenarios'][0]['p50'] = -1.0; + $this->assertIncomparable($negativeP50, 'p50'); + + $nonFiniteP50 = $this->candidateEvidence(); + $nonFiniteP50['results'][0]['baseline_result']['scenarios'][0]['p50'] = INF; + $this->assertIncomparable($nonFiniteP50, 'p50'); + + $unavailableP50Status = $this->candidateEvidence(); + $unavailableP50Status['results'][0]['baseline_result']['scenarios'][0]['p50_status'] = 'insufficient_samples'; + $this->assertIncomparable($unavailableP50Status, 'p50 status'); + + $wrongRepeatedWarmUnit = $this->candidateEvidence(); + foreach ($wrongRepeatedWarmUnit['results'] as &$result) { + if ($result['scenario_id'] === 'http_repeated_warm') { + $result['baseline_result']['scenarios'][0]['unit'] = 'microseconds'; + } + } + unset($result); + $this->assertIncomparable($wrongRepeatedWarmUnit, 'unit'); + } + + public function testMalformedBudgetReferenceDataIsAValidationError(): void + { + $budget = $this->budget(); + unset($budget['scenarios']['http_static']['reference_p50_microseconds']); + + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('performance budget'); + + (new PerformanceBudgetEvaluator())->evaluate($budget, $this->candidateEvidence()); + } + + public function testUnsupportedAndIncoherentBudgetPolicyIsAValidationError(): void + { + $unsupportedSchema = $this->budget(); + $unsupportedSchema['schema_version'] = 'evolvephp.performance-budget.v0'; + $this->assertInvalidBudget($unsupportedSchema); + + $wrongCalibrationCount = $this->budget(); + $wrongCalibrationCount['calibration']['run_count'] = 2; + $this->assertInvalidBudget($wrongCalibrationCount); + + $referenceAboveMaximum = $this->budget(); + $referenceAboveMaximum['scenarios']['http_static']['reference_p50_microseconds'] = 31.0; + $this->assertInvalidBudget($referenceAboveMaximum); + + $wrongMedian = $this->budget(); + $wrongMedian['scenarios']['http_static']['reference_p50_microseconds'] = 30.7; + $this->assertInvalidBudget($wrongMedian); + + $wrongRange = $this->budget(); + $wrongRange['scenarios']['http_static']['cross_run_range_percent'] = 99.0; + $this->assertInvalidBudget($wrongRange); + + $blockingThresholdBelowObservedMax = $this->budget(); + $blockingThresholdBelowObservedMax['scenarios']['http_static']['blocking_threshold_p50_microseconds'] = 30.8; + $this->assertInvalidBudget($blockingThresholdBelowObservedMax); + } + + public function testBudgetMustContainExactlyTheSupportedScenarioSet(): void + { + $missingHttpStatic = $this->budget(); + unset($missingHttpStatic['scenarios']['http_static']); + $this->assertInvalidBudget($missingHttpStatic); + + $missingApplicationBoot = $this->budget(); + unset($missingApplicationBoot['scenarios']['application_boot']); + $this->assertInvalidBudget($missingApplicationBoot); + + $extraScenario = $this->budget(); + $extraScenario['scenarios']['unknown_scenario'] = $extraScenario['scenarios']['http_static']; + $this->assertInvalidBudget($extraScenario); + } + + public function testBudgetComparatorIdentityContractIsPinned(): void + { + $manifestSchemaDrift = $this->budget(); + $manifestSchemaDrift['comparison_identity']['manifest_schema_version'] = 'evolvephp.comparator.evidence-manifest.v2'; + $this->assertInvalidBudget($manifestSchemaDrift); + + $resultSchemaDrift = $this->budget(); + $resultSchemaDrift['comparison_identity']['comparator_result_schema_version'] = 'evolvephp.comparator.result.v2'; + $this->assertInvalidBudget($resultSchemaDrift); + + $comparatorIdDrift = $this->budget(); + $comparatorIdDrift['comparison_identity']['comparator_id'] = 'laravel'; + $this->assertInvalidBudget($comparatorIdDrift); + } + + public function testAggregateStatePrecedenceIsDeterministic(): void + { + $warn = $this->candidateEvidence(['http_static' => 31.00]); + self::assertSame('warn', (new PerformanceBudgetEvaluator())->evaluate($this->budget(), $warn)['status']); + + $fail = $this->candidateEvidence(['http_static' => 32.35, 'application_boot' => 1200.00]); + self::assertSame('fail', (new PerformanceBudgetEvaluator())->evaluate($this->budget(), $fail)['status']); + + $incomparable = $this->candidateEvidence(['http_static' => 32.35]); + $incomparable['manifest']['execution_environment_fingerprint'] = str_repeat('f', 64); + self::assertSame('incomparable', (new PerformanceBudgetEvaluator())->evaluate($this->budget(), $incomparable)['status']); + } + + public function testCliCandidateOptionAcceptsDocumentedSpaceSeparatedDirectory(): void + { + $budgetPath = $this->writeBudgetFile($this->budget()); + $candidateDir = $this->writeCandidateDirectory($this->candidateEvidence()); + + $result = $this->runBudgetCli([ + '--budget', + $budgetPath, + '--candidate', + $candidateDir, + ]); + + self::assertSame(0, $result['exit_code'], $result['stderr']); + self::assertStringContainsString('"status": "pass"', $result['stdout']); + } + + public function testCliReferenceValidationFailsWhenDuplicatedSummaryPolicyDrifts(): void + { + $thresholdDrift = $this->trackedBudget(); + $thresholdDrift['scenarios']['http_static']['blocking_threshold_p50_microseconds'] = 32.35; + $thresholdResult = $this->runBudgetCli([ + '--budget', + $this->writeBudgetFile($thresholdDrift), + '--validate-reference', + ]); + self::assertSame(2, $thresholdResult['exit_code']); + + $protocolDrift = $this->trackedBudget(); + $protocolDrift['sample_protocol']['request_count'] = 24; + $protocolDrift['scenarios']['http_repeated_warm']['observed_p50_microseconds'] = [26.662, 26.652, 26.54]; + $protocolResult = $this->runBudgetCli([ + '--budget', + $this->writeBudgetFile($protocolDrift), + '--validate-reference', + ]); + self::assertSame(2, $protocolResult['exit_code']); + + $runDrift = $this->trackedBudget(); + $runDrift['scenarios']['http_static']['observed_p50_microseconds'] = [30.9, 30.8, 30.8]; + $runDrift['scenarios']['http_static']['reference_p50_microseconds'] = 30.8; + $runDrift['scenarios']['http_static']['observed_maximum_p50_microseconds'] = 30.9; + $runDrift['scenarios']['http_static']['cross_run_range_percent'] = 0.32467532467532; + $runResult = $this->runBudgetCli([ + '--budget', + $this->writeBudgetFile($runDrift), + '--validate-reference', + ]); + self::assertSame(2, $runResult['exit_code']); + } + + public function testCliRejectsNormalizedArtifactHashMismatchAndTraversal(): void + { + $budgetPath = $this->writeBudgetFile($this->budget()); + + $mismatchedHashDir = $this->writeCandidateDirectory($this->candidateEvidence(), corruptFirstHash: true); + $mismatchedHash = $this->runBudgetCli([ + '--budget', + $budgetPath, + '--candidate', + $mismatchedHashDir, + ]); + self::assertSame(2, $mismatchedHash['exit_code']); + + $traversalDir = $this->writeCandidateDirectory($this->candidateEvidence(), traversalFirstPath: true); + $traversal = $this->runBudgetCli([ + '--budget', + $budgetPath, + '--candidate', + $traversalDir, + ]); + self::assertSame(2, $traversal['exit_code']); + } + + /** + * @return array + */ + private function evaluateWithScenarioP50(string $scenarioId, float $p50): array + { + return (new PerformanceBudgetEvaluator())->evaluate($this->budget(), $this->candidateEvidence([$scenarioId => $p50])); + } + + /** + * @param array $candidate + */ + private function assertIncomparable(array $candidate, string $reasonFragment): void + { + $result = (new PerformanceBudgetEvaluator())->evaluate($this->budget(), $candidate); + + self::assertSame('incomparable', $result['status']); + self::assertFalse($result['blocking']); + self::assertStringContainsString($reasonFragment, implode(' ', $result['reasons'])); + } + + /** + * @param array $budget + */ + private function assertInvalidBudget(array $budget): void + { + try { + (new PerformanceBudgetEvaluator())->evaluate($budget, $this->candidateEvidence()); + } catch (InvalidArgumentException $exception) { + self::assertStringContainsString('performance budget', $exception->getMessage()); + + return; + } + + self::fail('Expected invalid performance budget.'); + } + + /** + * @return array + */ + private function budget(): array + { + return [ + 'schema_version' => 'evolvephp.performance-budget.v1', + 'baseline_source_sha' => str_repeat('a', 40), + 'calibration' => [ + 'source' => 'accepted controlled EvolvePHP-only regression calibration', + 'run_count' => 3, + 'sample_count' => 100, + 'primary_metric' => 'p50', + 'raw_evidence_policy' => 'full raw controlled evidence is retained outside this repository', + ], + 'canonical_runtime_policy' => [ + 'php_version' => '8.4.25', + 'opcache_cli_enabled' => true, + 'jit_enabled' => false, + ], + 'warmup_policy' => [ + 'warm_http_subject_warmups' => 5, + 'application_boot_measured_worker_in_process_warmups' => 0, + 'application_boot_discarded_worker_processes_per_measured_sample' => 1, + ], + 'comparison_identity' => [ + 'manifest_schema_version' => 'evolvephp.comparator.evidence-manifest.v1', + 'comparator_result_schema_version' => 'evolvephp.comparator.result.v1', + 'comparator_id' => 'evolvephp', + 'execution_environment_fingerprint' => str_repeat('b', 64), + 'matrix_sha256' => str_repeat('c', 64), + 'comparator_lock_sha256' => str_repeat('d', 64), + 'fixture_identity_hash' => str_repeat('e', 64), + ], + 'sample_protocol' => [ + 'sample_count' => 100, + 'warmups' => 5, + 'request_count' => 25, + 'process_isolation_model' => 'subprocess_per_measured_sample', + 'repeated_warm_operations_per_sample' => 25, + 'boot_protocol' => [ + 'discarded_worker_processes_per_measured_sample' => 1, + 'sample_order' => 'rotating_round_robin', + 'measured_worker_in_process_warmups' => 0, + 'outlier_policy' => 'retain_all_measured_samples', + 'primary_central_statistic' => 'p50', + ], + ], + 'primary_metric' => 'p50', + 'scenarios' => [ + 'application_boot' => [ + 'mode' => 'monitor', + 'observed_p50_microseconds' => [1068.25, 1042.40, 1044.05], + 'reference_p50_microseconds' => 1044.05, + 'observed_maximum_p50_microseconds' => 1068.25, + 'cross_run_range_percent' => 2.4798541826554, + 'observation_threshold_p50_microseconds' => 1148.455, + 'diagnostic_rsd_percent' => [29.76929761631, 18.666326692293, 25.872643772215], + ], + 'http_static' => [ + 'mode' => 'blocking', + 'observed_p50_microseconds' => [30.90, 30.80, 30.70], + 'reference_p50_microseconds' => 30.80, + 'observed_maximum_p50_microseconds' => 30.90, + 'cross_run_range_percent' => 0.6514657980456, + 'blocking_threshold_p50_microseconds' => 32.34, + ], + 'http_parameterized' => [ + 'mode' => 'blocking', + 'observed_p50_microseconds' => [34.40, 34.40, 34.50], + 'reference_p50_microseconds' => 34.40, + 'observed_maximum_p50_microseconds' => 34.50, + 'cross_run_range_percent' => 0.29069767441861, + 'blocking_threshold_p50_microseconds' => 36.12, + ], + 'http_middleware' => [ + 'mode' => 'blocking', + 'observed_p50_microseconds' => [37.20, 37.20, 37.30], + 'reference_p50_microseconds' => 37.20, + 'observed_maximum_p50_microseconds' => 37.30, + 'cross_run_range_percent' => 0.26881720430106, + 'blocking_threshold_p50_microseconds' => 39.06, + ], + 'http_not_found' => [ + 'mode' => 'blocking', + 'observed_p50_microseconds' => [23.90, 23.80, 23.80], + 'reference_p50_microseconds' => 23.80, + 'observed_maximum_p50_microseconds' => 23.90, + 'cross_run_range_percent' => 0.42016806722688, + 'blocking_threshold_p50_microseconds' => 24.99, + ], + 'http_repeated_warm' => [ + 'mode' => 'blocking', + 'observed_p50_microseconds' => [26.662, 26.652, 26.540], + 'reference_p50_microseconds' => 26.652, + 'observed_maximum_p50_microseconds' => 26.662, + 'cross_run_range_percent' => 0.45968349660889, + 'blocking_threshold_p50_microseconds' => 27.9846, + ], + ], + ]; + } + + /** + * @return array + */ + private function trackedBudget(): array + { + $budget = json_decode((string) file_get_contents(dirname(__DIR__) . DIRECTORY_SEPARATOR . 'budgets' . DIRECTORY_SEPARATOR . 'performance-budget.json'), true, flags: JSON_THROW_ON_ERROR); + self::assertIsArray($budget); + + return $budget; + } + + /** + * @param array $p50Overrides + * @return array + */ + private function candidateEvidence(array $p50Overrides = []): array + { + $budget = $this->budget(); + $results = []; + + foreach ($budget['scenarios'] as $scenarioId => $policy) { + $results[] = $this->scenarioResult( + $scenarioId, + $p50Overrides[$scenarioId] ?? (float) $policy['observed_maximum_p50_microseconds'], + ); + } + + return [ + 'manifest' => [ + 'schema_version' => 'evolvephp.comparator.evidence-manifest.v1', + 'status' => 'completed', + 'source' => ['git_sha' => str_repeat('f', 40), 'dirty' => false], + 'execution_environment_fingerprint' => str_repeat('b', 64), + 'matrix' => ['sha256' => str_repeat('c', 64)], + 'process_isolation' => ['model' => 'subprocess_per_measured_sample'], + 'samples' => 100, + 'warmups' => 5, + 'request_count' => 25, + 'boot_protocol' => $budget['sample_protocol']['boot_protocol'], + ], + 'results' => $results, + ]; + } + + /** + * @return array + */ + private function scenarioResult(string $scenarioId, float $p50): array + { + return [ + 'comparator_schema_version' => 'evolvephp.comparator.result.v1', + 'comparator_id' => 'evolvephp', + 'scenario_id' => $scenarioId, + 'availability' => 'available', + 'source_evolvephp_sha' => str_repeat('f', 40), + 'source_dirty' => false, + 'execution_environment_fingerprint' => str_repeat('b', 64), + 'matrix_sha256' => str_repeat('c', 64), + 'comparator_lock_sha256' => str_repeat('d', 64), + 'fixture_identity_hash' => str_repeat('e', 64), + 'baseline_result' => [ + 'schema_version' => 'evolvephp.benchmark.results.v1', + 'source_sha' => str_repeat('f', 40), + 'scenarios' => [ + [ + 'id' => $scenarioId, + 'sample_count' => 100, + 'unit' => $scenarioId === 'http_repeated_warm' ? 'per_operation_microseconds' : 'microseconds', + 'p50' => $p50, + 'p50_status' => 'available', + 'p95' => $p50 + 1.0, + 'p99' => $p50 + 2.0, + 'mean' => $p50, + 'relative_standard_deviation_percent' => 1.0, + 'throughput_per_second' => 1_000_000 / $p50, + 'operations_per_sample' => $scenarioId === 'http_repeated_warm' ? 25 : 1, + 'memory' => ['peak_bytes' => 1024], + ], + ], + ], + ]; + } + + /** + * @param array $budget + */ + private function writeBudgetFile(array $budget): string + { + $directory = $this->temporaryDirectory(); + $path = $directory . DIRECTORY_SEPARATOR . 'performance-budget.json'; + file_put_contents($path, json_encode($budget, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR) . PHP_EOL); + + return $path; + } + + /** + * @param array $candidate + */ + private function writeCandidateDirectory( + array $candidate, + bool $corruptFirstHash = false, + bool $traversalFirstPath = false, + ): string { + $directory = $this->temporaryDirectory(); + $normalizedDirectory = $directory . DIRECTORY_SEPARATOR . 'normalized'; + mkdir($normalizedDirectory, 0777, true); + $manifest = $candidate['manifest']; + $manifest['results'] = []; + + foreach ($candidate['results'] as $index => $result) { + $path = 'normalized/' . $result['scenario_id'] . '.json'; + $fullPath = $normalizedDirectory . DIRECTORY_SEPARATOR . $result['scenario_id'] . '.json'; + + if ($index === 0 && $traversalFirstPath) { + $outsideDirectory = $this->temporaryDirectory(); + $path = '../' . basename($outsideDirectory) . '/outside-normalized.json'; + $fullPath = $outsideDirectory . DIRECTORY_SEPARATOR . 'outside-normalized.json'; + } + + file_put_contents($fullPath, json_encode($result, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR) . PHP_EOL); + + $manifest['results'][] = [ + 'scenario_id' => $result['scenario_id'], + 'normalized_result' => [ + 'path' => $path, + 'sha256' => $index === 0 && $corruptFirstHash + ? str_repeat('0', 64) + : hash_file('sha256', $fullPath), + ], + ]; + } + + file_put_contents($directory . DIRECTORY_SEPARATOR . 'manifest.json', json_encode($manifest, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR) . PHP_EOL); + + return $directory; + } + + /** + * @param list $arguments + * @return array{exit_code: int, stdout: string, stderr: string} + */ + private function runBudgetCli(array $arguments): array + { + $command = array_merge([PHP_BINARY, dirname(__DIR__) . DIRECTORY_SEPARATOR . 'bin' . DIRECTORY_SEPARATOR . 'performance-budget.php'], $arguments); + $process = proc_open($command, [1 => ['pipe', 'w'], 2 => ['pipe', 'w']], $pipes, dirname(__DIR__)); + self::assertIsResource($process); + + $stdout = (string) stream_get_contents($pipes[1]); + $stderr = (string) stream_get_contents($pipes[2]); + fclose($pipes[1]); + fclose($pipes[2]); + + return [ + 'exit_code' => proc_close($process), + 'stdout' => $stdout, + 'stderr' => $stderr, + ]; + } + + private function temporaryDirectory(): string + { + $directory = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'evolvephp-budget-test-' . bin2hex(random_bytes(6)); + mkdir($directory, 0777, true); + $this->temporaryDirectories[] = $directory; + + return $directory; + } + + private function removeDirectory(string $path): void + { + if (!is_dir($path)) { + return; + } + + $iterator = new \RecursiveIteratorIterator( + new \RecursiveDirectoryIterator($path, \FilesystemIterator::SKIP_DOTS), + \RecursiveIteratorIterator::CHILD_FIRST, + ); + + foreach ($iterator as $file) { + if ($file->isDir()) { + rmdir($file->getPathname()); + continue; + } + + unlink($file->getPathname()); + } + + rmdir($path); + } +} diff --git a/tests/Architecture/EvolvePhp2BenchmarkHarnessTest.php b/tests/Architecture/EvolvePhp2BenchmarkHarnessTest.php index 0290a66..d877ca8 100644 --- a/tests/Architecture/EvolvePhp2BenchmarkHarnessTest.php +++ b/tests/Architecture/EvolvePhp2BenchmarkHarnessTest.php @@ -21,11 +21,18 @@ public function testBenchmarkToolingIsIsolatedUnderBenchmarksComposerRoot(): voi $this->assertFileExists($this->projectPath('benchmarks/bin/comparator-preflight.php')); $this->assertFileExists($this->projectPath('benchmarks/bin/capture-environment.php')); $this->assertFileExists($this->projectPath('benchmarks/bin/normalize-results.php')); + $this->assertFileExists($this->projectPath('benchmarks/bin/performance-budget.php')); + $this->assertFileExists($this->projectPath('benchmarks/budgets/performance-budget.json')); $this->assertFileExists($this->projectPath('benchmarks/results/README.md')); + $this->assertFileExists($this->projectPath('benchmarks/results/reference/performance-summary.json')); + $this->assertFileExists($this->projectPath('benchmarks/results/reference/performance-report.md')); $benchmarkManifest = $this->readJsonFile('benchmarks/composer.json'); $this->assertArrayHasKey('phpbench/phpbench', $benchmarkManifest['require-dev']); $this->assertArrayHasKey('nyholm/psr7', $benchmarkManifest['require']); + $this->assertArrayHasKey('budget:validate', $benchmarkManifest['scripts']); + $this->assertArrayHasKey('test:budget', $benchmarkManifest['scripts']); + $this->assertArrayHasKey('ci:policy', $benchmarkManifest['scripts']); $rootManifest = $this->readJsonFile('composer.json'); $this->assertPackageAbsentFromManifest('phpbench/phpbench', $rootManifest, 'composer.json'); @@ -90,22 +97,90 @@ public function testBenchmarkDocumentationRecordsProtocolAndNoMarketingClaim(): 'Canonical reference evidence', 'Phalcon Micro before handlers', 'non-ranking policy', + 'performance-budget.php', + 'pass', + 'warn', + 'fail', + 'incomparable', + 'application_boot', + 'monitor-only', + 'warm HTTP p50', + 'Shared GitHub-hosted runners', + 'Raw 100-sample process records', ] as $phrase) { $this->assertStringContainsString($phrase, $readme); } } + public function testPerformanceBudgetReferenceUsesDerivedComparatorIdentities(): void + { + $budget = $this->readJsonFile('benchmarks/budgets/performance-budget.json'); + $matrixPath = $this->projectPath('benchmarks/comparators/matrix.json'); + $matrix = $this->readJsonFile('benchmarks/comparators/matrix.json'); + $evolvePhpComparator = $this->evolvePhpComparator($matrix); + + $this->assertSame('evolvephp.performance-budget.v1', $budget['schema_version']); + $this->assertSame('c62cecb16cb4fcdc93bfbb0188a4b63d8cf704ce', $budget['baseline_source_sha']); + $this->assertSame('9c06a992d7f01cb7096a60b893f33a34aff7b2a86fba157c8b879d9ac55457a2', $budget['comparison_identity']['execution_environment_fingerprint']); + $this->assertSame(hash_file('sha256', $matrixPath), $budget['comparison_identity']['matrix_sha256']); + $this->assertSame($evolvePhpComparator['lock_sha256'], $budget['comparison_identity']['comparator_lock_sha256']); + $this->assertSame( + hash_file('sha256', $this->projectPath('benchmarks/comparators/evolvephp/composer.lock')), + $budget['comparison_identity']['comparator_lock_sha256'], + ); + $this->assertSame($this->fixtureIdentityHash($evolvePhpComparator), $budget['comparison_identity']['fixture_identity_hash']); + + $this->assertSame('8.4.25', $budget['canonical_runtime_policy']['php_version']); + $this->assertTrue($budget['canonical_runtime_policy']['opcache_cli_enabled']); + $this->assertFalse($budget['canonical_runtime_policy']['jit_enabled']); + $this->assertSame(100, $budget['sample_protocol']['sample_count']); + $this->assertSame(5, $budget['sample_protocol']['warmups']); + $this->assertSame(25, $budget['sample_protocol']['request_count']); + $this->assertSame(25, $budget['sample_protocol']['repeated_warm_operations_per_sample']); + $this->assertSame(0, $budget['warmup_policy']['application_boot_measured_worker_in_process_warmups']); + $this->assertSame(1, $budget['warmup_policy']['application_boot_discarded_worker_processes_per_measured_sample']); + $this->assertSame(1, $budget['sample_protocol']['boot_protocol']['discarded_worker_processes_per_measured_sample']); + $this->assertSame('rotating_round_robin', $budget['sample_protocol']['boot_protocol']['sample_order']); + $this->assertSame(0, $budget['sample_protocol']['boot_protocol']['measured_worker_in_process_warmups']); + $this->assertSame('retain_all_measured_samples', $budget['sample_protocol']['boot_protocol']['outlier_policy']); + $this->assertSame('p50', $budget['sample_protocol']['boot_protocol']['primary_central_statistic']); + $this->assertSame('p50', $budget['primary_metric']); + + $this->assertScenarioPolicy($budget, 'application_boot', 'monitor', 1044.05, 1068.25, 1148.455); + $this->assertScenarioPolicy($budget, 'http_static', 'blocking', 30.8, 30.9, 32.34); + $this->assertScenarioPolicy($budget, 'http_parameterized', 'blocking', 34.4, 34.5, 36.12); + $this->assertScenarioPolicy($budget, 'http_middleware', 'blocking', 37.2, 37.3, 39.06); + $this->assertScenarioPolicy($budget, 'http_not_found', 'blocking', 23.8, 23.9, 24.99); + $this->assertScenarioPolicy($budget, 'http_repeated_warm', 'blocking', 26.652, 26.662, 27.9846); + } + + public function testQualityWorkflowHasCiSafeBenchmarkPolicyJob(): void + { + $workflow = $this->readProjectFile('.github/workflows/quality.yml'); + + $this->assertStringContainsString('Benchmark policy (PHP 8.4)', $workflow); + $this->assertStringContainsString('composer validate --working-dir=benchmarks --strict --check-lock', $workflow); + $this->assertStringContainsString('composer install --working-dir=benchmarks --no-interaction --no-progress --prefer-dist', $workflow); + $this->assertStringContainsString('composer --working-dir=benchmarks ci:policy', $workflow); + $this->assertStringNotContainsString('benchmarks/bin/comparator-run.php', $workflow); + $this->assertStringNotContainsString('comparator:run', $workflow); + } + public function testBenchmarkPublicDocsDoNotContainInternalReviewProvenance(): void { foreach ([ 'benchmarks/README.md', 'benchmarks/results/README.md', + 'benchmarks/results/reference/performance-report.md', ] as $path) { $content = $this->readProjectFile($path); foreach ([ + '/\bphase\b/i', + '/\bmaintainer\b/i', '/Codex/i', '/ChatGPT/i', + '/\bagent\b/i', '/AI-generated/i', '/assistant review/i', '/credits/i', @@ -133,12 +208,91 @@ private function packageManifests(): array ]; } + /** + * @param array $manifest + */ private function assertPackageAbsentFromManifest(string $package, array $manifest, string $path): void { $this->assertArrayNotHasKey($package, $manifest['require'] ?? [], $path . ' must not require ' . $package . '.'); $this->assertArrayNotHasKey($package, $manifest['require-dev'] ?? [], $path . ' must not require-dev ' . $package . '.'); } + /** + * @param array $budget + */ + private function assertScenarioPolicy( + array $budget, + string $scenarioId, + string $mode, + float $referenceP50, + float $observedMaximumP50, + float $thresholdP50, + ): void { + $this->assertArrayHasKey($scenarioId, $budget['scenarios']); + $scenario = $budget['scenarios'][$scenarioId]; + + $this->assertSame($mode, $scenario['mode']); + $this->assertSame($referenceP50, $scenario['reference_p50_microseconds']); + $this->assertSame($observedMaximumP50, $scenario['observed_maximum_p50_microseconds']); + + if ($mode === 'monitor') { + $this->assertSame($thresholdP50, $scenario['observation_threshold_p50_microseconds']); + + return; + } + + $this->assertSame($thresholdP50, $scenario['blocking_threshold_p50_microseconds']); + } + + /** + * @param array $matrix + * @return array + */ + private function evolvePhpComparator(array $matrix): array + { + foreach ($matrix['comparators'] ?? [] as $comparator) { + if (is_array($comparator) && ($comparator['id'] ?? null) === 'evolvephp') { + return $comparator; + } + } + + $this->fail('EvolvePHP comparator should exist in the comparator matrix.'); + } + + /** + * @param array $comparator + */ + private function fixtureIdentityHash(array $comparator): string + { + $fields = [ + 'comparator_id' => $comparator['id'] ?? null, + 'configuration' => $comparator['configuration'] ?? [], + 'fixture_version' => $comparator['fixture_version'] ?? null, + 'framework_name' => $comparator['name'] ?? null, + 'framework_version' => $comparator['framework_version'] ?? null, + 'lock_hash' => $comparator['lock_sha256'] ?? null, + ]; + $this->sortRecursive($fields); + + return hash('sha256', json_encode($fields, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES)); + } + + /** + * @param array $value + */ + private function sortRecursive(array &$value): void + { + foreach ($value as &$entry) { + if (is_array($entry)) { + $this->sortRecursive($entry); + } + } + + if (!array_is_list($value)) { + ksort($value); + } + } + /** * @return list */ diff --git a/tests/Architecture/EvolvePhp2ContinuousIntegrationTest.php b/tests/Architecture/EvolvePhp2ContinuousIntegrationTest.php index d608a66..bc1b6f2 100644 --- a/tests/Architecture/EvolvePhp2ContinuousIntegrationTest.php +++ b/tests/Architecture/EvolvePhp2ContinuousIntegrationTest.php @@ -4,9 +4,9 @@ final class EvolvePhp2ContinuousIntegrationTest extends TestCase { - private $root; - private $workflowPath; - private $workflow; + private string $root; + private string $workflowPath; + private string $workflow; protected function setUp(): void { @@ -15,8 +15,10 @@ protected function setUp(): void $this->assertFileExists($this->workflowPath, 'The canonical EvolvePHP 2 quality workflow should exist.'); - $this->workflow = file_get_contents($this->workflowPath); - $this->assertNotFalse($this->workflow, 'The canonical EvolvePHP 2 quality workflow should be readable.'); + $workflow = file_get_contents($this->workflowPath); + $this->assertNotFalse($workflow, 'The canonical EvolvePHP 2 quality workflow should be readable.'); + + $this->workflow = $workflow; } public function testCanonicalWorkflowNameAndTriggersTargetOnlyEvolvePhp2(): void @@ -44,7 +46,7 @@ public function testWorkflowUsesLeastPrivilegePermissionsAndCancelsSupersededRun public function testWorkflowUsesOnlyApprovedUbuntuRunnerAndPhpMatrix(): void { - $this->assertSame(2, preg_match_all('/^\s{4}runs-on:\s*ubuntu-24\.04\s*$/m', $this->workflow, $matches)); + $this->assertSame(3, preg_match_all('/^\s{4}runs-on:\s*ubuntu-24\.04\s*$/m', $this->workflow, $matches)); $this->assertDoesNotMatchPattern('/ubuntu-latest|windows-|macos-/', $this->workflow); $this->assertMatchesPattern('/fail-fast:\s*false/', $this->workflow); $this->assertMatchesPattern('/php:\s*\R\s{10}- \'8\.4\'\s*\R\s{10}- \'8\.5\'/m', $this->workflow); @@ -57,12 +59,12 @@ public function testWorkflowPinsOnlyReviewedActionReleaseCommits(): void $checkoutSha = '3d3c42e5aac5ba805825da76410c181273ba90b1'; $setupPhpSha = 'f3e473d116dcccaddc5834248c87452386958240'; - $this->assertSame(2, preg_match_all('/uses:\s*actions\/checkout@' . $checkoutSha . '\s+# v7\.0\.1/', $this->workflow, $matches)); - $this->assertSame(2, preg_match_all('/uses:\s*shivammathur\/setup-php@' . $setupPhpSha . '\s+# 2\.37\.2/', $this->workflow, $matches)); - $this->assertSame(4, preg_match_all('/uses:\s*[^@\s]+@[0-9a-f]{40}\s+# (?:v7\.0\.1|2\.37\.2)/', $this->workflow, $matches)); + $this->assertSame(3, preg_match_all('/uses:\s*actions\/checkout@' . $checkoutSha . '\s+# v7\.0\.1/', $this->workflow, $matches)); + $this->assertSame(3, preg_match_all('/uses:\s*shivammathur\/setup-php@' . $setupPhpSha . '\s+# 2\.37\.2/', $this->workflow, $matches)); + $this->assertSame(6, preg_match_all('/uses:\s*[^@\s]+@[0-9a-f]{40}\s+# (?:v7\.0\.1|2\.37\.2)/', $this->workflow, $matches)); $this->assertMatchesPattern('/persist-credentials:\s*false/', $this->workflow); - $this->assertSame(2, preg_match_all('/tools:\s*composer:v2/', $this->workflow, $matches)); - $this->assertSame(2, preg_match_all('/coverage:\s*none/', $this->workflow, $matches)); + $this->assertSame(3, preg_match_all('/tools:\s*composer:v2/', $this->workflow, $matches)); + $this->assertSame(3, preg_match_all('/coverage:\s*none/', $this->workflow, $matches)); $this->assertDoesNotMatchPattern('/actions\/checkout@(?:v[0-9]+|main)|shivammathur\/setup-php@(?:v[0-9]+|main)/', $this->workflow); $this->assertDoesNotMatchPattern('/uses:\s*(?!actions\/checkout@|shivammathur\/setup-php@)[^@\s]+@/', $this->workflow); $this->assertDoesNotMatchPattern('/@[0-9a-f]{7,39}(?:\s|$)/', $this->workflow); @@ -87,9 +89,47 @@ public function testPolicyJobRunsRootPolicySuitesWithRootPhpUnitOnPhp84(): void $this->assertDoesNotMatchPattern('/phpunit.*(?:core|components|helpers|index\.php|route\.php)/i', $job); } + public function testBenchmarkPolicyJobRunsOnlyIsolatedBenchmarkPolicyChecksOnPhp84(): void + { + $job = $this->extractJob('benchmark-policy'); + + $this->assertMatchesPattern('/name:\s*Benchmark policy \(PHP 8\.4\)/', $job); + $this->assertMatchesPattern('/runs-on:\s*ubuntu-24\.04/', $job); + $this->assertMatchesPattern('/php-version:\s*\'8\.4\'/', $job); + $this->assertStringContainsString('uses: actions/checkout@', $this->extractStep($job, 'Checkout repository')); + $this->assertStringContainsString('uses: shivammathur/setup-php@', $this->extractStep($job, 'Set up PHP')); + $this->assertStringContainsString('composer validate --working-dir=benchmarks --strict --check-lock', $job); + $this->assertStringContainsString('composer install --working-dir=benchmarks --no-interaction --no-progress --prefer-dist', $job); + $this->assertStringContainsString('composer --working-dir=benchmarks ci:policy', $job); + + foreach ([ + '/benchmarks\/bin\/comparator-run\.php/', + '/comparator:run/', + '/100-sample|--samples(?:=|\s+)100|samples:\s*100|sample(?:s)?(?:=|\s+)100/i', + '/tools\/(?:release-validation|validate-release|validate-prerelease-consumers)|validate-prerelease-consumers\.php/i', + '/release:split:validate/', + '/release:skeleton:validate/', + '/release:consumer:validate/', + '/release:validate/', + '/composer update/', + '/--ignore-platform-reqs?/', + '/continue-on-error/', + '/secrets\./', + '/actions\/cache/', + '/cache:/', + '/deploy(?:ment)?/i', + '/publish/i', + '/upload-artifact/', + '/gh\s+release/i', + ] as $pattern) { + $this->assertDoesNotMatchPattern($pattern, $job); + } + } + public function testPolicyCheckoutFetchesCompleteHistoryForReleaseSplitValidation(): void { $policyCheckout = $this->extractStep($this->extractJob('policy'), 'Checkout repository'); + $benchmarkPolicyCheckout = $this->extractStep($this->extractJob('benchmark-policy'), 'Checkout repository'); $workspaceQualityCheckout = $this->extractStep($this->extractJob('workspace-quality'), 'Checkout repository'); $this->assertSame(1, substr_count($this->workflow, 'fetch-depth: 0')); @@ -97,6 +137,10 @@ public function testPolicyCheckoutFetchesCompleteHistoryForReleaseSplitValidatio $this->assertMatchesPattern('/fetch-depth:\s*0/', $policyCheckout); $this->assertStringContainsString('composer release:split:validate', $this->extractJob('policy')); + $this->assertMatchesPattern('/persist-credentials:\s*false/', $benchmarkPolicyCheckout); + $this->assertDoesNotMatchPattern('/fetch-depth:\s*0/', $benchmarkPolicyCheckout); + $this->assertStringNotContainsString('release:split:validate', $this->extractJob('benchmark-policy')); + $this->assertMatchesPattern('/persist-credentials:\s*false/', $workspaceQualityCheckout); $this->assertDoesNotMatchPattern('/fetch-depth:\s*0/', $workspaceQualityCheckout); $this->assertStringNotContainsString('release:split:validate', $this->extractJob('workspace-quality')); @@ -118,7 +162,7 @@ public function testWorkspaceQualityMatrixUsesLockfileInstallAndApprovedAggregat public function testWorkflowExcludesReleasePublishingSecretsCachesAndDeployments(): void { - foreach (array( + foreach ([ '/secrets\./', '/deploy(?:ment)?/i', '/publish/i', @@ -139,7 +183,7 @@ public function testWorkflowExcludesReleasePublishingSecretsCachesAndDeployments '/sudo\b/', '/--ignore-platform-reqs?/', '/config\.platform\.php/', - ) as $pattern) { + ] as $pattern) { $this->assertDoesNotMatchPattern($pattern, $this->workflow); } } @@ -149,7 +193,7 @@ public function testDocumentationRecordsContinuousIntegrationCompatibilityEviden $workspaceReadme = $this->readProjectFile('DEVELOPMENT.md'); $changelog = $this->readProjectFile('CHANGELOG.md'); - foreach (array( + foreach ([ '/## Continuous Integration/', '/\.github\/workflows\/quality\.yml/', '/EvolvePHP 2 Quality/', @@ -175,7 +219,7 @@ public function testDocumentationRecordsContinuousIntegrationCompatibilityEviden '/current.*(?:workspace|tooling|package foundation)|(?:workspace|tooling|package foundation).*current/i', '/EvolvePHP 1 runtime.*(?:not part|excluded)|(?:not part|excluded).*EvolvePHP 1 runtime/i', '/runtime implementation.*(?:incomplete|not complete)|(?:incomplete|not complete).*runtime implementation/i', - ) as $pattern) { + ] as $pattern) { $this->assertMatchesPattern($pattern, $workspaceReadme); } @@ -188,7 +232,7 @@ public function testDocumentationRecordsContinuousIntegrationCompatibilityEviden $this->assertDoesNotMatchPattern('/branch protection.*active|required checks|deployment|publishing|runtime implementation.*complete|legacy EvolvePHP 1 runtime.*PHP 8\.5/i', $changelog); } - private function extractTopLevelBlock($heading) + private function extractTopLevelBlock(string $heading): string { $pattern = '/^' . preg_quote($heading, '/') . ':\s*\R(?P.*?)(?=^[a-zA-Z_-]+:\s*|\z)/ms'; @@ -197,7 +241,7 @@ private function extractTopLevelBlock($heading) return $matches['block']; } - private function extractJob($job) + private function extractJob(string $job): string { $jobs = $this->extractTopLevelBlock('jobs'); $pattern = '/^\s{2}' . preg_quote($job, '/') . ':\s*\R(?P.*?)(?=^\s{2}[a-zA-Z0-9_-]+:\s*|\z)/ms'; @@ -207,7 +251,7 @@ private function extractJob($job) return $matches['job']; } - private function extractStep($job, $stepName) + private function extractStep(string $job, string $stepName): string { $pattern = '/^\s{6}- name:\s*' . preg_quote($stepName, '/') . '\s*\R(?P.*?)(?=^\s{6}- name:\s*|\z)/ms'; @@ -216,25 +260,28 @@ private function extractStep($job, $stepName) return $matches['step']; } - private function projectPath($path) + private function projectPath(string $path): string { return $this->root . DIRECTORY_SEPARATOR . str_replace('/', DIRECTORY_SEPARATOR, $path); } - private function readProjectFile($path) + private function readProjectFile(string $path): string { $fullPath = $this->projectPath($path); $this->assertFileExists($fullPath, $path . ' should exist before it is read.'); - return file_get_contents($fullPath); + $content = file_get_contents($fullPath); + $this->assertNotFalse($content, $path . ' should be readable.'); + + return $content; } - private function assertMatchesPattern($pattern, $content) + private function assertMatchesPattern(string $pattern, string $content): void { $this->assertSame(1, preg_match($pattern, $content), 'Failed asserting that content matches ' . $pattern); } - private function assertDoesNotMatchPattern($pattern, $content) + private function assertDoesNotMatchPattern(string $pattern, string $content): void { $this->assertSame(0, preg_match($pattern, $content), 'Failed asserting that content does not match ' . $pattern); } diff --git a/tests/Documentation/EvolvePhp2BenchmarkDocumentationTest.php b/tests/Documentation/EvolvePhp2BenchmarkDocumentationTest.php index 284d08a..2df82d4 100644 --- a/tests/Documentation/EvolvePhp2BenchmarkDocumentationTest.php +++ b/tests/Documentation/EvolvePhp2BenchmarkDocumentationTest.php @@ -35,11 +35,68 @@ public function testBenchmarkResultsDocumentationDescribesControlledEvidenceArti 'operations_per_sample', 'Candidate output from a dirty or uncommitted worktree is not canonical reference evidence', 'controlled PHP 8.4.25 lane', + 'results/reference/performance-summary.json', + 'results/reference/performance-report.md', + 'performance-budget.php', + 'pass', + 'warn', + 'fail', + 'incomparable', + 'monitor-only', + 'Warm HTTP p50 thresholds are blocking only for controlled evidence', + 'must not run the canonical 100-sample comparator timing suite', ] as $phrase) { $this->assertStringContainsString($phrase, $content); } } + public function testPublicPerformanceReportDocumentsNonRankingReferencePolicy(): void + { + $content = $this->readProjectFile('benchmarks/results/reference/performance-report.md'); + + foreach ([ + 'Methodology', + 'Regression Calibration', + 'Cross-Framework Reference', + 'CI And Regression Policy', + 'Limitations', + 'c62cecb16cb4fcdc93bfbb0188a4b63d8cf704ce', + 'debfb4228c4d652a5f6d0bdc4ff0f3a9c0a6c1c2', + '9c06a992d7f01cb7096a60b893f33a34aff7b2a86fba157c8b879d9ac55457a2', + 'The warm HTTP p50 results are competitive in this controlled matrix.', + 'Cold boot is the visible comparative gap', + 'The current data does not prove that framework architecture is the definitive cause', + 'This is a non-ranking report.', + ] as $phrase) { + $this->assertStringContainsString($phrase, $content); + } + + foreach ([ + 'EvolvePHP is generally faster', + 'EvolvePHP is the fastest framework', + 'top-three framework overall', + 'architecture is definitively the cause', + ] as $forbiddenClaim) { + $this->assertStringNotContainsString($forbiddenClaim, $content); + } + } + + public function testReferenceSummaryMatchesBudgetScenarioPolicy(): void + { + $budget = $this->readJsonFile('benchmarks/budgets/performance-budget.json'); + $summary = $this->readJsonFile('benchmarks/results/reference/performance-summary.json'); + + $this->assertSame($budget['baseline_source_sha'], $summary['regression_baseline_source_sha']); + $this->assertSame($budget['comparison_identity']['execution_environment_fingerprint'], $summary['canonical_environment_fingerprint']); + + foreach ($budget['scenarios'] as $scenarioId => $policy) { + $this->assertArrayHasKey($scenarioId, $summary['scenarios']); + $this->assertSame($policy['reference_p50_microseconds'], $summary['scenarios'][$scenarioId]['reference_median_p50_microseconds']); + $this->assertSame($policy['observed_maximum_p50_microseconds'], $summary['scenarios'][$scenarioId]['observed_maximum_p50_microseconds']); + $this->assertSame($policy['mode'], $summary['scenarios'][$scenarioId]['budget_classification']); + } + } + private function readProjectFile(string $path): string { $fullPath = dirname(__DIR__, 2) . DIRECTORY_SEPARATOR . str_replace('/', DIRECTORY_SEPARATOR, $path); @@ -50,4 +107,17 @@ private function readProjectFile(string $path): string return $content; } + + /** + * @return array + */ + private function readJsonFile(string $path): array + { + $json = json_decode($this->readProjectFile($path), true); + + $this->assertSame(JSON_ERROR_NONE, json_last_error(), $path . ' should contain valid JSON: ' . json_last_error_msg()); + $this->assertIsArray($json, $path . ' should decode to a JSON object.'); + + return $json; + } }