diff --git a/docs/QUALITY_SNAPSHOTS.md b/docs/QUALITY_SNAPSHOTS.md index 6be9feec..0fcb933a 100644 --- a/docs/QUALITY_SNAPSHOTS.md +++ b/docs/QUALITY_SNAPSHOTS.md @@ -16,18 +16,20 @@ A metric blocks when it exceeds both its absolute and relative budget: | --- | ---: | ---: | ---: | | Server startup P50 | 5 ms / 100% | 10 ms / 50% | 100 ms / 50% | | Server startup P95 | 50 ms / 200% | 50 ms / 100% | 750 ms / 100% | -| Full refresh P50 | 25 ms / 30% | 50 ms / 30% | 100 ms / 50% | +| Full refresh P50 | 25 ms / 30% | 150 ms / 50% | 100 ms / 50% | | Full refresh P95 | 50 ms / 50% | 250 ms / 100% | 300 ms / 100% | | Time to first environment P50 | 20 ms / 100% | 25 ms / 50% | 150 ms / 50% | | Time to first environment P95 | 25 ms / 100% | 100 ms / 100% | 250 ms / 100% | | Cold refresh P50 | 100 ms / 50% | 150 ms / 50% | 250 ms / 50% | -Each cell is `absolute / relative`. The warm P50 and server-startup budgets reflect observed GitHub-hosted runner variance from 11 consecutive main-branch baselines. Tighten them when a noisy path is fixed rather than normalizing a known regression into the baseline. +Each cell is `absolute / relative`. The Linux/macOS warm P50 and all server-startup budgets reflect observed GitHub-hosted runner variance from 11 consecutive main-branch baselines. Tighten them when a noisy path is fixed rather than normalizing a known regression into the baseline. The macOS server-startup P95 budget recalibration is tracked by issue #507 and follows PR #506's fix for issue #504. It uses three unchanged-content pull-request runs and the exact merged baseline at `f0c62d9`; the resulting absolute headroom is four to six times the observed post-fix run-to-run range. The warm refresh and warm time-to-first P95 budgets were recalibrated in issue #511 after PR #510 separated cold and warm samples. Three unchanged-code PR runs plus the exact schema-v2 baseline at `ad7ca14` retain at least 2.5 times the observed absolute run-to-run range. +The Windows warm full-refresh P50 budget was recalibrated in issue #513 from five unchanged-code pull-request measurements plus the exact schema-v2 baseline at `ad7ca14` (six measurements total). It retains nearly twice the observed absolute range while blocking a sustained median above 255ms against that baseline. + Schema v2 records `full_refresh` and `time_to_first_env` from the warm member of each pair and adds cold refresh/time-to-first distributions. During its one-time rollout, comparisons against a schema-v1 base checked cold P50 against explicit absolute ceilings of 500ms on Linux, 750ms on Windows, and 1,000ms on macOS. Schema-v2-to-v2 comparisons use the table's dual budgets. The cold P50 budgets were calibrated in issue #509 using two unchanged-head all-platform runs and the final pull-request validation. diff --git a/scripts/quality_snapshot.py b/scripts/quality_snapshot.py index 2de6c6df..02d8176e 100644 --- a/scripts/quality_snapshot.py +++ b/scripts/quality_snapshot.py @@ -91,7 +91,7 @@ def regressed(self) -> bool: 'windows': ( RegressionBudget(10, 50), RegressionBudget(50, 100), - RegressionBudget(50, 30), + RegressionBudget(150, 50), RegressionBudget(250, 100), RegressionBudget(25, 50), RegressionBudget(100, 100), diff --git a/scripts/tests/test_quality_snapshot.py b/scripts/tests/test_quality_snapshot.py index f1cb4647..024f7d0c 100644 --- a/scripts/tests/test_quality_snapshot.py +++ b/scripts/tests/test_quality_snapshot.py @@ -73,7 +73,7 @@ def test_unchanged_snapshot_passes(self): self.assertEqual(failures, []) def test_p50_regression_fails_when_both_budgets_are_exceeded(self): - current = performance_snapshot(refresh_p50=180) + current = performance_snapshot(refresh_p50=300) _, failures = compare_performance(current, performance_snapshot(refresh_p50=100), 'Windows') self.assertTrue(any('Full refresh P50' in failure for failure in failures)) @@ -158,6 +158,24 @@ def test_newer_performance_schema_is_invalid(self): 'Windows', ) + def test_schema_v2_windows_warm_p50_variance_passes(self): + baseline = performance_snapshot(schema_version=2, refresh_p50=105) + current = performance_snapshot(schema_version=2, refresh_p50=182) + + _, failures = compare_performance(current, baseline, 'Windows') + + self.assertEqual(failures, []) + + def test_schema_v2_windows_warm_p50_material_regression_fails(self): + baseline = performance_snapshot(schema_version=2, refresh_p50=105) + current = performance_snapshot(schema_version=2, refresh_p50=300) + + _, failures = compare_performance(current, baseline, 'Windows') + + self.assertTrue(any('Full refresh P50' in failure for failure in failures)) + self.assertFalse(any('Full refresh P95' in failure for failure in failures)) + self.assertFalse(any('Cold refresh P50' in failure for failure in failures)) + def test_schema_v2_warm_p95_variance_passes_on_all_platforms(self): cases = ( ('Linux', 60, 69, 16, 16), @@ -233,7 +251,7 @@ def test_noise_inside_absolute_budget_passes(self): def test_relative_budget_must_also_be_exceeded(self): - current = performance_snapshot(refresh_p50=1_060) + current = performance_snapshot(refresh_p50=1_160) _, failures = compare_performance(current, performance_snapshot(refresh_p50=1_000), 'Windows') self.assertEqual(failures, [])