test: safety net for decomposing the largest files (PR 1)

.github/CODEOWNERS

@@ -11,7 +11,9 @@
 /packages/flow/openadapt_flow/policy.py                    @abrichr
 /packages/flow/openadapt_flow/policies/                    @abrichr
 /packages/flow/openadapt_flow/qualification.py             @abrichr
+/packages/flow/openadapt_flow/qualification/             @abrichr
 /packages/flow/openadapt_flow/runtime/replayer.py          @abrichr
+/packages/flow/openadapt_flow/runtime/replay/            @abrichr
 /packages/flow/openadapt_flow/runtime/resolver.py          @abrichr
 /packages/flow/openadapt_flow/submit_binding.py            @abrichr
 /packages/flow/openadapt_flow/validation/                  @abrichr

.github/ci-tools/pyproject.toml

@@ -17,6 +17,9 @@ build = ["build>=1.6.1", "pip>=25.3"]
 pip = ["pip>=25.3"]
 lint = ["ruff==0.17.0"]
 claims = ["pyyaml>=6.0"]
+# Combines the flow-unit shards' coverage data (flow-coverage in packages.yml);
+# keep it at the version packages/flow/uv.lock pins, so the data format matches.
+coverage = ["coverage[toml]==7.15.1"]
 release = ["build>=1.6.1", "twine>=7.0.0"]
 # The in-guest Windows agent's screen and input modules, for the hosted
 # Windows replay (desktop-replay.yml), which runs Python 3.12. CI only: Flow

.github/workflows/packages.yml

@@ -435,7 +435,9 @@ jobs:
     # the group.
     if: github.event.pull_request.draft != true && (needs.scope.outputs.flow == 'true' && github.event_name != 'merge_group')
     runs-on: ubuntu-latest
-    timeout-minutes: 40
+    # The nightly tier (a schedule or a dispatch) also measures coverage,
+    # which slows the suite by about a third; it gets 60 minutes.
+    timeout-minutes: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && 60 || 40 }}
     strategy:
       fail-fast: false
       matrix:
@@ -565,13 +567,19 @@ jobs:
           find tests -name 'test_*.py' "${e2e[@]}" \
             -not -path 'tests/test_install_playwright_browser.py' \
             -not -path 'tests/test_qualification_gate_campaign.py' \
+            -not -path 'tests/golden_runs/test_golden_runs.py' \
             | python ../../scripts/ci/flow_shards.py --shard "$SHARD" --of "$SHARDS" > "$RUNNER_TEMP/shard.txt"
           wc -l < "$RUNNER_TEMP/shard.txt"
       - name: Test
         # A pull request runs the fast lane: tests/nightly_only.txt lists the
         # slow tests (over about 5 s on CI) that main and nightly run instead.
         env:
           OPENADAPT_TEST_LANE: ${{ github.event_name == 'pull_request' && 'fast' || 'full' }}
+          # Each shard keeps its coverage data; flow-coverage combines them.
+          COVERAGE_FILE: runs/coverage-shard-${{ matrix.shard }}.data
+          # Coverage only on the nightly tier, so it adds no time to a pull
+          # request's ten-minute `ci` or to main's post-merge run.
+          COVERAGE_ARGS: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && '--cov --cov-report=' || '' }}
         # -n logical: one xdist worker per vCPU. (-n auto counts physical
         # cores and started two workers on the four-vCPU runner.) --dist
         # loadfile keeps each file on one worker, so module fixtures and
@@ -581,8 +589,18 @@ jobs:
           set -euo pipefail
           mkdir -p runs
           mapfile -t files < "$RUNNER_TEMP/shard.txt"
+          # shellcheck disable=SC2086 # COVERAGE_ARGS is empty or two flags.
           pytest -n logical --dist loadfile -v --durations=20 --basetemp=runs/ci \
-            --junitxml=runs/flow-unit-shard-${{ matrix.shard }}-of-${{ matrix.of }}.xml "${files[@]}"
+            $COVERAGE_ARGS --junitxml=runs/flow-unit-shard-${{ matrix.shard }}-of-${{ matrix.of }}.xml "${files[@]}"
+      # Before the claims upload, which tests expect to be the last step.
+      - name: Upload coverage data
+        if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: flow-coverage-shard-${{ matrix.shard }}
+          path: packages/flow/runs/coverage-shard-${{ matrix.shard }}.data
+          if-no-files-found: warn
+          retention-days: 14
       - name: Retain this shard's claims evidence
         if: always()
         uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
@@ -592,6 +610,64 @@ jobs:
           if-no-files-found: error
           retention-days: 7
 
+  flow-coverage:
+    # Measures safety-path coverage and reports it in the job summary. It does
+    # not gate yet: the floor is set at the measured value once it is known
+    # (docs/design/DECOMPOSITION.md, section 9). Keep the include list in sync
+    # with the strict mypy list above and CODEOWNERS. It is not in
+    # packages-gate, and runs on the nightly tier only, where flow-unit
+    # collects the data.
+    name: flow safety-path coverage (reported, not gating)
+    needs: [scope, flow-unit]
+    if: always() && needs.scope.outputs.flow == 'true' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    defaults:
+      run:
+        working-directory: packages/flow
+    env:
+      SAFETY_PATH: >-
+        openadapt_flow/compiler/*,openadapt_flow/identity_signals.py,openadapt_flow/runtime/identity*,openadapt_flow/runtime/replayer.py,openadapt_flow/runtime/replay/*,openadapt_flow/runtime/resolver.py,openadapt_flow/submit_binding.py,openadapt_flow/policy.py,openadapt_flow/qualification.py,openadapt_flow/qualification/*,openadapt_flow/runtime/effects/*
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
+        with:
+          python-version: "3.12"
+      - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
+        with:
+          enable-cache: true
+          cache-dependency-glob: .github/ci-tools/uv.lock
+          cache-suffix: flow-coverage
+      - name: Install coverage from the CI tools lock
+        run: python ../../scripts/ci/locked_install.py --lock .github/ci-tools:coverage
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+        with:
+          pattern: flow-coverage-shard-*
+          path: packages/flow/runs/coverage
+          merge-multiple: true
+      - name: Combine and report
+        run: |
+          set -euo pipefail
+          ls runs/coverage
+          coverage combine --keep runs/coverage/*.data
+          {
+            echo "## Flow safety-path coverage (reported, not gating)"
+            echo
+            echo '```'
+            coverage report --include="$SAFETY_PATH" --skip-covered --sort=cover | tail -40
+            echo '```'
+          } >> "$GITHUB_STEP_SUMMARY"
+          coverage report --include="$SAFETY_PATH" --format=total
+          coverage json --include="$SAFETY_PATH" -o runs/safety-path-coverage.json
+      - name: Upload the report
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: flow-safety-path-coverage
+          path: packages/flow/runs/safety-path-coverage.json
+          retention-days: 30
+
   flow-critical:
     name: flow core critical (golden VERIFIED, zero model calls)
     needs: scope
@@ -655,6 +731,25 @@ jobs:
             tests/e2e/test_browser_teach_write_guard_e2e.py \
             tests/e2e/test_agent_guard_e2e.py \
             --basetemp=runs/ci-critical-e2e --junitxml=runs/critical-e2e-junit.xml
+      - name: Golden runs (report, artifacts and actions unchanged)
+        # docs/design/DECOMPOSITION.md, section 3.1. The baselines are Linux
+        # results; OPENADAPT_GOLDEN_WRITE=1 regenerates them (upload below).
+        # One process: the scenarios share one MockMed server per module.
+        env:
+          OPENADAPT_GOLDEN_OUT: runs/golden
+        run: |
+          pytest -vv --durations=20 tests/golden_runs \
+            --basetemp=runs/ci-goldens --junitxml=runs/goldens-junit.xml
+      - name: Upload golden-run results
+        if: always()
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: flow-golden-runs
+          path: |
+            packages/flow/runs/golden
+            packages/flow/tests/golden_runs/baselines
+          if-no-files-found: warn
+          retention-days: 14
       - name: Read-back and zero-model-call tests, by name
         run: |
           pytest -n logical --dist loadfile -v --durations=20 \
@@ -687,7 +782,7 @@ jobs:
           import sys
           import xml.etree.ElementTree as ET
           bad = False
-          for name in ("critical-e2e", "critical", "onprem-egress", "onprem-boundary"):
+          for name in ("critical-e2e", "goldens", "critical", "onprem-egress", "onprem-boundary"):
               root = ET.parse(f"runs/{name}-junit.xml").getroot()
               suites = [root] if root.tag == "testsuite" else list(root.iter("testsuite"))
               tests = sum(int(s.get("tests", 0)) for s in suites)

packages/flow/docs/design/DECOMPOSITION.md

@@ -594,16 +594,16 @@ that:
 <!-- size-ratchet:begin -->
 | File | Line budget |
 |---|---:|
-| `packages/flow/openadapt_flow/runtime/replayer.py` | 18258 |
-| `packages/flow/openadapt_flow/__main__.py` | 11970 |
-| `packages/flow/openadapt_flow/qualification.py` | 5775 |
-| `packages/flow/openadapt_flow/ir.py` | 5765 |
-| `packages/desktop/openadapt_desktop/dispatch.py` | 5686 |
-| `packages/capture/openadapt_capture/recorder.py` | 4532 |
-| `packages/flow/openadapt_flow/interactive_recorder.py` | 4524 |
-| `packages/flow/openadapt_flow/runtime/durable/attended.py` | 4271 |
+| `packages/flow/openadapt_flow/runtime/replayer.py` | 19861 |
+| `packages/flow/openadapt_flow/__main__.py` | 14487 |
+| `packages/flow/openadapt_flow/qualification.py` | 5882 |
+| `packages/flow/openadapt_flow/ir.py` | 6578 |
+| `packages/desktop/openadapt_desktop/dispatch.py` | 6524 |
+| `packages/capture/openadapt_capture/recorder.py` | 4836 |
+| `packages/flow/openadapt_flow/interactive_recorder.py` | 4620 |
+| `packages/flow/openadapt_flow/runtime/durable/attended.py` | 4367 |
 | `packages/flow/openadapt_flow/terminal_verification_v2.py` | 3663 |
-| `packages/flow/openadapt_flow/backends/playwright_backend.py` | 3636 |
+| `packages/flow/openadapt_flow/backends/playwright_backend.py` | 5996 |
 <!-- size-ratchet:end -->
 
 ## 7. Coordination with work in flight

packages/flow/public-artifacts.json

@@ -699,6 +699,38 @@
     "source-policy.public.json",
     "tests/extension_support/chrome_extension_harness.js",
     "tests/extension_support/network_harness.js",
-    "tests/golden/tutorial_run.json"
+    "tests/golden/tutorial_run.json",
+    "tests/golden_runs/baselines/bundles/mockmed-triage/manifest.json",
+    "tests/golden_runs/baselines/bundles/mockmed-triage/param_proposals.json",
+    "tests/golden_runs/baselines/bundles/mockmed-triage/sections.json",
+    "tests/golden_runs/baselines/bundles/mockmed-triage/templates/layout_fingerprint.json",
+    "tests/golden_runs/baselines/bundles/mockmed-triage/templates/step_000.png",
+    "tests/golden_runs/baselines/bundles/mockmed-triage/templates/step_000_expect.png",
+    "tests/golden_runs/baselines/bundles/mockmed-triage/templates/step_001_expect.png",
+    "tests/golden_runs/baselines/bundles/mockmed-triage/templates/step_002.png",
+    "tests/golden_runs/baselines/bundles/mockmed-triage/templates/step_002_expect.png",
+    "tests/golden_runs/baselines/bundles/mockmed-triage/templates/step_003_expect.png",
+    "tests/golden_runs/baselines/bundles/mockmed-triage/templates/step_004.png",
+    "tests/golden_runs/baselines/bundles/mockmed-triage/templates/step_004_expect.png",
+    "tests/golden_runs/baselines/bundles/mockmed-triage/templates/step_005.png",
+    "tests/golden_runs/baselines/bundles/mockmed-triage/templates/step_005_expect.png",
+    "tests/golden_runs/baselines/bundles/mockmed-triage/templates/step_006.png",
+    "tests/golden_runs/baselines/bundles/mockmed-triage/templates/step_006_expect.png",
+    "tests/golden_runs/baselines/bundles/mockmed-triage/templates/step_007.png",
+    "tests/golden_runs/baselines/bundles/mockmed-triage/templates/step_007_expect.png",
+    "tests/golden_runs/baselines/bundles/mockmed-triage/templates/step_008.png",
+    "tests/golden_runs/baselines/bundles/mockmed-triage/templates/step_008_expect.png",
+    "tests/golden_runs/baselines/bundles/mockmed-triage/templates/step_010.png",
+    "tests/golden_runs/baselines/bundles/mockmed-triage/templates/step_010_expect.png",
+    "tests/golden_runs/baselines/bundles/mockmed-triage/workflow.json",
+    "tests/golden_runs/baselines/mockmed_data_growth.json",
+    "tests/golden_runs/baselines/mockmed_lookalike_row.json",
+    "tests/golden_runs/baselines/mockmed_missing_row.json",
+    "tests/golden_runs/baselines/mockmed_modal.json",
+    "tests/golden_runs/baselines/mockmed_pass.json",
+    "tests/golden_runs/baselines/mockmed_pass_structural.json",
+    "tests/golden_runs/baselines/mockmed_rename.json",
+    "tests/golden_runs/baselines/mockmed_sort.json",
+    "tests/golden_runs/baselines/mockmed_theme.json"
   ]
 }

packages/flow/pyproject.toml

@@ -524,7 +524,7 @@ line-length = 88
 # Generated / non-package trees at the repo root are not part of the lint
 # surface. The benchmark harness (benchmark/harness/) is linted by naming it
 # explicitly in CI (`ruff check openadapt_flow benchmark/harness`).
-extend-exclude = ["/benchmark", "/docs", "/runs", "/dist"]
+extend-exclude = ["/benchmark", "/docs", "/runs", "/dist", "tests/golden_runs/baselines"]
 
 [tool.ruff.lint]
 # Start pragmatic: pyflakes (real bugs), the safe pycodestyle subset, warnings,